diff --git a/.gemini/config.yaml b/.gemini/config.yaml new file mode 100644 index 0000000000000..d2d13e0f46c67 --- /dev/null +++ b/.gemini/config.yaml @@ -0,0 +1,9 @@ +$schema: "http://json-schema.org/draft-07/schema#" +title: RepoConfig +description: Configuration for Gemini Code Assist on a repository. + +have_fun: false # Disables fun features like poems in PR summaries + +code_review: + disable: false # Enables code reviews + comment_severity_threshold: LOW ## Allows low severity comments such as grammar and spelling mistakes to be added to the PR \ No newline at end of file diff --git a/.gemini/styleguide.md b/.gemini/styleguide.md new file mode 100644 index 0000000000000..bbc06d75e3b84 --- /dev/null +++ b/.gemini/styleguide.md @@ -0,0 +1,67 @@ +# Documentation Review Style Guide + +## Behavior instruction + +You are acting as a **senior technical writer** who is reviewing TiDB documentation pull requests and you always provide ready-to-commit doc suggestions so the PR author can commit them directly. + +## Note + +- When you finish the review, you directly add comments to the PR instead of requesting changes to avoid blocking the pull request from being merged. +- If the PR author is ti-chi-bot, you only correct English grammar, spelling, and punctuation mistakes, if any. + +## Review aspects + +- Clarity, simplicity, completeness, and readability +- Logical flow and sentence structure +- Technical accuracy and terminology consistency + +## General writing principles + +- Correct English grammar, spelling, and punctuation mistakes, if any. +- Make sure the documentation is easy to understand for TiDB users. +- Write in **second person** ("you") when addressing users. +- Prefer **present tense** unless describing historical behavior. +- Avoid unnecessary words and repetition. +- Use **consistent terminology**. For example: + + - ❌ Do not mix "database" and "instance" + - ✅ Use "replicate" instead of "synchronize" when referring to replicating data from one TiDB cluster to another. + +## Structure and format + +- Use sentence case for headings (e.g., `## Configure the cluster`). +- Use ordered lists (`1.`, `2.`) for steps. +- Code snippets, command names, options, and paths should be in backticks (`` ` ``). + +## Markdown style + +- Add a blank line before and after headings and lists. +- Use proper heading hierarchy (no jumping from `##` to `####`). + +## Common issues to flag + +- Passive voice overuse + + _"The cluster is started by TiUP"_ → _"TiUP starts the cluster"_ + +- Inconsistent use of technical terms + + _"TiDB Cloud Serverless clusters" vs. "TiDB Serverless clusters"_ – pick one. + +- Unclear step instructions + + _"Do it like before"_ → _"Repeat step 3 using the updated config file"_ + +- Grammar and spelling issues + + _"recieve"_ → _"receive"_, _"an TiDB instance"_ → _"a TiDB instance"_ + +## Special notes + +- Follow any existing terminology in our glossary (`/glossary.md` if available). +- When in doubt, favor clarity over cleverness. +- If something might confuse a new user, suggest a reword. + +## Purposes of this style guide + +This guide helps Gemini Code Assist provide actionable, high-quality suggestions for improving technical documentation, especially for PRs related to user guides, how-to articles, and product reference material. \ No newline at end of file diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 45d885dc8abae..84db54b81e70b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -59,14 +59,14 @@ jobs: - name: Checkout uses: actions/checkout@v4 - name: Vale Linter - uses: errata-ai/vale-action@v2.0.1 + uses: errata-ai/vale-action@reviewdog with: # Optional # Specify file path to lint #files: . # Optional - onlyAnnotateModifiedLines: true + filter_mode: added env: # Required, set by GitHub actions automatically: # https://docs.github.com/en/actions/security-guides/automatic-token-authentication#about-the-github_token-secret diff --git a/.github/workflows/dispatch.yml b/.github/workflows/dispatch.yml index 35c5683680f50..7e72c5e4f3f8f 100644 --- a/.github/workflows/dispatch.yml +++ b/.github/workflows/dispatch.yml @@ -6,6 +6,7 @@ on: - ".github/**" branches: - master + - release-8.1 - release-7.2 - release-7.1 - release-7.0 diff --git a/.github/workflows/link-fail-fast.yaml b/.github/workflows/link-fail-fast.yaml index a5e4677d08972..04ac603a65fe2 100644 --- a/.github/workflows/link-fail-fast.yaml +++ b/.github/workflows/link-fail-fast.yaml @@ -17,10 +17,6 @@ jobs: CHANGED_FILES=$(git diff-tree --name-only --diff-filter 'AM' -r HEAD^1 HEAD -- "*.md" | sed -z "s/\n$//;s/\n/' '/g") echo "all_changed_files=${CHANGED_FILES}" >> $GITHUB_OUTPUT - - name: Download Exclude Path - run: | - curl https://raw.githubusercontent.com/pingcap/docs/master/.lycheeignore -O - - name: Link Checker if: ${{ steps.changed-files.outputs.all_changed_files }} uses: lycheeverse/lychee-action@v1.6.1 diff --git a/.github/workflows/media.yml b/.github/workflows/media.yml index bb69844588170..1f4ed8a9567e4 100644 --- a/.github/workflows/media.yml +++ b/.github/workflows/media.yml @@ -7,7 +7,7 @@ on: paths: - media/** jobs: - run: + upload: name: Upload media files runs-on: ubuntu-latest steps: @@ -34,3 +34,42 @@ jobs: # printf "%s\n" ${{ secrets.AWS_ACCESS_KEY }} ${{ secrets.AWS_SECRET_KEY }} ${{ secrets.AWS_REGION }} "json" | aws configure - name: Upload run: cloud-assets-utils verify-and-sync -qiniu true -qiniu-bucket ${{ secrets.QINIU_BUCKET_NAME }} media -replace-first-path-to images/docs -cdn-refresh https://download.pingcap.com/ + + - name: Install coscli + run: | + wget https://cosbrowser.cloud.tencent.com/software/coscli/coscli-linux-amd64 + mv coscli-linux-amd64 coscli + chmod 755 coscli + + - name: Upload to COS + run: | + ./coscli sync media/ cos://${{ secrets.TENCENTCLOUD_BUCKET_ID }}/media/images/docs \ + --init-skip \ + --recursive \ + --routines 16 \ + --secret-id ${{ secrets.TENCENTCLOUD_SECRET_ID }} \ + --secret-key ${{ secrets.TENCENTCLOUD_SECRET_KEY }} \ + --endpoint cos.ap-beijing.myqcloud.com + + cdn-refresh: + needs: upload + runs-on: ubuntu-latest + name: Refresh CDN Cache + env: + TENCENTCLOUD_SECRET_ID: ${{ secrets.TENCENTCLOUD_SECRET_ID }} + TENCENTCLOUD_SECRET_KEY: ${{ secrets.TENCENTCLOUD_SECRET_KEY }} + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python environment + uses: actions/setup-python@v5 + with: + python-version: "3.12" + architecture: "x64" + + - name: Install Tencent Cloud CLI + run: pipx install tccli + + - name: Purge production CDN cache + run: tccli cdn PurgePathCache --Paths '["https://docs-download.pingcap.com/media/images/docs/"]' --FlushType delete diff --git a/.lycheeignore b/.lycheeignore index 3ead57f29a342..2ee9f997efd6f 100644 --- a/.lycheeignore +++ b/.lycheeignore @@ -14,4 +14,22 @@ file://.*?http:/\$%7BPD_IP%7D:\$%7BPD_PORT%7D/dashboard.* http://\{grafana-ip\}:3000 http://\{pd-ip\}:2379/dashboard http://localhost:\d+/ -https://github\.com/\$user/(docs|docs-cn) \ No newline at end of file +https://github\.com/\$user/(docs|docs-cn) +https://linux\.die\.net/man.* +https://dev\.mysql\.com/doc/.+/5.7/en/.* +https://dev\.mysql\.com/doc/.+/8\.0/en/.* +https://dev\.mysql\.com/doc/.+/8\.4/en/.* +https://dev\.mysql\.com/doc/[a-z\-]+/en/.* +https://dev\.mysql\.com/doc/relnotes/[a-z\-]+/en/.* +https://dev\.mysql\.com/doc/dev/mysql-server/.* +https://dev\.mysql\.com/downloads/.* +https://bugs\.mysql\.com/bug\.php.* +https://www\.mysql\.com/products/.* +https://platform\.openai\.com/docs/.* +https://openai\.com/.* +https://jwt\.io/ +https://typeorm\.io/.* +https://dash\.cloudflare\.com/.* +https://centminmod\.com/mydumper\.html +https://docs\.pingcap\.com/tidb/v6\.6/system-variables#tidb_pessimistic_txn_aggressive_locking-new-in-v660 +https://docs\.pingcap\.com/tidb/v7\.6/system-variables#tidb_ddl_version-new-in-v760 diff --git a/OWNERS b/OWNERS index 928908987c355..6b9b126ae31bc 100644 --- a/OWNERS +++ b/OWNERS @@ -1,31 +1,32 @@ # See the OWNERS docs at https://go.k8s.io/owners approvers: - breezewish - - CaitinChen - - CharLotteiu - - cofyc - csuzhangxc - - DanielZhangQD - - dcalvin - - dragonly - - en-jin19 - hfxsd - Icemap - jackysp - kissmydb - lance6716 - - lichunzhu - lilin90 - - Liuxiaozhen12 - - morgo - Oreoxmt - overvenus - qiancai + - tangenta +emeritus_approvers: + - CaitinChen + - CharLotteiu + - cofyc + - DanielZhangQD + - dcalvin + - dragonly + - en-jin19 + - lichunzhu + - Liuxiaozhen12 + - morgo - queenypingcap - ran-huang - shichun-0415 - SunRunAway - - tangenta - TomShawn - toutdesuite - WangXiangUSTC diff --git a/README.md b/README.md index fb0f88abf30e7..69b20ba79eb64 100644 --- a/README.md +++ b/README.md @@ -28,6 +28,7 @@ Currently, we maintain the following versions of TiDB documentation in different | Branch name | TiDB docs version | | :---------|:----------| | [`master`](https://github.com/pingcap/docs/tree/master) | The latest development version | +| [`release-8.1`](https://github.com/pingcap/docs/tree/release-8.1) | 8.1 LTS (Long-Term Support) | | [`release-8.0`](https://github.com/pingcap/docs/tree/release-8.0) | 8.0 Development Milestone Release | | [`release-7.6`](https://github.com/pingcap/docs/tree/release-7.6) | 7.6 Development Milestone Release | | [`release-7.5`](https://github.com/pingcap/docs/tree/release-7.5) | 7.5 LTS (Long-Term Support) | diff --git a/TOC-tidb-cloud.md b/TOC-tidb-cloud.md index f94cf4d8b4e02..e7ad633cdf1a0 100644 --- a/TOC-tidb-cloud.md +++ b/TOC-tidb-cloud.md @@ -1,143 +1,167 @@ -- [Docs Home](https://docs.pingcap.com/) -- About TiDB Cloud - - [Why TiDB Cloud](/tidb-cloud/tidb-cloud-intro.md) - - [Architecture](/tidb-cloud/tidb-cloud-intro.md#architecture) - - [High Availability](/tidb-cloud/high-availability-with-multi-az.md) +# Table of Contents + +## GET STARTED + +- Why TiDB Cloud + - [TiDB Cloud Introduction](/tidb-cloud/tidb-cloud-intro.md) - [MySQL Compatibility](/mysql-compatibility.md) - - [Roadmap](/tidb-cloud/tidb-cloud-roadmap.md) -- Get Started - - [Try Out TiDB Cloud](/tidb-cloud/tidb-cloud-quickstart.md) +- Get Started with TiDB Cloud + - [Try Out TiDB Cloud Serverless](/tidb-cloud/tidb-cloud-quickstart.md) + - [Try Out TiDB + AI](/tidb-cloud/vector-search-get-started-using-python.md) - [Try Out HTAP](/tidb-cloud/tidb-cloud-htap-quickstart.md) - [Try Out TiDB Cloud CLI](/tidb-cloud/get-started-with-cli.md) - [Perform a PoC](/tidb-cloud/tidb-cloud-poc.md) -- Develop Applications - - [Overview](/develop/dev-guide-overview.md) - - Quick Start - - [Build a TiDB Serverless Cluster in TiDB Cloud](/develop/dev-guide-build-cluster-in-cloud.md) - - [CRUD SQL in TiDB](/develop/dev-guide-tidb-crud-sql.md) - - Connect to TiDB Cloud - - GUI Database Tools - - [JetBrains DataGrip](/develop/dev-guide-gui-datagrip.md) - - [DBeaver](/develop/dev-guide-gui-dbeaver.md) - - [VS Code](/develop/dev-guide-gui-vscode-sqltools.md) - - [MySQL Workbench](/develop/dev-guide-gui-mysql-workbench.md) - - [Navicat](/develop/dev-guide-gui-navicat.md) - - [Choose Driver or ORM](/develop/dev-guide-choose-driver-or-orm.md) - - BI - - [Looker Studio](/tidb-cloud/dev-guide-bi-looker-studio.md) - - Java - - [JDBC](/develop/dev-guide-sample-application-java-jdbc.md) - - [MyBatis](/develop/dev-guide-sample-application-java-mybatis.md) - - [Hibernate](/develop/dev-guide-sample-application-java-hibernate.md) - - [Spring Boot](/develop/dev-guide-sample-application-java-spring-boot.md) - - [Connection Pools and Connection Parameters](/develop/dev-guide-connection-parameters.md) - - Go - - [Go-MySQL-Driver](/develop/dev-guide-sample-application-golang-sql-driver.md) - - [GORM](/develop/dev-guide-sample-application-golang-gorm.md) - - Python - - [mysqlclient](/develop/dev-guide-sample-application-python-mysqlclient.md) - - [MySQL Connector/Python](/develop/dev-guide-sample-application-python-mysql-connector.md) - - [PyMySQL](/develop/dev-guide-sample-application-python-pymysql.md) - - [SQLAlchemy](/develop/dev-guide-sample-application-python-sqlalchemy.md) - - [peewee](/develop/dev-guide-sample-application-python-peewee.md) - - [Django](/develop/dev-guide-sample-application-python-django.md) - - Node.js - - [node-mysql2](/develop/dev-guide-sample-application-nodejs-mysql2.md) - - [mysql.js](/develop/dev-guide-sample-application-nodejs-mysqljs.md) - - [Prisma](/develop/dev-guide-sample-application-nodejs-prisma.md) - - [Sequelize](/develop/dev-guide-sample-application-nodejs-sequelize.md) - - [TypeORM](/develop/dev-guide-sample-application-nodejs-typeorm.md) - - [Next.js](/develop/dev-guide-sample-application-nextjs.md) - - [AWS Lambda](/develop/dev-guide-sample-application-aws-lambda.md) - - Ruby - - [mysql2](/develop/dev-guide-sample-application-ruby-mysql2.md) - - [Rails](/develop/dev-guide-sample-application-ruby-rails.md) - - [WordPress](/tidb-cloud/dev-guide-wordpress.md) - - Serverless Driver (Beta) - - [TiDB Cloud Serverless Driver](/tidb-cloud/serverless-driver.md) - - [Node.js Example](/tidb-cloud/serverless-driver-node-example.md) - - [Prisma Example](/tidb-cloud/serverless-driver-prisma-example.md) - - [Kysely Example](/tidb-cloud/serverless-driver-kysely-example.md) +- Key Concepts + - [Overview](/tidb-cloud/key-concepts.md) + - [Architecture](/tidb-cloud/architecture-concepts.md) + - [Database Schema](/tidb-cloud/database-schema-concepts.md) + - [Transactions](/tidb-cloud/transaction-concepts.md) + - [SQL](/tidb-cloud/sql-concepts.md) + - [AI Features](/tidb-cloud/ai-feature-concepts.md) + - [Data Service](/tidb-cloud/data-service-concepts.md) ![BETA](/media/tidb-cloud/blank_transparent_placeholder.png) + - [Scalability](/tidb-cloud/scalability-concepts.md) + - High Availability + - [High Availability in TiDB Cloud Serverless](/tidb-cloud/serverless-high-availability.md) + - [High Availability in TiDB Cloud Dedicated](/tidb-cloud/high-availability-with-multi-az.md) + - [Monitoring](/tidb-cloud/monitoring-concepts.md) + - [Data Streaming](/tidb-cloud/data-streaming-concepts.md) + - [Backup & Restore](/tidb-cloud/backup-and-restore-concepts.md) + - [Security](/tidb-cloud/security-concepts.md) + +## DEVELOP + +- Development Quick Start + - [Developer Guide Overview](/develop/dev-guide-overview.md) + - [Build a TiDB Cloud Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md) + - [CRUD SQL in TiDB](/develop/dev-guide-tidb-crud-sql.md) +- Connect to TiDB Cloud + - GUI Database Tools + - [JetBrains DataGrip](/develop/dev-guide-gui-datagrip.md) + - [DBeaver](/develop/dev-guide-gui-dbeaver.md) + - [VS Code](/develop/dev-guide-gui-vscode-sqltools.md) + - [MySQL Workbench](/develop/dev-guide-gui-mysql-workbench.md) + - [Navicat](/develop/dev-guide-gui-navicat.md) + - [Choose Driver or ORM](/develop/dev-guide-choose-driver-or-orm.md) + - BI + - [Looker Studio](/tidb-cloud/dev-guide-bi-looker-studio.md) + - Java + - [JDBC](/develop/dev-guide-sample-application-java-jdbc.md) + - [MyBatis](/develop/dev-guide-sample-application-java-mybatis.md) + - [Hibernate](/develop/dev-guide-sample-application-java-hibernate.md) + - [Spring Boot](/develop/dev-guide-sample-application-java-spring-boot.md) + - [Connection Pools and Connection Parameters](/develop/dev-guide-connection-parameters.md) + - Go + - [Go-MySQL-Driver](/develop/dev-guide-sample-application-golang-sql-driver.md) + - [GORM](/develop/dev-guide-sample-application-golang-gorm.md) + - Python + - [mysqlclient](/develop/dev-guide-sample-application-python-mysqlclient.md) + - [MySQL Connector/Python](/develop/dev-guide-sample-application-python-mysql-connector.md) + - [PyMySQL](/develop/dev-guide-sample-application-python-pymysql.md) + - [SQLAlchemy](/develop/dev-guide-sample-application-python-sqlalchemy.md) + - [peewee](/develop/dev-guide-sample-application-python-peewee.md) + - [Django](/develop/dev-guide-sample-application-python-django.md) + - Node.js + - [node-mysql2](/develop/dev-guide-sample-application-nodejs-mysql2.md) + - [mysql.js](/develop/dev-guide-sample-application-nodejs-mysqljs.md) + - [Prisma](/develop/dev-guide-sample-application-nodejs-prisma.md) + - [Sequelize](/develop/dev-guide-sample-application-nodejs-sequelize.md) + - [TypeORM](/develop/dev-guide-sample-application-nodejs-typeorm.md) + - [Next.js](/develop/dev-guide-sample-application-nextjs.md) + - [AWS Lambda](/develop/dev-guide-sample-application-aws-lambda.md) + - Ruby + - [mysql2](/develop/dev-guide-sample-application-ruby-mysql2.md) + - [Rails](/develop/dev-guide-sample-application-ruby-rails.md) + - [WordPress](/tidb-cloud/dev-guide-wordpress.md) + - Serverless Driver ![BETA](/media/tidb-cloud/blank_transparent_placeholder.png) + - [TiDB Cloud Serverless Driver](/tidb-cloud/serverless-driver.md) + - [Node.js Example](/tidb-cloud/serverless-driver-node-example.md) + - [Prisma Example](/tidb-cloud/serverless-driver-prisma-example.md) + - [Kysely Example](/tidb-cloud/serverless-driver-kysely-example.md) + - [Drizzle Example](/tidb-cloud/serverless-driver-drizzle-example.md) +- Development Reference + - Design Database Schema + - [Overview](/develop/dev-guide-schema-design-overview.md) + - [Create a Database](/develop/dev-guide-create-database.md) + - [Create a Table](/develop/dev-guide-create-table.md) + - [Create a Secondary Index](/develop/dev-guide-create-secondary-indexes.md) + - Write Data + - [Insert Data](/develop/dev-guide-insert-data.md) + - [Update Data](/develop/dev-guide-update-data.md) + - [Delete Data](/develop/dev-guide-delete-data.md) + - [Periodically Delete Expired Data Using TTL (Time to Live)](/time-to-live.md) + - [Prepared Statements](/develop/dev-guide-prepared-statement.md) + - Read Data + - [Query Data from a Single Table](/develop/dev-guide-get-data-from-single-table.md) + - [Multi-table Join Queries](/develop/dev-guide-join-tables.md) + - [Subquery](/develop/dev-guide-use-subqueries.md) + - [Paginate Results](/develop/dev-guide-paginate-results.md) + - [Views](/develop/dev-guide-use-views.md) + - [Temporary Tables](/develop/dev-guide-use-temporary-tables.md) + - [Common Table Expression](/develop/dev-guide-use-common-table-expression.md) + - Read Replica Data + - [Follower Read](/develop/dev-guide-use-follower-read.md) + - [Stale Read](/develop/dev-guide-use-stale-read.md) + - [HTAP Queries](/develop/dev-guide-hybrid-oltp-and-olap-queries.md) + - Transaction + - [Overview](/develop/dev-guide-transaction-overview.md) + - [Optimistic and Pessimistic Transactions](/develop/dev-guide-optimistic-and-pessimistic-transaction.md) + - [Transaction Restraints](/develop/dev-guide-transaction-restraints.md) + - [Handle Transaction Errors](/develop/dev-guide-transaction-troubleshoot.md) + - Optimize + - [Overview](/develop/dev-guide-optimize-sql-overview.md) + - [SQL Performance Tuning](/develop/dev-guide-optimize-sql.md) + - [Best Practices for Performance Tuning](/develop/dev-guide-optimize-sql-best-practices.md) + - [Best Practices for Indexing](/develop/dev-guide-index-best-practice.md) + - Other Optimization Methods + - [Avoid Implicit Type Conversions](/develop/dev-guide-implicit-type-conversion.md) + - [Unique Serial Number Generation](/develop/dev-guide-unique-serial-number-generation.md) + - Troubleshoot + - [SQL or Transaction Issues](/develop/dev-guide-troubleshoot-overview.md) + - [Unstable Result Set](/develop/dev-guide-unstable-result-set.md) + - [Timeouts](/develop/dev-guide-timeouts-in-tidb.md) + - Development Guidelines + - [Object Naming Convention](/develop/dev-guide-object-naming-guidelines.md) + - [SQL Development Specifications](/develop/dev-guide-sql-development-specification.md) + - [Bookshop Example Application](/develop/dev-guide-bookshop-schema-design.md) - Third-Party Support - [Third-Party Tools Supported by TiDB](/develop/dev-guide-third-party-support.md) - [Known Incompatibility Issues with Third-Party Tools](/develop/dev-guide-third-party-tools-compatibility.md) - - Development Reference - - Design Database Schema - - [Overview](/develop/dev-guide-schema-design-overview.md) - - [Create a Database](/develop/dev-guide-create-database.md) - - [Create a Table](/develop/dev-guide-create-table.md) - - [Create a Secondary Index](/develop/dev-guide-create-secondary-indexes.md) - - Write Data - - [Insert Data](/develop/dev-guide-insert-data.md) - - [Update Data](/develop/dev-guide-update-data.md) - - [Delete Data](/develop/dev-guide-delete-data.md) - - [Periodically Delete Expired Data Using TTL (Time to Live)](/time-to-live.md) - - [Prepared Statements](/develop/dev-guide-prepared-statement.md) - - Read Data - - [Query Data from a Single Table](/develop/dev-guide-get-data-from-single-table.md) - - [Multi-table Join Queries](/develop/dev-guide-join-tables.md) - - [Subquery](/develop/dev-guide-use-subqueries.md) - - [Paginate Results](/develop/dev-guide-paginate-results.md) - - [Views](/develop/dev-guide-use-views.md) - - [Temporary Tables](/develop/dev-guide-use-temporary-tables.md) - - [Common Table Expression](/develop/dev-guide-use-common-table-expression.md) - - Read Replica Data - - [Follower Read](/develop/dev-guide-use-follower-read.md) - - [Stale Read](/develop/dev-guide-use-stale-read.md) - - [HTAP Queries](/develop/dev-guide-hybrid-oltp-and-olap-queries.md) - - Transaction - - [Overview](/develop/dev-guide-transaction-overview.md) - - [Optimistic and Pessimistic Transactions](/develop/dev-guide-optimistic-and-pessimistic-transaction.md) - - [Transaction Restraints](/develop/dev-guide-transaction-restraints.md) - - [Handle Transaction Errors](/develop/dev-guide-transaction-troubleshoot.md) - - Optimize - - [Overview](/develop/dev-guide-optimize-sql-overview.md) - - [SQL Performance Tuning](/develop/dev-guide-optimize-sql.md) - - [Best Practices for Performance Tuning](/develop/dev-guide-optimize-sql-best-practices.md) - - [Best Practices for Indexing](/develop/dev-guide-index-best-practice.md) - - Other Optimization Methods - - [Avoid Implicit Type Conversions](/develop/dev-guide-implicit-type-conversion.md) - - [Unique Serial Number Generation](/develop/dev-guide-unique-serial-number-generation.md) - - Troubleshoot - - [SQL or Transaction Issues](/develop/dev-guide-troubleshoot-overview.md) - - [Unstable Result Set](/develop/dev-guide-unstable-result-set.md) - - [Timeouts](/develop/dev-guide-timeouts-in-tidb.md) - - Development Guidelines - - [Object Naming Convention](/develop/dev-guide-object-naming-guidelines.md) - - [SQL Development Specifications](/develop/dev-guide-sql-development-specification.md) - - [Bookshop Example Application](/develop/dev-guide-bookshop-schema-design.md) + +## GUIDES + - Manage Cluster - Plan Your Cluster - [Select Your Cluster Tier](/tidb-cloud/select-cluster-tier.md) - [Determine Your TiDB Size](/tidb-cloud/size-your-cluster.md) - [TiDB Cloud Performance Reference](/tidb-cloud/tidb-cloud-performance-reference.md) - - Manage TiDB Serverless Clusters - - [Create a TiDB Serverless Cluster](/tidb-cloud/create-tidb-cluster-serverless.md) - - Connect to Your TiDB Serverless Cluster + - Manage TiDB Cloud Serverless Clusters + - [Create a TiDB Cloud Serverless Cluster](/tidb-cloud/create-tidb-cluster-serverless.md) + - Connect to Your TiDB Cloud Serverless Cluster - [Connection Overview](/tidb-cloud/connect-to-tidb-cluster-serverless.md) - [Connect via Public Endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md) - [Connect via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md) - - Branch (Beta) + - Branch ![BETA](/media/tidb-cloud/blank_transparent_placeholder.png) - [Overview](/tidb-cloud/branch-overview.md) - [Manage Branches](/tidb-cloud/branch-manage.md) - [GitHub Integration](/tidb-cloud/branch-github-integration.md) - [Manage Spending Limit](/tidb-cloud/manage-serverless-spend-limit.md) - - [Back Up and Restore TiDB Serverless Data](/tidb-cloud/backup-and-restore-serverless.md) - - Manage TiDB Dedicated Clusters - - [Create a TiDB Dedicated Cluster](/tidb-cloud/create-tidb-cluster.md) - - Connect to Your TiDB Dedicated Cluster + - [Back Up and Restore TiDB Cloud Serverless Data](/tidb-cloud/backup-and-restore-serverless.md) + - [Export Data from TiDB Cloud Serverless](/tidb-cloud/serverless-export.md) + - Manage TiDB Cloud Dedicated Clusters + - [Create a TiDB Cloud Dedicated Cluster](/tidb-cloud/create-tidb-cluster.md) + - Connect to Your TiDB Cloud Dedicated Cluster - [Connection Method Overview](/tidb-cloud/connect-to-tidb-cluster.md) - - [Connect via Standard Connection](/tidb-cloud/connect-via-standard-connection.md) + - [Connect via Public Connection](/tidb-cloud/connect-via-standard-connection.md) - [Connect via Private Endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md) - [Connect via Private Endpoint (Private Service Connect) with Google Cloud](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md) - [Connect via VPC Peering](/tidb-cloud/set-up-vpc-peering-connections.md) - [Connect via SQL Shell](/tidb-cloud/connect-via-sql-shell.md) - - [Scale a TiDB Dedicated Cluster](/tidb-cloud/scale-tidb-cluster.md) - - [Back Up and Restore TiDB Dedicated Data](/tidb-cloud/backup-and-restore.md) - - [Pause or Resume a TiDB Dedicated Cluster](/tidb-cloud/pause-or-resume-tidb-cluster.md) + - [Scale a TiDB Cloud Dedicated Cluster](/tidb-cloud/scale-tidb-cluster.md) + - [Back Up and Restore TiDB Cloud Dedicated Data](/tidb-cloud/backup-and-restore.md) + - [Pause or Resume a TiDB Cloud Dedicated Cluster](/tidb-cloud/pause-or-resume-tidb-cluster.md) - [Configure Maintenance Window](/tidb-cloud/configure-maintenance-window.md) - Use an HTAP Cluster with TiFlash - [TiFlash Overview](/tiflash/tiflash-overview.md) @@ -154,13 +178,18 @@ - [Overview](/tidb-cloud/monitor-tidb-cluster.md) - [Built-in Metrics](/tidb-cloud/built-in-monitoring.md) - [Built-in Alerting](/tidb-cloud/monitor-built-in-alerting.md) + - Subscribe to Alert Notifications + - [Subscribe via Email](/tidb-cloud/monitor-alert-email.md) + - [Subscribe via Slack](/tidb-cloud/monitor-alert-slack.md) + - [Subscribe via Zoom](/tidb-cloud/monitor-alert-zoom.md) - [Cluster Events](/tidb-cloud/tidb-cloud-events.md) - - [Third-Party Metrics Integrations (Beta)](/tidb-cloud/third-party-monitoring-integrations.md) + - [Third-Party Metrics Integrations](/tidb-cloud/third-party-monitoring-integrations.md) ![BETA](/media/tidb-cloud/blank_transparent_placeholder.png) + - [TiDB Cloud Clinic](/tidb-cloud/tidb-cloud-clinic.md) - Tune Performance - [Overview](/tidb-cloud/tidb-cloud-tune-performance-overview.md) - Analyze Performance - [Use the Diagnosis Tab](/tidb-cloud/tune-performance.md) - - [Use Index Insight (Beta)](/tidb-cloud/index-insight.md) + - [Use Index Insight](/tidb-cloud/index-insight.md) ![BETA](/media/tidb-cloud/blank_transparent_placeholder.png) - [Use Statement Summary Tables](/statement-summary-tables.md) - SQL Tuning - [Overview](/tidb-cloud/tidb-cloud-sql-tuning-overview.md) @@ -211,39 +240,86 @@ - [Overview](/garbage-collection-overview.md) - [Configuration](/garbage-collection-configuration.md) - [Tune TiFlash Performance](/tiflash/tune-tiflash-performance.md) + - Optimize Resource Allocation + - [Overview of Resource Allocation](/tidb-cloud/optimize-resource-allocation.md) + - [Resource Control](/tidb-resource-control.md) + - TiDB Node Group + - [Overview of TiDB Node Group](/tidb-cloud/tidb-node-group-overview.md) + - [Manage TiDB Node Groups](/tidb-cloud/tidb-node-group-management.md) - [Upgrade a TiDB Cluster](/tidb-cloud/upgrade-tidb-cluster.md) - [Delete a TiDB Cluster](/tidb-cloud/delete-tidb-cluster.md) - Migrate or Import Data - [Overview](/tidb-cloud/tidb-cloud-migration-overview.md) - Migrate Data into TiDB Cloud - - [Migrate Existing and Incremental Data Using Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md) - - [Migrate Incremental Data Using Data Migration](/tidb-cloud/migrate-incremental-data-from-mysql-using-data-migration.md) - - [Migrate and Merge MySQL Shards of Large Datasets](/tidb-cloud/migrate-sql-shards.md) - - [Migrate from On-Premises TiDB to TiDB Cloud](/tidb-cloud/migrate-from-op-tidb.md) - - [Migrate from MySQL-Compatible Databases Using AWS DMS](/tidb-cloud/migrate-from-mysql-using-aws-dms.md) - - [Migrate from Amazon RDS for Oracle Using AWS DMS](/tidb-cloud/migrate-from-oracle-using-aws-dms.md) - - Import Data into TiDB Cloud - - [Import Local Files](/tidb-cloud/tidb-cloud-import-local-files.md) - - [Import Sample Data (SQL File)](/tidb-cloud/import-sample-data.md) - - [Import CSV Files from Amazon S3 or GCS](/tidb-cloud/import-csv-files.md) - - [Import Apache Parquet Files from Amazon S3 or GCS](/tidb-cloud/import-parquet-files.md) - - [Import with MySQL CLI](/tidb-cloud/import-with-mysql-cli.md) + - [Migrate Existing and Incremental Data Using Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md) + - [Migrate Incremental Data Using Data Migration](/tidb-cloud/migrate-incremental-data-from-mysql-using-data-migration.md) + - [Migrate and Merge MySQL Shards of Large Datasets](/tidb-cloud/migrate-sql-shards.md) + - [Migrate from TiDB Self-Managed to TiDB Cloud](/tidb-cloud/migrate-from-op-tidb.md) + - [Migrate from MySQL-Compatible Databases Using AWS DMS](/tidb-cloud/migrate-from-mysql-using-aws-dms.md) + - [Migrate from Amazon RDS for Oracle Using AWS DMS](/tidb-cloud/migrate-from-oracle-using-aws-dms.md) + - Import Data into TiDB Cloud Dedicated + - [Import Sample Data](/tidb-cloud/import-sample-data.md) + - [Import CSV Files from Amazon S3 or GCS](/tidb-cloud/import-csv-files.md) + - [Import Parquet Files from Amazon S3 or GCS](/tidb-cloud/import-parquet-files.md) + - [Import with MySQL CLI](/tidb-cloud/import-with-mysql-cli.md) + - Import Data into TiDB Cloud Serverless + - [Import Sample Data](/tidb-cloud/import-sample-data-serverless.md) + - [Import Local Files](/tidb-cloud/tidb-cloud-import-local-files.md) + - [Import CSV Files from Cloud Storage](/tidb-cloud/import-csv-files-serverless.md) + - [Import Parquet Files from Cloud Storage](/tidb-cloud/import-parquet-files-serverless.md) + - [Import with MySQL CLI](/tidb-cloud/import-with-mysql-cli-serverless.md) - Reference - - [Configure Amazon S3 Access and GCS Access](/tidb-cloud/config-s3-and-gcs-access.md) - - [Naming Conventions for Data Import](/tidb-cloud/naming-conventions-for-data-import.md) - - [CSV Configurations for Importing Data](/tidb-cloud/csv-config-for-import-data.md) - - [Troubleshoot Access Denied Errors during Data Import from Amazon S3](/tidb-cloud/troubleshoot-import-access-denied-error.md) - - [Precheck Errors, Migration Errors, and Alerts for Data Migration](/tidb-cloud/tidb-cloud-dm-precheck-and-troubleshooting.md) - - [Connect AWS DMS to TiDB Cloud clusters](/tidb-cloud/tidb-cloud-connect-aws-dms.md) + - [Configure External Storage Access for TiDB Cloud Dedicated](/tidb-cloud/config-s3-and-gcs-access.md) + - [Configure External Storage Access for TiDB Cloud Serverless](/tidb-cloud/serverless-external-storage.md) + - [Naming Conventions for Data Import](/tidb-cloud/naming-conventions-for-data-import.md) + - [CSV Configurations for Importing Data](/tidb-cloud/csv-config-for-import-data.md) + - [Troubleshoot Access Denied Errors during Data Import from Amazon S3](/tidb-cloud/troubleshoot-import-access-denied-error.md) + - [Precheck Errors, Migration Errors, and Alerts for Data Migration](/tidb-cloud/tidb-cloud-dm-precheck-and-troubleshooting.md) + - [Connect AWS DMS to TiDB Cloud clusters](/tidb-cloud/tidb-cloud-connect-aws-dms.md) - Explore Data - - [Chat2Query (Beta)](/tidb-cloud/explore-data-with-chat2query.md) -- Data Service (Beta) + - [Chat2Query in SQL Editor](/tidb-cloud/explore-data-with-chat2query.md) ![BETA](/media/tidb-cloud/blank_transparent_placeholder.png) + - [SQL Proxy Account](/tidb-cloud/sql-proxy-account.md) +- Vector Search ![BETA](/media/tidb-cloud/blank_transparent_placeholder.png) + - [Overview](/tidb-cloud/vector-search-overview.md) + - Get Started + - [Get Started with SQL](/tidb-cloud/vector-search-get-started-using-sql.md) + - [Get Started with Python](/tidb-cloud/vector-search-get-started-using-python.md) + - Integrations + - [Overview](/tidb-cloud/vector-search-integration-overview.md) + - AI Frameworks + - [LlamaIndex](/tidb-cloud/vector-search-integrate-with-llamaindex.md) + - [Langchain](/tidb-cloud/vector-search-integrate-with-langchain.md) + - AI Services + - [Amazon Bedrock](/tidb-cloud/vector-search-integrate-with-amazon-bedrock.md) + - Embedding Models/Services + - [Jina AI](/tidb-cloud/vector-search-integrate-with-jinaai-embedding.md) + - ORM Libraries + - [SQLAlchemy](/tidb-cloud/vector-search-integrate-with-sqlalchemy.md) + - [peewee](/tidb-cloud/vector-search-integrate-with-peewee.md) + - [Django ORM](/tidb-cloud/vector-search-integrate-with-django-orm.md) + - Text Search + - [Full-Text Search with SQL](/tidb-cloud/vector-search-full-text-search-sql.md) + - [Full-Text Search with Python](/tidb-cloud/vector-search-full-text-search-python.md) + - [Hybrid Search](/tidb-cloud/vector-search-hybrid-search.md) + - Reference + - [Vector Data Types](/tidb-cloud/vector-search-data-types.md) + - [Vector Functions and Operators](/tidb-cloud/vector-search-functions-and-operators.md) + - [Vector Index](/tidb-cloud/vector-search-index.md) + - [Improve Performance](/tidb-cloud/vector-search-improve-performance.md) + - [Limitations](/tidb-cloud/vector-search-limitations.md) + - [Changelogs](/tidb-cloud/vector-search-changelogs.md) +- Data Service ![BETA](/media/tidb-cloud/blank_transparent_placeholder.png) - [Overview](/tidb-cloud/data-service-overview.md) - [Get Started](/tidb-cloud/data-service-get-started.md) - - [Try Out Chat2Query API](/tidb-cloud/use-chat2query-api.md) + - Chat2Query API + - [Get Started](/tidb-cloud/use-chat2query-api.md) + - [Start Multi-round Chat2Query](/tidb-cloud/use-chat2query-sessions.md) + - [Use Knowledge Bases](/tidb-cloud/use-chat2query-knowledge.md) - [Manage Data App](/tidb-cloud/data-service-manage-data-app.md) - [Manage Endpoint](/tidb-cloud/data-service-manage-endpoint.md) - [API Key](/tidb-cloud/data-service-api-key.md) + - [Custom Domain](/tidb-cloud/data-service-custom-domain.md) + - [Integrations](/tidb-cloud/data-service-integrations.md) - [Run in Postman](/tidb-cloud/data-service-postman-integration.md) - [Deploy Automatically with GitHub](/tidb-cloud/data-service-manage-github-connection.md) - [Use OpenAPI Specification with Next.js](/tidb-cloud/data-service-oas-with-nextjs.md) @@ -255,48 +331,55 @@ - [To Kafka Sink](/tidb-cloud/changefeed-sink-to-apache-kafka.md) - [To TiDB Cloud Sink](/tidb-cloud/changefeed-sink-to-tidb-cloud.md) - [To Cloud Storage](/tidb-cloud/changefeed-sink-to-cloud-storage.md) + - Reference + - [Set Up Self-Hosted Kafka Private Link Service in AWS](/tidb-cloud/setup-self-hosted-kafka-private-link-service.md) + - [Set Up Self-Hosted Kafka Private Service Connect in Google Cloud](/tidb-cloud/setup-self-hosted-kafka-private-service-connect.md) +- Disaster Recovery + - [Recovery Group Overview](/tidb-cloud/recovery-group-overview.md) + - [Get Started](/tidb-cloud/recovery-group-get-started.md) + - [Failover and Reprotect Databases](/tidb-cloud/recovery-group-failover.md) + - [Delete a Recovery Group](/tidb-cloud/recovery-group-delete.md) - Security - Identity Access Control - [Password Authentication](/tidb-cloud/tidb-cloud-password-authentication.md) - - [Basic SSO Authentication](/tidb-cloud/tidb-cloud-sso-authentication.md) + - [Standard SSO Authentication](/tidb-cloud/tidb-cloud-sso-authentication.md) - [Organization SSO Authentication](/tidb-cloud/tidb-cloud-org-sso-authentication.md) - [Identity Access Management](/tidb-cloud/manage-user-access.md) + - [OAuth 2.0](/tidb-cloud/oauth2.md) - Network Access Control - - TiDB Serverless + - TiDB Cloud Serverless - [Connect via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md) - - [TLS Connections to TiDB Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md) - - TiDB Dedicated + - [Configure Firewall Rules for Public Endpoints](/tidb-cloud/configure-serverless-firewall-rules-for-public-endpoints.md) + - [TLS Connections to TiDB Cloud Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md) + - TiDB Cloud Dedicated - [Configure an IP Access List](/tidb-cloud/configure-ip-access-list.md) - - [Connect via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md) + - [Connect via Private Endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md) + - [Connect via Private Endpoint (Private Service Connect) with Google Cloud](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md) - [Connect via VPC Peering](/tidb-cloud/set-up-vpc-peering-connections.md) - - [TLS Connections to TiDB Dedicated](/tidb-cloud/tidb-cloud-tls-connect-to-dedicated.md) + - [TLS Connections to TiDB Cloud Dedicated](/tidb-cloud/tidb-cloud-tls-connect-to-dedicated.md) - Data Access Control - [Encryption at Rest Using Customer-Managed Encryption Keys](/tidb-cloud/tidb-cloud-encrypt-cmek.md) - Database Access Control - - [Configure Cluster Security Settings](/tidb-cloud/configure-security-settings.md) + - [Configure Cluster Password Settings](/tidb-cloud/configure-security-settings.md) - Audit Management - [Database Audit Logging](/tidb-cloud/tidb-cloud-auditing.md) - [Console Audit Logging](/tidb-cloud/tidb-cloud-console-auditing.md) - Billing - [Invoices](/tidb-cloud/tidb-cloud-billing.md#invoices) - [Billing Details](/tidb-cloud/tidb-cloud-billing.md#billing-details) + - [Cost Explorer](/tidb-cloud/tidb-cloud-billing.md#cost-explorer) + - [Billing Profile](/tidb-cloud/tidb-cloud-billing.md#billing-profile) - [Credits](/tidb-cloud/tidb-cloud-billing.md#credits) - [Payment Method Setting](/tidb-cloud/tidb-cloud-billing.md#payment-method) - [Billing from AWS or GCP Marketplace](/tidb-cloud/tidb-cloud-billing.md#billing-from-aws-marketplace-or-google-cloud-marketplace) - [Billing for Changefeed](/tidb-cloud/tidb-cloud-billing-ticdc-rcu.md) - [Billing for Data Migration](/tidb-cloud/tidb-cloud-billing-dm.md) -- API -- API - - [API Overview](/tidb-cloud/api-overview.md) - - API Reference - - v1beta1 - - [Billing](https://docs.pingcap.com/tidbcloud/api/v1beta1/billing) - - [IAM](https://docs.pingcap.com/tidbcloud/api/v1beta1/apikey) - - [MSP](https://docs.pingcap.com/tidbcloud/api/msp/v1beta1) - - [v1beta](https://docs.pingcap.com/tidbcloud/api/v1beta) + - [Billing for Recovery Groups](/tidb-cloud/tidb-cloud-billing-recovery-group.md) + - [Manage Budgets](/tidb-cloud/tidb-cloud-budget.md) - Integrations - [Airbyte](/tidb-cloud/integrate-tidbcloud-with-airbyte.md) - [Amazon AppFlow](/develop/dev-guide-aws-appflow-integration.md) + - [AWS Lambda](/tidb-cloud/integrate-tidbcloud-with-aws-lambda.md) - [Cloudflare](/tidb-cloud/integrate-tidbcloud-with-cloudflare.md) - [Datadog](/tidb-cloud/monitor-datadog-integration.md) - [dbt](/tidb-cloud/integrate-tidbcloud-with-dbt.md) @@ -315,361 +398,454 @@ - [Use Import Resource](/tidb-cloud/terraform-use-import-resource.md) - [Vercel](/tidb-cloud/integrate-tidbcloud-with-vercel.md) - [Zapier](/tidb-cloud/integrate-tidbcloud-with-zapier.md) -- Reference - - TiDB Cluster Architecture - - [Overview](/tidb-architecture.md) - - [Storage](/tidb-storage.md) - - [Computing](/tidb-computing.md) - - [Scheduling](/tidb-scheduling.md) - - [TSO](/tso.md) - - [TiDB Dedicated Limitations and Quotas](/tidb-cloud/limitations-and-quotas.md) - - [TiDB Serverless Limitations](/tidb-cloud/serverless-limitations.md) - - [Limited SQL Features on TiDB Cloud](/tidb-cloud/limited-sql-features.md) - - [TiDB Limitations](/tidb-limitations.md) - - TiDB Distributed eXecution Framework (DXF) - - [Introduction](/tidb-distributed-execution-framework.md) - - [TiDB Global Sort](/tidb-global-sort.md) - - Benchmarks - - TiDB v7.5 - - [TPC-C Performance Test Report](/tidb-cloud/v7.5.0-performance-benchmarking-with-tpcc.md) - - [Sysbench Performance Test Report](/tidb-cloud/v7.5.0-performance-benchmarking-with-sysbench.md) - - TiDB v7.1 - - [TPC-C Performance Test Report](/tidb-cloud/v7.1.0-performance-benchmarking-with-tpcc.md) - - [Sysbench Performance Test Report](/tidb-cloud/v7.1.0-performance-benchmarking-with-sysbench.md) - - SQL - - [Explore SQL with TiDB](/basic-sql-operations.md) - - SQL Language Structure and Syntax - - Attributes - - [AUTO_INCREMENT](/auto-increment.md) - - [AUTO_RANDOM](/auto-random.md) - - [SHARD_ROW_ID_BITS](/shard-row-id-bits.md) - - [Literal Values](/literal-values.md) - - [Schema Object Names](/schema-object-names.md) - - [Keywords and Reserved Words](/keywords.md) - - [User-Defined Variables](/user-defined-variables.md) - - [Expression Syntax](/expression-syntax.md) - - [Comment Syntax](/comment-syntax.md) - - SQL Statements + +## REFERENCE + +- SQL Reference + - [Explore SQL with TiDB](/basic-sql-operations.md) + - SQL Language Structure and Syntax + - Attributes + - [AUTO_INCREMENT](/auto-increment.md) + - [AUTO_RANDOM](/auto-random.md) + - [SHARD_ROW_ID_BITS](/shard-row-id-bits.md) + - [Literal Values](/literal-values.md) + - [Schema Object Names](/schema-object-names.md) + - [Keywords and Reserved Words](/keywords.md) + - [User-Defined Variables](/user-defined-variables.md) + - [Expression Syntax](/expression-syntax.md) + - [Comment Syntax](/comment-syntax.md) + - SQL Statements + - [Overview](/sql-statements/sql-statement-overview.md) + - [`ADMIN`](/sql-statements/sql-statement-admin.md) + - [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) + - [`ADMIN CHECKSUM TABLE`](/sql-statements/sql-statement-admin-checksum-table.md) + - [`ADMIN CHECK [TABLE|INDEX]`](/sql-statements/sql-statement-admin-check-table-index.md) + - [`ADMIN CLEANUP INDEX`](/sql-statements/sql-statement-admin-cleanup.md) + - [`ADMIN PAUSE DDL`](/sql-statements/sql-statement-admin-pause-ddl.md) + - [`ADMIN RECOVER INDEX`](/sql-statements/sql-statement-admin-recover.md) + - [`ADMIN RESUME DDL`](/sql-statements/sql-statement-admin-resume-ddl.md) + - [`ADMIN SHOW DDL [JOBS|JOB QUERIES]`](/sql-statements/sql-statement-admin-show-ddl.md) + - [`ALTER DATABASE`](/sql-statements/sql-statement-alter-database.md) + - [`ALTER INSTANCE`](/sql-statements/sql-statement-alter-instance.md) + - [`ALTER PLACEMENT POLICY`](/sql-statements/sql-statement-alter-placement-policy.md) + - [`ALTER RANGE`](/sql-statements/sql-statement-alter-range.md) + - [`ALTER RESOURCE GROUP`](/sql-statements/sql-statement-alter-resource-group.md) + - [`ALTER SEQUENCE`](/sql-statements/sql-statement-alter-sequence.md) + - `ALTER TABLE` + - [Overview](/sql-statements/sql-statement-alter-table.md) - [`ADD COLUMN`](/sql-statements/sql-statement-add-column.md) - [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) - - [`ADMIN`](/sql-statements/sql-statement-admin.md) - - [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) - - [`ADMIN CHECKSUM TABLE`](/sql-statements/sql-statement-admin-checksum-table.md) - - [`ADMIN CHECK [TABLE|INDEX]`](/sql-statements/sql-statement-admin-check-table-index.md) - - [`ADMIN CLEANUP INDEX`](/sql-statements/sql-statement-admin-cleanup.md) - - [`ADMIN PAUSE DDL`](/sql-statements/sql-statement-admin-pause-ddl.md) - - [`ADMIN RECOVER INDEX`](/sql-statements/sql-statement-admin-recover.md) - - [`ADMIN RESUME DDL`](/sql-statements/sql-statement-admin-resume-ddl.md) - - [`ADMIN SHOW DDL [JOBS|JOB QUERIES]`](/sql-statements/sql-statement-admin-show-ddl.md) - - [`ALTER DATABASE`](/sql-statements/sql-statement-alter-database.md) - [`ALTER INDEX`](/sql-statements/sql-statement-alter-index.md) - - [`ALTER INSTANCE`](/sql-statements/sql-statement-alter-instance.md) - - [`ALTER PLACEMENT POLICY`](/sql-statements/sql-statement-alter-placement-policy.md) - - [`ALTER RANGE`](/sql-statements/sql-statement-alter-range.md) - - [`ALTER RESOURCE GROUP`](/sql-statements/sql-statement-alter-resource-group.md) - - [`ALTER SEQUENCE`](/sql-statements/sql-statement-alter-sequence.md) - - [`ALTER TABLE`](/sql-statements/sql-statement-alter-table.md) - - [`ALTER TABLE COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) - - [`ALTER USER`](/sql-statements/sql-statement-alter-user.md) - - [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) - - [`BACKUP`](/sql-statements/sql-statement-backup.md) - - [`BATCH`](/sql-statements/sql-statement-batch.md) - - [`BEGIN`](/sql-statements/sql-statement-begin.md) - - [`CANCEL IMPORT JOB`](/sql-statements/sql-statement-cancel-import-job.md) - [`CHANGE COLUMN`](/sql-statements/sql-statement-change-column.md) - - [`COMMIT`](/sql-statements/sql-statement-commit.md) - - [`CREATE [GLOBAL|SESSION] BINDING`](/sql-statements/sql-statement-create-binding.md) - - [`CREATE DATABASE`](/sql-statements/sql-statement-create-database.md) - - [`CREATE INDEX`](/sql-statements/sql-statement-create-index.md) - - [`CREATE PLACEMENT POLICY`](/sql-statements/sql-statement-create-placement-policy.md) - - [`CREATE RESOURCE GROUP`](/sql-statements/sql-statement-create-resource-group.md) - - [`CREATE ROLE`](/sql-statements/sql-statement-create-role.md) - - [`CREATE SEQUENCE`](/sql-statements/sql-statement-create-sequence.md) - - [`CREATE TABLE LIKE`](/sql-statements/sql-statement-create-table-like.md) - - [`CREATE TABLE`](/sql-statements/sql-statement-create-table.md) - - [`CREATE USER`](/sql-statements/sql-statement-create-user.md) - - [`CREATE VIEW`](/sql-statements/sql-statement-create-view.md) - - [`DEALLOCATE`](/sql-statements/sql-statement-deallocate.md) - - [`DELETE`](/sql-statements/sql-statement-delete.md) - - [`DESC`](/sql-statements/sql-statement-desc.md) - - [`DESCRIBE`](/sql-statements/sql-statement-describe.md) - - [`DO`](/sql-statements/sql-statement-do.md) - - [`DROP [GLOBAL|SESSION] BINDING`](/sql-statements/sql-statement-drop-binding.md) + - [`COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) - [`DROP COLUMN`](/sql-statements/sql-statement-drop-column.md) - - [`DROP DATABASE`](/sql-statements/sql-statement-drop-database.md) - [`DROP INDEX`](/sql-statements/sql-statement-drop-index.md) - - [`DROP PLACEMENT POLICY`](/sql-statements/sql-statement-drop-placement-policy.md) - - [`DROP RESOURCE GROUP`](/sql-statements/sql-statement-drop-resource-group.md) - - [`DROP ROLE`](/sql-statements/sql-statement-drop-role.md) - - [`DROP SEQUENCE`](/sql-statements/sql-statement-drop-sequence.md) - - [`DROP STATS`](/sql-statements/sql-statement-drop-stats.md) - - [`DROP TABLE`](/sql-statements/sql-statement-drop-table.md) - - [`DROP USER`](/sql-statements/sql-statement-drop-user.md) - - [`DROP VIEW`](/sql-statements/sql-statement-drop-view.md) - - [`EXECUTE`](/sql-statements/sql-statement-execute.md) - - [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) - - [`EXPLAIN`](/sql-statements/sql-statement-explain.md) - - [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md) - - [`FLASHBACK DATABASE`](/sql-statements/sql-statement-flashback-database.md) - - [`FLASHBACK TABLE`](/sql-statements/sql-statement-flashback-table.md) - - [`FLUSH PRIVILEGES`](/sql-statements/sql-statement-flush-privileges.md) - - [`FLUSH STATUS`](/sql-statements/sql-statement-flush-status.md) - - [`FLUSH TABLES`](/sql-statements/sql-statement-flush-tables.md) - - [`GRANT `](/sql-statements/sql-statement-grant-privileges.md) - - [`GRANT `](/sql-statements/sql-statement-grant-role.md) - - [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) - - [`INSERT`](/sql-statements/sql-statement-insert.md) - - [`KILL [TIDB]`](/sql-statements/sql-statement-kill.md) - - [`LOAD DATA`](/sql-statements/sql-statement-load-data.md) - - [`LOAD STATS`](/sql-statements/sql-statement-load-stats.md) - - [`LOCK STATS`](/sql-statements/sql-statement-lock-stats.md) - - [`LOCK TABLES` and `UNLOCK TABLES`](/sql-statements/sql-statement-lock-tables-and-unlock-tables.md) - [`MODIFY COLUMN`](/sql-statements/sql-statement-modify-column.md) - - [`PREPARE`](/sql-statements/sql-statement-prepare.md) - - [`QUERY WATCH`](/sql-statements/sql-statement-query-watch.md) - - [`RECOVER TABLE`](/sql-statements/sql-statement-recover-table.md) - [`RENAME INDEX`](/sql-statements/sql-statement-rename-index.md) - - [`RENAME TABLE`](/sql-statements/sql-statement-rename-table.md) - - [`RENAME USER`](/sql-statements/sql-statement-rename-user.md) - - [`REPLACE`](/sql-statements/sql-statement-replace.md) - - [`RESTORE`](/sql-statements/sql-statement-restore.md) - - [`REVOKE `](/sql-statements/sql-statement-revoke-privileges.md) - - [`REVOKE `](/sql-statements/sql-statement-revoke-role.md) - - [`ROLLBACK`](/sql-statements/sql-statement-rollback.md) - - [`SAVEPOINT`](/sql-statements/sql-statement-savepoint.md) - - [`SELECT`](/sql-statements/sql-statement-select.md) - - [`SET DEFAULT ROLE`](/sql-statements/sql-statement-set-default-role.md) - - [`SET [NAMES|CHARACTER SET]`](/sql-statements/sql-statement-set-names.md) - - [`SET PASSWORD`](/sql-statements/sql-statement-set-password.md) - - [`SET RESOURCE GROUP`](/sql-statements/sql-statement-set-resource-group.md) - - [`SET ROLE`](/sql-statements/sql-statement-set-role.md) - - [`SET TRANSACTION`](/sql-statements/sql-statement-set-transaction.md) - - [`SET [GLOBAL|SESSION] `](/sql-statements/sql-statement-set-variable.md) - - [`SHOW ANALYZE STATUS`](/sql-statements/sql-statement-show-analyze-status.md) - - [`SHOW [BACKUPS|RESTORES]`](/sql-statements/sql-statement-show-backups.md) - - [`SHOW [GLOBAL|SESSION] BINDINGS`](/sql-statements/sql-statement-show-bindings.md) - - [`SHOW BUILTINS`](/sql-statements/sql-statement-show-builtins.md) - - [`SHOW CHARACTER SET`](/sql-statements/sql-statement-show-character-set.md) - - [`SHOW COLLATION`](/sql-statements/sql-statement-show-collation.md) - - [`SHOW [FULL] COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md) - - [`SHOW CREATE DATABASE`](/sql-statements/sql-statement-show-create-database.md) - - [`SHOW CREATE PLACEMENT POLICY`](/sql-statements/sql-statement-show-create-placement-policy.md) - - [`SHOW CREATE RESOURCE GROUP`](/sql-statements/sql-statement-show-create-resource-group.md) - - [`SHOW CREATE SEQUENCE`](/sql-statements/sql-statement-show-create-sequence.md) - - [`SHOW CREATE TABLE`](/sql-statements/sql-statement-show-create-table.md) - - [`SHOW CREATE USER`](/sql-statements/sql-statement-show-create-user.md) - - [`SHOW DATABASES`](/sql-statements/sql-statement-show-databases.md) - - [`SHOW ENGINES`](/sql-statements/sql-statement-show-engines.md) - - [`SHOW ERRORS`](/sql-statements/sql-statement-show-errors.md) - - [`SHOW [FULL] FIELDS FROM`](/sql-statements/sql-statement-show-fields-from.md) - - [`SHOW GRANTS`](/sql-statements/sql-statement-show-grants.md) - - [`SHOW IMPORT JOB`](/sql-statements/sql-statement-show-import-job.md) - - [`SHOW INDEXES [FROM|IN]`](/sql-statements/sql-statement-show-indexes.md) - - [`SHOW MASTER STATUS`](/sql-statements/sql-statement-show-master-status.md) - - [`SHOW PLACEMENT`](/sql-statements/sql-statement-show-placement.md) - - [`SHOW PLACEMENT FOR`](/sql-statements/sql-statement-show-placement-for.md) - - [`SHOW PLACEMENT LABELS`](/sql-statements/sql-statement-show-placement-labels.md) - - [`SHOW PLUGINS`](/sql-statements/sql-statement-show-plugins.md) - - [`SHOW PRIVILEGES`](/sql-statements/sql-statement-show-privileges.md) - - [`SHOW [FULL] PROCESSSLIST`](/sql-statements/sql-statement-show-processlist.md) - - [`SHOW PROFILES`](/sql-statements/sql-statement-show-profiles.md) - - [`SHOW SCHEMAS`](/sql-statements/sql-statement-show-schemas.md) - - [`SHOW STATS_HEALTHY`](/sql-statements/sql-statement-show-stats-healthy.md) - - [`SHOW STATS_HISTOGRAMS`](/sql-statements/sql-statement-show-stats-histograms.md) - - [`SHOW STATS_LOCKED`](/sql-statements/sql-statement-show-stats-locked.md) - - [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md) - - [`SHOW STATUS`](/sql-statements/sql-statement-show-status.md) - - [`SHOW TABLE NEXT_ROW_ID`](/sql-statements/sql-statement-show-table-next-rowid.md) - - [`SHOW TABLE REGIONS`](/sql-statements/sql-statement-show-table-regions.md) - - [`SHOW TABLE STATUS`](/sql-statements/sql-statement-show-table-status.md) - - [`SHOW [FULL] TABLES`](/sql-statements/sql-statement-show-tables.md) - - [`SHOW [GLOBAL|SESSION] VARIABLES`](/sql-statements/sql-statement-show-variables.md) - - [`SHOW WARNINGS`](/sql-statements/sql-statement-show-warnings.md) - - [`SPLIT REGION`](/sql-statements/sql-statement-split-region.md) - - [`START TRANSACTION`](/sql-statements/sql-statement-start-transaction.md) - - [`TABLE`](/sql-statements/sql-statement-table.md) - - [`TRACE`](/sql-statements/sql-statement-trace.md) - - [`TRUNCATE`](/sql-statements/sql-statement-truncate.md) - - [`UNLOCK STATS`](/sql-statements/sql-statement-unlock-stats.md) - - [`UPDATE`](/sql-statements/sql-statement-update.md) - - [`USE`](/sql-statements/sql-statement-use.md) - - [`WITH`](/sql-statements/sql-statement-with.md) - - Data Types - - [Overview](/data-type-overview.md) - - [Default Values](/data-type-default-values.md) - - [Numeric Types](/data-type-numeric.md) - - [Date and Time Types](/data-type-date-and-time.md) - - [String Types](/data-type-string.md) - - [JSON Type](/data-type-json.md) - - Functions and Operators - - [Overview](/functions-and-operators/functions-and-operators-overview.md) - - [Type Conversion in Expression Evaluation](/functions-and-operators/type-conversion-in-expression-evaluation.md) - - [Operators](/functions-and-operators/operators.md) - - [Control Flow Functions](/functions-and-operators/control-flow-functions.md) - - [String Functions](/functions-and-operators/string-functions.md) - - [Numeric Functions and Operators](/functions-and-operators/numeric-functions-and-operators.md) - - [Date and Time Functions](/functions-and-operators/date-and-time-functions.md) - - [Bit Functions and Operators](/functions-and-operators/bit-functions-and-operators.md) - - [Cast Functions and Operators](/functions-and-operators/cast-functions-and-operators.md) - - [Encryption and Compression Functions](/functions-and-operators/encryption-and-compression-functions.md) - - [Locking Functions](/functions-and-operators/locking-functions.md) - - [Information Functions](/functions-and-operators/information-functions.md) - - [JSON Functions](/functions-and-operators/json-functions.md) - - [Aggregate (GROUP BY) Functions](/functions-and-operators/aggregate-group-by-functions.md) - - [GROUP BY Modifiers](/functions-and-operators/group-by-modifier.md) - - [Window Functions](/functions-and-operators/window-functions.md) - - [Miscellaneous Functions](/functions-and-operators/miscellaneous-functions.md) - - [Precision Math](/functions-and-operators/precision-math.md) - - [Set Operations](/functions-and-operators/set-operators.md) - - [Sequence Functions](/functions-and-operators/sequence-functions.md) - - [List of Expressions for Pushdown](/functions-and-operators/expressions-pushed-down.md) - - [TiDB Specific Functions](/functions-and-operators/tidb-functions.md) - - [Clustered Indexes](/clustered-indexes.md) - - [Constraints](/constraints.md) - - [Generated Columns](/generated-columns.md) - - [SQL Mode](/sql-mode.md) - - [Table Attributes](/table-attributes.md) - - Transactions - - [Overview](/transaction-overview.md) - - [Isolation Levels](/transaction-isolation-levels.md) - - [Optimistic Transactions](/optimistic-transaction.md) - - [Pessimistic Transactions](/pessimistic-transaction.md) - - [Non-Transactional DML Statements](/non-transactional-dml.md) - - [Views](/views.md) - - [Partitioning](/partitioned-table.md) - - [Temporary Tables](/temporary-tables.md) - - [Cached Tables](/cached-tables.md) - - [FOREIGN KEY Constraints](/foreign-key.md) - - Character Set and Collation - - [Overview](/character-set-and-collation.md) - - [GBK](/character-set-gbk.md) - - Read Historical Data - - Use Stale Read (Recommended) - - [Usage Scenarios of Stale Read](/stale-read.md) - - [Perform Stale Read Using `As OF TIMESTAMP`](/as-of-timestamp.md) - - [Perform Stale Read Using `tidb_read_staleness`](/tidb-read-staleness.md) - - [Perform Stale Read Using `tidb_external_ts`](/tidb-external-ts.md) - - [Use the `tidb_snapshot` System Variable](/read-historical-data.md) - - [Placement Rules in SQL](/placement-rules-in-sql.md) - - System Tables - - [`mysql`](/mysql-schema.md) - - INFORMATION_SCHEMA - - [Overview](/information-schema/information-schema.md) - - [`ANALYZE_STATUS`](/information-schema/information-schema-analyze-status.md) - - [`CHECK_CONSTRAINTS`](/information-schema/information-schema-check-constraints.md) - - [`CLIENT_ERRORS_SUMMARY_BY_HOST`](/information-schema/client-errors-summary-by-host.md) - - [`CLIENT_ERRORS_SUMMARY_BY_USER`](/information-schema/client-errors-summary-by-user.md) - - [`CLIENT_ERRORS_SUMMARY_GLOBAL`](/information-schema/client-errors-summary-global.md) - - [`CHARACTER_SETS`](/information-schema/information-schema-character-sets.md) - - [`CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) - - [`COLLATIONS`](/information-schema/information-schema-collations.md) - - [`COLLATION_CHARACTER_SET_APPLICABILITY`](/information-schema/information-schema-collation-character-set-applicability.md) - - [`COLUMNS`](/information-schema/information-schema-columns.md) - - [`DATA_LOCK_WAITS`](/information-schema/information-schema-data-lock-waits.md) - - [`DDL_JOBS`](/information-schema/information-schema-ddl-jobs.md) - - [`DEADLOCKS`](/information-schema/information-schema-deadlocks.md) - - [`ENGINES`](/information-schema/information-schema-engines.md) - - [`KEYWORDS`](/information-schema/information-schema-keywords.md) - - [`KEY_COLUMN_USAGE`](/information-schema/information-schema-key-column-usage.md) - - [`MEMORY_USAGE`](/information-schema/information-schema-memory-usage.md) - - [`MEMORY_USAGE_OPS_HISTORY`](/information-schema/information-schema-memory-usage-ops-history.md) - - [`PARTITIONS`](/information-schema/information-schema-partitions.md) - - [`PLACEMENT_POLICIES`](/information-schema/information-schema-placement-policies.md) - - [`PROCESSLIST`](/information-schema/information-schema-processlist.md) - - [`REFERENTIAL_CONSTRAINTS`](/information-schema/information-schema-referential-constraints.md) - - [`RESOURCE_GROUPS`](/information-schema/information-schema-resource-groups.md) - - [`RUNAWAY_WATCHES`](/information-schema/information-schema-runaway-watches.md) - - [`SCHEMATA`](/information-schema/information-schema-schemata.md) - - [`SEQUENCES`](/information-schema/information-schema-sequences.md) - - [`SESSION_VARIABLES`](/information-schema/information-schema-session-variables.md) - - [`SLOW_QUERY`](/information-schema/information-schema-slow-query.md) - - [`STATISTICS`](/information-schema/information-schema-statistics.md) - - [`TABLES`](/information-schema/information-schema-tables.md) - - [`TABLE_CONSTRAINTS`](/information-schema/information-schema-table-constraints.md) - - [`TABLE_STORAGE_STATS`](/information-schema/information-schema-table-storage-stats.md) - - [`TIDB_HOT_REGIONS_HISTORY`](/information-schema/information-schema-tidb-hot-regions-history.md) - - [`TIDB_INDEXES`](/information-schema/information-schema-tidb-indexes.md) - - [`TIDB_INDEX_USAGE`](/information-schema/information-schema-tidb-index-usage.md) - - [`TIDB_SERVERS_INFO`](/information-schema/information-schema-tidb-servers-info.md) - - [`TIDB_TRX`](/information-schema/information-schema-tidb-trx.md) - - [`TIFLASH_REPLICA`](/information-schema/information-schema-tiflash-replica.md) - - [`TIFLASH_SEGMENTS`](/information-schema/information-schema-tiflash-segments.md) - - [`TIFLASH_TABLES`](/information-schema/information-schema-tiflash-tables.md) - - [`TIKV_REGION_PEERS`](/information-schema/information-schema-tikv-region-peers.md) - - [`TIKV_REGION_STATUS`](/information-schema/information-schema-tikv-region-status.md) - - [`TIKV_STORE_STATUS`](/information-schema/information-schema-tikv-store-status.md) - - [`USER_ATTRIBUTES`](/information-schema/information-schema-user-attributes.md) - - [`USER_PRIVILEGES`](/information-schema/information-schema-user-privileges.md) - - [`VARIABLES_INFO`](/information-schema/information-schema-variables-info.md) - - [`VIEWS`](/information-schema/information-schema-views.md) - - PERFORMANCE_SCHEMA - - [Overview](/performance-schema/performance-schema.md) - - [`SESSION_CONNECT_ATTRS`](/performance-schema/performance-schema-session-connect-attrs.md) - - SYS - - [Overview](/sys-schema/sys-schema.md) - - [`schema_unused_indexes`](/sys-schema/sys-schema-unused-indexes.md) - - [Metadata Lock](/metadata-lock.md) - - [Use UUIDs](/best-practices/uuid.md) - - [TiDB Accelerated Table Creation](/accelerated-table-creation.md) - - [System Variables](/system-variables.md) - - [Server Status Variables](/status-variables.md) - - Storage Engines - - TiKV - - [TiKV Overview](/tikv-overview.md) - - [RocksDB Overview](/storage-engine/rocksdb-overview.md) - - TiFlash - - [TiFlash Overview](/tiflash/tiflash-overview.md) - - [Spill to Disk](/tiflash/tiflash-spill-disk.md) - - CLI - - [Overview](/tidb-cloud/cli-reference.md) - - cluster - - [create](/tidb-cloud/ticloud-cluster-create.md) - - [delete](/tidb-cloud/ticloud-cluster-delete.md) - - [describe](/tidb-cloud/ticloud-cluster-describe.md) - - [list](/tidb-cloud/ticloud-cluster-list.md) - - [connect-info](/tidb-cloud/ticloud-cluster-connect-info.md) + - [`ALTER USER`](/sql-statements/sql-statement-alter-user.md) + - [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) + - [`BACKUP`](/sql-statements/sql-statement-backup.md) + - [`BATCH`](/sql-statements/sql-statement-batch.md) + - [`BEGIN`](/sql-statements/sql-statement-begin.md) + - [`CANCEL IMPORT JOB`](/sql-statements/sql-statement-cancel-import-job.md) + - [`COMMIT`](/sql-statements/sql-statement-commit.md) + - [`CREATE [GLOBAL|SESSION] BINDING`](/sql-statements/sql-statement-create-binding.md) + - [`CREATE DATABASE`](/sql-statements/sql-statement-create-database.md) + - [`CREATE INDEX`](/sql-statements/sql-statement-create-index.md) + - [`CREATE PLACEMENT POLICY`](/sql-statements/sql-statement-create-placement-policy.md) + - [`CREATE RESOURCE GROUP`](/sql-statements/sql-statement-create-resource-group.md) + - [`CREATE ROLE`](/sql-statements/sql-statement-create-role.md) + - [`CREATE SEQUENCE`](/sql-statements/sql-statement-create-sequence.md) + - [`CREATE TABLE LIKE`](/sql-statements/sql-statement-create-table-like.md) + - [`CREATE TABLE`](/sql-statements/sql-statement-create-table.md) + - [`CREATE USER`](/sql-statements/sql-statement-create-user.md) + - [`CREATE VIEW`](/sql-statements/sql-statement-create-view.md) + - [`DEALLOCATE`](/sql-statements/sql-statement-deallocate.md) + - [`DELETE`](/sql-statements/sql-statement-delete.md) + - [`DESC`](/sql-statements/sql-statement-desc.md) + - [`DESCRIBE`](/sql-statements/sql-statement-describe.md) + - [`DO`](/sql-statements/sql-statement-do.md) + - [`DROP [GLOBAL|SESSION] BINDING`](/sql-statements/sql-statement-drop-binding.md) + - [`DROP DATABASE`](/sql-statements/sql-statement-drop-database.md) + - [`DROP PLACEMENT POLICY`](/sql-statements/sql-statement-drop-placement-policy.md) + - [`DROP RESOURCE GROUP`](/sql-statements/sql-statement-drop-resource-group.md) + - [`DROP ROLE`](/sql-statements/sql-statement-drop-role.md) + - [`DROP SEQUENCE`](/sql-statements/sql-statement-drop-sequence.md) + - [`DROP STATS`](/sql-statements/sql-statement-drop-stats.md) + - [`DROP TABLE`](/sql-statements/sql-statement-drop-table.md) + - [`DROP USER`](/sql-statements/sql-statement-drop-user.md) + - [`DROP VIEW`](/sql-statements/sql-statement-drop-view.md) + - [`EXECUTE`](/sql-statements/sql-statement-execute.md) + - [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) + - [`EXPLAIN`](/sql-statements/sql-statement-explain.md) + - [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md) + - [`FLASHBACK DATABASE`](/sql-statements/sql-statement-flashback-database.md) + - [`FLASHBACK TABLE`](/sql-statements/sql-statement-flashback-table.md) + - [`FLUSH PRIVILEGES`](/sql-statements/sql-statement-flush-privileges.md) + - [`FLUSH STATUS`](/sql-statements/sql-statement-flush-status.md) + - [`FLUSH TABLES`](/sql-statements/sql-statement-flush-tables.md) + - [`GRANT `](/sql-statements/sql-statement-grant-privileges.md) + - [`GRANT `](/sql-statements/sql-statement-grant-role.md) + - [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) + - [`INSERT`](/sql-statements/sql-statement-insert.md) + - [`KILL [TIDB]`](/sql-statements/sql-statement-kill.md) + - [`LOAD DATA`](/sql-statements/sql-statement-load-data.md) + - [`LOAD STATS`](/sql-statements/sql-statement-load-stats.md) + - [`LOCK STATS`](/sql-statements/sql-statement-lock-stats.md) + - [`LOCK TABLES` and `UNLOCK TABLES`](/sql-statements/sql-statement-lock-tables-and-unlock-tables.md) + - [`PREPARE`](/sql-statements/sql-statement-prepare.md) + - [`QUERY WATCH`](/sql-statements/sql-statement-query-watch.md) + - [`RECOVER TABLE`](/sql-statements/sql-statement-recover-table.md) + - [`RENAME TABLE`](/sql-statements/sql-statement-rename-table.md) + - [`RENAME USER`](/sql-statements/sql-statement-rename-user.md) + - [`REPLACE`](/sql-statements/sql-statement-replace.md) + - [`RESTORE`](/sql-statements/sql-statement-restore.md) + - [`REVOKE `](/sql-statements/sql-statement-revoke-privileges.md) + - [`REVOKE `](/sql-statements/sql-statement-revoke-role.md) + - [`ROLLBACK`](/sql-statements/sql-statement-rollback.md) + - [`SAVEPOINT`](/sql-statements/sql-statement-savepoint.md) + - [`SELECT`](/sql-statements/sql-statement-select.md) + - [`SET DEFAULT ROLE`](/sql-statements/sql-statement-set-default-role.md) + - [`SET [NAMES|CHARACTER SET]`](/sql-statements/sql-statement-set-names.md) + - [`SET PASSWORD`](/sql-statements/sql-statement-set-password.md) + - [`SET RESOURCE GROUP`](/sql-statements/sql-statement-set-resource-group.md) + - [`SET ROLE`](/sql-statements/sql-statement-set-role.md) + - [`SET TRANSACTION`](/sql-statements/sql-statement-set-transaction.md) + - [`SET [GLOBAL|SESSION] `](/sql-statements/sql-statement-set-variable.md) + - [`SHOW ANALYZE STATUS`](/sql-statements/sql-statement-show-analyze-status.md) + - [`SHOW [BACKUPS|RESTORES]`](/sql-statements/sql-statement-show-backups.md) + - [`SHOW [GLOBAL|SESSION] BINDINGS`](/sql-statements/sql-statement-show-bindings.md) + - [`SHOW BUILTINS`](/sql-statements/sql-statement-show-builtins.md) + - [`SHOW CHARACTER SET`](/sql-statements/sql-statement-show-character-set.md) + - [`SHOW COLLATION`](/sql-statements/sql-statement-show-collation.md) + - [`SHOW COLUMN_STATS_USAGE`](/sql-statements/sql-statement-show-column-stats-usage.md) + - [`SHOW COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md) + - [`SHOW CREATE DATABASE`](/sql-statements/sql-statement-show-create-database.md) + - [`SHOW CREATE PLACEMENT POLICY`](/sql-statements/sql-statement-show-create-placement-policy.md) + - [`SHOW CREATE RESOURCE GROUP`](/sql-statements/sql-statement-show-create-resource-group.md) + - [`SHOW CREATE SEQUENCE`](/sql-statements/sql-statement-show-create-sequence.md) + - [`SHOW CREATE TABLE`](/sql-statements/sql-statement-show-create-table.md) + - [`SHOW CREATE USER`](/sql-statements/sql-statement-show-create-user.md) + - [`SHOW DATABASES`](/sql-statements/sql-statement-show-databases.md) + - [`SHOW ENGINES`](/sql-statements/sql-statement-show-engines.md) + - [`SHOW ERRORS`](/sql-statements/sql-statement-show-errors.md) + - [`SHOW FIELDS FROM`](/sql-statements/sql-statement-show-fields-from.md) + - [`SHOW GRANTS`](/sql-statements/sql-statement-show-grants.md) + - [`SHOW IMPORT JOB`](/sql-statements/sql-statement-show-import-job.md) + - [`SHOW INDEXES [FROM|IN]`](/sql-statements/sql-statement-show-indexes.md) + - [`SHOW MASTER STATUS`](/sql-statements/sql-statement-show-master-status.md) + - [`SHOW PLACEMENT`](/sql-statements/sql-statement-show-placement.md) + - [`SHOW PLACEMENT FOR`](/sql-statements/sql-statement-show-placement-for.md) + - [`SHOW PLACEMENT LABELS`](/sql-statements/sql-statement-show-placement-labels.md) + - [`SHOW PLUGINS`](/sql-statements/sql-statement-show-plugins.md) + - [`SHOW PRIVILEGES`](/sql-statements/sql-statement-show-privileges.md) + - [`SHOW PROCESSLIST`](/sql-statements/sql-statement-show-processlist.md) + - [`SHOW PROFILES`](/sql-statements/sql-statement-show-profiles.md) + - [`SHOW SCHEMAS`](/sql-statements/sql-statement-show-schemas.md) + - [`SHOW STATS_BUCKETS`](/sql-statements/sql-statement-show-stats-buckets.md) + - [`SHOW STATS_HEALTHY`](/sql-statements/sql-statement-show-stats-healthy.md) + - [`SHOW STATS_HISTOGRAMS`](/sql-statements/sql-statement-show-stats-histograms.md) + - [`SHOW STATS_LOCKED`](/sql-statements/sql-statement-show-stats-locked.md) + - [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md) + - [`SHOW STATS_TOPN`](/sql-statements/sql-statement-show-stats-topn.md) + - [`SHOW STATUS`](/sql-statements/sql-statement-show-status.md) + - [`SHOW TABLE NEXT_ROW_ID`](/sql-statements/sql-statement-show-table-next-rowid.md) + - [`SHOW TABLE REGIONS`](/sql-statements/sql-statement-show-table-regions.md) + - [`SHOW TABLE STATUS`](/sql-statements/sql-statement-show-table-status.md) + - [`SHOW TABLES`](/sql-statements/sql-statement-show-tables.md) + - [`SHOW [GLOBAL|SESSION] VARIABLES`](/sql-statements/sql-statement-show-variables.md) + - [`SHOW WARNINGS`](/sql-statements/sql-statement-show-warnings.md) + - [`SPLIT REGION`](/sql-statements/sql-statement-split-region.md) + - [`START TRANSACTION`](/sql-statements/sql-statement-start-transaction.md) + - [`TABLE`](/sql-statements/sql-statement-table.md) + - [`TRACE`](/sql-statements/sql-statement-trace.md) + - [`TRUNCATE`](/sql-statements/sql-statement-truncate.md) + - [`UNLOCK STATS`](/sql-statements/sql-statement-unlock-stats.md) + - [`UPDATE`](/sql-statements/sql-statement-update.md) + - [`USE`](/sql-statements/sql-statement-use.md) + - [`WITH`](/sql-statements/sql-statement-with.md) + - Data Types + - [Overview](/data-type-overview.md) + - [Default Values](/data-type-default-values.md) + - [Numeric Types](/data-type-numeric.md) + - [Date and Time Types](/data-type-date-and-time.md) + - [String Types](/data-type-string.md) + - [JSON Type](/data-type-json.md) + - Functions and Operators + - [Overview](/functions-and-operators/functions-and-operators-overview.md) + - [Type Conversion in Expression Evaluation](/functions-and-operators/type-conversion-in-expression-evaluation.md) + - [Operators](/functions-and-operators/operators.md) + - [Control Flow Functions](/functions-and-operators/control-flow-functions.md) + - [String Functions](/functions-and-operators/string-functions.md) + - [Numeric Functions and Operators](/functions-and-operators/numeric-functions-and-operators.md) + - [Date and Time Functions](/functions-and-operators/date-and-time-functions.md) + - [Bit Functions and Operators](/functions-and-operators/bit-functions-and-operators.md) + - [Cast Functions and Operators](/functions-and-operators/cast-functions-and-operators.md) + - [Encryption and Compression Functions](/functions-and-operators/encryption-and-compression-functions.md) + - [Locking Functions](/functions-and-operators/locking-functions.md) + - [Information Functions](/functions-and-operators/information-functions.md) + - JSON Functions + - [Overview](/functions-and-operators/json-functions.md) + - [Functions That Create JSON](/functions-and-operators/json-functions/json-functions-create.md) + - [Functions That Search JSON](/functions-and-operators/json-functions/json-functions-search.md) + - [Functions That Modify JSON](/functions-and-operators/json-functions/json-functions-modify.md) + - [Functions That Return JSON](/functions-and-operators/json-functions/json-functions-return.md) + - [JSON Utility Functions](/functions-and-operators/json-functions/json-functions-utility.md) + - [Functions That Aggregate JSON](/functions-and-operators/json-functions/json-functions-aggregate.md) + - [Aggregate (GROUP BY) Functions](/functions-and-operators/aggregate-group-by-functions.md) + - [GROUP BY Modifiers](/functions-and-operators/group-by-modifier.md) + - [Window Functions](/functions-and-operators/window-functions.md) + - [Miscellaneous Functions](/functions-and-operators/miscellaneous-functions.md) + - [Precision Math](/functions-and-operators/precision-math.md) + - [Set Operations](/functions-and-operators/set-operators.md) + - [Sequence Functions](/functions-and-operators/sequence-functions.md) + - [List of Expressions for Pushdown](/functions-and-operators/expressions-pushed-down.md) + - [TiDB Specific Functions](/functions-and-operators/tidb-functions.md) + - [Clustered Indexes](/clustered-indexes.md) + - [Constraints](/constraints.md) + - [Generated Columns](/generated-columns.md) + - [SQL Mode](/sql-mode.md) + - [Table Attributes](/table-attributes.md) + - Transactions + - [Overview](/transaction-overview.md) + - [Isolation Levels](/transaction-isolation-levels.md) + - [Optimistic Transactions](/optimistic-transaction.md) + - [Pessimistic Transactions](/pessimistic-transaction.md) + - [Non-Transactional DML Statements](/non-transactional-dml.md) + - [Views](/views.md) + - [Partitioning](/partitioned-table.md) + - [Temporary Tables](/temporary-tables.md) + - [Cached Tables](/cached-tables.md) + - [FOREIGN KEY Constraints](/foreign-key.md) + - Character Set and Collation + - [Overview](/character-set-and-collation.md) + - [GBK](/character-set-gbk.md) + - Read Historical Data + - Use Stale Read (Recommended) + - [Usage Scenarios of Stale Read](/stale-read.md) + - [Perform Stale Read Using `As OF TIMESTAMP`](/as-of-timestamp.md) + - [Perform Stale Read Using `tidb_read_staleness`](/tidb-read-staleness.md) + - [Perform Stale Read Using `tidb_external_ts`](/tidb-external-ts.md) + - [Use the `tidb_snapshot` System Variable](/read-historical-data.md) + - [Placement Rules in SQL](/placement-rules-in-sql.md) + - System Tables + - `mysql` Schema + - [Overview](/mysql-schema/mysql-schema.md) + - [`user`](/mysql-schema/mysql-schema-user.md) + - INFORMATION_SCHEMA + - [Overview](/information-schema/information-schema.md) + - [`ANALYZE_STATUS`](/information-schema/information-schema-analyze-status.md) + - [`CHECK_CONSTRAINTS`](/information-schema/information-schema-check-constraints.md) + - [`CLIENT_ERRORS_SUMMARY_BY_HOST`](/information-schema/client-errors-summary-by-host.md) + - [`CLIENT_ERRORS_SUMMARY_BY_USER`](/information-schema/client-errors-summary-by-user.md) + - [`CLIENT_ERRORS_SUMMARY_GLOBAL`](/information-schema/client-errors-summary-global.md) + - [`CHARACTER_SETS`](/information-schema/information-schema-character-sets.md) + - [`CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) + - [`COLLATIONS`](/information-schema/information-schema-collations.md) + - [`COLLATION_CHARACTER_SET_APPLICABILITY`](/information-schema/information-schema-collation-character-set-applicability.md) + - [`COLUMNS`](/information-schema/information-schema-columns.md) + - [`DATA_LOCK_WAITS`](/information-schema/information-schema-data-lock-waits.md) + - [`DDL_JOBS`](/information-schema/information-schema-ddl-jobs.md) + - [`DEADLOCKS`](/information-schema/information-schema-deadlocks.md) + - [`ENGINES`](/information-schema/information-schema-engines.md) + - [`KEYWORDS`](/information-schema/information-schema-keywords.md) + - [`KEY_COLUMN_USAGE`](/information-schema/information-schema-key-column-usage.md) + - [`MEMORY_USAGE`](/information-schema/information-schema-memory-usage.md) + - [`MEMORY_USAGE_OPS_HISTORY`](/information-schema/information-schema-memory-usage-ops-history.md) + - [`PARTITIONS`](/information-schema/information-schema-partitions.md) + - [`PLACEMENT_POLICIES`](/information-schema/information-schema-placement-policies.md) + - [`PROCESSLIST`](/information-schema/information-schema-processlist.md) + - [`REFERENTIAL_CONSTRAINTS`](/information-schema/information-schema-referential-constraints.md) + - [`RESOURCE_GROUPS`](/information-schema/information-schema-resource-groups.md) + - [`RUNAWAY_WATCHES`](/information-schema/information-schema-runaway-watches.md) + - [`SCHEMATA`](/information-schema/information-schema-schemata.md) + - [`SEQUENCES`](/information-schema/information-schema-sequences.md) + - [`SESSION_VARIABLES`](/information-schema/information-schema-session-variables.md) + - [`SLOW_QUERY`](/information-schema/information-schema-slow-query.md) + - [`STATISTICS`](/information-schema/information-schema-statistics.md) + - [`TABLES`](/information-schema/information-schema-tables.md) + - [`TABLE_CONSTRAINTS`](/information-schema/information-schema-table-constraints.md) + - [`TABLE_STORAGE_STATS`](/information-schema/information-schema-table-storage-stats.md) + - [`TIDB_CHECK_CONSTRAINTS`](/information-schema/information-schema-tidb-check-constraints.md) + - [`TIDB_HOT_REGIONS_HISTORY`](/information-schema/information-schema-tidb-hot-regions-history.md) + - [`TIDB_INDEXES`](/information-schema/information-schema-tidb-indexes.md) + - [`TIDB_INDEX_USAGE`](/information-schema/information-schema-tidb-index-usage.md) + - [`TIDB_SERVERS_INFO`](/information-schema/information-schema-tidb-servers-info.md) + - [`TIDB_TRX`](/information-schema/information-schema-tidb-trx.md) + - [`TIFLASH_REPLICA`](/information-schema/information-schema-tiflash-replica.md) + - [`TIFLASH_SEGMENTS`](/information-schema/information-schema-tiflash-segments.md) + - [`TIFLASH_TABLES`](/information-schema/information-schema-tiflash-tables.md) + - [`TIKV_REGION_PEERS`](/information-schema/information-schema-tikv-region-peers.md) + - [`TIKV_REGION_STATUS`](/information-schema/information-schema-tikv-region-status.md) + - [`TIKV_STORE_STATUS`](/information-schema/information-schema-tikv-store-status.md) + - [`USER_ATTRIBUTES`](/information-schema/information-schema-user-attributes.md) + - [`USER_PRIVILEGES`](/information-schema/information-schema-user-privileges.md) + - [`VARIABLES_INFO`](/information-schema/information-schema-variables-info.md) + - [`VIEWS`](/information-schema/information-schema-views.md) + - PERFORMANCE_SCHEMA + - [Overview](/performance-schema/performance-schema.md) + - [`SESSION_CONNECT_ATTRS`](/performance-schema/performance-schema-session-connect-attrs.md) + - SYS + - [Overview](/sys-schema/sys-schema.md) + - [`schema_unused_indexes`](/sys-schema/sys-schema-unused-indexes.md) + - [Metadata Lock](/metadata-lock.md) + - [Use UUIDs](/best-practices/uuid.md) + - [TiDB Accelerated Table Creation](/accelerated-table-creation.md) +- API Reference ![BETA](/media/tidb-cloud/blank_transparent_placeholder.png) + - [Overview](/tidb-cloud/api-overview.md) + - v1beta1 + - [Billing](https://docs.pingcap.com/tidbcloud/api/v1beta1/billing) + - [Data Service](https://docs.pingcap.com/tidbcloud/api/v1beta1/dataservice) + - [IAM](https://docs.pingcap.com/tidbcloud/api/v1beta1/iam) + - [MSP (Deprecated)](https://docs.pingcap.com/tidbcloud/api/v1beta1/msp) + - [v1beta](https://docs.pingcap.com/tidbcloud/api/v1beta) +- CLI Reference ![BETA](/media/tidb-cloud/blank_transparent_placeholder.png) + - [Overview](/tidb-cloud/cli-reference.md) + - auth + - [login](/tidb-cloud/ticloud-auth-login.md) + - [logout](/tidb-cloud/ticloud-auth-logout.md) + - [whoami](/tidb-cloud/ticloud-auth-whoami.md) + - serverless + - [create](/tidb-cloud/ticloud-cluster-create.md) + - [delete](/tidb-cloud/ticloud-cluster-delete.md) + - [describe](/tidb-cloud/ticloud-cluster-describe.md) + - [list](/tidb-cloud/ticloud-cluster-list.md) + - [update](/tidb-cloud/ticloud-serverless-update.md) + - [spending-limit](/tidb-cloud/ticloud-serverless-spending-limit.md) + - [region](/tidb-cloud/ticloud-serverless-region.md) + - [shell](/tidb-cloud/ticloud-serverless-shell.md) - branch - [create](/tidb-cloud/ticloud-branch-create.md) - [delete](/tidb-cloud/ticloud-branch-delete.md) - [describe](/tidb-cloud/ticloud-branch-describe.md) - [list](/tidb-cloud/ticloud-branch-list.md) - - [connect-info](/tidb-cloud/ticloud-branch-connect-info.md) - - config - - [create](/tidb-cloud/ticloud-config-create.md) - - [delete](/tidb-cloud/ticloud-config-delete.md) - - [describe](/tidb-cloud/ticloud-config-describe.md) - - [edit](/tidb-cloud/ticloud-config-edit.md) - - [list](/tidb-cloud/ticloud-config-list.md) - - [set](/tidb-cloud/ticloud-config-set.md) - - [use](/tidb-cloud/ticloud-config-use.md) - - [connect](/tidb-cloud/ticloud-connect.md) + - [shell](/tidb-cloud/ticloud-branch-shell.md) - import - [cancel](/tidb-cloud/ticloud-import-cancel.md) - [describe](/tidb-cloud/ticloud-import-describe.md) - [list](/tidb-cloud/ticloud-import-list.md) - - start - - [local](/tidb-cloud/ticloud-import-start-local.md) - - [s3](/tidb-cloud/ticloud-import-start-s3.md) - - [mysql](/tidb-cloud/ticloud-import-start-mysql.md) - - project - - [list](/tidb-cloud/ticloud-project-list.md) - - [update](/tidb-cloud/ticloud-update.md) + - [start](/tidb-cloud/ticloud-import-start.md) + - export + - [create](/tidb-cloud/ticloud-serverless-export-create.md) + - [describe](/tidb-cloud/ticloud-serverless-export-describe.md) + - [list](/tidb-cloud/ticloud-serverless-export-list.md) + - [cancel](/tidb-cloud/ticloud-serverless-export-cancel.md) + - [download](/tidb-cloud/ticloud-serverless-export-download.md) + - sql-user + - [create](/tidb-cloud/ticloud-serverless-sql-user-create.md) + - [delete](/tidb-cloud/ticloud-serverless-sql-user-delete.md) + - [list](/tidb-cloud/ticloud-serverless-sql-user-list.md) + - [update](/tidb-cloud/ticloud-serverless-sql-user-update.md) + - authorized-network + - [create](/tidb-cloud/ticloud-serverless-authorized-network-create.md) + - [delete](/tidb-cloud/ticloud-serverless-authorized-network-delete.md) + - [list](/tidb-cloud/ticloud-serverless-authorized-network-list.md) + - [update](/tidb-cloud/ticloud-serverless-authorized-network-update.md) + - [ai](/tidb-cloud/ticloud-ai.md) + - [completion](/tidb-cloud/ticloud-completion.md) + - config + - [create](/tidb-cloud/ticloud-config-create.md) + - [delete](/tidb-cloud/ticloud-config-delete.md) + - [describe](/tidb-cloud/ticloud-config-describe.md) + - [edit](/tidb-cloud/ticloud-config-edit.md) + - [list](/tidb-cloud/ticloud-config-list.md) + - [set](/tidb-cloud/ticloud-config-set.md) + - [use](/tidb-cloud/ticloud-config-use.md) + - project + - [list](/tidb-cloud/ticloud-project-list.md) + - [upgrade](/tidb-cloud/ticloud-upgrade.md) + - [help](/tidb-cloud/ticloud-help.md) +- General Reference + - TiDB Cluster Architecture + - [Overview](/tidb-architecture.md) + - [Storage](/tidb-storage.md) + - [Computing](/tidb-computing.md) + - [Scheduling](/tidb-scheduling.md) + - [TSO](/tso.md) + - Storage Engines + - TiKV + - [TiKV Overview](/tikv-overview.md) + - [RocksDB Overview](/storage-engine/rocksdb-overview.md) + - TiFlash + - [TiFlash Overview](/tiflash/tiflash-overview.md) + - [Spill to Disk](/tiflash/tiflash-spill-disk.md) + - TiDB Cloud Partner Web Console + - [TiDB Cloud Partners](/tidb-cloud/tidb-cloud-partners.md) + - [MSP Customer](/tidb-cloud/managed-service-provider-customer.md) + - [Reseller's Customer](/tidb-cloud/cppo-customer.md) + - TiDB Distributed eXecution Framework (DXF) + - [Introduction](/tidb-distributed-execution-framework.md) + - [TiDB Global Sort](/tidb-global-sort.md) + - [TiDB Cloud Dedicated Limitations and Quotas](/tidb-cloud/limitations-and-quotas.md) + - [TiDB Cloud Serverless Limitations](/tidb-cloud/serverless-limitations.md) + - [Limited SQL Features on TiDB Cloud](/tidb-cloud/limited-sql-features.md) + - [TiDB Limitations](/tidb-limitations.md) + - Benchmarks + - TiDB v8.5 + - [Performance Highlights](/tidb-cloud/v8.5-performance-highlights.md) + - [TPC-C Performance Test Report](/tidb-cloud/v8.5-performance-benchmarking-with-tpcc.md) + - [Sysbench Performance Test Report](/tidb-cloud/v8.5-performance-benchmarking-with-sysbench.md) + - TiDB v8.1 + - [TPC-C Performance Test Report](/tidb-cloud/v8.1-performance-benchmarking-with-tpcc.md) + - [Sysbench Performance Test Report](/tidb-cloud/v8.1-performance-benchmarking-with-sysbench.md) + - TiDB v7.5 + - [TPC-C Performance Test Report](/tidb-cloud/v7.5-performance-benchmarking-with-tpcc.md) + - [Sysbench Performance Test Report](/tidb-cloud/v7.5-performance-benchmarking-with-sysbench.md) + - TiDB v7.1 + - [TPC-C Performance Test Report](/tidb-cloud/v7.1-performance-benchmarking-with-tpcc.md) + - [Sysbench Performance Test Report](/tidb-cloud/v7.1-performance-benchmarking-with-sysbench.md) + - TiDB v6.5 + - [TPC-C Performance Test Report](/tidb-cloud/v6.5-performance-benchmarking-with-tpcc.md) + - [Sysbench Performance Test Report](/tidb-cloud/v6.5-performance-benchmarking-with-sysbench.md) + - [System Variables](/system-variables.md) + - [Server Status Variables](/status-variables.md) - [Table Filter](/table-filter.md) - - [Resource Control](/tidb-resource-control.md) + - Optimize Resource Allocation + - [Overview of Resource Allocation](/tidb-cloud/optimize-resource-allocation.md) + - [Resource Control](/tidb-resource-control.md) + - TiDB Node Group + - [Overview of TiDB Node Group](/tidb-cloud/tidb-node-group-overview.md) + - [Manage TiDB Node Groups](/tidb-cloud/tidb-node-group-management.md) - [URI Formats of External Storage Services](/external-storage-uri.md) - [DDL Execution Principles and Best Practices](/ddl-introduction.md) - [Troubleshoot Inconsistency Between Data and Indexes](/troubleshoot-data-inconsistency-errors.md) - - [Support](/tidb-cloud/tidb-cloud-support.md) + - [Notifications](/tidb-cloud/notifications.md) - [Glossary](/tidb-cloud/tidb-cloud-glossary.md) +- Support Plan + - [Connected Care Overview](/tidb-cloud/connected-care-overview.md) + - [Connected Care Details](/tidb-cloud/connected-care-detail.md) + - Connected Care Support Service Features + - [Connected: Clinic Service](/tidb-cloud/tidb-cloud-clinic.md) + - [Connected: AI Chat in IM](/tidb-cloud/connected-ai-chat-in-im.md) + - Connected: IM Subscription for TiDB Cloud Alerts + - [Subscribe via Slack](/tidb-cloud/monitor-alert-slack.md) + - [Subscribe via Zoom](/tidb-cloud/monitor-alert-zoom.md) + - Connected: IM Ticket Creation and Update Subscription + - [Create Tickets and Subscribe to Ticket Updates via Slack](/tidb-cloud/connected-slack-ticket-creation.md) + - [Create Tickets and Subscribe to Ticket Updates via Lark](/tidb-cloud/connected-lark-ticket-creation.md) + - Connected: IM Interaction for Support Tickets + - [Interact with Support Tickets via Slack](/tidb-cloud/connected-slack-ticket-interaction.md) + - [Interact with Support Tickets via Lark](/tidb-cloud/connected-lark-ticket-interaction.md) + - [Get Support](/tidb-cloud/tidb-cloud-support.md) - FAQs - [TiDB Cloud FAQs](/tidb-cloud/tidb-cloud-faq.md) - - [TiDB Serverless FAQs](/tidb-cloud/serverless-faqs.md) + - [TiDB Cloud Serverless FAQs](/tidb-cloud/serverless-faqs.md) + +## RELEASES + - Release Notes - - [2023](/tidb-cloud/tidb-cloud-release-notes.md) + - [2025](/tidb-cloud/tidb-cloud-release-notes.md) + - [2024](/tidb-cloud/release-notes-2024.md) + - [2023](/tidb-cloud/release-notes-2023.md) - [2022](/tidb-cloud/release-notes-2022.md) - [2021](/tidb-cloud/release-notes-2021.md) - [2020](/tidb-cloud/release-notes-2020.md) - Maintenance Notification - - [[2023-11-14] TiDB Dedicated Scale Feature Maintenance Notification](/tidb-cloud/notification-2023-11-14-scale-feature-maintenance.md) + - [[2024-09-15] TiDB Cloud Console Maintenance Notification](/tidb-cloud/notification-2024-09-15-console-maintenance.md) + - [[2024-04-18] TiDB Cloud Data Migration (DM) Feature Maintenance Notification](/tidb-cloud/notification-2024-04-18-dm-feature-maintenance.md) + - [[2024-04-16] TiDB Cloud Monitoring Features Maintenance Notification](/tidb-cloud/notification-2024-04-16-monitoring-features-maintenance.md) + - [[2024-04-11] TiDB Cloud Data Migration (DM) Feature Maintenance Notification](/tidb-cloud/notification-2024-04-11-dm-feature-maintenance.md) + - [[2024-04-09] TiDB Cloud Monitoring Features Maintenance Notification](/tidb-cloud/notification-2024-04-09-monitoring-features-maintenance.md) + - [[2023-11-14] TiDB Cloud Dedicated Scale Feature Maintenance Notification](/tidb-cloud/notification-2023-11-14-scale-feature-maintenance.md) - [[2023-09-26] TiDB Cloud Console Maintenance Notification](/tidb-cloud/notification-2023-09-26-console-maintenance.md) - [[2023-08-31] TiDB Cloud Console Maintenance Notification](/tidb-cloud/notification-2023-08-31-console-maintenance.md) diff --git a/TOC.md b/TOC.md index 627b2bbe4e4fb..525efcc86a9dc 100644 --- a/TOC.md +++ b/TOC.md @@ -2,14 +2,14 @@ - [Docs Home](https://docs.pingcap.com/) -- About TiDB - - [TiDB Introduction](/overview.md) - - [TiDB 8.0 Release Notes](/releases/release-8.0.0.md) +- About TiDB Self-Managed + - [What is TiDB Self-Managed](/overview.md) + - [TiDB 8.1 Release Notes](/releases/release-8.1.0.md) - [Features](/basic-features.md) - [MySQL Compatibility](/mysql-compatibility.md) - [TiDB Limitations](/tidb-limitations.md) - [Credits](/credits.md) - - [Roadmap](/tidb-roadmap.md) + - [Roadmap](https://docs.pingcap.com/tidb/dev/tidb-roadmap) - Quick Start - [Try Out TiDB](/quick-start-with-tidb.md) - [Try Out HTAP](/quick-start-with-htap.md) @@ -19,7 +19,7 @@ - Develop - [Overview](/develop/dev-guide-overview.md) - Quick Start - - [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md) + - [Build a TiDB Cloud Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md) - [CRUD SQL in TiDB](/develop/dev-guide-tidb-crud-sql.md) - Example Applications - Java @@ -157,10 +157,19 @@ - [Integrate with Confluent and Snowflake](/ticdc/integrate-confluent-using-ticdc.md) - [Integrate with Apache Kafka and Apache Flink](/replicate-data-to-kafka.md) - Maintain + - Security + - [Best Practices for TiDB Security Configuration](/best-practices-for-security-configuration.md) + - [Enable TLS Between TiDB Clients and Servers](/enable-tls-between-clients-and-servers.md) + - [Enable TLS Between TiDB Components](/enable-tls-between-components.md) + - [Generate Self-signed Certificates](/generate-self-signed-certificates.md) + - [Encryption at Rest](/encryption-at-rest.md) + - [Enable Encryption for Disk Spill](/enable-disk-spill-encrypt.md) + - [Log Redaction](/log-redaction.md) - Upgrade - [Use TiUP](/upgrade-tidb-using-tiup.md) - [Use TiDB Operator](https://docs.pingcap.com/tidb-in-kubernetes/stable/upgrade-a-tidb-cluster) - [TiDB Smooth Upgrade](/smooth-upgrade-tidb.md) + - [Migrate and Upgrade a TiDB Cluster](/tidb-upgrade-migration-guide.md) - [TiFlash Upgrade Guide](/tiflash-upgrade-guide.md) - Scale - [Use TiUP (Recommended)](/scale-tidb-using-tiup.md) @@ -208,6 +217,37 @@ - [Monitoring API](/tidb-monitoring-api.md) - [Deploy Monitoring Services](/deploy-monitoring-services.md) - [Upgrade Monitoring Services](/upgrade-monitoring-services.md) + - TiDB Dashboard + - [Overview](/dashboard/dashboard-intro.md) + - Maintain + - [Deploy](/dashboard/dashboard-ops-deploy.md) + - [Reverse Proxy](/dashboard/dashboard-ops-reverse-proxy.md) + - [User Management](/dashboard/dashboard-user.md) + - [Secure](/dashboard/dashboard-ops-security.md) + - [Access](/dashboard/dashboard-access.md) + - [Overview Page](/dashboard/dashboard-overview.md) + - [Cluster Info Page](/dashboard/dashboard-cluster-info.md) + - [Top SQL Page](/dashboard/top-sql.md) + - [Key Visualizer Page](/dashboard/dashboard-key-visualizer.md) + - [Metrics Relation Graph](/dashboard/dashboard-metrics-relation.md) + - SQL Statements Analysis + - [SQL Statements Page](/dashboard/dashboard-statement-list.md) + - [SQL Details Page](/dashboard/dashboard-statement-details.md) + - [Slow Queries Page](/dashboard/dashboard-slow-query.md) + - Cluster Diagnostics + - [Access Cluster Diagnostics Page](/dashboard/dashboard-diagnostics-access.md) + - [View Diagnostics Report](/dashboard/dashboard-diagnostics-report.md) + - [Use Diagnostics](/dashboard/dashboard-diagnostics-usage.md) + - [Monitoring Page](/dashboard/dashboard-monitoring.md) + - [Search Logs Page](/dashboard/dashboard-log-search.md) + - [Resource Manager Page](/dashboard/dashboard-resource-manager.md) + - Instance Profiling + - [Manual Profiling](/dashboard/dashboard-profiling.md) + - [Continuous Profiling](/dashboard/continuous-profiling.md) + - Session Management and Configuration + - [Share Session](/dashboard/dashboard-session-share.md) + - [Configure SSO](/dashboard/dashboard-session-sso.md) + - [FAQ](/dashboard/dashboard-faq.md) - [Export Grafana Snapshots](/exporting-grafana-snapshots.md) - [TiDB Cluster Alert Rules](/alert-rules.md) - [TiFlash Alert Rules](/tiflash/tiflash-alert-rules.md) @@ -240,11 +280,14 @@ - Performance Tuning - Tuning Guide - [Performance Tuning Overview](/performance-tuning-overview.md) - - [Performance Analysis and Tuning](/performance-tuning-methods.md) - - [Performance Tuning Practices for OLTP Scenarios](/performance-tuning-practices.md) - - [TiFlash Performance Analysis Methods](/tiflash-performance-tuning-methods.md) - - [TiCDC Performance Analysis Methods](/ticdc-performance-tuning-methods.md) - - [Latency Breakdown](/latency-breakdown.md) + - [Configure TiDB for Optimal Performance](/tidb-performance-tuning-config.md) + - [A Practical Guide for SQL Tuning](/sql-tuning-best-practice.md) + - Tuning by Metrics + - [Performance Analysis and Tuning](/performance-tuning-methods.md) + - [Performance Tuning Practices for OLTP Scenarios](/performance-tuning-practices.md) + - [TiFlash Performance Analysis Methods](/tiflash-performance-tuning-methods.md) + - [TiCDC Performance Analysis Methods](/ticdc-performance-tuning-methods.md) + - [Latency Breakdown](/latency-breakdown.md) - [TiDB Best Practices on Public Cloud](/best-practices-on-public-cloud.md) - Configuration Tuning - [Tune Operating System Performance](/tune-operating-system.md) @@ -423,6 +466,7 @@ - TiUP Components - [tiup-playground](/tiup/tiup-playground.md) - [tiup-cluster](/tiup/tiup-cluster.md) + - [No-sudo Mode](/tiup/tiup-cluster-no-sudo-mode.md) - [tiup-mirror](/tiup/tiup-mirror.md) - [tiup-bench](/tiup/tiup-bench.md) - [TiDB Operator](/tidb-operator-overview.md) @@ -521,6 +565,8 @@ - [Release Notes](/dm/dm-release-notes.md) - TiDB Lightning - [Overview](/tidb-lightning/tidb-lightning-overview.md) + - [`IMPORT INTO` vs. TiDB Lightning](/tidb-lightning/import-into-vs-tidb-lightning.md) + - [Compatibility of TiDB Lightning and `IMPORT INTO` with TiCDC and Log Backup](/tidb-lightning/tidb-lightning-compatibility-and-scenarios.md) - [Get Started](/get-started-with-tidb-lightning.md) - [Deploy TiDB Lightning](/tidb-lightning/deploy-tidb-lightning.md) - [Target Database Requirements](/tidb-lightning/tidb-lightning-requirements.md) @@ -570,7 +616,7 @@ - [Bidirectional Replication](/ticdc/ticdc-bidirectional-replication.md) - [Data Integrity Validation for Single-Row Data](/ticdc/ticdc-integrity-check.md) - [Data Consistency Validation for TiDB Upstream/Downstream Clusters](/ticdc/ticdc-upstream-downstream-check.md) - - [TiCDC Behavior Changes](/ticdc/ticdc-behavior-change.md) + - [TiCDC Behavior in Splitting UPDATE Events](/ticdc/ticdc-split-update-behavior.md) - Monitor and Alert - [Monitoring Metrics Summary](/ticdc/ticdc-summary-monitor.md) - [Monitoring Metrics Details](/ticdc/monitor-ticdc.md) @@ -632,6 +678,7 @@ - [Configuration](/tiproxy/tiproxy-configuration.md) - [Command Line Parameters](/tiproxy/tiproxy-command-line-flags.md) - [Monitoring Metrics](/tiproxy/tiproxy-grafana.md) + - [API](/tiproxy/tiproxy-api.md) - [Troubleshooting](/tiproxy/troubleshoot-tiproxy.md) - [Performance Test](/tiproxy/tiproxy-performance-test.md) - Reference @@ -660,6 +707,7 @@ - [TiFlash Late Materialization](/tiflash/tiflash-late-materialization.md) - [Spill to Disk](/tiflash/tiflash-spill-disk.md) - [Data Validation](/tiflash/tiflash-data-validation.md) + - [MinTSO Scheduler](/tiflash/tiflash-mintso-scheduler.md) - [Compatibility](/tiflash/tiflash-compatibility.md) - [Pipeline Execution Model](/tiflash/tiflash-pipeline-model.md) - TiDB Distributed eXecution Framework (DXF) @@ -691,13 +739,6 @@ - [TiFlash](/tiflash/monitor-tiflash.md) - [TiCDC](/ticdc/monitor-ticdc.md) - [Resource Control](/grafana-resource-control-dashboard.md) - - Security - - [Enable TLS Between TiDB Clients and Servers](/enable-tls-between-clients-and-servers.md) - - [Enable TLS Between TiDB Components](/enable-tls-between-components.md) - - [Generate Self-signed Certificates](/generate-self-signed-certificates.md) - - [Encryption at Rest](/encryption-at-rest.md) - - [Enable Encryption for Disk Spill](/enable-disk-spill-encrypt.md) - - [Log Redaction](/log-redaction.md) - Privileges - [Security Compatibility with MySQL](/security-compatibility-with-mysql.md) - [Privilege Management](/privilege-management.md) @@ -718,6 +759,7 @@ - [Expression Syntax](/expression-syntax.md) - [Comment Syntax](/comment-syntax.md) - SQL Statements + - [Overview](/sql-statements/sql-statement-overview.md) - [`ADD COLUMN`](/sql-statements/sql-statement-add-column.md) - [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) - [`ADMIN`](/sql-statements/sql-statement-admin.md) @@ -730,7 +772,6 @@ - [`ADMIN RESUME DDL`](/sql-statements/sql-statement-admin-resume-ddl.md) - [`ADMIN [SET|SHOW|UNSET] BDR ROLE`](/sql-statements/sql-statement-admin-bdr-role.md) - [`ADMIN SHOW DDL [JOBS|JOB QUERIES]`](/sql-statements/sql-statement-admin-show-ddl.md) - - [`ADMIN SHOW TELEMETRY`](/sql-statements/sql-statement-admin-show-telemetry.md) - [`ALTER DATABASE`](/sql-statements/sql-statement-alter-database.md) - [`ALTER INDEX`](/sql-statements/sql-statement-alter-index.md) - [`ALTER INSTANCE`](/sql-statements/sql-statement-alter-instance.md) @@ -824,6 +865,7 @@ - [`SHOW BUILTINS`](/sql-statements/sql-statement-show-builtins.md) - [`SHOW CHARACTER SET`](/sql-statements/sql-statement-show-character-set.md) - [`SHOW COLLATION`](/sql-statements/sql-statement-show-collation.md) + - [`SHOW COLUMN_STATS_USAGE`](/sql-statements/sql-statement-show-column-stats-usage.md) - [`SHOW COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md) - [`SHOW CONFIG`](/sql-statements/sql-statement-show-config.md) - [`SHOW CREATE DATABASE`](/sql-statements/sql-statement-show-create-database.md) @@ -846,14 +888,16 @@ - [`SHOW PLACEMENT LABELS`](/sql-statements/sql-statement-show-placement-labels.md) - [`SHOW PLUGINS`](/sql-statements/sql-statement-show-plugins.md) - [`SHOW PRIVILEGES`](/sql-statements/sql-statement-show-privileges.md) - - [`SHOW PROCESSSLIST`](/sql-statements/sql-statement-show-processlist.md) + - [`SHOW PROCESSLIST`](/sql-statements/sql-statement-show-processlist.md) - [`SHOW PROFILES`](/sql-statements/sql-statement-show-profiles.md) - [`SHOW PUMP STATUS`](/sql-statements/sql-statement-show-pump-status.md) - [`SHOW SCHEMAS`](/sql-statements/sql-statement-show-schemas.md) + - [`SHOW STATS_BUCKETS`](/sql-statements/sql-statement-show-stats-buckets.md) - [`SHOW STATS_HEALTHY`](/sql-statements/sql-statement-show-stats-healthy.md) - [`SHOW STATS_HISTOGRAMS`](/sql-statements/sql-statement-show-stats-histograms.md) - [`SHOW STATS_LOCKED`](/sql-statements/sql-statement-show-stats-locked.md) - [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md) + - [`SHOW STATS_TOPN`](/sql-statements/sql-statement-show-stats-topn.md) - [`SHOW STATUS`](/sql-statements/sql-statement-show-status.md) - [`SHOW TABLE NEXT_ROW_ID`](/sql-statements/sql-statement-show-table-next-rowid.md) - [`SHOW TABLE REGIONS`](/sql-statements/sql-statement-show-table-regions.md) @@ -891,7 +935,14 @@ - [Encryption and Compression Functions](/functions-and-operators/encryption-and-compression-functions.md) - [Locking Functions](/functions-and-operators/locking-functions.md) - [Information Functions](/functions-and-operators/information-functions.md) - - [JSON Functions](/functions-and-operators/json-functions.md) + - JSON Functions + - [Overview](/functions-and-operators/json-functions.md) + - [Functions That Create JSON](/functions-and-operators/json-functions/json-functions-create.md) + - [Functions That Search JSON](/functions-and-operators/json-functions/json-functions-search.md) + - [Functions That Modify JSON](/functions-and-operators/json-functions/json-functions-modify.md) + - [Functions That Return JSON](/functions-and-operators/json-functions/json-functions-return.md) + - [JSON Utility Functions](/functions-and-operators/json-functions/json-functions-utility.md) + - [Functions That Aggregate JSON](/functions-and-operators/json-functions/json-functions-aggregate.md) - [Aggregate (GROUP BY) Functions](/functions-and-operators/aggregate-group-by-functions.md) - [GROUP BY Modifiers](/functions-and-operators/group-by-modifier.md) - [Window Functions](/functions-and-operators/window-functions.md) @@ -923,7 +974,9 @@ - [GBK](/character-set-gbk.md) - [Placement Rules in SQL](/placement-rules-in-sql.md) - System Tables - - [`mysql`](/mysql-schema.md) + - `mysql` Schema + - [Overview](/mysql-schema/mysql-schema.md) + - [`user`](/mysql-schema/mysql-schema-user.md) - INFORMATION_SCHEMA - [Overview](/information-schema/information-schema.md) - [`ANALYZE_STATUS`](/information-schema/information-schema-analyze-status.md) @@ -968,6 +1021,7 @@ - [`TABLES`](/information-schema/information-schema-tables.md) - [`TABLE_CONSTRAINTS`](/information-schema/information-schema-table-constraints.md) - [`TABLE_STORAGE_STATS`](/information-schema/information-schema-table-storage-stats.md) + - [`TIDB_CHECK_CONSTRAINTS`](/information-schema/information-schema-tidb-check-constraints.md) - [`TIDB_HOT_REGIONS`](/information-schema/information-schema-tidb-hot-regions.md) - [`TIDB_HOT_REGIONS_HISTORY`](/information-schema/information-schema-tidb-hot-regions-history.md) - [`TIDB_INDEXES`](/information-schema/information-schema-tidb-indexes.md) @@ -993,38 +1047,6 @@ - [`schema_unused_indexes`](/sys-schema/sys-schema-unused-indexes.md) - [Metadata Lock](/metadata-lock.md) - [TiDB Accelerated Table Creation](/accelerated-table-creation.md) - - UI - - TiDB Dashboard - - [Overview](/dashboard/dashboard-intro.md) - - Maintain - - [Deploy](/dashboard/dashboard-ops-deploy.md) - - [Reverse Proxy](/dashboard/dashboard-ops-reverse-proxy.md) - - [User Management](/dashboard/dashboard-user.md) - - [Secure](/dashboard/dashboard-ops-security.md) - - [Access](/dashboard/dashboard-access.md) - - [Overview Page](/dashboard/dashboard-overview.md) - - [Cluster Info Page](/dashboard/dashboard-cluster-info.md) - - [Top SQL Page](/dashboard/top-sql.md) - - [Key Visualizer Page](/dashboard/dashboard-key-visualizer.md) - - [Metrics Relation Graph](/dashboard/dashboard-metrics-relation.md) - - SQL Statements Analysis - - [SQL Statements Page](/dashboard/dashboard-statement-list.md) - - [SQL Details Page](/dashboard/dashboard-statement-details.md) - - [Slow Queries Page](/dashboard/dashboard-slow-query.md) - - Cluster Diagnostics - - [Access Cluster Diagnostics Page](/dashboard/dashboard-diagnostics-access.md) - - [View Diagnostics Report](/dashboard/dashboard-diagnostics-report.md) - - [Use Diagnostics](/dashboard/dashboard-diagnostics-usage.md) - - [Monitoring Page](/dashboard/dashboard-monitoring.md) - - [Search Logs Page](/dashboard/dashboard-log-search.md) - - [Resource Manager Page](/dashboard/dashboard-resource-manager.md) - - Instance Profiling - - [Manual Profiling](/dashboard/dashboard-profiling.md) - - [Continuous Profiling](/dashboard/continuous-profiling.md) - - Session Management and Configuration - - [Share Session](/dashboard/dashboard-session-share.md) - - [Configure SSO](/dashboard/dashboard-session-sso.md) - - [FAQ](/dashboard/dashboard-faq.md) - [Telemetry](/telemetry.md) - [Error Codes](/error-codes.md) - [Table Filter](/table-filter.md) @@ -1046,12 +1068,22 @@ - [All Releases](/releases/release-notes.md) - [Release Timeline](/releases/release-timeline.md) - [TiDB Versioning](/releases/versioning.md) + - [Release Support Policy](https://www.pingcap.com/tidb-release-support-policy/) - [TiDB Installation Packages](/binary-package.md) + - v8.1 + - [8.1.2](/releases/release-8.1.2.md) + - [8.1.1](/releases/release-8.1.1.md) + - [8.1.0](/releases/release-8.1.0.md) - v8.0 - [8.0.0-DMR](/releases/release-8.0.0.md) - v7.6 - [7.6.0-DMR](/releases/release-7.6.0.md) - v7.5 + - [7.5.6](/releases/release-7.5.6.md) + - [7.5.5](/releases/release-7.5.5.md) + - [7.5.4](/releases/release-7.5.4.md) + - [7.5.3](/releases/release-7.5.3.md) + - [7.5.2](/releases/release-7.5.2.md) - [7.5.1](/releases/release-7.5.1.md) - [7.5.0](/releases/release-7.5.0.md) - v7.4 @@ -1061,6 +1093,7 @@ - v7.2 - [7.2.0-DMR](/releases/release-7.2.0.md) - v7.1 + - [7.1.6](/releases/release-7.1.6.md) - [7.1.5](/releases/release-7.1.5.md) - [7.1.4](/releases/release-7.1.4.md) - [7.1.3](/releases/release-7.1.3.md) @@ -1072,6 +1105,9 @@ - v6.6 - [6.6.0-DMR](/releases/release-6.6.0.md) - v6.5 + - [6.5.12](/releases/release-6.5.12.md) + - [6.5.11](/releases/release-6.5.11.md) + - [6.5.10](/releases/release-6.5.10.md) - [6.5.9](/releases/release-6.5.9.md) - [6.5.8](/releases/release-6.5.8.md) - [6.5.7](/releases/release-6.5.7.md) diff --git a/_docHome.md b/_docHome.md index db74f2c1a4b1c..aa5782aa68b21 100644 --- a/_docHome.md +++ b/_docHome.md @@ -3,10 +3,10 @@ title: PingCAP Documentation hide_sidebar: true hide_commit: true hide_leftNav: true -summary: PingCAP Documentation provides how-to guides and references for using TiDB Cloud and TiDB, including data migration and application building. TiDB Cloud is a fully-managed Database-as-a-Service, offering easy access to the power of a cloud-native, distributed SQL database. TiDB is an open-source distributed SQL database with MySQL compatibility, horizontal scalability, and high availability. Developers can access documentation for application development and explore additional resources such as TiDB Playground, PingCAP Education, and community engagement opportunities. +summary: TiDB Documentation provides how-to guides and references for using TiDB Cloud and TiDB Self-Managed, including data migration and application building. TiDB Cloud is a fully-managed Database-as-a-Service, offering easy access to the power of a cloud-native, distributed SQL database. TiDB is an open-source distributed SQL database with MySQL compatibility, horizontal scalability, and high availability. Developers can access documentation for application development and explore additional resources such as TiDB Playground, PingCAP Education, and community engagement opportunities. --- - + @@ -14,33 +14,35 @@ TiDB Cloud is a fully-managed Database-as-a-Service (DBaaS) that brings everythi - + -View the documentation for TiDB Cloud. +Learn what TiDB Cloud is as an easy-to-use database and its key features. - + Guide for an easy way to get started with TiDB Cloud. - + -Helps you quickly complete a Proof of Concept (PoC) with TiDB Cloud. +Connect your application with the languages and frameworks you prefer. - + + +Explore native support of Vector Search in TiDB Cloud Serverless to build your AI application. -Get the power of a cloud-native, distributed SQL database built for real-time analytics in a fully-managed service. + -Try Free + - + -TiDB is an open-source distributed SQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. It is MySQL compatible and features horizontal scalability, strong consistency, and high availability. You can deploy TiDB in a self-hosted environment or in the cloud. +TiDB is an open-source distributed SQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. It is MySQL compatible and features horizontal scalability, strong consistency, and high availability. You can deploy TiDB in a self-hosted environment or on the cloud. - + -View the documentation for TiDB. +Learn what is TiDB Self-Managed and its key features. - + -Walks you through the quickest way to get started with TiDB. +Walks you through the quickest way to get started with TiDB Self-Managed. - + Learn how to deploy TiDB locally in a production environment. - - -The open-source TiDB platform is released under the Apache 2.0 license and is supported by the community. + -View on GitHub +For application developers using TiDB Self-Managed. - - - - - + - + -Documentation for TiDB application developers. +TiDB is highly compatible with the MySQL protocol and the common features and syntax of MySQL 5.7 and MySQL 8.0. - + -Documentation for TiDB Cloud application developers. +Planned features and release dates for TiDB Self-Managed. +The open-source TiDB platform is released under the Apache 2.0 license and is supported by the community. [View on GitHub](https://github.com/pingcap/tidb) + - + - + -Experience the capabilities of TiDB WITHOUT registration. +Learn TiDB Cloud and TiDB Self-Managed through well-designed online courses and instructor-led training. - + -Learn TiDB and TiDB Cloud through well-designed online courses and instructor-led training. +Read great articles about TiDB Cloud and TiDB Self-Managed. - + -Join us on Discord or become a contributor. +Learn about events hosted by PingCAP and the community. - + -Read great articles about TiDB and TiDB Cloud. +Download eBooks and papers. - + Watch a compilation of short videos describing TiDB and various use cases. - + -Learn about events hosted by PingCAP and the community. +A powerful insight tool that provides in-depth analysis of any GitHub repository, powered by TiDB Cloud. - + -Download eBooks and papers. +Experience the capabilities of TiDB without registration. - + -A powerful insight tool that analyzes any GitHub repository in depth, powered by TiDB Cloud. - - - - - -Let's work together to improve the documentation! +Join us on Discord or become a contributor. diff --git a/_index.md b/_index.md index e03c61375038f..ebc71a83c493a 100644 --- a/_index.md +++ b/_index.md @@ -1,12 +1,11 @@ --- title: TiDB Introduction -aliases: ["/docs/dev/", "/docs/dev/adopters/", "/tidb/dev/adopters"] hide_sidebar: true hide_commit: true summary: TiDB is an open-source distributed SQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. The guide provides information on features, TiFlash, development, deployment, migration, maintenance, monitoring, tuning, tools, and references. It covers everything from quick start to advanced configurations and tools for TiDB. --- - + @@ -66,21 +65,6 @@ This section gives the alert rules for the TiDB component. Same as [`TiDB_schema_error`](#tidb_schema_error). -#### `TiDB_monitor_keep_alive` - -* Alert rule: - - `increase(tidb_monitor_keep_alive_total[10m]) < 100` - -* Description: - - Indicates whether the TiDB process still exists. If the number of times for `tidb_monitor_keep_alive_total` increases less than 100 in 10 minutes, the TiDB process might already exit and an alert is triggered. - -* Solution: - - * Check whether the TiDB process is out of memory. - * Check whether the machine has restarted. - ### Critical-level alerts #### `TiDB_server_panic_total` @@ -121,7 +105,7 @@ This section gives the alert rules for the TiDB component. * Description: - The latency of handling a request in TiDB. If the ninety-ninth percentile latency exceeds 1 second, an alert is triggered. + The latency of handling a request in TiDB. The response time for 99% of requests should be within 1 second; otherwise, an alert is triggered. * Solution: @@ -578,11 +562,13 @@ This section gives the alert rules for the TiKV component. * Alert rule: - `sum(rate(tikv_thread_cpu_seconds_total{name=~"raftstore_.*"}[1m])) by (instance, name) > 1.6` + `sum(rate(tikv_thread_cpu_seconds_total{name=~"raftstore_.*"}[1m])) by (instance) > 1.6` * Description: - The pressure on the Raftstore thread is too high. + This rule monitors CPU usage by Raftstore. If the value is high, it indicates pressure on Raftstore threads is heavy. + + The alert threshold is 80% of the [`raftstore.store-pool-size`](/tikv-configuration-file.md#store-pool-size) value. `raftstore.store-pool-size` is 2 by default, so the alert threshold is 1.6. * Solution: diff --git a/as-of-timestamp.md b/as-of-timestamp.md index daaf9ba060ed8..76872ff5c13bc 100644 --- a/as-of-timestamp.md +++ b/as-of-timestamp.md @@ -23,7 +23,7 @@ You can use the `AS OF TIMESTAMP` clause in the following three ways: If you want to specify an exact point of time, you can set a datetime value or use a time function in the `AS OF TIMESTAMP` clause. The format of datetime is like "2016-10-08 16:45:26.999", with millisecond as the minimum time unit, but for most of the time, the time unit of second is enough for specifying a datetime, such as "2016-10-08 16:45:26". You can also get the current time to the millisecond using the `NOW(3)` function. If you want to read the data of several seconds ago, it is **recommended** to use an expression such as `NOW() - INTERVAL 10 SECOND`. -If you want to specify a time range, you can use the `TIDB_BOUNDED_STALENESS()` function in the clause. When this function is used, TiDB selects a suitable timestamp within the specified time range. "Suitable" means there are no transactions that start before this timestamp and have not been committed on the accessed replica, that is, TiDB can perform read operations on the accessed replica and the read operations are not blocked. You need to use `TIDB_BOUNDED_STALENESS(t1, t2)` to call this function. `t1` and `t2` are the two ends of the time range, which can be specified using either datetime values or time functions. +If you want to specify a time range, you can use the [`TIDB_BOUNDED_STALENESS()`](/functions-and-operators/tidb-functions.md#tidb_bounded_staleness) function in the clause. When this function is used, TiDB selects a suitable timestamp within the specified time range. "Suitable" means there are no transactions that start before this timestamp and have not been committed on the accessed replica, that is, TiDB can perform read operations on the accessed replica and the read operations are not blocked. You need to use `TIDB_BOUNDED_STALENESS(t1, t2)` to call this function. `t1` and `t2` are the two ends of the time range, which can be specified using either datetime values or time functions. Here are some examples of the `AS OF TIMESTAMP` clause: diff --git a/auto-increment.md b/auto-increment.md index a227a8e5e1e22..00bfa14c09c4e 100644 --- a/auto-increment.md +++ b/auto-increment.md @@ -1,7 +1,6 @@ --- title: AUTO_INCREMENT summary: Learn the `AUTO_INCREMENT` column attribute of TiDB. -aliases: ['/docs/dev/auto-increment/'] --- # AUTO_INCREMENT @@ -304,46 +303,116 @@ After the value `2030000` is inserted, the next value is `2060001`. This jump in In earlier versions of TiDB, the cache size of the auto-increment ID was transparent to users. Starting from v3.0.14, v3.1.2, and v4.0.rc-2, TiDB has introduced the `AUTO_ID_CACHE` table option to allow users to set the cache size for allocating the auto-increment ID. ```sql -mysql> CREATE TABLE t(a int AUTO_INCREMENT key) AUTO_ID_CACHE 100; +CREATE TABLE t(a int AUTO_INCREMENT key) AUTO_ID_CACHE 100; Query OK, 0 rows affected (0.02 sec) -mysql> INSERT INTO t values(); +INSERT INTO t values(); Query OK, 1 row affected (0.00 sec) -Records: 1 Duplicates: 0 Warnings: 0 -mysql> SELECT * FROM t; +SELECT * FROM t; +---+ | a | +---+ | 1 | +---+ 1 row in set (0.01 sec) + +SHOW CREATE TABLE t; ++-------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Table | Create Table | ++-------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| t | CREATE TABLE `t` ( + `a` int(11) NOT NULL AUTO_INCREMENT, + PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=101 /*T![auto_id_cache] AUTO_ID_CACHE=100 */ | ++-------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +1 row in set (0.00 sec) ``` -At this time, if you invalidate the auto-increment cache of this column and redo the implicit insertion, the result is as follows: +At this time, if you restart TiDB, the auto-increment ID cache will be lost, and new insert operations will allocate IDs starting from a higher value beyond the previously cached range. ```sql -mysql> DELETE FROM t; -Query OK, 1 row affected (0.01 sec) - -mysql> RENAME TABLE t to t1; -Query OK, 0 rows affected (0.01 sec) - -mysql> INSERT INTO t1 values() +INSERT INTO t VALUES(); Query OK, 1 row affected (0.00 sec) -mysql> SELECT * FROM t; +SELECT * FROM t; +-----+ | a | +-----+ +| 1 | | 101 | +-----+ -1 row in set (0.00 sec) +2 rows in set (0.01 sec) +``` + +The newly allocated value is `101`. This shows that the size of cache for allocating auto-increment IDs is `100`. + +In addition, when the length of consecutive IDs in a batch `INSERT` statement exceeds the length of `AUTO_ID_CACHE`, TiDB increases the cache size accordingly to ensure that the statement can insert data properly. + +### Clear the auto-increment ID cache + +In some scenarios, you might need to clear the auto-increment ID cache to ensure data consistency. For example: + + + +- In the scenario of incremental replication using [Data Migration (DM)](/dm/dm-overview.md), once the replication is complete, data writing to the downstream TiDB switches from DM to your application's write operations. Meanwhile, the ID writing mode of the auto-increment column usually switches from explicit insertion to implicit allocation. + + + + +- In the scenario of incremental replication using the [Data Migration](/tidb-cloud/migrate-incremental-data-from-mysql-using-data-migration.md) feature, once the replication is complete, data writing to the downstream TiDB switches from DM to your application's write operations. Meanwhile, the ID writing mode of the auto-increment column usually switches from explicit insertion to implicit allocation. + + + +- When your application involves both explicit ID insertion and implicit ID allocation, you need to clear the auto-increment ID cache to avoid conflicts between future implicitly allocated IDs and previously explicitly inserted IDs, which could result in primary key conflict errors. For more information, see [Uniqueness](/auto-increment.md#uniqueness). + +To clear the auto-increment ID cache on all TiDB nodes in the cluster, you can execute the `ALTER TABLE` statement with `AUTO_INCREMENT = 0`. For example: + +```sql +CREATE TABLE t(a int AUTO_INCREMENT key) AUTO_ID_CACHE 100; +Query OK, 0 rows affected (0.02 sec) + +INSERT INTO t VALUES(); +Query OK, 1 row affected (0.02 sec) + +INSERT INTO t VALUES(50); +Query OK, 1 row affected (0.00 sec) + +SELECT * FROM t; ++----+ +| a | ++----+ +| 1 | +| 50 | ++----+ +2 rows in set (0.01 sec) ``` -The re-assigned value is `101`. This shows that the size of cache for allocating the auto-increment ID is `100`. +```sql +ALTER TABLE t AUTO_INCREMENT = 0; +Query OK, 0 rows affected, 1 warning (0.07 sec) + +SHOW WARNINGS; ++---------+------+-------------------------------------------------------------------------+ +| Level | Code | Message | ++---------+------+-------------------------------------------------------------------------+ +| Warning | 1105 | Can't reset AUTO_INCREMENT to 0 without FORCE option, using 101 instead | ++---------+------+-------------------------------------------------------------------------+ +1 row in set (0.01 sec) -In addition, when the length of consecutive IDs in a batch `INSERT` statement exceeds the length of `AUTO_ID_CACHE`, TiDB increases the cache size accordingly to ensure that the statement can be inserted properly. +INSERT INTO t VALUES(); +Query OK, 1 row affected (0.02 sec) + +SELECT * FROM t; ++-----+ +| a | ++-----+ +| 1 | +| 50 | +| 101 | ++-----+ +3 rows in set (0.01 sec) +``` ### Auto-increment step size and offset @@ -355,25 +424,49 @@ The value (ID) implicitly assigned to auto-increment columns satisfies the follo ## MySQL compatibility mode -TiDB v6.4.0 introduces a centralized auto-increment ID allocating service. In each request, an auto-increment ID is allocated from this service instead of caching data in TiDB instances. +TiDB provides a MySQL-compatible mode for auto-increment columns that ensures strictly increasing IDs with minimal gaps. To enable this mode, set `AUTO_ID_CACHE` to `1` when creating a table: + +```sql +CREATE TABLE t(a int AUTO_INCREMENT key) AUTO_ID_CACHE 1; +``` -Currently, the centralized allocating service is in the TiDB process and works like DDL Owner. One TiDB instance allocates IDs as the primary node and other TiDB instances work as secondary nodes. To ensure high availability, when the primary instance fails, TiDB starts automatic failover. +When `AUTO_ID_CACHE` is set to `1`, IDs are strictly increasing across all TiDB instances, each ID is guaranteed to be unique, and gaps between IDs are minimal compared to the default cache mode (`AUTO_ID_CACHE 0` with 30000 cached values). -To use the MySQL compatibility mode, you can set `AUTO_ID_CACHE` to `1` when creating a table: +For example, with `AUTO_ID_CACHE 1`, you might see a sequence as follows: ```sql -CREATE TABLE t(a int AUTO_INCREMENT key) AUTO_ID_CACHE 1; +INSERT INTO t VALUES (); -- Returns ID 1 +INSERT INTO t VALUES (); -- Returns ID 2 +INSERT INTO t VALUES (); -- Returns ID 3 +-- After failover +INSERT INTO t VALUES (); -- Might return ID 5 ``` +In contrast, with the default cache (`AUTO_ID_CACHE 0`), larger gaps can occur: + +```sql +INSERT INTO t VALUES (); -- Returns ID 1 +INSERT INTO t VALUES (); -- Returns ID 2 +-- New TiDB instance allocates next batch +INSERT INTO t VALUES (); -- Returns ID 30001 +``` + +While IDs are always increasing and without significant gaps like those seen with `AUTO_ID_CACHE 0`, small gaps in the sequence might still occur in the following scenarios. These gaps are necessary to maintain both uniqueness and the strictly increasing property of the IDs. + +- During failover when the primary instance exits or crashes + + After you enable the MySQL compatibility mode, the allocated IDs are **unique** and **monotonically increasing**, and the behavior is almost the same as MySQL. Even when accessing across multiple TiDB instances, ID monotonicity is maintained. However, if the primary instance of the centralized service crashes, a few IDs might become non-continuous. This occurs because the secondary instance discards some IDs allocated by the primary instance during failover to ensure ID uniqueness. + +- During rolling upgrades of TiDB nodes +- During normal concurrent transactions (similar to MySQL) + > **Note:** > -> In TiDB, setting `AUTO_ID_CACHE` to `1` means that TiDB no longer caches IDs. But the implementation varies with TiDB versions: +> The behavior and performance of `AUTO_ID_CACHE 1` has evolved across TiDB versions: > -> - Before TiDB v6.4.0, since allocating ID requires a TiKV transaction to persist the `AUTO_INCREMENT` value for each request, setting `AUTO_ID_CACHE` to `1` causes performance degradation. -> - Since TiDB v6.4.0, the modification of the `AUTO_INCREMENT` value is faster because it is only an in-memory operation in the TiDB process as the centralized allocating service is introduced. -> - Setting `AUTO_ID_CACHE` to `0` means that TiDB uses the default cache size `30000`. - -After you enable the MySQL compatibility mode, the allocated IDs are **unique** and **monotonically increasing**, and the behavior is almost the same as MySQL. Even if you access across TiDB instances, the IDs will keep monotonic. Only when the primary instance of the centralized service crashes, there might be a few IDs that are not continuous. This is because the secondary instance discards some IDs that are supposed to have been allocated by the primary instance during the failover to ensure ID uniqueness. +> - Before v6.4.0, each ID allocation requires a TiKV transaction, which affects performance. +> - In v6.4.0, TiDB introduces a centralized allocating service that performs ID allocation as an in-memory operation, significantly improving performance. +> - Starting from v8.1.0, TiDB removes the automatic `forceRebase` operation during primary node exits to enable faster restarts. While this might result in additional non-consecutive IDs during failover, it prevents potential write blocking when many tables use `AUTO_ID_CACHE 1`. ## Restrictions diff --git a/auto-random.md b/auto-random.md index bd331ed4370c9..2790648b98a94 100644 --- a/auto-random.md +++ b/auto-random.md @@ -1,7 +1,6 @@ --- title: AUTO_RANDOM summary: Learn the AUTO_RANDOM attribute. -aliases: ['/docs/dev/auto-random/','/docs/dev/reference/sql/attributes/auto-random/'] --- # AUTO_RANDOM New in v3.1.0 @@ -47,7 +46,7 @@ When you execute an `INSERT` statement: - If you do not explicitly specify the value of the `AUTO_RANDOM` column, TiDB generates a random value and inserts it into the table. ```sql -tidb> CREATE TABLE t (a BIGINT PRIMARY KEY AUTO_RANDOM, b VARCHAR(255)); +tidb> CREATE TABLE t (a BIGINT PRIMARY KEY AUTO_RANDOM, b VARCHAR(255)) /*T! PRE_SPLIT_REGIONS=2 */ ; Query OK, 0 rows affected, 1 warning (0.01 sec) tidb> INSERT INTO t(a, b) VALUES (1, 'string'); @@ -76,6 +75,29 @@ tidb> SELECT * FROM t; | 4899916394579099651 | string3 | +---------------------+---------+ 3 rows in set (0.00 sec) + +tidb> SHOW CREATE TABLE t; ++-------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Table | Create Table | ++-------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| t | CREATE TABLE `t` ( + `a` bigint(20) NOT NULL /*T![auto_rand] AUTO_RANDOM(5) */, + `b` varchar(255) DEFAULT NULL, + PRIMARY KEY (`a`) /*T![clustered_index] CLUSTERED */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin /*T! PRE_SPLIT_REGIONS=2 */ | ++-------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +1 row in set (0.00 sec) + +tidb> SHOW TABLE t REGIONS; ++-----------+-----------------------------+-----------------------------+-----------+-----------------+---------------------+------------+---------------+------------+----------------------+------------------+------------------------+------------------+ +| REGION_ID | START_KEY | END_KEY | LEADER_ID | LEADER_STORE_ID | PEERS | SCATTERING | WRITTEN_BYTES | READ_BYTES | APPROXIMATE_SIZE(MB) | APPROXIMATE_KEYS | SCHEDULING_CONSTRAINTS | SCHEDULING_STATE | ++-----------+-----------------------------+-----------------------------+-----------+-----------------+---------------------+------------+---------------+------------+----------------------+------------------+------------------------+------------------+ +| 62798 | t_158_ | t_158_r_2305843009213693952 | 62810 | 28 | 62811, 62812, 62810 | 0 | 151 | 0 | 1 | 0 | | | +| 62802 | t_158_r_2305843009213693952 | t_158_r_4611686018427387904 | 62803 | 1 | 62803, 62804, 62805 | 0 | 39 | 0 | 1 | 0 | | | +| 62806 | t_158_r_4611686018427387904 | t_158_r_6917529027641081856 | 62813 | 4 | 62813, 62814, 62815 | 0 | 160 | 0 | 1 | 0 | | | +| 9289 | t_158_r_6917529027641081856 | 78000000 | 48268 | 1 | 48268, 58951, 62791 | 0 | 10628 | 43639 | 2 | 7999 | | | ++-----------+-----------------------------+-----------------------------+-----------+-----------------+---------------------+------------+---------------+------------+----------------------+------------------+------------------------+------------------+ +4 rows in set (0.00 sec) ``` The `AUTO_RANDOM(S, R)` column value automatically assigned by TiDB has a total of 64 bits: @@ -83,16 +105,25 @@ The `AUTO_RANDOM(S, R)` column value automatically assigned by TiDB has a total - `S` is the number of shard bits. The value ranges from `1` to `15`. The default value is `5`. - `R` is the total length of the automatic allocation range. The value ranges from `32` to `64`. The default value is `64`. -The structure of an `AUTO_RANDOM` value is as follows: +The structure of an `AUTO_RANDOM` value with a signed bit is as follows: + +| Signed bit | Reserved bits | Shard bits | Auto-increment bits | +|---------|-------------|--------|--------------| +| 1 bit | `64-R` bits | `S` bits | `R-1-S` bits | + +The structure of an `AUTO_RANDOM` value without a signed bit is as follows: -| Total number of bits | Sign bit | Reserved bits | Shard bits | Auto-increment bits | -|---------|---------|-------------|--------|--------------| -| 64 bits | 0/1 bit | (64-R) bits | S bits | (R-1-S) bits | +| Reserved bits | Shard bits | Auto-increment bits | +|-------------|--------|--------------| +| `64-R` bits | `S` bits | `R-S` bits | +- Whether a value has a signed bit depends on whether the corresponding column has the `UNSIGNED` attribute. - The length of the sign bit is determined by the existence of an `UNSIGNED` attribute. If there is an `UNSIGNED` attribute, the length is `0`. Otherwise, the length is `1`. - The length of the reserved bits is `64-R`. The reserved bits are always `0`. - The content of the shard bits is obtained by calculating the hash value of the starting time of the current transaction. To use a different length of shard bits (such as 10), you can specify `AUTO_RANDOM(10)` when creating the table. - The value of the auto-increment bits is stored in the storage engine and allocated sequentially. Each time a new value is allocated, the value is incremented by 1. The auto-increment bits ensure that the values of `AUTO_RANDOM` are unique globally. When the auto-increment bits are exhausted, an error `Failed to read auto-increment value from storage engine` is reported when the value is allocated again. +- Value range: the maximum number of bits for the final generated value = shard bits + auto-increment bits. The range of a signed column is `[-(2^(R-1))+1, (2^(R-1))-1]`, and the range of an unsigned column is `[0, (2^R)-1]`. +- You can use `AUTO_RANDOM` with `PRE_SPLIT_REGIONS`. When a table is created successfully, `PRE_SPLIT_REGIONS` pre-splits data in the table into the number of Regions as specified by `2^(PRE_SPLIT_REGIONS)`. > **Note:** > @@ -104,7 +135,7 @@ The structure of an `AUTO_RANDOM` value is as follows: > Selection of range (`R`): > > - Typically, the `R` parameter needs to be set when the numeric type of the application cannot represent a full 64-bit integer. -> - For example, the range of JSON number is `[-2^53+1, 2^53-1]`. TiDB can easily assign an integer outside this range to a column of `AUTO_RANDOM(5)`, causing unexpected behaviors when the application reads the column. In this case, you can replace `AUTO_RANDOM(5)` with `AUTO_RANDOM(5, 54)` and TiDB does not assign an integer greater than `9007199254740991` (2^53-1) to the column. +> - For example, the range of JSON number is `[-(2^53)+1, (2^53)-1]`. TiDB can easily assign an integer beyond this range to a column defined as `AUTO_RANDOM(5)`, causing unexpected behaviors when the application reads the column. In such cases, you can replace `AUTO_RANDOM(5)` with `AUTO_RANDOM(5, 54)` for signed columns, and replace `AUTO_RANDOM(5)` with `AUTO_RANDOM(5, 53)` for unsigned columns, ensuring that TiDB does not assign integers greater than `9007199254740991` (2^53-1) to the column. Values allocated implicitly to the `AUTO_RANDOM` column affect `last_insert_id()`. To get the ID that TiDB last implicitly allocates, you can use the `SELECT last_insert_id ()` statement. diff --git a/basic-features.md b/basic-features.md index a56ab968b3a4f..b79ee56e8f249 100644 --- a/basic-features.md +++ b/basic-features.md @@ -1,7 +1,6 @@ --- title: TiDB Features summary: Learn about the feature overview of TiDB. -aliases: ['/docs/dev/basic-features/','/tidb/dev/experimental-features-4.0/'] --- # TiDB Features @@ -22,245 +21,245 @@ You can try out TiDB features on [TiDB Playground](https://play.tidbcloud.com/?u ## Data types, functions, and operators -| Data types, functions, and operators | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Numeric types](/data-type-numeric.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Date and time types](/data-type-date-and-time.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [String types](/data-type-string.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [JSON type](/data-type-json.md) | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [Control flow functions](/functions-and-operators/control-flow-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [String functions](/functions-and-operators/string-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Numeric functions and operators](/functions-and-operators/numeric-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Date and time functions](/functions-and-operators/date-and-time-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Bit functions and operators](/functions-and-operators/bit-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Cast functions and operators](/functions-and-operators/cast-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Encryption and compression functions](/functions-and-operators/encryption-and-compression-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Information functions](/functions-and-operators/information-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [JSON functions](/functions-and-operators/json-functions.md) | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [Aggregation functions](/functions-and-operators/aggregate-group-by-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Window functions](/functions-and-operators/window-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Miscellaneous functions](/functions-and-operators/miscellaneous-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Operators](/functions-and-operators/operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Character sets and collations](/character-set-and-collation.md) [^1] | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [User-level lock](/functions-and-operators/locking-functions.md) | Y | Y | Y | Y | Y | Y | N | N | N | N | N | N | +| Data types, functions, and operators | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Numeric types](/data-type-numeric.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Date and time types](/data-type-date-and-time.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [String types](/data-type-string.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [JSON type](/data-type-json.md) | Y | Y | Y | Y | E | E | E | E | E | +| [Control flow functions](/functions-and-operators/control-flow-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [String functions](/functions-and-operators/string-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Numeric functions and operators](/functions-and-operators/numeric-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Date and time functions](/functions-and-operators/date-and-time-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Bit functions and operators](/functions-and-operators/bit-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Cast functions and operators](/functions-and-operators/cast-functions-and-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Encryption and compression functions](/functions-and-operators/encryption-and-compression-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Information functions](/functions-and-operators/information-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [JSON functions](/functions-and-operators/json-functions.md) | Y | Y | Y | Y | E | E | E | E | E | +| [Aggregation functions](/functions-and-operators/aggregate-group-by-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Window functions](/functions-and-operators/window-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Miscellaneous functions](/functions-and-operators/miscellaneous-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Operators](/functions-and-operators/operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Character sets and collations](/character-set-and-collation.md) [^1] | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [User-level lock](/functions-and-operators/locking-functions.md) | Y | Y | Y | Y | Y | N | N | N | N | ## Indexing and constraints -| Indexing and constraints | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Expression indexes](/sql-statements/sql-statement-create-index.md#expression-index) [^2] | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [Columnar storage (TiFlash)](/tiflash/tiflash-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Use FastScan to accelerate queries in OLAP scenarios](/tiflash/use-fastscan.md) | Y | Y | Y | Y | E | N | N | N | N | N | N | N | -| [RocksDB engine](/storage-engine/rocksdb-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Titan plugin](/storage-engine/titan-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Titan Level Merge](/storage-engine/titan-configuration.md#level-merge-experimental) | E | E | E | E | E | E | E | E | E | E | E | E | -| [Use buckets to improve scan concurrency](/tune-region-performance.md#use-bucket-to-increase-concurrency) | E | E | E | E | E | E | N | N | N | N | N | N | -| [Invisible indexes](/sql-statements/sql-statement-create-index.md#invisible-index) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| [Composite `PRIMARY KEY`](/constraints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`CHECK` constraints](/constraints.md#check) | Y | Y | Y | N | N | N | N | N | N | N | N | N | -| [Unique indexes](/constraints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Clustered index on integer `PRIMARY KEY`](/clustered-indexes.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Clustered index on composite or non-integer key](/clustered-indexes.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| [Multi-valued indexes](/sql-statements/sql-statement-create-index.md#multi-valued-indexes) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [Foreign key](/constraints.md#foreign-key) | E | E | E | E | N | N | N | N | N | N | N | N | -| [TiFlash late materialization](/tiflash/tiflash-late-materialization.md) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | +| Indexing and constraints | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Expression indexes](/sql-statements/sql-statement-create-index.md#expression-index) [^2] | Y | Y | Y | Y | E | E | E | E | E | +| [Columnar storage (TiFlash)](/tiflash/tiflash-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Use FastScan to accelerate queries in OLAP scenarios](/tiflash/use-fastscan.md) | Y | Y | Y | E | N | N | N | N | N | +| [RocksDB engine](/storage-engine/rocksdb-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Titan plugin](/storage-engine/titan-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Titan Level Merge](/storage-engine/titan-configuration.md#level-merge-experimental) | E | E | E | E | E | E | E | E | E | +| [Use buckets to improve scan concurrency](/tune-region-performance.md#use-bucket-to-increase-concurrency) | E | E | E | E | E | N | N | N | N | +| [Invisible indexes](/sql-statements/sql-statement-create-index.md#invisible-index) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Composite `PRIMARY KEY`](/constraints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`CHECK` constraints](/constraints.md#check) | Y | Y | N | N | N | N | N | N | N | +| [Unique indexes](/constraints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Clustered index on integer `PRIMARY KEY`](/clustered-indexes.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Clustered index on composite or non-integer key](/clustered-indexes.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Multi-valued indexes](/sql-statements/sql-statement-create-index.md#multi-valued-indexes) | Y | Y | Y | N | N | N | N | N | N | +| [Foreign key](/constraints.md#foreign-key) | E | E | E | N | N | N | N | N | N | +| [TiFlash late materialization](/tiflash/tiflash-late-materialization.md) | Y | Y | Y | N | N | N | N | N | N | ## SQL statements -| SQL statements [^3] | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| Basic `SELECT`, `INSERT`, `UPDATE`, `DELETE`, `REPLACE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| `INSERT ON DUPLICATE KEY UPDATE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| `LOAD DATA INFILE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| `SELECT INTO OUTFILE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| `INNER JOIN`, LEFT\|RIGHT [OUTER] JOIN | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| `UNION`, `UNION ALL` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`EXCEPT` and `INTERSECT` operators](/functions-and-operators/set-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| `GROUP BY`, `ORDER BY` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Window Functions](/functions-and-operators/window-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Common Table Expressions (CTE)](/sql-statements/sql-statement-with.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | -| `START TRANSACTION`, `COMMIT`, `ROLLBACK` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`EXPLAIN`](/sql-statements/sql-statement-explain.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [User-defined variables](/user-defined-variables.md) | E | E | E | E | E | E | E | E | E | E | E | E | -| [`BATCH [ON COLUMN] LIMIT INTEGER DELETE`](/sql-statements/sql-statement-batch.md) | Y | Y | Y | Y | Y | Y | N | N | N | N | N | N | -| [`BATCH [ON COLUMN] LIMIT INTEGER INSERT/UPDATE/REPLACE`](/sql-statements/sql-statement-batch.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [`ALTER TABLE ... COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) | Y | Y | Y | Y | Y | E | N | N | N | N | N | N | -| [Table Lock](/sql-statements/sql-statement-lock-tables-and-unlock-tables.md) | E | E | E | E | E | E | E | E | E | E | E | E | -| [TiFlash Query Result Materialization](/tiflash/tiflash-results-materialization.md) | Y | Y | Y | Y | E | N | N | N | N | N | N | N | +| SQL statements [^3] | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| Basic `SELECT`, `INSERT`, `UPDATE`, `DELETE`, `REPLACE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `INSERT ON DUPLICATE KEY UPDATE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `LOAD DATA INFILE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `SELECT INTO OUTFILE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `INNER JOIN`, LEFT\|RIGHT [OUTER] JOIN | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `UNION`, `UNION ALL` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`EXCEPT` and `INTERSECT` operators](/functions-and-operators/set-operators.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `GROUP BY`, `ORDER BY` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Window Functions](/functions-and-operators/window-functions.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Common Table Expressions (CTE)](/sql-statements/sql-statement-with.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `START TRANSACTION`, `COMMIT`, `ROLLBACK` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`EXPLAIN`](/sql-statements/sql-statement-explain.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [User-defined variables](/user-defined-variables.md) | E | E | E | E | E | E | E | E | E | +| [`BATCH [ON COLUMN] LIMIT INTEGER DELETE`](/sql-statements/sql-statement-batch.md) | Y | Y | Y | Y | Y | N | N | N | N | +| [`BATCH [ON COLUMN] LIMIT INTEGER INSERT/UPDATE/REPLACE`](/sql-statements/sql-statement-batch.md) | Y | Y | Y | Y | N | N | N | N | N | +| [`ALTER TABLE ... COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) | Y | Y | Y | Y | E | N | N | N | N | +| [Table Lock](/sql-statements/sql-statement-lock-tables-and-unlock-tables.md) | E | E | E | E | E | E | E | E | E | +| [TiFlash Query Result Materialization](/tiflash/tiflash-results-materialization.md) | Y | Y | Y | E | N | N | N | N | N | ## Advanced SQL features -| Advanced SQL features | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Prepared statement cache](/sql-prepared-plan-cache.md) | Y | Y | Y | Y | Y | Y | Y | Y | E | E | E | E | -| [Non-prepared statement cache](/sql-non-prepared-plan-cache.md) | Y | Y | Y | E | N | N | N | N | N | N | N | N | -| [SQL binding](/sql-plan-management.md#sql-binding) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Cross-database binding](/sql-plan-management.md#cross-database-binding) | Y | Y | N | N | N | N | N | N | N | N | N | N | -| [Create bindings according to historical execution plans](/sql-plan-management.md#create-a-binding-according-to-a-historical-execution-plan) | Y | Y | Y | Y | E | N | N | N | N | N | N | N | -| [Coprocessor cache](/coprocessor-cache.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | E | -| [Stale Read](/stale-read.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | -| [Follower reads](/follower-read.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Read historical data (tidb_snapshot)](/read-historical-data.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Optimizer hints](/optimizer-hints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [MPP execution engine](/explain-mpp.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| [MPP execution engine - compression exchange](/explain-mpp.md#mpp-version-and-exchange-data-compression) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [TiFlash Pipeline Model](/tiflash/tiflash-pipeline-model.md) | Y | Y | Y | N | N | N | N | N | N | N | N | N | -| [TiFlash replica selection strategy](/system-variables.md#tiflash_replica_read-new-in-v730) | Y | Y | Y | N | N | N | N | N | N | N | N | N | -| [Index Merge](/explain-index-merge.md) | Y | Y | Y | Y | Y | Y | Y | E | E | E | E | E | -| [Placement Rules in SQL](/placement-rules-in-sql.md) | Y | Y | Y | Y | Y | Y | E | E | N | N | N | N | -| [Cascades Planner](/system-variables.md#tidb_enable_cascades_planner) | E | E | E | E | E | E | E | E | E | E | E | E | -| [Runtime Filter](/runtime-filter.md) | Y | Y | Y | N | N | N | N | N | N | N | N | N | +| Advanced SQL features | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Prepared statement cache](/sql-prepared-plan-cache.md) | Y | Y | Y | Y | Y | Y | Y | E | E | +| [Non-prepared statement cache](/sql-non-prepared-plan-cache.md) | Y | Y | E | N | N | N | N | N | N | +| [SQL binding](/sql-plan-management.md#sql-binding) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Cross-database binding](/sql-plan-management.md#cross-database-binding) | Y | N | N | N | N | N | N | N | N | +| [Create bindings according to historical execution plans](/sql-plan-management.md#create-a-binding-according-to-a-historical-execution-plan) | Y | Y | Y | E | N | N | N | N | N | +| [Coprocessor cache](/coprocessor-cache.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Stale Read](/stale-read.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Follower reads](/follower-read.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Read historical data (tidb_snapshot)](/read-historical-data.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Optimizer hints](/optimizer-hints.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [MPP execution engine](/explain-mpp.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [MPP execution engine - compression exchange](/explain-mpp.md#mpp-version-and-exchange-data-compression) | Y | Y | Y | N | N | N | N | N | N | +| [TiFlash Pipeline Model](/tiflash/tiflash-pipeline-model.md) | Y | Y | N | N | N | N | N | N | N | +| [TiFlash replica selection strategy](/system-variables.md#tiflash_replica_read-new-in-v730) | Y | Y | N | N | N | N | N | N | N | +| [Index Merge](/explain-index-merge.md) | Y | Y | Y | Y | Y | Y | E | E | E | +| [Placement Rules in SQL](/placement-rules-in-sql.md) | Y | Y | Y | Y | Y | E | E | N | N | +| [Cascades Planner](/system-variables.md#tidb_enable_cascades_planner) | E | E | E | E | E | E | E | E | E | +| [Runtime Filter](/runtime-filter.md) | Y | Y | N | N | N | N | N | N | N | ## Data definition language (DDL) -| Data definition language (DDL) | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| Basic `CREATE`, `DROP`, `ALTER`, `RENAME`, `TRUNCATE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Generated columns](/generated-columns.md) | Y | Y | Y | Y | E | E | E | E | E | E | E | E | -| [Views](/views.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Sequences](/sql-statements/sql-statement-create-sequence.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Auto increment](/auto-increment.md) | Y | Y | Y | Y | Y[^4] | Y | Y | Y | Y | Y | Y | Y | -| [Auto random](/auto-random.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [TTL (Time to Live)](/time-to-live.md) | Y | Y | Y | Y | E | N | N | N | N | N | N | N | -| [DDL algorithm assertions](/sql-statements/sql-statement-alter-table.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| Multi-schema change: add columns | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [Change column type](/sql-statements/sql-statement-modify-column.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | -| [Temporary tables](/temporary-tables.md) | Y | Y | Y | Y | Y | Y | Y | Y | N | N | N | N | -| Concurrent DDL statements | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Acceleration of `ADD INDEX` and `CREATE INDEX`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Metadata lock](/metadata-lock.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Pause](/sql-statements/sql-statement-admin-pause-ddl.md)/[Resume](/sql-statements/sql-statement-admin-resume-ddl.md) DDL | Y | Y | Y | N | N | N | N | N | N | N | N | N | -| [TiDB Accelerated Table Creation](/accelerated-table-creation.md) | E | N | N | N | N | N | N | N | N | N | N | N | -| [Configure BDR role to replicate DDL statements in BDR mode](/sql-statements/sql-statement-admin-bdr-role.md#admin-setshowunset-bdr-role) | E | E | N | N | N | N | N | N | N | N | N | N | +| Data definition language (DDL) | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| Basic `CREATE`, `DROP`, `ALTER`, `RENAME`, `TRUNCATE` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Generated columns](/generated-columns.md) | Y | Y | Y | E | E | E | E | E | E | +| [Views](/views.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Sequences](/sql-statements/sql-statement-create-sequence.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Auto increment](/auto-increment.md) | Y | Y | Y | Y[^4] | Y | Y | Y | Y | Y | +| [Auto random](/auto-random.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [TTL (Time to Live)](/time-to-live.md) | Y | Y | Y | E | N | N | N | N | N | +| [DDL algorithm assertions](/sql-statements/sql-statement-alter-table.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| Multi-schema change: add columns | Y | Y | Y | Y | E | E | E | E | E | +| [Change column type](/sql-statements/sql-statement-modify-column.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Temporary tables](/temporary-tables.md) | Y | Y | Y | Y | Y | Y | Y | N | N | +| Concurrent DDL statements | Y | Y | Y | Y | N | N | N | N | N | +| [Acceleration of `ADD INDEX` and `CREATE INDEX`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) | Y | Y | Y | Y | N | N | N | N | N | +| [Metadata lock](/metadata-lock.md) | Y | Y | Y | Y | N | N | N | N | N | +| [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md) | Y | Y | Y | Y | N | N | N | N | N | +| [Pause](/sql-statements/sql-statement-admin-pause-ddl.md)/[Resume](/sql-statements/sql-statement-admin-resume-ddl.md) DDL | Y | Y | N | N | N | N | N | N | N | +| [TiDB Accelerated Table Creation](/accelerated-table-creation.md) | E | N | N | N | N | N | N | N | N | +| [Configure BDR role to replicate DDL statements in BDR mode](/sql-statements/sql-statement-admin-bdr-role.md#admin-setshowunset-bdr-role) | E | N | N | N | N | N | N | N | N | ## Transactions -| Transactions | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Async commit](/system-variables.md#tidb_enable_async_commit-new-in-v50) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| [1PC](/system-variables.md#tidb_enable_1pc-new-in-v50) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | -| [Large transactions (10GB)](/transaction-overview.md#transaction-size-limit) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Pessimistic transactions](/pessimistic-transaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Optimistic transactions](/optimistic-transaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Repeatable-read isolation (snapshot isolation)](/transaction-isolation-levels.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Read-committed isolation](/transaction-isolation-levels.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Automatically terminating long-running idle transactions](/system-variables.md#tidb_idle_transaction_timeout-new-in-v760) | Y | Y | N | N | N | N | N | N | N | N | N | N | +| Transactions | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Async commit](/system-variables.md#tidb_enable_async_commit-new-in-v50) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [1PC](/system-variables.md#tidb_enable_1pc-new-in-v50) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Large transactions (10GB)](/transaction-overview.md#transaction-size-limit) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Pessimistic transactions](/pessimistic-transaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Optimistic transactions](/optimistic-transaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Repeatable-read isolation (snapshot isolation)](/transaction-isolation-levels.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Read-committed isolation](/transaction-isolation-levels.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Automatically terminating long-running idle transactions](/system-variables.md#tidb_idle_transaction_timeout-new-in-v760) | Y | N | N | N | N | N | N | N | N | +| [Bulk DML execution mode (`tidb_dml_type = "bulk"`)](/system-variables.md#tidb_dml_type-new-in-v800) | E | N | N | N | N | N | N | N | N | ## Partitioning -| Partitioning | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Range partitioning](/partitioned-table.md#range-partitioning) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Hash partitioning](/partitioned-table.md#hash-partitioning) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Key partitioning](/partitioned-table.md#key-partitioning) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [List partitioning](/partitioned-table.md#list-partitioning) | Y | Y | Y | Y | Y | Y | E | E | E | E | E | N | -| [List COLUMNS partitioning](/partitioned-table.md) | Y | Y | Y | Y | Y | Y | E | E | E | E | E | N | -| [Default partition for List and List COLUMNS partitioned tables](/partitioned-table.md#default-list-partition) | Y | Y | Y | N | N | N | N | N | N | N | N | N | -| [`EXCHANGE PARTITION`](/partitioned-table.md) | Y | Y | Y | Y | Y | E | E | E | E | E | E | N | -| [`REORGANIZE PARTITION`](/partitioned-table.md#reorganize-partitions) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [`COALESCE PARTITION`](/partitioned-table.md#decrease-the-number-of-partitions) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [Dynamic pruning](/partitioned-table.md#dynamic-pruning-mode) | Y | Y | Y | Y | Y | Y | E | E | E | E | N | N | -| [Range COLUMNS partitioning](/partitioned-table.md#range-columns-partitioning) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Range INTERVAL partitioning](/partitioned-table.md#range-interval-partitioning) | Y | Y | Y | Y | E | N | N | N | N | N | N | N | -| [Convert a partitioned table to a non-partitioned table](/partitioned-table.md#convert-a-partitioned-table-to-a-non-partitioned-table) | Y | Y | Y | N | N | N | N | N | N | N | N | N | -| [Partition an existing table](/partitioned-table.md#partition-an-existing-table) | Y | Y | Y | N | N | N | N | N | N | N | N | N | +| Partitioning | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Range partitioning](/partitioned-table.md#range-partitioning) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Hash partitioning](/partitioned-table.md#hash-partitioning) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Key partitioning](/partitioned-table.md#key-partitioning) | Y | Y | Y | N | N | N | N | N | N | +| [List partitioning](/partitioned-table.md#list-partitioning) | Y | Y | Y | Y | Y | E | E | E | E | +| [List COLUMNS partitioning](/partitioned-table.md) | Y | Y | Y | Y | Y | E | E | E | E | +| [Default partition for List and List COLUMNS partitioned tables](/partitioned-table.md#default-list-partition) | Y | Y | N | N | N | N | N | N | N | +| [`EXCHANGE PARTITION`](/partitioned-table.md) | Y | Y | Y | Y | E | E | E | E | E | +| [`REORGANIZE PARTITION`](/partitioned-table.md#reorganize-partitions) | Y | Y | Y | N | N | N | N | N | N | +| [`COALESCE PARTITION`](/partitioned-table.md#decrease-the-number-of-partitions) | Y | Y | Y | N | N | N | N | N | N | +| [Dynamic pruning](/partitioned-table.md#dynamic-pruning-mode) | Y | Y | Y | Y | Y | E | E | E | E | +| [Range COLUMNS partitioning](/partitioned-table.md#range-columns-partitioning) | Y | Y | Y | Y | N | N | N | N | N | +| [Range INTERVAL partitioning](/partitioned-table.md#range-interval-partitioning) | Y | Y | Y | E | N | N | N | N | N | +| [Convert a partitioned table to a non-partitioned table](/partitioned-table.md#convert-a-partitioned-table-to-a-non-partitioned-table) | Y | Y | N | N | N | N | N | N | N | +| [Partition an existing table](/partitioned-table.md#partition-an-existing-table) | Y | Y | N | N | N | N | N | N | N | ## Statistics -| Statistics | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [CMSketch](/statistics.md) | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Y | Y | Y |Y | -| [Histograms](/statistics.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Extended statistics](/extended-statistics.md) | E | E | E | E | E | E | E | E | E | E | E | N | -| Statistics feedback | N | N | N | N | N | Deprecated | Deprecated | E | E | E | E | E | -| [Automatically update statistics](/statistics.md#automatic-update) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Fast Analyze](/system-variables.md#tidb_enable_fast_analyze) | Deprecated | Deprecated | Deprecated | E | E | E | E | E | E | E | E | E | -| [Dynamic pruning](/partitioned-table.md#dynamic-pruning-mode) | Y | Y | Y | Y | Y | Y | E | E | E | E | N | N | -| [Collect statistics for `PREDICATE COLUMNS`](/statistics.md#collect-statistics-on-some-columns) | E | E | E | E | E | E | E | N | N | N | N | N | -| [Control the memory quota for collecting statistics](/statistics.md#the-memory-quota-for-collecting-statistics) | E | E | E | E | E | N | N | N | N | N | N | N | -| [Randomly sample about 10000 rows of data to quickly build statistics](/system-variables.md#tidb_enable_fast_analyze) | Deprecated | Deprecated | Deprecated | E | E | E | E | E | E | E | E | E | -| [Lock statistics](/statistics.md#lock-statistics) | Y | Y | Y | E | E | N | N | N | N | N | N | N | -| [Lightweight statistics initialization](/statistics.md#load-statistics) | Y | Y | Y | E | N | N | N | N | N | N | N | N | -| [Show the progress of collecting statistics](/sql-statements/sql-statement-show-analyze-status.md) | Y | Y | Y | N | N | N | N | N | N | N | N | N | +| Statistics | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|---|---|---|---|---|---|---|---|---| +| [CMSketch](/statistics.md) | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Disabled by default | Y | Y | +| [Histograms](/statistics.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Extended statistics](/extended-statistics.md) | E | E | E | E | E | E | E | E | E | +| Statistics feedback | N | N | N | N | Deprecated | Deprecated | E | E | E | +| [Automatically update statistics](/statistics.md#automatic-update) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Dynamic pruning](/partitioned-table.md#dynamic-pruning-mode) | Y | Y | Y | Y | Y | E | E | E | E | +| [Collect statistics for `PREDICATE COLUMNS`](/statistics.md#collect-statistics-on-some-columns) | E | E | E | E | E | E | N | N | N | +| [Control the memory quota for collecting statistics](/statistics.md#the-memory-quota-for-collecting-statistics) | E | E | E | E | N | N | N | N | N | +| [Randomly sample about 10000 rows of data to quickly build statistics](/system-variables.md#tidb_enable_fast_analyze) | Deprecated | Deprecated | E | E | E | E | E | E | E | +| [Lock statistics](/statistics.md#lock-statistics) | Y | Y | E | E | N | N | N | N | N | +| [Lightweight statistics initialization](/statistics.md#load-statistics) | Y | Y | E | N | N | N | N | N | N | +| [Show the progress of collecting statistics](/sql-statements/sql-statement-show-analyze-status.md) | Y | Y | N | N | N | N | N | N | N | ## Security -| Security | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Transparent layer security (TLS)](/enable-tls-between-clients-and-servers.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Encryption at rest (TDE)](/encryption-at-rest.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Role-based authentication (RBAC)](/role-based-access-control.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Certificate-based authentication](/certificate-authentication.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`caching_sha2_password` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | N | -| [`tidb_sm3_password` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [`tidb_auth_token` authentication](/security-compatibility-with-mysql.md#tidb_auth_token) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [`authentication_ldap_sasl` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | N | N | N | N | N | N | N | N | -| [`authentication_ldap_simple` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [Password management](/password-management.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [MySQL compatible `GRANT` system](/privilege-management.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Dynamic Privileges](/privilege-management.md#dynamic-privileges) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | -| [Security Enhanced Mode](/system-variables.md#tidb_enable_enhanced_security) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | N | -| [Redacted Log Files](/log-redaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N | +| Security | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [Transparent layer security (TLS)](/enable-tls-between-clients-and-servers.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Encryption at rest (TDE)](/encryption-at-rest.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Role-based authentication (RBAC)](/role-based-access-control.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Certificate-based authentication](/certificate-authentication.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`caching_sha2_password` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | Y | Y | Y | Y | Y | N | +| [`tidb_sm3_password` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | Y | N | N | N | N | N | +| [`tidb_auth_token` authentication](/security-compatibility-with-mysql.md#tidb_auth_token) | Y | Y | Y | Y | N | N | N | N | N | +| [`authentication_ldap_sasl` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | N | N | N | N | N | N | N | +| [`authentication_ldap_simple` authentication](/system-variables.md#default_authentication_plugin) | Y | Y | Y | N | N | N | N | N | N | +| [Password management](/password-management.md) | Y | Y | Y | Y | N | N | N | N | N | +| [MySQL compatible `GRANT` system](/privilege-management.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Dynamic Privileges](/privilege-management.md#dynamic-privileges) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Security Enhanced Mode](/system-variables.md#tidb_enable_enhanced_security) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Redacted Log Files](/log-redaction.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | ## Data import and export -| Data import and export | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [Fast import using TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Fast import using the `IMPORT INTO` statement](/sql-statements/sql-statement-import-into.md) | Y | Y | Y | N | N | N | N | N | N | N | N | N | -| mydumper logical dumper | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | -| [Dumpling logical dumper](/dumpling-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Transactional `LOAD DATA`](/sql-statements/sql-statement-load-data.md) [^5] | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | N [^6] | -| [Database migration toolkit (DM)](/migration-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md) [^7] | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Change data capture (CDC)](/ticdc/ticdc-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Stream data to Amazon S3, GCS, Azure Blob Storage, and NFS through TiCDC](/ticdc/ticdc-sink-to-cloud-storage.md) | Y | Y | Y | Y | E | N | N | N | N | N | N | N | -| [TiCDC supports bidirectional replication between two TiDB clusters](/ticdc/ticdc-bidirectional-replication.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [TiCDC OpenAPI v2](/ticdc/ticdc-open-api-v2.md) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [DM](/dm/dm-overview.md) supports migrating MySQL 8.0 | Y | Y | E | E | E | E | N | N | N | N | N | N | +| Data import and export | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|---|---|---|---|---|---|---|---|---| +| [Fast import using TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Fast import using the `IMPORT INTO` statement](/sql-statements/sql-statement-import-into.md) | Y | Y | N | N | N | N | N | N | N | +| mydumper logical dumper | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | Deprecated | +| [Dumpling logical dumper](/dumpling-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Transactional `LOAD DATA`](/sql-statements/sql-statement-load-data.md) [^5] | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Database migration toolkit (DM)](/migration-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md) [^6] | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Change data capture (CDC)](/ticdc/ticdc-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Stream data to Amazon S3, GCS, Azure Blob Storage, and NFS through TiCDC](/ticdc/ticdc-sink-to-cloud-storage.md) | Y | Y | Y | E | N | N | N | N | N | +| [TiCDC supports bidirectional replication between two TiDB clusters](/ticdc/ticdc-bidirectional-replication.md) | Y | Y | Y | Y | N | N | N | N | N | +| [TiCDC OpenAPI v2](/ticdc/ticdc-open-api-v2.md) | Y | Y | Y | N | N | N | N | N | N | +| [DM](/dm/dm-overview.md) supports migrating MySQL 8.0 | Y | E | E | E | E | N | N | N | N | ## Management, observability, and tools -| Management, observability, and tools | 8.0 | 7.6 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | 5.0 | 4.0 | -|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| -| [TiDB Dashboard UI](/dashboard/dashboard-intro.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [TiDB Dashboard Continuous Profiling](/dashboard/continuous-profiling.md) | Y | Y | Y | Y | Y | Y | E | E | N | N | N | N | -| [TiDB Dashboard Top SQL](/dashboard/top-sql.md) | Y | Y | Y | Y | Y | Y | E | N | N | N | N | N | -| [TiDB Dashboard SQL Diagnostics](/information-schema/information-schema-sql-diagnostics.md) | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [TiDB Dashboard Cluster Diagnostics](/dashboard/dashboard-diagnostics-access.md) | Y | Y | Y | Y | Y | E | E | E | E | E | E | E | -| [TiKV-FastTune dashboard](/grafana-tikv-dashboard.md#tikv-fasttune-dashboard) | E | E | E | E | E | E | E | E | E | E | E | E | -| [Information schema](/information-schema/information-schema.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Metrics schema](/metrics-schema.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Statements summary tables](/statement-summary-tables.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Statements summary tables - summary persistence](/statement-summary-tables.md#persist-statements-summary) | E | E | E | E | N | N | N | N | N | N | N | N | -| [Slow query log](/identify-slow-queries.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [TiUP deployment](/tiup/tiup-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Kubernetes operator](https://docs.pingcap.com/tidb-in-kubernetes/) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Built-in physical backup](/br/backup-and-restore-use-cases.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [Global Kill](/sql-statements/sql-statement-kill.md) | Y | Y | Y | Y | Y | Y | E | E | E | E | E | E | -| [Lock View](/information-schema/information-schema-data-lock-waits.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | E | E | E | -| [`SHOW CONFIG`](/sql-statements/sql-statement-show-config.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | Y | -| [`SET CONFIG`](/dynamic-config.md) | Y | Y | Y | Y | Y | Y | E | E | E | E | E | E | -| [DM WebUI](/dm/dm-webui-guide.md) | E | E | E | E | E | E | N | N | N | N | N | N | -| [Foreground Quota Limiter](/tikv-configuration-file.md#foreground-quota-limiter) | Y | Y | Y | Y | Y | E | N | N | N | N | N | N | -| [Background Quota Limiter](/tikv-configuration-file.md#background-quota-limiter) | E | E | E | E | E | N | N | N | N | N | N | N | -| [EBS volume snapshot backup and restore](https://docs.pingcap.com/tidb-in-kubernetes/v1.4/backup-to-aws-s3-by-snapshot) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [PITR](/br/backup-and-restore-overview.md) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Global memory control](/configure-memory-usage.md#configure-the-memory-usage-threshold-of-a-tidb-server-instance) | Y | Y | Y | Y | Y | N | N | N | N | N | N | N | -| [Cross-cluster RawKV replication](/tikv-configuration-file.md#api-version-new-in-v610) | E | E | E | E | E | N | N | N | N | N | N | N | -| [Green GC](/system-variables.md#tidb_gc_scan_lock_mode-new-in-v50) | E | E | E | E | E | E | E | E | E | E | E | N | -| [Resource control](/tidb-resource-control.md) | Y | Y | Y | Y | N | N | N | N | N | N | N | N | -| [Runaway Queries management](/tidb-resource-control.md#manage-queries-that-consume-more-resources-than-expected-runaway-queries) | E | E | E | N | N | N | N | N | N | N | N | N | -| [Background tasks management](/tidb-resource-control.md#manage-background-tasks) | E | E | E | N | N | N | N | N | N | N | N | N | -| [TiFlash Disaggregated Storage and Compute Architecture and S3 Support](/tiflash/tiflash-disaggregated-and-s3.md) | Y | Y | Y | E | N | N | N | N | N | N | N | N | -| [Selecting TiDB nodes for the Distributed eXecution Framework (DXF) tasks](/system-variables.md#tidb_service_scope-new-in-v740) | Y | Y | Y | N | N | N | N | N | N | N | N | N | -| PD Follower Proxy (controlled by [`tidb_enable_tso_follower_proxy`](/system-variables.md#tidb_enable_tso_follower_proxy-new-in-v530)) | Y | Y | Y | Y | Y | Y | Y | Y | N | N | N | N | -| [Active PD Follower](/tune-region-performance.md#use-the-active-pd-follower-feature-to-enhance-the-scalability-of-pds-region-information-query-service) (controlled by [`pd_enable_follower_handle_region`](/system-variables.md#pd_enable_follower_handle_region-new-in-v760)) | E | E | N | N | N | N | N | N | N | N | N | N | -| [PD microservices](/pd-microservices.md) | E | N | N | N | N | N | N | N | N | N | N | N | -| [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md) | Y | Y | Y | E | N | N | N | N | N | N | N | N | -| [Global Sort](/tidb-global-sort.md) | Y | E | E | N | N | N | N | N | N | N | N | N | +| Management, observability, and tools | 8.1 | 7.5 | 7.1 | 6.5 | 6.1 | 5.4 | 5.3 | 5.2 | 5.1 | +|---|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:|:---:| +| [TiDB Dashboard UI](/dashboard/dashboard-intro.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [TiDB Dashboard Continuous Profiling](/dashboard/continuous-profiling.md) | Y | Y | Y | Y | Y | E | E | N | N | +| [TiDB Dashboard Top SQL](/dashboard/top-sql.md) | Y | Y | Y | Y | Y | E | N | N | N | +| [TiDB Dashboard SQL Diagnostics](/information-schema/information-schema-sql-diagnostics.md) | Y | Y | Y | Y | E | E | E | E | E | +| [TiDB Dashboard Cluster Diagnostics](/dashboard/dashboard-diagnostics-access.md) | Y | Y | Y | Y | E | E | E | E | E | +| [TiKV-FastTune dashboard](/grafana-tikv-dashboard.md#tikv-fasttune-dashboard) | E | E | E | E | E | E | E | E | E | +| [Information schema](/information-schema/information-schema.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Metrics schema](/metrics-schema.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Statements summary tables](/statement-summary-tables.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Statements summary tables - summary persistence](/statement-summary-tables.md#persist-statements-summary) | E | E | E | N | N | N | N | N | N | +| [Slow query log](/identify-slow-queries.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [TiUP deployment](/tiup/tiup-overview.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Kubernetes operator](https://docs.pingcap.com/tidb-in-kubernetes/) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Built-in physical backup](/br/backup-and-restore-use-cases.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [Global Kill](/sql-statements/sql-statement-kill.md) | Y | Y | Y | Y | Y | E | E | E | E | +| [Lock View](/information-schema/information-schema-data-lock-waits.md) | Y | Y | Y | Y | Y | Y | Y | Y | E | +| [`SHOW CONFIG`](/sql-statements/sql-statement-show-config.md) | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| [`SET CONFIG`](/dynamic-config.md) | Y | Y | Y | Y | Y | E | E | E | E | +| [DM WebUI](/dm/dm-webui-guide.md) | E | E | E | E | E | N | N | N | N | +| [Foreground Quota Limiter](/tikv-configuration-file.md#foreground-quota-limiter) | Y | Y | Y | Y | E | N | N | N | N | +| [Background Quota Limiter](/tikv-configuration-file.md#background-quota-limiter) | E | E | E | E | N | N | N | N | N | +| [EBS volume snapshot backup and restore](https://docs.pingcap.com/tidb-in-kubernetes/v1.4/backup-to-aws-s3-by-snapshot) | Y | Y | Y | Y | N | N | N | N | N | +| [PITR](/br/backup-and-restore-overview.md) | Y | Y | Y | Y | N | N | N | N | N | +| [Global memory control](/configure-memory-usage.md#configure-the-memory-usage-threshold-of-a-tidb-server-instance) | Y | Y | Y | Y | N | N | N | N | N | +| [Cross-cluster RawKV replication](/tikv-configuration-file.md#api-version-new-in-v610) | E | E | E | E | N | N | N | N | N | +| [Green GC](/system-variables.md#tidb_gc_scan_lock_mode-new-in-v50) | E | E | E | E | E | E | E | E | E | +| [Resource control](/tidb-resource-control.md) | Y | Y | Y | N | N | N | N | N | N | +| [Runaway Queries management](/tidb-resource-control.md#manage-queries-that-consume-more-resources-than-expected-runaway-queries) | Y | E | N | N | N | N | N | N | N | +| [Background tasks management](/tidb-resource-control.md#manage-background-tasks) | E | E | N | N | N | N | N | N | N | +| [TiFlash Disaggregated Storage and Compute Architecture and S3 Support](/tiflash/tiflash-disaggregated-and-s3.md) | Y | Y | E | N | N | N | N | N | N | +| [Selecting TiDB nodes for the Distributed eXecution Framework (DXF) tasks](/system-variables.md#tidb_service_scope-new-in-v740) | Y | Y | N | N | N | N | N | N | N | +| PD Follower Proxy (controlled by [`tidb_enable_tso_follower_proxy`](/system-variables.md#tidb_enable_tso_follower_proxy-new-in-v530)) | Y | Y | Y | Y | Y | Y | Y | N | N | +| [Active PD Follower](/tune-region-performance.md#use-the-active-pd-follower-feature-to-enhance-the-scalability-of-pds-region-information-query-service) (controlled by [`pd_enable_follower_handle_region`](/system-variables.md#pd_enable_follower_handle_region-new-in-v760)) | E | N | N | N | N | N | N | N | N | +| [PD microservices](/pd-microservices.md) | E | N | N | N | N | N | N | N | N | +| [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md) | Y | Y | E | N | N | N | N | N | N | +| [Global Sort](/tidb-global-sort.md) | Y | E | N | N | N | N | N | N | N | [^1]: TiDB incorrectly treats latin1 as a subset of utf8. See [TiDB #18955](https://github.com/pingcap/tidb/issues/18955) for more details. @@ -272,6 +271,4 @@ You can try out TiDB features on [TiDB Playground](https://play.tidbcloud.com/?u [^5]: Starting from [TiDB v7.0.0](/releases/release-7.0.0.md), the new parameter `FIELDS DEFINED NULL BY` and support for importing data from S3 and GCS are experimental features. Starting from [v7.6.0](/releases/release-7.6.0.md), TiDB processes `LOAD DATA` in transactions in the same way as MySQL. The `LOAD DATA` statement in a transaction no longer automatically commits the current transaction or starts a new transaction. Moreover, you can explicitly commit or roll back the `LOAD DATA` statement in a transaction. Additionally, the `LOAD DATA` statement is affected by the TiDB transaction mode setting (optimistic or pessimistic transaction). -[^6]: For TiDB v4.0, the `LOAD DATA` transaction does not guarantee atomicity. - -[^7]: Starting from TiDB v7.5.0, technical support for the data replication feature of [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md) is no longer provided. It is strongly recommended to use [TiCDC](/ticdc/ticdc-overview.md) as an alternative solution for data replication. Although TiDB Binlog v7.5.0 still supports the Point-in-Time Recovery (PITR) scenario, this component will be completely deprecated in future versions. It is recommended to use [PITR](/br/br-pitr-guide.md) as an alternative solution for data recovery. +[^6]: Starting from TiDB v7.5.0, technical support for the data replication feature of [TiDB Binlog](/tidb-binlog/tidb-binlog-overview.md) is no longer provided. It is strongly recommended to use [TiCDC](/ticdc/ticdc-overview.md) as an alternative solution for data replication. Although TiDB Binlog v7.5.0 still supports the Point-in-Time Recovery (PITR) scenario, this component will be completely deprecated in future versions. It is recommended to use [PITR](/br/br-pitr-guide.md) as an alternative solution for data recovery. diff --git a/basic-sql-operations.md b/basic-sql-operations.md index 6bd04f4324776..f2e1240a271e6 100644 --- a/basic-sql-operations.md +++ b/basic-sql-operations.md @@ -1,7 +1,6 @@ --- title: Explore SQL with TiDB summary: Learn about the basic SQL statements for the TiDB database. -aliases: ['/docs/dev/basic-sql-operations/','/docs/dev/how-to/get-started/explore-sql/'] --- # Explore SQL with TiDB @@ -14,7 +13,7 @@ To experiment with SQL and test out TiDB compatibility with MySQL queries, you c -This page walks you through the basic TiDB SQL statements such as DDL, DML and CRUD operations. For a complete list of TiDB statements, see [TiDB SQL Syntax Diagram](https://pingcap.github.io/sqlgram/). +This page walks you through the basic TiDB SQL statements such as DDL, DML and CRUD operations. For a complete list of TiDB statements, see [SQL Statement Overview](/sql-statements/sql-statement-overview.md). ## Category diff --git a/benchmark/benchmark-sysbench-v2.md b/benchmark/benchmark-sysbench-v2.md index 2fdc749b3aa00..93168caab39fc 100644 --- a/benchmark/benchmark-sysbench-v2.md +++ b/benchmark/benchmark-sysbench-v2.md @@ -1,6 +1,5 @@ --- title: TiDB Sysbench Performance Test Report -- v2.0.0 vs. v1.0.0 -aliases: ['/docs/dev/benchmark/benchmark-sysbench-v2/','/docs/dev/benchmark/sysbench-v2/'] summary: TiDB 2.0 GA outperforms TiDB 1.0 GA in `Select` and `Insert` tests, with a 10% increase in `Select` query performance and a slight improvement in `Insert` query performance. However, the OLTP performance of both versions is almost the same. --- diff --git a/benchmark/benchmark-sysbench-v3.md b/benchmark/benchmark-sysbench-v3.md index fd583653db0d4..9aa795659eaec 100644 --- a/benchmark/benchmark-sysbench-v3.md +++ b/benchmark/benchmark-sysbench-v3.md @@ -1,6 +1,5 @@ --- title: TiDB Sysbench Performance Test Report -- v2.1 vs. v2.0 -aliases: ['/docs/dev/benchmark/benchmark-sysbench-v3/','/docs/dev/benchmark/sysbench-v3/'] summary: TiDB 2.1 outperforms TiDB 2.0 in the `Point Select` test, with a 50% increase in query performance. However, the `Update Non-Index` and `Update Index` tests show similar performance between the two versions. The test was conducted in September 2018 in Beijing, China, using a specific test environment and configuration. --- diff --git a/benchmark/benchmark-sysbench-v4-vs-v3.md b/benchmark/benchmark-sysbench-v4-vs-v3.md index 41fd3a544de22..1c997eb2b8316 100644 --- a/benchmark/benchmark-sysbench-v4-vs-v3.md +++ b/benchmark/benchmark-sysbench-v4-vs-v3.md @@ -1,7 +1,6 @@ --- title: TiDB Sysbench Performance Test Report -- v4.0 vs. v3.0 summary: Compare the Sysbench performance of TiDB 4.0 and TiDB 3.0. -aliases: ['/docs/dev/benchmark/benchmark-sysbench-v4-vs-v3/'] --- # TiDB Sysbench Performance Test Report -- v4.0 vs. v3.0 diff --git a/benchmark/benchmark-tidb-using-sysbench.md b/benchmark/benchmark-tidb-using-sysbench.md index f821bdb1ff03f..38dca1f519f68 100644 --- a/benchmark/benchmark-tidb-using-sysbench.md +++ b/benchmark/benchmark-tidb-using-sysbench.md @@ -1,6 +1,5 @@ --- title: How to Test TiDB Using Sysbench -aliases: ['/docs/dev/benchmark/benchmark-tidb-using-sysbench/','/docs/dev/benchmark/how-to-run-sysbench/'] summary: TiDB performance can be optimized by using Sysbench 1.0 or later. Configure TiDB and TiKV with higher log levels for better performance. Adjust Sysbench configuration and import data to optimize performance. Address common issues related to proxy use and CPU utilization rates. --- diff --git a/benchmark/benchmark-tidb-using-tpcc.md b/benchmark/benchmark-tidb-using-tpcc.md index 72a3c270f1d1c..f920dbea71016 100644 --- a/benchmark/benchmark-tidb-using-tpcc.md +++ b/benchmark/benchmark-tidb-using-tpcc.md @@ -1,6 +1,5 @@ --- title: How to Run TPC-C Test on TiDB -aliases: ['/docs/dev/benchmark/benchmark-tidb-using-tpcc/','/docs/dev/benchmark/how-to-run-tpcc/'] summary: This document describes how to test TiDB using TPC-C, an online transaction processing benchmark. It specifies the initial state of the database, provides commands for loading data, running the test, and cleaning up test data. The test measures the maximum qualified throughput using tpmC (transactions per minute). --- @@ -24,7 +23,7 @@ Before testing, TPC-C Benchmark specifies the initial state of the database, whi * The `DISTRICT` table has W \* 10 records (Each warehouse provides services to 10 districts) * The `CUSTOMER` table has W \* 10 \* 3,000 records (Each district has 3,000 customers) * The `HISTORY` table has W \* 10 \* 3,000 records (Each customer has one transaction history) -* The `ORDER` table has W \* 10 \* 3,000 records (Each district has 3,000 orders and the last 900 orders generated are added to the `NEW-ORDER` table. Each order randomly generates 5 ~ 15 ORDER-LINE records. +* The `ORDER` table has W \* 10 \* 3,000 records (Each district has 3,000 orders and the last 900 orders generated are added to the `NEW-ORDER` table. Each order randomly generates 5 ~ 15 ORDER-LINE records.) In this document, the testing uses 1,000 warehouses as an example to test TiDB. diff --git a/benchmark/benchmark-tpch.md b/benchmark/benchmark-tpch.md index 2689a52021d3d..20c740294d503 100644 --- a/benchmark/benchmark-tpch.md +++ b/benchmark/benchmark-tpch.md @@ -1,6 +1,5 @@ --- title: TiDB TPC-H 50G Performance Test Report V2.0 -aliases: ['/docs/dev/benchmark/benchmark-tpch/','/docs/dev/benchmark/tpch/'] summary: TiDB TPC-H 50G Performance Test compared TiDB 1.0 and TiDB 2.0 in an OLAP scenario. Test results show that TiDB 2.0 outperformed TiDB 1.0 in most queries, with significant improvements in query processing time. Some queries in TiDB 1.0 did not return results, while others had high memory consumption. Future releases plan to support VIEW and address these issues. --- diff --git a/benchmark/online-workloads-and-add-index-operations.md b/benchmark/online-workloads-and-add-index-operations.md index 366cf3adb3fb7..dafc6462c39c6 100644 --- a/benchmark/online-workloads-and-add-index-operations.md +++ b/benchmark/online-workloads-and-add-index-operations.md @@ -1,7 +1,6 @@ --- title: Interaction Test on Online Workloads and `ADD INDEX` Operations summary: This document tests the interaction effects between online workloads and `ADD INDEX` operations. -aliases: ['/docs/dev/benchmark/online-workloads-and-add-index-operations/','/docs/dev/benchmark/add-index-with-load/'] --- # Interaction Test on Online Workloads and `ADD INDEX` Operations diff --git a/benchmark/v3.0-performance-benchmarking-with-sysbench.md b/benchmark/v3.0-performance-benchmarking-with-sysbench.md index 30afbc7bb9e54..085203dc773d9 100644 --- a/benchmark/v3.0-performance-benchmarking-with-sysbench.md +++ b/benchmark/v3.0-performance-benchmarking-with-sysbench.md @@ -1,6 +1,5 @@ --- title: TiDB Sysbench Performance Test Report -- v3.0 vs. v2.1 -aliases: ['/docs/dev/benchmark/v3.0-performance-benchmarking-with-sysbench/','/docs/dev/benchmark/sysbench-v4/'] summary: TiDB v3.0 outperformed v2.1 in all tests, with higher QPS and lower latency. Configuration changes in v3.0 contributed to the improved performance. --- diff --git a/benchmark/v3.0-performance-benchmarking-with-tpcc.md b/benchmark/v3.0-performance-benchmarking-with-tpcc.md index 3e0daae7b5dd1..0d902d207d405 100644 --- a/benchmark/v3.0-performance-benchmarking-with-tpcc.md +++ b/benchmark/v3.0-performance-benchmarking-with-tpcc.md @@ -1,6 +1,5 @@ --- title: TiDB TPC-C Performance Test Report -- v3.0 vs. v2.1 -aliases: ['/docs/dev/benchmark/v3.0-performance-benchmarking-with-tpcc/','/docs/dev/benchmark/tpcc/'] summary: TiDB v3.0 outperforms v2.1 in TPC-C performance test. With 1000 warehouses, v3.0 achieved 450% higher performance than v2.1. --- diff --git a/benchmark/v4.0-performance-benchmarking-with-tpcc.md b/benchmark/v4.0-performance-benchmarking-with-tpcc.md index 1169a9a92f492..bc7ad745b8824 100644 --- a/benchmark/v4.0-performance-benchmarking-with-tpcc.md +++ b/benchmark/v4.0-performance-benchmarking-with-tpcc.md @@ -1,7 +1,6 @@ --- title: TiDB TPC-C Performance Test Report -- v4.0 vs. v3.0 summary: Compare the TPC-C performance of TiDB 4.0 and TiDB 3.0 using BenchmarkSQL. -aliases: ['/docs/dev/benchmark/v4.0-performance-benchmarking-with-tpcc/'] --- # TiDB TPC-C Performance Test Report -- v4.0 vs. v3.0 diff --git a/benchmark/v4.0-performance-benchmarking-with-tpch.md b/benchmark/v4.0-performance-benchmarking-with-tpch.md index 801db52f05b69..e7364e64b85e7 100644 --- a/benchmark/v4.0-performance-benchmarking-with-tpch.md +++ b/benchmark/v4.0-performance-benchmarking-with-tpch.md @@ -1,7 +1,6 @@ --- title: TiDB TPC-H Performance Test Report -- v4.0 vs. v3.0 summary: Compare the TPC-H performance of TiDB 4.0 and TiDB 3.0. -aliases: ['/docs/dev/benchmark/v4.0-performance-benchmarking-with-tpch/'] --- # TiDB TPC-H Performance Test Report -- v4.0 vs. v3.0 diff --git a/best-practices-for-security-configuration.md b/best-practices-for-security-configuration.md new file mode 100644 index 0000000000000..1b684ce86f96d --- /dev/null +++ b/best-practices-for-security-configuration.md @@ -0,0 +1,124 @@ +--- +title: Best Practices for TiDB Security Configuration +summary: Learn the best practices for TiDB security configuration to help mitigate potential security risks. +--- + +# Best Practices for TiDB Security Configuration + +The security of TiDB is crucial for protecting data integrity and confidentiality. This document provides guidelines for configuring TiDB clusters securely during deployment. By following these best practices, you can effectively reduce potential security risks, prevent data breaches, and ensure the continuous, stable, and reliable operation of your TiDB database system. + +> **Note:** +> +> This document offers general recommendations on TiDB security configurations. PingCAP does not guarantee the completeness or accuracy of the information, and it assumes no responsibility for any issues arising from the use of this guide. Users should assess these recommendations based on their specific needs and consult professionals for tailored advice. + +## Set the initial password for the root user + +By default, the root user in a newly created TiDB cluster has no password, which poses a potential security risk. If a password is not set, anyone can attempt to log in to the TiDB database as the root user, potentially gaining access to and modifying data. + +To avoid this risk, it is recommended to set a root password during deployment: + +- For deployments using TiUP, refer to [Deploy TiDB Cluster Using TiUP](/production-deployment-using-tiup.md#step-7-start-a-tidb-cluster) to generate a random password for the root user. +- For deployments using TiDB Operator, refer to [Set initial account and password](https://docs.pingcap.com/tidb-in-kubernetes/stable/initialize-a-cluster#set-initial-account-and-password) to set the root password. + +## Enable password complexity checks + +By default, TiDB does not enforce password complexity policies, which might lead to the use of weak or empty passwords, increasing security risks. + +To ensure that database users create strong passwords, it is recommended to configure a reasonable [password complexity policy](/password-management.md#password-complexity-policy). For example, configure a policy that requires passwords to include a combination of uppercase letters, lowercase letters, numbers, and special characters. By enforcing password complexity checks, you can improve database security, prevent brute force attacks, reduce internal threats, ensure compliance with regulations, and lower the risk of data breaches, thus enhancing overall security. + +## Change the default Grafana password + +TiDB installation includes the Grafana component by default, and the default username and password are typically `admin`/`admin`. If the password is not changed promptly, attackers could exploit this to gain control of the system. + +It is recommended to immediately change the Grafana password to a strong one during the TiDB deployment, and regularly update the password to ensure system security. Here are the steps to change the Grafana password: + +- Upon first login to Grafana, follow the prompts to change the password. + + ![Grafana Password Reset Guide](/media/grafana-password-reset1.png) + +- Access the Grafana personal configuration center to change the password. + + ![Grafana Password Reset Guide](/media/grafana-password-reset2.png) + +## Enhance TiDB Dashboard security + +### Use a least privilege user + +TiDB Dashboard shares the account system with TiDB SQL users, and TiDB Dashboard authorization is based on TiDB SQL user permissions. TiDB Dashboard requires minimal permissions and can even operate with read-only access. + +To enhance security, it is recommended to create a [least-privilege SQL user](/dashboard/dashboard-user.md) for accessing the TiDB Dashboard and to avoid using high-privilege users. + +### Restrict access control + +By default, TiDB Dashboard is designed for trusted users. The default port includes additional API interfaces besides TiDB Dashboard. If you want to allow access to TiDB Dashboard from external networks or untrusted users, take the following measures to avoid security vulnerabilities: + +- Use a firewall or other mechanisms to restrict the default `2379` port to trusted domains, preventing access by external users. + + > **Note:** + > + > TiDB, TiKV, and other components need to communicate with the PD component via the PD client port. Do not block internal network access between components, which will make the cluster unavailable. + +- [Configure a reverse proxy](/dashboard/dashboard-ops-reverse-proxy.md#use-tidb-dashboard-behind-a-reverse-proxy) to securely provide TiDB Dashboard services to external users on a different port. + +## Protect internal ports + +By default, TiDB installation includes several privileged interfaces for inter-component communication. These ports typically do not need to be accessible to users, because they are primarily for internal communication. Exposing these ports on public networks increases the attack surface, violates the principle of least privilege, and raises the risk of security vulnerabilities. The following table lists the default listening ports in a TiDB cluster: + +| Component | Default port | Protocol | +|-------------------|-------------|------------| +| TiDB | 4000 | MySQL | +| TiDB | 10080 | HTTP | +| TiKV | 20160 | Protocol | +| TiKV | 20180 | HTTP | +| PD | 2379 | HTTP/Protocol| +| PD | 2380 | Protocol | +| TiFlash | 3930 | Protocol | +| TiFlash | 20170 | Protocol | +| TiFlash | 20292 | HTTP | +| TiFlash | 8234 | HTTP | +| TiFlow | 8261 | HTTP | +| TiFlow | 8291 | HTTP | +| TiFlow | 8262 | HTTP | +| TiFlow | 8300 | HTTP | +| TiDB Lightning | 8289 | HTTP | +| TiDB Operator | 6060 | HTTP | +| TiDB Dashboard | 2379 | HTTP | +| TiDB Binlog | 8250 | HTTP | +| TiDB Binlog | 8249 | HTTP | +| TMS | 8082 | HTTP | +| TEM | 8080 | HTTP | +| TEM | 8000 | HTTP | +| TEM | 4110 | HTTP | +| TEM | 4111 | HTTP | +| TEM | 4112 | HTTP | +| TEM | 4113 | HTTP | +| TEM | 4124 | HTTP | +| Prometheus | 9090 | HTTP | +| Grafana | 3000 | HTTP | +| AlertManager | 9093 | HTTP | +| AlertManager | 9094 | Protocol | +| Node Exporter | 9100 | HTTP | +| Blackbox Exporter | 9115 | HTTP | +| NG Monitoring | 12020 | HTTP | + +It is recommended to only expose the `4000` port for the database and the `9000` port for the Grafana dashboard to ordinary users, while restricting access to other ports using network security policies or firewalls. The following is an example of using `iptables` to restrict port access: + +```shell +# Allow internal port communication from the whitelist of component IP addresses +sudo iptables -A INPUT -s internal IP address range -j ACCEPT + +# Only open ports 4000 and 9000 to external users +sudo iptables -A INPUT -p tcp --dport 4000 -j ACCEPT +sudo iptables -A INPUT -p tcp --dport 9000 -j ACCEPT + +# Deny all other traffic by default +sudo iptables -P INPUT DROP +``` + +If you need to access TiDB Dashboard, it is recommended to [configure a reverse proxy](/dashboard/dashboard-ops-reverse-proxy.md#use-tidb-dashboard-behind-a-reverse-proxy) to securely provide services to external networks on a separate port. + +## Resolving false positives from third-party MySQL vulnerability scanners + +Most vulnerability scanners detect MySQL vulnerabilities based on version information. Because TiDB is MySQL protocol-compatible but not MySQL itself, version-based vulnerability scans might lead to false positives. It is recommended to focus vulnerability scans on principle-based assessments. If compliance scanning tools require a specific MySQL version, you can [modify the server version number](/faq/high-reliability-faq.md#does-tidb-support-modifying-the-mysql-version-string-of-the-server-to-a-specific-one-that-is-required-by-the-security-vulnerability-scanning-tool) to meet the requirement. + +By changing the server version number, you can avoid false positives from vulnerability scanners. The [`server-version`](/tidb-configuration-file.md#server-version) value is used by TiDB nodes to verify the current TiDB version. Before upgrading the TiDB cluster, ensure that the `server-version` value is either empty or the actual version of TiDB to avoid unexpected behavior. \ No newline at end of file diff --git a/best-practices-on-public-cloud.md b/best-practices-on-public-cloud.md index 06497bfaf2f3b..acb163f13bb55 100644 --- a/best-practices-on-public-cloud.md +++ b/best-practices-on-public-cloud.md @@ -7,7 +7,40 @@ summary: Learn about the best practices for deploying TiDB on public cloud. Public cloud infrastructure has become an increasingly popular choice for deploying and managing TiDB. However, deploying TiDB on public cloud requires careful consideration of several critical factors, including performance tuning, cost optimization, reliability, and scalability. -This document covers various essential best practices for deploying TiDB on public cloud, such as using a dedicated disk for Raft Engine, reducing compaction I/O flow in KV RocksDB, optimizing costs for cross-AZ traffic, mitigating Google Cloud live migration events, and fine-tuning the PD server in large clusters. By following these best practices, you can maximize the performance, cost efficiency, reliability, and scalability of your TiDB deployment on public cloud. +This document covers various essential best practices for deploying TiDB on public cloud, such as reducing compaction I/O flow in KV RocksDB, using a dedicated disk for Raft Engine, optimizing costs for cross-AZ traffic, mitigating Google Cloud live migration events, and fine-tuning the PD server in large clusters. By following these best practices, you can maximize the performance, cost efficiency, reliability, and scalability of your TiDB deployment on public cloud. + +## Reduce compaction I/O flow in KV RocksDB + +As the storage engine of TiKV, [RocksDB](https://rocksdb.org/) is used to store user data. Because the provisioned IO throughput on cloud EBS is usually limited due to cost considerations, RocksDB might exhibit high write amplification, and the disk throughput might become the bottleneck for the workload. As a result, the total number of pending compaction bytes grows over time and triggers flow control, which indicates that TiKV lacks sufficient disk bandwidth to keep up with the foreground write flow. + +To alleviate the bottleneck caused by limited disk throughput, you can improve performance by [enabling Titan](#enable-titan). If your average row size is smaller than 512 bytes, Titan is not applicable. In this case, you can improve performance by [increasing all the compression levels](#increase-all-the-compression-levels). + +### Enable Titan + +[Titan](/storage-engine/titan-overview.md) is a high-performance [RocksDB](https://github.com/facebook/rocksdb) plugin for key-value separation, which can reduce write amplification in RocksDB when large values are used. + +If your average row size is larger than 512 bytes, you can enable Titan to reduce the compaction I/O flow as follows, with `min-blob-size` set to `"512B"` or `"1KB"` and `blob-file-compression` set to `"zstd"`: + +```toml +[rocksdb.titan] +enabled = true +[rocksdb.defaultcf.titan] +min-blob-size = "1KB" +blob-file-compression = "zstd" +``` + +> **Note:** +> +> When Titan is enabled, there might be a slight performance degradation for range scans on the primary key. For more information, see [Impact of `min-blob-size` on performance](/storage-engine/titan-overview.md#impact-of-min-blob-size-on-performance). + +### Increase all the compression levels + +If your average row size is smaller than 512 bytes, you can increase all the compression levels of the default column family to `"zstd"` as follows: + +```toml +[rocksdb.defaultcf] +compression-per-level = ["zstd", "zstd", "zstd", "zstd", "zstd", "zstd", "zstd"] +``` ## Use a dedicated disk for Raft Engine @@ -97,17 +130,6 @@ tikv: storageSize: 512Gi ``` -## Reduce compaction I/O flow in KV RocksDB - -As the storage engine of TiKV, [RocksDB](https://rocksdb.org/) is used to store user data. Because the provisioned IO throughput on cloud EBS is usually limited due to cost considerations, RocksDB might exhibit high write amplification, and the disk throughput might become the bottleneck for the workload. As a result, the total number of pending compaction bytes grows over time and triggers flow control, which indicates that TiKV lacks sufficient disk bandwidth to keep up with the foreground write flow. - -To alleviate the bottleneck caused by limited disk throughput, you can improve performance by increasing the compression level for RocksDB and reducing the disk throughput. For example, you can refer to the following example to increase all the compression levels of the default column family to `zstd`. - -``` -[rocksdb.defaultcf] -compression-per-level = ["zstd", "zstd", "zstd", "zstd", "zstd", "zstd", "zstd"] -``` - ## Optimize cost for cross-AZ network traffic Deploying TiDB across multiple availability zones (AZs) can lead to increased costs due to cross-AZ data transfer fees. To optimize costs, it is important to reduce cross-AZ network traffic. diff --git a/best-practices/grafana-monitor-best-practices.md b/best-practices/grafana-monitor-best-practices.md index d399b359143b4..206b1f5334b60 100644 --- a/best-practices/grafana-monitor-best-practices.md +++ b/best-practices/grafana-monitor-best-practices.md @@ -1,7 +1,6 @@ --- title: Best Practices for Monitoring TiDB Using Grafana summary: Best Practices for Monitoring TiDB Using Grafana. Deploy a TiDB cluster using TiUP and add Grafana and Prometheus for monitoring. Use metrics to analyze cluster status and diagnose problems. Prometheus collects metrics from TiDB components, and Grafana displays them. Tips for efficient Grafana use include modifying query expressions, switching Y-axis scale, and using API for query results. The platform is powerful for analyzing and diagnosing TiDB cluster status. -aliases: ['/docs/dev/best-practices/grafana-monitor-best-practices/','/docs/dev/reference/best-practices/grafana-monitor/'] --- # Best Practices for Monitoring TiDB Using Grafana @@ -22,12 +21,12 @@ For TiDB 2.1.3 or later versions, TiDB monitoring supports the pull method. It i ## Source and display of monitoring data -The three core components of TiDB (TiDB server, TiKV server and PD server) obtain metrics through the HTTP interface. These metrics are collected from the program code, and the ports are as follows: +The three core components of TiDB (TiDB server, TiKV server and PD server) obtain metrics through the HTTP interface. These metrics are collected from the program code, and the default ports are as follows: | Component | Port | | :---------- |:----- | | TiDB server | 10080 | -| TiKV server | 20181 | +| TiKV server | 20180 | | PD server | 2379 | Execute the following command to check the QPS of a SQL statement through the HTTP interface. Take the TiDB server as an example: @@ -160,7 +159,7 @@ The API of Prometheus is shown as follows: {{< copyable "shell-regular" >}} ```bash -curl -u user:pass 'http://__grafana_ip__:3000/api/datasources/proxy/1/api/v1/query_range?query=sum(tikv_engine_size_bytes%7Binstancexxxxxxxxx20181%22%7D)%20by%20(instance)&start=1565879269&end=1565882869&step=30' |python -m json.tool +curl -u user:pass 'http://__grafana_ip__:3000/api/datasources/proxy/1/api/v1/query_range?query=sum(tikv_engine_size_bytes%7Binstancexxxxxxxxx20180%22%7D)%20by%20(instance)&start=1565879269&end=1565882869&step=30' |python -m json.tool ``` ``` @@ -169,7 +168,7 @@ curl -u user:pass 'http://__grafana_ip__:3000/api/datasources/proxy/1/api/v1/que "result": [ { "metric": { - "instance": "xxxxxxxxxx:20181" + "instance": "xxxxxxxxxx:20180" }, "values": [ [ diff --git a/best-practices/haproxy-best-practices.md b/best-practices/haproxy-best-practices.md index 061a2bdac305c..f9f012486af20 100644 --- a/best-practices/haproxy-best-practices.md +++ b/best-practices/haproxy-best-practices.md @@ -1,7 +1,6 @@ --- title: Best Practices for Using HAProxy in TiDB summary: HAProxy is a free, open-source load balancer and proxy server for TCP and HTTP-based applications. It provides high availability, load balancing, health checks, sticky sessions, SSL support, and monitoring. To deploy HAProxy, ensure hardware and software requirements are met, then install and configure it. Use the latest stable version for best results. -aliases: ['/docs/dev/best-practices/haproxy-best-practices/','/docs/dev/reference/best-practices/haproxy/'] --- # Best Practices for Using HAProxy in TiDB diff --git a/best-practices/high-concurrency-best-practices.md b/best-practices/high-concurrency-best-practices.md index bb296b8629df5..49c0ba3d42d25 100644 --- a/best-practices/high-concurrency-best-practices.md +++ b/best-practices/high-concurrency-best-practices.md @@ -1,7 +1,6 @@ --- title: Highly Concurrent Write Best Practices summary: This document provides best practices for handling highly-concurrent write-heavy workloads in TiDB. It addresses challenges and solutions for data distribution, hotspot cases, and complex hotspot problems. The article also discusses parameter configuration for optimizing performance. -aliases: ['/docs/dev/best-practices/high-concurrency-best-practices/','/docs/dev/reference/best-practices/high-concurrency/'] --- # Highly Concurrent Write Best Practices @@ -10,11 +9,11 @@ This document describes best practices for handling highly-concurrent write-heav ## Target audience -This document assumes that you have a basic understanding of TiDB. It is recommended that you first read the following three blog articles that explain TiDB fundamentals, and [TiDB Best Practices](https://en.pingcap.com/blog/tidb-best-practice/): +This document assumes that you have a basic understanding of TiDB. It is recommended that you first read the following three blog articles that explain TiDB fundamentals, and [TiDB Best Practices](https://www.pingcap.com/blog/tidb-best-practice/): -+ [Data Storage](https://en.pingcap.com/blog/tidb-internal-data-storage/) -+ [Computing](https://en.pingcap.com/blog/tidb-internal-computing/) -+ [Scheduling](https://en.pingcap.com/blog/tidb-internal-scheduling/) ++ [Data Storage](https://www.pingcap.com/blog/tidb-internal-data-storage/) ++ [Computing](https://www.pingcap.com/blog/tidb-internal-computing/) ++ [Scheduling](https://www.pingcap.com/blog/tidb-internal-scheduling/) ## Highly-concurrent write-intensive scenario @@ -33,7 +32,7 @@ For a distributed database, it is important to make full use of the capacity of ## Data distribution principles in TiDB -To address the above challenges, it is necessary to start with the data segmentation and scheduling principle of TiDB. Refer to [Scheduling](https://en.pingcap.com/blog/tidb-internal-scheduling/) for more details. +To address the above challenges, it is necessary to start with the data segmentation and scheduling principle of TiDB. Refer to [Scheduling](https://www.pingcap.com/blog/tidb-internal-scheduling/) for more details. TiDB splits data into Regions, each representing a range of data with a size limit of 96M by default. Each Region has multiple replicas, and each group of replicas is called a Raft Group. In a Raft Group, the Region Leader executes the read and write tasks (TiDB supports [Follower-Read](/follower-read.md)) within the data range. The Region Leader is automatically scheduled by the Placement Driver (PD) component to different physical nodes evenly to distribute the read and write pressure. diff --git a/best-practices/java-app-best-practices.md b/best-practices/java-app-best-practices.md index a62b7ceffcf72..d8e49dd9ec69d 100644 --- a/best-practices/java-app-best-practices.md +++ b/best-practices/java-app-best-practices.md @@ -1,7 +1,6 @@ --- title: Best Practices for Developing Java Applications with TiDB summary: This document introduces best practices for developing Java applications with TiDB, covering database-related components, JDBC usage, connection pool configuration, data access framework, Spring Transaction, and troubleshooting tools. TiDB is highly compatible with MySQL, so most MySQL-based Java application best practices also apply to TiDB. -aliases: ['/docs/dev/best-practices/java-app-best-practices/','/docs/dev/reference/best-practices/java-app/'] --- # Best Practices for Developing Java Applications with TiDB @@ -200,26 +199,62 @@ Through monitoring, you might notice that although the application only performs After it is configured, you can check the monitoring to see a decreased number of `SELECT` statements. +> **Note:** +> +> Enabling `useConfigs=maxPerformance` requires MySQL Connector/J version 8.0.33 or later. For more details, see [MySQL JDBC Bug](/develop/dev-guide-third-party-tools-compatibility.md#mysql-jdbc-bugs). + #### Timeout-related parameters TiDB provides two MySQL-compatible parameters that controls the timeout: `wait_timeout` and `max_execution_time`. These two parameters respectively control the connection idle timeout with the Java application and the timeout of the SQL execution in the connection; that is to say, these parameters control the longest idle time and the longest busy time for the connection between TiDB and the Java application. The default value of both parameters is `0`, which by default allows the connection to be infinitely idle and infinitely busy (an infinite duration for one SQL statement to execute). However, in an actual production environment, idle connections and SQL statements with excessively long execution time negatively affect databases and applications. To avoid idle connections and SQL statements that are executed for too long, you can configure these two parameters in your application's connection string. For example, set `sessionVariables=wait_timeout=3600` (1 hour) and `sessionVariables=max_execution_time=300000` (5 minutes). +#### Typical JDBC connection string parameters + +Combining the preceding parameter values, the JDBC connection string configuration is as follows: + +``` +jdbc:mysql://:/?characterEncoding=UTF-8&useSSL=false&useServerPrepStmts=true&cachePrepStmts=true&prepStmtCacheSqlLimit=10000&prepStmtCacheSize=1000&useConfigs=maxPerformance&rewriteBatchedStatements=true +``` + +> **Note:** +> +> If you are connecting over a public network, you need to set `useSSL=true` and [enable TLS between TiDB clients and servers](/enable-tls-between-clients-and-servers.md). + ## Connection pool Building TiDB (MySQL) connections is relatively expensive (for OLTP scenarios at least), because in addition to building a TCP connection, connection authentication is also required. Therefore, the client usually saves the TiDB (MySQL) connections to the connection pool for reuse. -Java has many connection pool implementations such as [HikariCP](https://github.com/brettwooldridge/HikariCP), [tomcat-jdbc](https://tomcat.apache.org/tomcat-10.1-doc/jdbc-pool.html), [druid](https://github.com/alibaba/druid), [c3p0](https://www.mchange.com/projects/c3p0/), and [dbcp](https://commons.apache.org/proper/commons-dbcp/). TiDB does not limit which connection pool you use, so you can choose whichever you like for your application. +TiDB supports the following Java connection pools: + +- [HikariCP](https://github.com/brettwooldridge/HikariCP) +- [tomcat-jdbc](https://tomcat.apache.org/tomcat-10.1-doc/jdbc-pool) +- [druid](https://github.com/alibaba/druid) +- [c3p0](https://www.mchange.com/projects/c3p0/) +- [dbcp](https://commons.apache.org/proper/commons-dbcp/) -### Configure the number of connections +In practice, some connection pools might persistently use specific active sessions. Although the total number of connections appears evenly distributed across TiDB compute nodes, uneven distribution of active connections can lead to actual load imbalance. In distributed scenarios, it is recommended to use HikariCP, which manages connection lifecycles effectively and helps prevent active connections from being fixed on certain nodes, achieving balanced load distribution. -It is a common practice that the connection pool size is well adjusted according to the application's own needs. Take HikariCP as an example: +### Typical connection pool configuration -- `maximumPoolSize`: The maximum number of connections in the connection pool. If this value is too large, TiDB consumes resources to maintain useless connections. If this value is too small, the application gets slow connections. So configure this value for your own good. For details, see [About Pool Sizing](https://github.com/brettwooldridge/HikariCP/wiki/About-Pool-Sizing). -- `minimumIdle`: The minimum number of idle connections in the connection pool. It is mainly used to reserve some connections to respond to sudden requests when the application is idle. You can also configure it according to your application needs. +The following is an example configuration for HikariCP: -The application needs to return the connection after finishing using it. It is also recommended that the application use the corresponding connection pool monitoring (such as `metricRegistry`) to locate the connection pool issue in time. +```yaml +hikari: + maximumPoolSize: 20 + poolName: hikariCP + connectionTimeout: 30000 + maxLifetime: 1200000 + keepaliveTime: 120000 +``` + +The parameter explanations are as follows. For more details, refer to the [official HikariCP documentation](https://github.com/brettwooldridge/HikariCP/blob/dev/README.md). + +- `maximumPoolSize`: the maximum number of connections in the pool. The default value is `10`. In containerized environments, it is recommended to set this to 4–10 times the number of CPU cores available to the Java application. Setting this value too high can lead to resource wastage, while setting it too low can slow down connection acquisition. See [About Pool Sizing](https://github.com/brettwooldridge/HikariCP/wiki/About-Pool-Sizing) for more details. +- `minimumIdle`: HikariCP recommends not setting this parameter. The default value is equal to the value of `maximumPoolSize`, which disables connection pool scaling. This ensures that connections are readily available during traffic spikes and avoids delays caused by connection creation. +- `connectionTimeout`: the maximum time (in milliseconds) that an application waits to acquire a connection from the pool. The default value is `30000` milliseconds (30 seconds). If no available connection is obtained within this time, a `SQLException` exception occurs. +- `maxLifetime`: the maximum lifetime (in milliseconds) of a connection in the pool. The default value is `1800000` milliseconds (30 minutes). Connections in use are not affected. After the connection is closed, it will be removed according to this setting. Setting this value too low can cause frequent reconnections. If you are using [`graceful-wait-before-shutdown`](/tidb-configuration-file.md#graceful-wait-before-shutdown-new-in-v50), ensure this value is less than the wait time. +- `keepaliveTime`: the interval (in milliseconds) between keepalive operations on connections in the pool. This setting helps prevent disconnections caused by database or network idle timeouts. The default value is `120000` milliseconds (2 minutes). The pool prefers using the JDBC4 `isValid()` method to keep idle connections alive. ### Probe configuration @@ -370,4 +405,6 @@ Obtaining flame graphs in Java applications is tedious. For details, see [Java F Based on commonly used Java components that interact with databases, this document describes the common problems and solutions for developing Java applications with TiDB. TiDB is highly compatible with the MySQL protocol, so most of the best practices for MySQL-based Java applications also apply to TiDB. -Join us at [TiDB Community slack channel](https://tidbcommunity.slack.com/archives/CH7TTLL7P), and share with broad TiDB user group about your experience or problems when you develop Java applications with TiDB. +## Need help? + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). \ No newline at end of file diff --git a/best-practices/massive-regions-best-practices.md b/best-practices/massive-regions-best-practices.md index 9619fe5cffcb7..8b1effde4b944 100644 --- a/best-practices/massive-regions-best-practices.md +++ b/best-practices/massive-regions-best-practices.md @@ -1,7 +1,6 @@ --- title: Best Practices for TiKV Performance Tuning with Massive Regions summary: TiKV performance tuning involves reducing the number of Regions and messages, increasing Raftstore concurrency, enabling Hibernate Region and Region Merge, adjusting Raft base tick interval, increasing TiKV instances, and adjusting Region size. Other issues include slow PD leader switching and outdated PD routing information. -aliases: ['/docs/dev/best-practices/massive-regions-best-practices/','/docs/dev/reference/best-practices/massive-regions/'] --- # Best Practices for TiKV Performance Tuning with Massive Regions @@ -51,6 +50,18 @@ You can check the following monitoring metrics in Grafana's **TiKV Dashboard**: ![Check Propose wait duration](/media/best-practices/propose-wait-duration.png) ++ `Commit log duration` in the **Raft IO** panel + + `Commit log duration` is the time Raftstore takes to commit Raft logs to the majority of members in the respective Region. The possible reasons for a high value of this metric with significant fluctuations include the following: + + - The workload on Raftstore is heavy. + - The append log operation is slow. + - Raft logs cannot be committed timely due to network congestion. + + Reference value: lower than 200-500 ms. + + ![Check Commit log duration](/media/best-practices/commit-log-duration.png) + ## Performance tuning methods After finding out the cause of a performance problem, try to solve it from the following two aspects: @@ -128,13 +139,17 @@ If Region followers have not received the heartbeat from the leader within the ` The default size of a Region is 96 MiB, and you can reduce the number of Regions by setting Regions to a larger size. For more information, see [Tune Region Performance](/tune-region-performance.md). -> **Warning:** +> **Note:** > -> Currently, customized Region size is an experimental feature introduced in TiDB v6.1.0. It is not recommended that you use it in production environments. The risks are as follows: +> Customized Region size is an experimental feature before TiDB v6.5.0. If you need to resize the Region size, it is recommended that you upgrade to v6.5.0 or a later version. + +### Method 7: Increase the maximum number of connections for Raft communication + +By default, the maximum number of connections used for Raft communication between TiKV nodes is 1. Increasing this number can help alleviate blockage issues caused by heavy communication workloads of a large number of Regions. For detailed instructions, see [`grpc-raft-conn-num`](/tikv-configuration-file.md#grpc-raft-conn-num). + +> **Note:** > -> + Performance jitter might be caused. -> + The query performance, especially for queries that deal with a large range of data, might decrease. -> + The Region scheduling slows down. +> To reduce unnecessary thread switching overhead and mitigate potential negative impacts from batch processing, it is recommended to set the number within the range of `[1, 4]`. ## Other problems and solutions diff --git a/best-practices/pd-scheduling-best-practices.md b/best-practices/pd-scheduling-best-practices.md index a86e17f0974d8..9c94198777b74 100644 --- a/best-practices/pd-scheduling-best-practices.md +++ b/best-practices/pd-scheduling-best-practices.md @@ -1,7 +1,6 @@ --- title: PD Scheduling Best Practices summary: This document summarizes PD scheduling best practices, including scheduling process, load balancing, hot regions scheduling, cluster topology awareness, scale-down and failure recovery, region merge, query scheduling status, and control scheduling strategy. It also covers common scenarios such as uneven distribution of leaders/regions, slow node recovery, and troubleshooting TiKV nodes. -aliases: ['/docs/dev/best-practices/pd-scheduling-best-practices/','/docs/dev/reference/best-practices/pd-scheduling/'] --- # PD Scheduling Best Practices @@ -254,7 +253,7 @@ Hot regions scheduling issues generally fall into the following categories: - The load of some nodes is significantly higher than that of other nodes from TiKV-related metrics, which becomes the bottleneck of the whole system. Currently, PD counts hotspots through traffic analysis only, so it is possible that PD fails to identify hotspots in certain scenarios. For example, when there are intensive point lookup requests for some regions, it might not be obvious to detect in traffic, but still the high QPS might lead to bottlenecks in key modules. - **Solutions**: Firstly, locate the table where hot regions are formed based on the specific business. Then add a `scatter-range-scheduler` scheduler to make all regions of this table evenly distributed. TiDB also provides an interface in its HTTP API to simplify this operation. Refer to [TiDB HTTP API](https://github.com/pingcap/tidb/blob/master/docs/tidb_http_api.md) for more details. + **Solutions**: Firstly, locate the table where hot regions are formed based on the specific business. Then add a `scatter-range-scheduler` scheduler to make all regions of this table evenly distributed. TiDB also provides an interface in its HTTP API to simplify this operation. Refer to [TiDB HTTP API](https://github.com/pingcap/tidb/blob/release-8.1/docs/tidb_http_api.md) for more details. ### Region merge is slow diff --git a/best-practices/three-nodes-hybrid-deployment.md b/best-practices/three-nodes-hybrid-deployment.md index 9c6cb82d7bcb8..cbd698c50de28 100644 --- a/best-practices/three-nodes-hybrid-deployment.md +++ b/best-practices/three-nodes-hybrid-deployment.md @@ -69,7 +69,7 @@ In this test, the value of this parameter is set to `2`. Observe the **gRPC poll #### `storage.scheduler-worker-pool-size` -When TiKV detects that the CPU core number of the machine is greater than or equal to `16`, this parameter value defaults to `8`. When the CPU core number is smaller than `16`, the parameter value defaults to `4`. This parameter is used when TiKV converts complex transaction requests to simple key-value reads or writes, but the scheduler thread pool does not performs any writes. +When TiKV detects that the CPU core number of the machine is greater than or equal to `16`, this parameter value defaults to `8`. When the CPU core number is smaller than `16`, the parameter value defaults to `4`. This parameter is used when TiKV converts complex transaction requests to simple key-value reads or writes, but the scheduler thread pool does not perform any writes. Ideally, the usage rate of the scheduler thread pool is kept between 50% and 75%. Similar to the gRPC thread pool, the `storage.scheduler-worker-pool-size` parameter defaults to a larger value during the hybrid deployment, which makes resource usage insufficient. In this test, the value of this parameter is set to `2`, which is in line with the best practices, a conclusion drawn by observing the corresponding metrics in the **Scheduler worker CPU** panel. diff --git a/best-practices/tidb-best-practices.md b/best-practices/tidb-best-practices.md index 9495c2e4f2a57..a941c4e56d88b 100644 --- a/best-practices/tidb-best-practices.md +++ b/best-practices/tidb-best-practices.md @@ -1,7 +1,6 @@ --- title: TiDB Best Practices summary: This document summarizes best practices for using TiDB, covering SQL use and optimization tips for OLAP and OLTP scenarios, with a focus on TiDB-specific optimization options. It also recommends reading three blog posts introducing TiDB's technical principles before diving into the best practices. -aliases: ['/docs/dev/tidb-best-practices/'] --- # TiDB Best Practices @@ -10,9 +9,9 @@ This document summarizes the best practices of using TiDB, including the use of Before you read this document, it is recommended that you read three blog posts that introduce the technical principles of TiDB: -* [TiDB Internal (I) - Data Storage](https://en.pingcap.com/blog/tidb-internal-data-storage/) -* [TiDB Internal (II) - Computing](https://en.pingcap.com/blog/tidb-internal-computing/) -* [TiDB Internal (III) - Scheduling](https://en.pingcap.com/blog/tidb-internal-scheduling/) +* [TiDB Internal (I) - Data Storage](https://www.pingcap.com/blog/tidb-internal-data-storage/) +* [TiDB Internal (II) - Computing](https://www.pingcap.com/blog/tidb-internal-computing/) +* [TiDB Internal (III) - Scheduling](https://www.pingcap.com/blog/tidb-internal-scheduling/) ## Preface @@ -34,7 +33,7 @@ To store three replicas, compared with the replication of Source-Replica, Raft i ### Distributed transactions -TiDB provides complete distributed transactions and the model has some optimizations on the basis of [Google Percolator](https://research.google.com/pubs/pub36726.html). This document introduces the following features: +TiDB provides complete distributed transactions and the model has some optimizations on the basis of [Google Percolator](https://research.google/pubs/large-scale-incremental-processing-using-distributed-transactions-and-notifications/). This document introduces the following features: * Optimistic transaction model @@ -68,7 +67,7 @@ Placement Driver (PD) balances the load of the cluster according to the status o ### SQL on KV -TiDB automatically maps the SQL structure into Key-Value structure. For details, see [TiDB Internal (II) - Computing](https://en.pingcap.com/blog/tidb-internal-computing/). +TiDB automatically maps the SQL structure into Key-Value structure. For details, see [TiDB Internal (II) - Computing](https://www.pingcap.com/blog/tidb-internal-computing/). Simply put, TiDB performs the following operations: diff --git a/best-practices/uuid.md b/best-practices/uuid.md index 406831b742ad3..04b420b2973f6 100644 --- a/best-practices/uuid.md +++ b/best-practices/uuid.md @@ -7,17 +7,17 @@ summary: UUIDs, when used as primary keys, offer benefits such as reduced networ ## Overview of UUIDs -When used as a primary key, instead of an `AUTO_INCREMENT` integer value, a universally unique identifier (UUID) delivers the following benefits: +When used as a primary key, instead of an [`AUTO_INCREMENT`](/auto-increment.md) integer value, a universally unique identifier (UUID) delivers the following benefits: - UUIDs can be generated on multiple systems without risking conflicts. In some cases, this means that the number of network trips to TiDB can be reduced, leading to improved performance. - UUIDs are supported by most programming languages and database systems. -- When used as a part of a URL, a UUID is not vulnerable to enumeration attacks. In comparison, with an `auto_increment` number, it is possible to guess the invoice IDs or user IDs. +- When used as a part of a URL, a UUID is not vulnerable to enumeration attacks. In comparison, with an `AUTO_INCREMENT` number, it is possible to guess the invoice IDs or user IDs. ## Best practices ### Store as binary -The textual UUID format looks like this: `ab06f63e-8fe7-11ec-a514-5405db7aad56`, which is a string of 36 characters. By using `UUID_TO_BIN()`, the textual format can be converted into a binary format of 16 bytes. This allows you to store the text in a `BINARY(16)` column. When retrieving the UUID, you can use the `BIN_TO_UUID()` function to get back to the textual format. +The textual UUID format looks like this: `ab06f63e-8fe7-11ec-a514-5405db7aad56`, which is a string of 36 characters. By using [`UUID_TO_BIN()`](/functions-and-operators/miscellaneous-functions.md#uuid_to_bin), the textual format can be converted into a binary format of 16 bytes. This allows you to store the text in a [`BINARY(16)`](/data-type-string.md#binary-type) column. When retrieving the UUID, you can use the [`BIN_TO_UUID()`](/functions-and-operators/miscellaneous-functions.md#bin_to_uuid) function to get back to the textual format. ### UUID format binary order and a clustered PK diff --git a/binary-package.md b/binary-package.md index bd4f3f135c3ad..7a77fe4f557a8 100644 --- a/binary-package.md +++ b/binary-package.md @@ -64,7 +64,6 @@ The `TiDB-community-toolkit` package contains the following contents. | binlogctl | New in v6.0.0 | | sync_diff_inspector | | | reparo | | -| arbiter | | | server-{version}-linux-{arch}.tar.gz | New in v6.2.0 | | grafana-{version}-linux-{arch}.tar.gz | New in v6.2.0 | | alertmanager-{version}-linux-{arch}.tar.gz | New in v6.2.0 | diff --git a/br/backup-and-restore-overview.md b/br/backup-and-restore-overview.md index 5e1b4d7cb5e92..09c277efca359 100644 --- a/br/backup-and-restore-overview.md +++ b/br/backup-and-restore-overview.md @@ -1,7 +1,6 @@ --- title: TiDB Backup & Restore Overview summary: TiDB Backup & Restore (BR) ensures high availability of clusters and data safety. It supports disaster recovery with a short RPO, handles misoperations, and provides history data auditing. It is recommended to perform backup operations during off-peak hours and store backup data to compatible storage systems. BR supports full backup and log backup, as well as restoring data to any point in time. It is important to use BR of the same major version as the TiDB cluster for backup and restoration. -aliases: ['/docs/dev/br/backup-and-restore-tool/','/docs/dev/reference/tools/br/br/','/docs/dev/how-to/maintain/backup-and-restore/br/','/tidb/dev/backup-and-restore-tool/','/tidb/dev/point-in-time-recovery/'] --- # TiDB Backup & Restore Overview @@ -17,7 +16,7 @@ BR satisfies the following requirements: ## Before you use -This section describes the prerequisites for using TiDB backup and restore, including restrictions, usage tips and compatibility issues. +This section describes the prerequisites for using TiDB backup and restore, including restrictions, usage tips and compatibility issues. For more information about the compatibility of the BR tool with other features or versions, see [Compatibility](#compatibility). ### Restrictions @@ -26,7 +25,7 @@ This section describes the prerequisites for using TiDB backup and restore, incl - PITR does not support restoring the data of user tables or privilege tables from system tables. - BR does not support running multiple backup tasks on a cluster **at the same time**. - BR does not support running snapshot backup tasks and data restore tasks on a cluster **at the same time**. -- When a PITR is running, you cannot run a log backup task or use TiCDC to replicate data to a downstream cluster. +- When restoring a cluster using PITR, you cannot run a log backup task or use TiCDC to replicate data to a downstream cluster. ### Some tips @@ -94,7 +93,7 @@ Corresponding to the backup features, you can perform two types of restore: full #### Restore performance and impact on TiDB clusters -- Data restore is performed at a scalable speed. Generally, the speed is 100 MiB/s per TiKV node. `br` only supports restoring data to a new cluster and uses the resources of the target cluster as much as possible. For more details, see [Restore performance and impact](/br/br-snapshot-guide.md#performance-and-impact-of-snapshot-restore). +- Data restore is performed at a scalable speed. Generally, the speed is 1 GiB/s per TiKV node. For more details, see [Restore performance and impact](/br/br-snapshot-guide.md#performance-and-impact-of-snapshot-restore). - On each TiKV node, PITR can restore log data at 30 GiB/h. For more details, see [PITR performance and impact](/br/br-pitr-guide.md#performance-capabilities-of-pitr). ## Backup storage @@ -128,6 +127,8 @@ Before performing backup and restore, BR compares the TiDB cluster version with Starting from v7.0.0, TiDB gradually supports performing backup and restore operations through SQL statements. Therefore, it is strongly recommended to use the BR tool of the same major version as the TiDB cluster when backing up and restoring cluster data, and avoid performing data backup and restore operations across major versions. This helps ensure smooth execution of restore operations and data consistency. Starting from v7.6.0, BR restores data in some `mysql` system tables by default, that is, the `--with-sys-table` option is set to `true` by default. When restoring data to a TiDB cluster with a different version, if you encounter an error similar to `[BR:Restore:ErrRestoreIncompatibleSys]incompatible system table` due to different schemas of system tables, you can set `--with-sys-table=false` to skip restoring the system tables and avoid this error. +#### BR version compatibility matrix before TiDB v6.6.0 + The compatibility information for BR before TiDB v6.6.0 is as follows: | Backup version (vertical) \ Restore version (horizontal) | Restore to TiDB v6.0 | Restore to TiDB v6.1 | Restore to TiDB v6.2 | Restore to TiDB v6.3, v6.4, or v6.5 | Restore to TiDB v6.6 | @@ -135,6 +136,38 @@ The compatibility information for BR before TiDB v6.6.0 is as follows: | TiDB v6.0, v6.1, v6.2, v6.3, v6.4, or v6.5 snapshot backup | Compatible (known issue [#36379](https://github.com/pingcap/tidb/issues/36379): if backup data contains an empty schema, BR might report an error.) | Compatible | Compatible | Compatible | Compatible (BR must be v6.6) | | TiDB v6.3, v6.4, v6.5, or v6.6 log backup| Incompatible | Incompatible | Incompatible | Compatible | Compatible | +#### BR version compatibility matrix between TiDB v6.5.0 and v8.1.0 + +This section introduces the BR compatibility information for all [Long-Term Support (LTS)](/releases/versioning.md#long-term-support-releases) versions between TiDB v6.5.0 and v8.1.0 (including v6.5.0, v7.1.0, v7.5.0, v8.1.0): + +> **Note:** +> +> Known issue: Starting from version v7.2.0, some system table fields in newly created clusters are case-insensitive. However, for clusters that are **upgraded online** from versions earlier than v7.2.0 to v7.2.0 or later, the corresponding system table fields remain case-sensitive. Backup and restore operations involving system tables between these two types of clusters might fail. For more details, see [Issue #43717](https://github.com/pingcap/tidb/issues/43717). + +The following table lists the compatibility matrix for full backups. Note that all information in the table applies to newly created clusters. For clusters upgraded from a version earlier than v7.2.0 to v7.2.0 or later, their behavior is consistent with that of backups from v7.1.0. + +| Backup version | Compatible restore versions | Incompatible restore versions | +|:--|:--|:--| +| v6.5.0 | 7.1.0 | v7.5.0 and later | +| v7.1.0 | - | v7.5.0 and later | +| v7.5.0 | v7.5.0 and later | - | +| v8.1.0 | v8.1.0 and later | - | + +The following table lists the compatibility matrix for log backups. Note that all information in the table applies to newly created clusters. For clusters upgraded from a version earlier than v7.2.0 to v7.2.0 or later, their behavior is consistent with that of backups from v7.1.0. + +| Backup version | Compatible restore versions | Incompatible restore versions | +|:--|:--|:--| +| v6.5.0 | 7.1.0 | v7.5.0 and later | +| v7.1.0 | - | v7.5.0 and later | +| v7.5.0 | v7.5.0 and later | - | +| v8.1.0 | v8.1.0 and later | - | + +> **Note:** +> +> - When only user data is backed up (full backup or log backup), all versions are compatible with each other. +> - In scenarios where restoring the `mysql` system table is incompatible, you can resolve the problem by setting `--with-sys-table=false` to skip restoring all system tables, or use a more fine-grained filter to just skip incompatible system tables, for example: `--filter '*.*' --filter "__TiDB_BR_Temporary_*.*" --filter '!mysql.*' --filter 'mysql.bind_info' --filter 'mysql.user' --filter 'mysql.global_priv' --filter 'mysql.global_grants' --filter 'mysql.default_roles' --filter 'mysql.role_edges' --filter '!sys.*' --filter '!INFORMATION_SCHEMA.*' --filter '!PERFORMANCE_SCHEMA.*' --filter '!METRICS_SCHEMA.*' --filter '!INSPECTION_SCHEMA.*'`. +> - `-` means that there are no compatibility restrictions for the corresponding scenario. + ## See also - [TiDB Snapshot Backup and Restore Guide](/br/br-snapshot-guide.md) diff --git a/br/backup-and-restore-storages.md b/br/backup-and-restore-storages.md index 8f589075995ef..b3de97c50d066 100644 --- a/br/backup-and-restore-storages.md +++ b/br/backup-and-restore-storages.md @@ -1,7 +1,6 @@ --- title: Backup Storages summary: TiDB supports backup storage to Amazon S3, Google Cloud Storage, Azure Blob Storage, and NFS. You can specify the URI and authentication for different storage services. BR sends credentials to TiKV by default when using S3, GCS, or Azure Blob Storage. You can disable this for cloud environments. The URI format for each storage service is specified, along with authentication methods. Server-side encryption is supported for Amazon S3 and Azure Blob Storage. BR v6.3.0 also supports AWS S3 Object Lock. -aliases: ['/docs/dev/br/backup-and-restore-storages/','/tidb/dev/backup-storage-S3/','/tidb/dev/backup-storage-azblob/','/tidb/dev/backup-storage-gcs/','/tidb/dev/external-storage/'] --- # Backup Storages @@ -10,9 +9,9 @@ TiDB supports storing backup data to Amazon S3, Google Cloud Storage (GCS), Azur ## Send credentials to TiKV -| CLI parameter | Description | Default value -|:----------|:-------|:-------| -| `--send-credentials-to-tikv` | Controls whether to send credentials obtained by BR to TiKV. | `true`| +| CLI parameter | Description | Default value | +|:-----------------------------|:-------------------------------------------------------------|:--------------| +| `--send-credentials-to-tikv` | Controls whether to send credentials obtained by BR to TiKV. | `true` | By default, BR sends a credential to each TiKV node when using Amazon S3, GCS, or Azure Blob Storage as the storage system. This behavior simplifies the configuration and is controlled by the parameter `--send-credentials-to-tikv`(or `-c` in short). diff --git a/br/backup-and-restore-use-cases.md b/br/backup-and-restore-use-cases.md index 12fedc828d464..160a8497a91bb 100644 --- a/br/backup-and-restore-use-cases.md +++ b/br/backup-and-restore-use-cases.md @@ -1,7 +1,6 @@ --- title: TiDB Backup and Restore Use Cases -summary: TiDB provides snapshot and log backup solutions for specific use cases, such as timely data recovery and business audits. To use point-in-time recovery (PITR), deploy a TiDB cluster >= v6.2.0 and update BR to v8.0.0. Configure backup storage on Amazon S3 and set a backup policy to meet data loss and recovery requirements. Run log and snapshot backups, and use PITR to restore data to a specific time point. Clean up outdated data regularly. For detailed steps, refer to TiDB documentation. -aliases: ['/docs/dev/br/backup-and-restore-use-cases/','/docs/dev/reference/tools/br/use-cases/','/tidb/dev/backup-and-restore-use-cases-for-maintain/'] +summary: TiDB provides snapshot and log backup solutions for specific use cases, such as timely data recovery and business audits. To use point-in-time recovery (PITR), deploy a TiDB cluster >= v6.2.0 and update BR to the same version as the TiDB cluster. Configure backup storage on Amazon S3 and set a backup policy to meet data loss and recovery requirements. Run log and snapshot backups, and use PITR to restore data to a specific time point. Clean up outdated data regularly. For detailed steps, refer to TiDB documentation. --- # TiDB Backup and Restore Use Cases @@ -17,7 +16,7 @@ With PITR, you can satisfy the preceding requirements. ## Deploy the TiDB cluster and BR -To use PITR, you need to deploy a TiDB cluster >= v6.2.0 and update BR to the same version as the TiDB cluster. This document uses v8.0.0 as an example. +To use PITR, you need to deploy a TiDB cluster >= v6.2.0 and update BR to the same version as the TiDB cluster. This document uses v8.1.2 as an example. The following table shows the recommended hardware resources for using PITR in a TiDB cluster. @@ -44,13 +43,13 @@ Install or upgrade BR using TiUP: - Install: ```shell - tiup install br:v8.0.0 + tiup install br:v8.1.2 ``` - Upgrade: ```shell - tiup update br:v8.0.0 + tiup update br:v8.1.2 ``` ## Configure backup storage (Amazon S3) @@ -122,7 +121,7 @@ The following are two snapshot backup examples: ```shell tiup br backup full --pd="${PD_IP}:2379" \ --storage='s3://tidb-pitr-bucket/backup-data/snapshot-20220514000000' \ - --backupts='2022/05/14 00:00:00' + --backupts='2022/05/14 00:00:00 +08:00' ``` - Run a snapshot backup at 2022/05/16 00:00:00 @@ -130,7 +129,7 @@ The following are two snapshot backup examples: ```shell tiup br backup full --pd="${PD_IP}:2379" \ --storage='s3://tidb-pitr-bucket/backup-data/snapshot-20220516000000' \ - --backupts='2022/05/16 00:00:00' + --backupts='2022/05/16 00:00:00 +08:00' ``` ## Run PITR diff --git a/br/br-auto-tune.md b/br/br-auto-tune.md index 2c15b1ea675d0..479dee8c597b4 100644 --- a/br/br-auto-tune.md +++ b/br/br-auto-tune.md @@ -13,7 +13,7 @@ To reduce the impact of backup tasks on the cluster, TiDB v5.4.0 introduces the If you want to reduce the impact of backup tasks on the cluster, you can enable the auto-tune feature. With this feature enabled, TiDB performs backup tasks as fast as possible without excessively affecting the cluster. -Alternatively, you can limit the backup speed by using the TiKV configuration item [`backup.num-threads`](/tikv-configuration-file.md#num-threads-1) or using the parameter `--ratelimit`. +Alternatively, you can limit the backup speed by using the TiKV configuration item [`backup.num-threads`](/tikv-configuration-file.md#num-threads-1) or using the parameter `--ratelimit`. When `--ratelimit` is set, to avoid too many tasks causing the speed limit to fail, the `concurrency` parameter of br is automatically adjusted to `1`. ## Use auto-tune diff --git a/br/br-checkpoint-backup.md b/br/br-checkpoint-backup.md index 286cbbbdd486b..46d39bbb22efa 100644 --- a/br/br-checkpoint-backup.md +++ b/br/br-checkpoint-backup.md @@ -1,7 +1,6 @@ --- title: Checkpoint Backup summary: TiDB v6.5.0 introduces checkpoint backup feature to continue interrupted backups, reducing the need to start from scratch. It records backed up shards to resume backup progress, but relies on GC mechanism and may require some data to be backed up again. The `br` tool periodically updates `gc-safepoint` to avoid data being garbage collected, and can extend retention period if needed. -aliases: ["/tidb/dev/br-checkpoint"] --- # Checkpoint Backup diff --git a/br/br-incremental-guide.md b/br/br-incremental-guide.md index a0dac0d3e6d48..56f159b5ef102 100644 --- a/br/br-incremental-guide.md +++ b/br/br-incremental-guide.md @@ -11,6 +11,12 @@ Incremental data of a TiDB cluster is differentiated data between the starting s > > Development for this feature has stopped. It is recommended that you use [log backup and PITR](/br/br-pitr-guide.md) as an alternative. +## Limitations + +Because restoring the incremental backup relies on the snapshot of the database table at the backup time point to filter incremental DDL statements, the tables deleted during the incremental backup process might still exist after the data restore and need to be manually deleted. + +The incremental backup does not support batch renaming of tables. If batch renaming of tables occurs during the incremental backup process, the data restore might fail. It is recommended to perform a full backup after batch renaming tables, and use the latest full backup to replace the incremental data during restore. + ## Back up incremental data To back up incremental data, run the `tiup br backup` command with **the last backup timestamp** `--lastbackupts` specified. In this way, br command-line tool automatically backs up incremental data generated between `lastbackupts` and the current time. To get `--lastbackupts`, run the `validate` command. The following is an example: diff --git a/br/br-log-architecture.md b/br/br-log-architecture.md index fe3f9e25b3810..c3c7a38ab44fb 100644 --- a/br/br-log-architecture.md +++ b/br/br-log-architecture.md @@ -108,18 +108,24 @@ Log backup generates the following types of files: . ├── v1 │   ├── backupmeta -│   │   ├── {min_restored_ts}-{uuid}.meta -│   │   ├── {checkpoint}-{uuid}.meta +│   │   ├── ... +│   │   └── {resolved_ts}-{uuid}.meta │   ├── global_checkpoint -│   │   ├── {store_id}.ts -│   ├── {date} -│   │   ├── {hour} -│   │   │   ├── {store_id} -│   │   │   │   ├── {min_ts}-{uuid}.log -│   │   │   │   ├── {min_ts}-{uuid}.log -├── v1_stream_truncate_safepoint.txt +│   │   └── {store_id}.ts +│   └── {date} +│      └── {hour} +│         └── {store_id} +│            ├── ... +│            └── {min_ts}-{uuid}.log +└── v1_stream_truncate_safepoint.txt ``` +Explanation of the backup file directory structure: + +- `backupmeta`: stores backup metadata. The `resolved_ts` in the filename indicates the backup progress, meaning that data before this TSO has been fully backed up. However, note that this TSO only reflects the progress of certain shards. +- `global_checkpoint`: represents the global backup progress. It records the latest point in time to which data can be restored using `br restore point`. +- `{date}/{hour}`: stores backup data for the corresponding date and hour. When cleaning up storage, always use `br log truncate` instead of manually deleting data. This is because the metadata references the data in this directory, and manual deletion might lead to restore failures or data inconsistencies after restore. + The following is an example: ``` @@ -129,24 +135,24 @@ The following is an example: │   │   ├── ... │   │   ├── 435213818858112001-e2569bda-a75a-4411-88de-f469b49d6256.meta │   │   ├── 435214043785779202-1780f291-3b8a-455e-a31d-8a1302c43ead.meta -│   │   ├── 435214443785779202-224f1408-fff5-445f-8e41-ca4fcfbd2a67.meta +│   │   └── 435214443785779202-224f1408-fff5-445f-8e41-ca4fcfbd2a67.meta │   ├── global_checkpoint │   │   ├── 1.ts │   │   ├── 2.ts -│   │   ├── 3.ts -│   ├── 20220811 -│   │   ├── 03 -│   │   │   ├── 1 -│   │   │   │   ├── ... -│   │   │   │   ├── 435213866703257604-60fcbdb6-8f55-4098-b3e7-2ce604dafe54.log -│   │   │   │   ├── 435214023989657606-72ce65ff-1fa8-4705-9fd9-cb4a1e803a56.log -│   │   │   ├── 2 -│   │   │   │   ├── ... -│   │   │   │   ├── 435214102632857605-11deba64-beff-4414-bc9c-7a161b6fb22c.log -│   │   │   │   ├── 435214417205657604-e6980303-cbaa-4629-a863-1e745d7b8aed.log -│   │   │   ├── 3 -│   │   │   │   ├── ... -│   │   │   │   ├── 435214495848857605-7bf65e92-8c43-427e-b81e-f0050bd40be0.log -│   │   │   │   ├── 435214574492057604-80d3b15e-3d9f-4b0c-b133-87ed3f6b2697.log -├── v1_stream_truncate_safepoint.txt +│   │   └── 3.ts +│   └── 20220811 +│      └── 03 +│         ├── 1 +│         │   ├── ... +│         │   ├── 435213866703257604-60fcbdb6-8f55-4098-b3e7-2ce604dafe54.log +│         │   └── 435214023989657606-72ce65ff-1fa8-4705-9fd9-cb4a1e803a56.log +│         ├── 2 +│         │   ├── ... +│         │   ├── 435214102632857605-11deba64-beff-4414-bc9c-7a161b6fb22c.log +│         │   └── 435214417205657604-e6980303-cbaa-4629-a863-1e745d7b8aed.log +│         └── 3 +│            ├── ... +│            ├── 435214495848857605-7bf65e92-8c43-427e-b81e-f0050bd40be0.log +│            └── 435214574492057604-80d3b15e-3d9f-4b0c-b133-87ed3f6b2697.log +└── v1_stream_truncate_safepoint.txt ``` diff --git a/br/br-monitoring-and-alert.md b/br/br-monitoring-and-alert.md index 288854109a56c..14c0ba9c86061 100644 --- a/br/br-monitoring-and-alert.md +++ b/br/br-monitoring-and-alert.md @@ -7,6 +7,10 @@ summary: This document describes monitoring and alert for backup and restore, in This document describes the monitoring and alert of the backup and restore feature, including how to deploy monitoring components, monitoring metrics, and common alerts. +## Snapshot backup and restore monitoring + +To view the snapshot backup and restore metrics, go to the [**TiKV-Details** > **Backup & Import** dashboard](/grafana-tikv-dashboard.md#backup--import) in Grafana. + ## Log backup monitoring Log backup supports using [Prometheus](https://prometheus.io/) to collect monitoring metrics. Currently all monitoring metrics are built into TiKV. @@ -19,13 +23,13 @@ Log backup supports using [Prometheus](https://prometheus.io/) to collect monito ### Grafana configuration - For clusters deployed using TiUP, the [Grafana](https://grafana.com/) dashboard contains the point-in-time recovery (PITR) panel. The **Backup Log** panel in the TiKV-Details dashboard is the PITR panel. -- For clusters deployed manually, refer to [Import a Grafana dashboard](/deploy-monitoring-services.md#step-2-import-a-grafana-dashboard) and upload the [tikv_details](https://github.com/tikv/tikv/blob/master/metrics/grafana/tikv_details.json) JSON file to Grafana. Then find the **Backup Log** panel in the TiKV-Details dashboard. +- For clusters deployed manually, refer to [Import a Grafana dashboard](/deploy-monitoring-services.md#step-2-import-a-grafana-dashboard) and upload the [tikv_details](https://github.com/tikv/tikv/blob/release-8.1/metrics/grafana/tikv_details.json) JSON file to Grafana. Then find the **Backup Log** panel in the TiKV-Details dashboard. ### Monitoring metrics | Metrics | Type | Description | |-------------------------------------------------------|-----------|---------------------------------------------------------------------------------------------------------------------------------------------------------| -| **tikv_log_backup_interal_actor_acting_duration_sec** | Histogram | The duration of handling all internal messages and events.
`message :: TaskType` | +| **tikv_log_backup_internal_actor_acting_duration_sec** | Histogram | The duration of handling all internal messages and events.
`message :: TaskType` | | **tikv_log_backup_initial_scan_reason** | Counter | Statistics of the reasons why initial scan is triggered. The main reason is leader transfer or Region version change.
`reason :: {"leader-changed", "region-changed", "retry"}` | | **tikv_log_backup_event_handle_duration_sec** | Histogram | The duration of handling KV events. Compared with `tikv_log_backup_on_event_duration_seconds`, this metric also includes the duration of internal conversion.
`stage :: {"to_stream_event", "save_to_temp_file"}` | | **tikv_log_backup_handle_kv_batch** | Histogram | Region-level statistics of the sizes of KV pair batches sent by Raftstore. | diff --git a/br/br-pitr-guide.md b/br/br-pitr-guide.md index 64418cd1a7198..6c0f95954f771 100644 --- a/br/br-pitr-guide.md +++ b/br/br-pitr-guide.md @@ -1,7 +1,6 @@ --- title: TiDB Log Backup and PITR Guide summary: TiDB Log Backup and PITR Guide explains how to back up and restore data using the br command-line tool. It includes instructions for starting log backup, running full backup regularly, and cleaning up outdated data. The guide also provides information on running PITR and the performance capabilities of PITR. -aliases: ['/tidb/dev/pitr-usage'] --- # TiDB Log Backup and PITR Guide @@ -105,17 +104,17 @@ The following steps describe how to clean up backup data that exceeds the backup ## Performance capabilities of PITR -- On each TiKV node, PITR can restore snapshot data at a speed of 280 GB/h and log data 30 GB/h. -- BR deletes outdated log backup data at a speed of 600 GB/h. +- On each TiKV node, PITR can restore snapshot data (full restore) at a speed of 2 TiB/h and log data (including meta files and KV files) at a speed of 30 GiB/h. +- BR deletes outdated log backup data (`tiup br log truncate`) at a speed of 600 GB/h. > **Note:** > > The preceding specifications are based on test results from the following two testing scenarios. The actual data might be different. > -> - Snapshot data restore speed = Snapshot data size / (duration * the number of TiKV nodes) -> - Log data restore speed = Restored log data size / (duration * the number of TiKV nodes) +> - Snapshot data restore speed = Total size of restored snapshot data on all TiKV nodes in the cluster / (duration * the number of TiKV nodes) +> - Log data restore speed = Total size of restored log data on all TiKV nodes in the cluster / (duration * the number of TiKV nodes) > -> The snapshot data size refers to the logical size of all KVs in a single replica, not the actual amount of restored data. BR restores all replicas according to the number of replicas configured for the cluster. The more replicas there are, the more data can be actually restored. +> External storage only contains KV data of a single replica. Therefore, the data size in external storage does not represent the actual data size restored in the cluster. BR restores all replicas according to the number of replicas configured for the cluster. The more replicas there are, the more data can be actually restored. > The default replica number for all clusters in the test is 3. > To improve the overall restore performance, you can modify the [`import.num-threads`](/tikv-configuration-file.md#import) item in the TiKV configuration file and the [`concurrency`](/br/use-br-command-line-tool.md#common-options) option in the BR command. @@ -128,7 +127,7 @@ Testing scenario 1 (on [TiDB Cloud](https://tidbcloud.com)) is as follows: - New log data created in the cluster: 10 GB/h - Write (INSERT/UPDATE/DELETE) QPS: 10,000 -Testing scenario 2 (on TiDB Self-Hosted) is as follows: +Testing scenario 2 (on TiDB Self-Managed) is as follows: - The number of TiKV nodes (8 core, 64 GB memory): 6 - TiKV configuration item `import.num-threads`: 8 @@ -137,8 +136,15 @@ Testing scenario 2 (on TiDB Self-Hosted) is as follows: - New log data created in the cluster: 10 GB/h - Write (INSERT/UPDATE/DELETE) QPS: 10,000 +## Monitoring and alert + +After log backup tasks are distributed, each TiKV node continuously writes data to external storage. You can view the monitoring data for this process in the **TiKV-Details > Backup Log** dashboard. + +To receive notifications metrics deviate from normal ranges, see [Log backup alerts](/br/br-monitoring-and-alert.md#log-backup-alerts) to configure alert rules. + ## See also * [TiDB Backup and Restore Use Cases](/br/backup-and-restore-use-cases.md) * [br Command-line Manual](/br/use-br-command-line-tool.md) * [Log Backup and PITR Architecture](/br/br-log-architecture.md) +* [Monitoring and Alert for Backup and Restore](/br/br-monitoring-and-alert.md) diff --git a/br/br-pitr-manual.md b/br/br-pitr-manual.md index d4c9076ea1949..758916a119f9a 100644 --- a/br/br-pitr-manual.md +++ b/br/br-pitr-manual.md @@ -1,7 +1,6 @@ --- title: TiDB Log Backup and PITR Command Manual summary: Introduce the commands used in TiDB log backup and point-in-time recovery (PITR). -aliases: ['/tidb/dev/br-log-command-line/'] --- # TiDB Log Backup and PITR Command Manual @@ -290,7 +289,7 @@ Usage example: ```shell tiup br log truncate --until='2022-07-26 21:20:00+0800' \ -–-storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}"' +--storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}' ``` Expected output: @@ -330,7 +329,7 @@ The `--storage` parameter is used to specify the backup storage address. Current Usage example: ```shell -tiup br log metadata –-storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}"' +tiup br log metadata --storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}' ``` Expected output: @@ -380,8 +379,8 @@ Usage example: ```shell tiup br restore point --pd="${PD_IP}:2379" ---storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}"' ---full-backup-storage='s3://backup-101/snapshot-202205120000?access-key=${access-key}&secret-access-key=${secret-access-key}"' +--storage='s3://backup-101/logbackup?access-key=${access-key}&secret-access-key=${secret-access-key}' +--full-backup-storage='s3://backup-101/snapshot-202205120000?access-key=${access-key}&secret-access-key=${secret-access-key}' Full Restore <--------------------------------------------------------------------------------------------------------------------------------------------------------> 100.00% *** ***["Full Restore success summary"] ****** [total-take=3.112928252s] [restore-data-size(after-compressed)=5.056kB] [Size=5056] [BackupTS=434693927394607136] [total-kv=4] [total-kv-size=290B] [average-speed=93.16B/s] diff --git a/br/br-snapshot-guide.md b/br/br-snapshot-guide.md index 06f34a8287564..5255c7af25164 100644 --- a/br/br-snapshot-guide.md +++ b/br/br-snapshot-guide.md @@ -1,7 +1,6 @@ --- title: Snapshot Backup and Restore Guide summary: This document describes how to back up and restore TiDB snapshots using the br command-line tool. It includes instructions for snapshot backup, restoring data of a specified time point, and restoring a database or table. The document also covers the performance and impact of snapshot backup and restore. -aliases: ['/tidb/dev/br-usage-backup/','/tidb/dev/br-usage-restore/','/tidb/dev/br-usage-restore-for-maintain/', '/tidb/dev/br-usage-backup-for-maintain/'] --- # Snapshot Backup and Restore Guide @@ -26,14 +25,14 @@ You can back up a TiDB cluster snapshot by running the `tiup br backup full` com ```shell tiup br backup full --pd "${PD_IP}:2379" \ - --backupts '2022-09-08 13:30:00' \ + --backupts '2022-09-08 13:30:00 +08:00' \ --storage "s3://backup-101/snapshot-202209081330?access-key=${access-key}&secret-access-key=${secret-access-key}" \ --ratelimit 128 \ ``` In the preceding command: -- `--backupts`: The time point of the snapshot. The format can be [TSO](/glossary.md#tso) or timestamp, such as `400036290571534337` or `2018-05-11 01:42:23`. If the data of this snapshot is garbage collected, the `tiup br backup` command returns an error and `br` exits. If you leave this parameter unspecified, `br` picks the snapshot corresponding to the backup start time. +- `--backupts`: The time point of the snapshot. The format can be [TSO](/glossary.md#tso) or timestamp, such as `400036290571534337` or `2018-05-11 01:42:23 +08:00`. If the data of this snapshot is garbage collected, the `tiup br backup` command returns an error and `br` exits. When backing up using a timestamp, it is recommended to specify the time zone as well. Otherwise, `br` uses the local time zone to construct the timestamp by default, which might lead to an incorrect backup time point. If you leave this parameter unspecified, `br` picks the snapshot corresponding to the backup start time. - `--storage`: The storage address of the backup data. Snapshot backup supports Amazon S3, Google Cloud Storage, and Azure Blob Storage as backup storage. The preceding command uses Amazon S3 as an example. For more details, see [URI Formats of External Storage Services](/external-storage-uri.md). - `--ratelimit`: The maximum speed **per TiKV** performing backup tasks. The unit is in MiB/s. @@ -77,12 +76,6 @@ tiup br restore full --pd "${PD_IP}:2379" \ --storage "s3://backup-101/snapshot-202209081330?access-key=${access-key}&secret-access-key=${secret-access-key}" ``` -> **Warning:** -> -> The coarse-grained Region scatter algorithm (enabled by setting `--granularity="coarse-grained"`) is experimental. It is recommended that you use this feature to accelerate data recovery in clusters with up to 1,000 tables. Note that this feature does not support checkpoint restore. - -To further improve the restore speed of large clusters, starting from v7.6.0, BR supports a coarse-grained Region scatter algorithm (experimental) for faster parallel recovery. You can enable this algorithm by specifying `--granularity="coarse-grained"`. After it is enabled, BR can quickly split the restore task into a large number of small tasks and scatter them to all TiKV nodes in batches, thus fully utilizing the resources of each TiKV node for fast recovery in parallel. - During restore, a progress bar is displayed in the terminal as shown below. When the progress bar advances to 100%, the restore task is completed and statistics such as total restore time, average restore speed, and total data size are displayed. ```shell @@ -157,7 +150,7 @@ tiup br restore full \ - Statistics tables (`mysql.stat_*`). But statistics can be restored. See [Back up statistics](/br/br-snapshot-manual.md#back-up-statistics). - System variable tables (`mysql.tidb` and `mysql.global_variables`) -- [Other system tables](https://github.com/pingcap/tidb/blob/master/br/pkg/restore/systable_restore.go#L31) +- [Other system tables](https://github.com/pingcap/tidb/blob/release-8.1/br/pkg/restore/systable_restore.go#L31) ``` +-----------------------------------------------------+ @@ -166,7 +159,6 @@ tiup br restore full \ | gc_delete_range | | gc_delete_range_done | | global_variables | -| schema_index_usage | | stats_buckets | | stats_extended | | stats_feedback | @@ -181,14 +173,11 @@ tiup br restore full \ +-----------------------------------------------------+ ``` -When you restore data related to system privilege, note the following: - -- Before v7.6.0, BR does not restore user data with `user` as `cloud_admin` and `host` as `'%'`. This user is reserved for TiDB Cloud. Starting from v7.6.0, BR supports restoring all user data (including `cloud_admin`) by default. -- Before restoring data, BR checks whether the system tables in the target cluster are compatible with those in the backup data. "Compatible" means that all the following conditions are met: +When you restore data related to system privilege, note that before restoring data, BR checks whether the system tables in the target cluster are compatible with those in the backup data. "Compatible" means that all the following conditions are met: - - The target cluster has the same system tables as the backup data. - - The **number of columns** in the system privilege table of the target cluster is the same as that in the backup data. The column order is not important. - - The columns in the system privilege table of the target cluster are compatible with that in the backup data. If the data type of the column is a type with a length (such as integer and string), the length in the target cluster must be >= the length in the backup data. If the data type of the column is an `ENUM` type, the number of `ENUM` values in the target cluster must be a superset of that in the backup data. +- The target cluster has the same system tables as the backup data. +- The **number of columns** in the system privilege table of the target cluster is the same as that in the backup data. The column order is not important. +- The columns in the system privilege table of the target cluster are compatible with that in the backup data. If the data type of the column is a type with a length (such as integer and string), the length in the target cluster must be >= the length in the backup data. If the data type of the column is an `ENUM` type, the number of `ENUM` values in the target cluster must be a superset of that in the backup data. ## Performance and impact @@ -205,7 +194,7 @@ To illustrate the impact of backup, this document lists the test conclusions of You can use the following methods to manually control the impact of backup tasks on cluster performance. However, these two methods also reduce the speed of backup tasks while reducing the impact of backup tasks on the cluster. -- Use the `--ratelimit` parameter to limit the speed of backup tasks. Note that this parameter limits the speed of **saving backup files to external storage**. When calculating the total size of backup files, use the `backup data size(after compressed)` as a benchmark. +- Use the `--ratelimit` parameter to limit the speed of backup tasks. Note that this parameter limits the speed of **saving backup files to external storage**. When calculating the total size of backup files, use the `backup data size(after compressed)` as a benchmark. When `--ratelimit` is set, to avoid too many tasks causing the speed limit to fail, the `concurrency` parameter of br is automatically adjusted to `1`. - Adjust the TiKV configuration item [`backup.num-threads`](/tikv-configuration-file.md#num-threads-1) to limit the number of threads used by backup tasks. According to internal tests, when BR uses no more than `8` threads for backup tasks, and the total CPU utilization of the cluster does not exceed 60%, the backup tasks have little impact on the cluster, regardless of the read and write workload. The impact of backup on cluster performance can be reduced by limiting the backup threads number, but this affects the backup performance. The preceding tests show that the backup speed is proportional to the number of backup threads. When the number of threads is small, the backup speed is about 20 MiB/thread. For example, 5 backup threads on a single TiKV node can reach a backup speed of 100 MiB/s. diff --git a/br/br-snapshot-manual.md b/br/br-snapshot-manual.md index 2906e05298dd5..e0c52559887a5 100644 --- a/br/br-snapshot-manual.md +++ b/br/br-snapshot-manual.md @@ -34,7 +34,7 @@ You can back up the latest or specified snapshot of the TiDB cluster using the ` ```shell tiup br backup full \ --pd "${PD_IP}:2379" \ - --backupts '2022-09-08 13:30:00' \ + --backupts '2024-06-28 13:30:00 +08:00' \ --storage "s3://${backup_collection_addr}/snapshot-${date}?access-key=${access-key}&secret-access-key=${secret-access-key}" \ --ratelimit 128 \ --log-file backupfull.log @@ -42,7 +42,7 @@ tiup br backup full \ In the preceding command: -- `--backupts`: The time point of the snapshot. The format can be [TSO](/glossary.md#tso) or timestamp, such as `400036290571534337` or `2018-05-11 01:42:23`. If the data of this snapshot is garbage collected, the `tiup br backup` command returns an error and 'br' exits. If you leave this parameter unspecified, `br` picks the snapshot corresponding to the backup start time. +- `--backupts`: The time point of the snapshot. The format can be [TSO](/glossary.md#tso) or timestamp, such as `400036290571534337` or `2024-06-28 13:30:00 +08:00`. If the data of this snapshot is garbage collected, the `tiup br backup` command returns an error and 'br' exits. If you leave this parameter unspecified, `br` picks the snapshot corresponding to the backup start time. - `--ratelimit`: The maximum speed **per TiKV** performing backup tasks. The unit is in MiB/s. - `--log-file`: The target file where `br` log is written. @@ -178,7 +178,7 @@ tiup br restore full \ In the preceding command: - `--with-sys-table`: BR restores **data in some system tables**, including account permission data and SQL bindings, and statistics (see [Back up statistics](/br/br-snapshot-manual.md#back-up-statistics)). However, it does not restore statistics tables (`mysql.stat_*`) and system variable tables (`mysql.tidb` and `mysql.global_variables`). For more information, see [Restore tables in the `mysql` schema](/br/br-snapshot-guide.md#restore-tables-in-the-mysql-schema). -- `--ratelimit`: The maximum speed **per TiKV** performing backup tasks. The unit is in MiB/s. +- `--ratelimit`: The maximum speed **per TiKV** performing restore tasks. The unit is in MiB/s. - `--log-file`: The target file where the `br` log is written. During restore, a progress bar is displayed in the terminal as shown below. When the progress bar advances to 100%, the restore task is completed. Then `br` will verify the restored data to ensure data security. diff --git a/br/br-use-overview.md b/br/br-use-overview.md index dd28eadd7b2d9..e97760cf88aa2 100644 --- a/br/br-use-overview.md +++ b/br/br-use-overview.md @@ -1,7 +1,6 @@ --- title: Usage Overview of TiDB Backup and Restore summary: TiDB Backup and Restore provides best practices for choosing backup methods, managing backup data, and deploying the tool. It recommends using both full and log backups, storing data in recommended storage systems, and setting backup retention periods. The tool can be deployed using the command-line tool, SQL statements, or TiDB Operator on Kubernetes. For detailed usage, refer to the provided documentation. -aliases: ['/tidb/dev/br-deployment/'] --- # Usage Overview of TiDB Backup and Restore diff --git a/br/use-br-command-line-tool.md b/br/use-br-command-line-tool.md index 3cfcc43cde70b..4bedae96478aa 100644 --- a/br/use-br-command-line-tool.md +++ b/br/use-br-command-line-tool.md @@ -32,6 +32,7 @@ A `tiup br` command consists of multiple layers of sub-commands. Currently, br c * `tiup br backup`: used to back up the data of the TiDB cluster. * `tiup br log`: used to start and manage log backup tasks. * `tiup br restore`: used to restore backup data of the TiDB cluster. +* `tiup br debug`: used to parse backup metadata, check backup data, and so on. `tiup br backup` and `tiup br restore` include the following sub-commands: @@ -39,6 +40,15 @@ A `tiup br` command consists of multiple layers of sub-commands. Currently, br c * `db`: used to back up or restore a specified database of the cluster. * `table`: used to back up or restore a single table in the specified database of the cluster. +`tiup br debug` includes the following sub-commands: + +* `checksum`: (hidden parameter) used to offline check the integrity of backup data to ensure that all backup files match the CRC64 checksum results calculated by [`ADMIN CHECKSUM TABLE`](/sql-statements/sql-statement-admin-checksum-table.md). +* `backupmeta`: used to check whether an intersection exists among backup data files. In normal cases, backup data files do not intersect. +* `decode`: used to parse the `backupmeta` metadata file of a full backup into JSON format. In addition, you can parse specific fields using the `--field` parameter. +* `encode`: used to encode the `backupmeta.json` metadata file of a full backup into the protobuf format that is used during data restore. +* `reset-pd-config-as-default`: (deprecated) used to restore the PD configurations that were changed during the data recovery process to default configurations. +* `search-log-backup`: used to search for specific key information in log backup data. + ### Common options * `--pd`: specifies the PD service address. For example, `"${PD_IP}:2379"`. @@ -48,8 +58,8 @@ A `tiup br` command consists of multiple layers of sub-commands. Currently, br c * `--key`: specifies the path to the SSL certificate key in the PEM format. * `--status-addr`: specifies the listening address through which `br` provides statistics to Prometheus. * `--concurrency`: the number of concurrent tasks during the backup or restore. -* `--compression`：determines the compression algorithm used for generating backup files. It supports `lz4`, `snappy`, and `zstd`, with the default being `zstd` (usually no need to modify). For guidance on choosing different compression algorithms, refer to [this document](https://github.com/EighteenZi/rocksdb_wiki/blob/master/Compression.md). -* `--compression-level`：sets the compression level corresponding to the chosen compression algorithm for backup. The default compression level for `zstd` is 3. In most cases there is no need to set this option. +* `--compression`: determines the compression algorithm used for generating backup files. It supports `lz4`, `snappy`, and `zstd`, with the default being `zstd` (usually no need to modify). For guidance on choosing different compression algorithms, refer to [this document](https://github.com/EighteenZi/rocksdb_wiki/blob/master/Compression.md). +* `--compression-level`: sets the compression level corresponding to the chosen compression algorithm for backup. The default compression level for `zstd` is 3. In most cases there is no need to set this option. ## Commands of full backup diff --git a/cached-tables.md b/cached-tables.md index ed3d9d39add6c..e255c41bef5c1 100644 --- a/cached-tables.md +++ b/cached-tables.md @@ -15,7 +15,7 @@ The cached table feature is suitable for tables with the following characteristi - The data volume of the table is small, for example, less than 4 MiB. - The table is read-only or rarely updated, for example, with a write QPS (queries per second) of less than 10 times per minute. -- The table is frequently accessed, and you expect a better read performance, for example, when encountering hotspots on small tables during direct reads from from TiKV. +- The table is frequently accessed, and you expect a better read performance, for example, when encountering hotspots on small tables during direct reads from TiKV. When the data volume of the table is small but the data is frequently accessed, the data is concentrated on a Region in TiKV and makes it a hotspot Region, which affects the performance. Therefore, the typical usage scenarios of cached tables are as follows: @@ -33,8 +33,6 @@ This section describes the usage of cached tables by examples. Suppose that there is a table `users`: -{{< copyable "sql" >}} - ```sql CREATE TABLE users ( id BIGINT, @@ -45,8 +43,6 @@ CREATE TABLE users ( To set this table to a cached table, use the `ALTER TABLE` statement: -{{< copyable "sql" >}} - ```sql ALTER TABLE users CACHE; ``` @@ -59,8 +55,6 @@ Query OK, 0 rows affected (0.01 sec) To verify a cached table, use the `SHOW CREATE TABLE` statement. If the table is cached, the returned result contains the `CACHED ON` attribute: -{{< copyable "sql" >}} - ```sql SHOW CREATE TABLE users; ``` @@ -78,9 +72,7 @@ SHOW CREATE TABLE users; 1 row in set (0.00 sec) ``` -After reading data from a cached table, TiDB loads the data in memory. You can use the `trace` statement to check whether the data is loaded into memory. When the cache is not loaded, the returned result contains the `regionRequest.SendReqCtx` attribute, which indicates that TiDB reads data from TiKV. - -{{< copyable "sql" >}} +After reading data from a cached table, TiDB loads the data in memory. You can use the [`TRACE`](/sql-statements/sql-statement-trace.md) statement to check whether the data is loaded into memory. When the cache is not loaded, the returned result contains the `regionRequest.SendReqCtx` attribute, which indicates that TiDB reads data from TiKV. ```sql TRACE SELECT * FROM users; @@ -106,9 +98,7 @@ TRACE SELECT * FROM users; 12 rows in set (0.01 sec) ``` -After executing `trace` again, the returned result no longer contains the `regionRequest.SendReqCtx` attribute, which indicates that TiDB no longer reads data from TiKV but reads data from the memory instead. - -{{< copyable "sql" >}} +After executing [`TRACE`](/sql-statements/sql-statement-trace.md) again, the returned result no longer contains the `regionRequest.SendReqCtx` attribute, which indicates that TiDB no longer reads data from TiKV but reads data from the memory instead. ```sql +----------------------------------------+-----------------+------------+ @@ -127,8 +117,6 @@ After executing `trace` again, the returned result no longer contains the `regio Note that the `UnionScan` operator is used to read the cached tables, so you can see `UnionScan` in the execution plan of the cached tables through `explain`: -{{< copyable "sql" >}} - ```sql +-------------------------+---------+-----------+---------------+--------------------------------+ | id | estRows | task | access object | operator info | @@ -144,8 +132,6 @@ Note that the `UnionScan` operator is used to read the cached tables, so you can Cached tables support data writes. For example, you can insert a record into the `users` table: -{{< copyable "sql" >}} - ```sql INSERT INTO users(id, name) VALUES(1001, 'Davis'); ``` @@ -154,8 +140,6 @@ INSERT INTO users(id, name) VALUES(1001, 'Davis'); Query OK, 1 row affected (0.00 sec) ``` -{{< copyable "sql" >}} - ```sql SELECT * FROM users; ``` @@ -201,8 +185,6 @@ Create Table: CREATE TABLE `table_cache_meta` ( > > Executing DDL statements on a cached table will fail. Before executing DDL statements on a cached table, you need to remove the cache attribute first and set the cached table back to a normal table. -{{< copyable "sql" >}} - ```sql TRUNCATE TABLE users; ``` @@ -211,8 +193,6 @@ TRUNCATE TABLE users; ERROR 8242 (HY000): 'Truncate Table' is unsupported on cache tables. ``` -{{< copyable "sql" >}} - ```sql mysql> ALTER TABLE users ADD INDEX k_id(id); ``` @@ -223,8 +203,6 @@ ERROR 8242 (HY000): 'Alter Table' is unsupported on cache tables. To revert a cached table to a normal table, use `ALTER TABLE t NOCACHE`: -{{< copyable "sql" >}} - ```sql ALTER TABLE users NOCACHE; ``` diff --git a/certificate-authentication.md b/certificate-authentication.md index 56e6d22296b91..9ad89494872b1 100644 --- a/certificate-authentication.md +++ b/certificate-authentication.md @@ -1,7 +1,6 @@ --- title: Certificate-Based Authentication for Login summary: Learn the certificate-based authentication used for login. -aliases: ['/docs/dev/certificate-authentication/','/docs/dev/reference/security/cert-based-authentication/'] --- # Certificate-Based Authentication for Login @@ -499,3 +498,7 @@ Also replace the old CA certificate with the combined certificate so that the cl ``` 3. Configure the TiDB server to use the new server key and certificate. See [Configure TiDB server](#configure-tidb-and-the-client-to-use-certificates) for details. + +## Policy-based access control for certificates + +TiDB supports policy-based access control (PBAC) for certificates, leveraging policies defined by the underlying key management server. This enables fine-grained control over access based on various criteria, such as time-based policies (for example, certificates only valid during specific hours), location-based policies (for example, restricting access to certain geographic locations), and other customizable conditions, ensuring enhanced security and flexibility in certificate management. \ No newline at end of file diff --git a/character-set-and-collation.md b/character-set-and-collation.md index d3b750a6c7b76..65cc92677b33d 100644 --- a/character-set-and-collation.md +++ b/character-set-and-collation.md @@ -1,7 +1,6 @@ --- title: Character Set and Collation summary: Learn about the supported character sets and collations in TiDB. -aliases: ['/docs/dev/character-set-and-collation/','/docs/dev/reference/sql/characterset-and-collation/','/docs/dev/reference/sql/character-set/'] --- # Character Set and Collation @@ -10,7 +9,7 @@ This document introduces the character sets and collations supported by TiDB. ## Concepts -A character set is a set of symbols and encodings. The default character set in TiDB is utf8mb4, which matches the default in MySQL 8.0 and above. +A character set is a set of symbols and encodings. The default character set in TiDB is `utf8mb4`, which matches the default character set in MySQL 8.0 and later. A collation is a set of rules for comparing characters in a character set, and the sorting order of characters. For example in a binary collation `A` and `a` do not compare as equal: @@ -73,7 +72,7 @@ SHOW CHARACTER SET; +---------+-------------------------------------+-------------------+--------+ | ascii | US ASCII | ascii_bin | 1 | | binary | binary | binary | 1 | -| gbk | Chinese Internal Code Specification | gbk_bin | 2 | +| gbk | Chinese Internal Code Specification | gbk_chinese_ci | 2 | | latin1 | Latin1 | latin1_bin | 1 | | utf8 | UTF-8 Unicode | utf8_bin | 3 | | utf8mb4 | UTF-8 Unicode | utf8mb4_bin | 4 | @@ -413,7 +412,7 @@ You can use the following statement to set the character set and collation that ```sql SET character_set_client = charset_name; SET character_set_results = charset_name; - SET charset_connection = @@charset_database; + SET character_set_connection=@@character_set_database; SET collation_connection = @@collation_database; ``` diff --git a/character-set-gbk.md b/character-set-gbk.md index 68b5d9d088d22..2da2454100e8f 100644 --- a/character-set-gbk.md +++ b/character-set-gbk.md @@ -5,14 +5,16 @@ summary: This document provides details about the TiDB support of the GBK charac # GBK -Since v5.4.0, TiDB supports the GBK character set. This document provides the TiDB support and compatibility information of the GBK character set. +Starting from v5.4.0, TiDB supports the GBK character set. This document provides the TiDB support and compatibility information of the GBK character set. + +Starting from v6.0.0, TiDB enables the [new framework for collations](/character-set-and-collation.md#new-framework-for-collations) by default. The default collation for TiDB GBK character set is `gbk_chinese_ci`, which is consistent with MySQL. ```sql SHOW CHARACTER SET WHERE CHARSET = 'gbk'; +---------+-------------------------------------+-------------------+--------+ | Charset | Description | Default collation | Maxlen | +---------+-------------------------------------+-------------------+--------+ -| gbk | Chinese Internal Code Specification | gbk_bin | 2 | +| gbk | Chinese Internal Code Specification | gbk_chinese_ci | 2 | +---------+-------------------------------------+-------------------+--------+ 1 row in set (0.00 sec) @@ -21,8 +23,9 @@ SHOW COLLATION WHERE CHARSET = 'gbk'; | Collation | Charset | Id | Default | Compiled | Sortlen | +----------------+---------+------+---------+----------+---------+ | gbk_bin | gbk | 87 | | Yes | 1 | +| gbk_chinese_ci | gbk | 28 | Yes | Yes | 1 | +----------------+---------+------+---------+----------+---------+ -1 rows in set (0.00 sec) +2 rows in set (0.00 sec) ``` ## MySQL compatibility @@ -31,40 +34,22 @@ This section provides the compatibility information between MySQL and TiDB. ### Collations -The default collation of the GBK character set in MySQL is `gbk_chinese_ci`. Unlike MySQL, the default collation of the GBK character set in TiDB is `gbk_bin`. Additionally, because TiDB converts GBK to UTF8MB4 and then uses a binary collation, the `gbk_bin` collation in TiDB is not the same as the `gbk_bin` collation in MySQL. - -To make TiDB compatible with the collations of MySQL GBK character set, when you first initialize the TiDB cluster, you need to set the TiDB option [`new_collations_enabled_on_first_bootstrap`](/tidb-configuration-file.md#new_collations_enabled_on_first_bootstrap) to `true` to enable the [new framework for collations](/character-set-and-collation.md#new-framework-for-collations). +The default collation of the GBK character set in MySQL is `gbk_chinese_ci`. The default collation for the GBK character set in TiDB depends on the value of the TiDB configuration item [`new_collations_enabled_on_first_bootstrap`](/tidb-configuration-file.md#new_collations_enabled_on_first_bootstrap): + +- By default, the TiDB configuration item [`new_collations_enabled_on_first_bootstrap`](/tidb-configuration-file.md#new_collations_enabled_on_first_bootstrap) is set to `true`, which means that the [new framework for collations](/character-set-and-collation.md#new-framework-for-collations) is enabled and the default collation for the GBK character set is `gbk_chinese_ci`. +- When the TiDB configuration item [`new_collations_enabled_on_first_bootstrap`](/tidb-configuration-file.md#new_collations_enabled_on_first_bootstrap) is set to `false`, the [new framework for collations](/character-set-and-collation.md#new-framework-for-collations) is disabled, and the default collation for the GBK character set is `gbk_bin`. -To make TiDB compatible with the collations of MySQL GBK character set, when you first initialize the TiDB cluster, TiDB Cloud enables the [new framework for collations](/character-set-and-collation.md#new-framework-for-collations) by default. +By default, TiDB Cloud enables the [new framework for collations](/character-set-and-collation.md#new-framework-for-collations) and the default collation for the GBK character set is `gbk_chinese_ci`. -After enabling the new framework for collations, if you check the collations corresponding to the GBK character set, you can see that the TiDB GBK default collation is changed to `gbk_chinese_ci`. - -```sql -SHOW CHARACTER SET WHERE CHARSET = 'gbk'; -+---------+-------------------------------------+-------------------+--------+ -| Charset | Description | Default collation | Maxlen | -+---------+-------------------------------------+-------------------+--------+ -| gbk | Chinese Internal Code Specification | gbk_chinese_ci | 2 | -+---------+-------------------------------------+-------------------+--------+ -1 row in set (0.00 sec) - -SHOW COLLATION WHERE CHARSET = 'gbk'; -+----------------+---------+------+---------+----------+---------+ -| Collation | Charset | Id | Default | Compiled | Sortlen | -+----------------+---------+------+---------+----------+---------+ -| gbk_bin | gbk | 87 | | Yes | 1 | -| gbk_chinese_ci | gbk | 28 | Yes | Yes | 1 | -+----------------+---------+------+---------+----------+---------+ -2 rows in set (0.00 sec) -``` +Additionally, because TiDB converts GBK to `utf8mb4` and then uses a binary collation, the `gbk_bin` collation in TiDB is not the same as the `gbk_bin` collation in MySQL. ### Illegal character compatibility diff --git a/check-before-deployment.md b/check-before-deployment.md index 0b8f957ca52e3..acbccf703ea5f 100644 --- a/check-before-deployment.md +++ b/check-before-deployment.md @@ -1,7 +1,6 @@ --- title: TiDB Environment and System Configuration Check summary: Learn the environment check operations before deploying TiDB. -aliases: ['/docs/dev/check-before-deployment/'] --- # TiDB Environment and System Configuration Check @@ -42,6 +41,13 @@ Take the `/dev/nvme0n1` data disk as an example: parted -s -a optimal /dev/nvme0n1 mklabel gpt -- mkpart primary ext4 1 -1 ``` + For large NVMe devices, you can create multiple partitions: + + ```bash + parted -s -a optimal /dev/nvme0n1 mklabel gpt -- mkpart primary ext4 1 2000GB + parted -s -a optimal /dev/nvme0n1 -- mkpart primary ext4 2000GB -1 + ``` + > **Note:** > > Use the `lsblk` command to view the device number of the partition: for a NVMe disk, the generated device number is usually `nvme0n1p1`; for a regular disk (for example, `/dev/sdb`), the generated device number is usually `sdb1`. @@ -93,6 +99,7 @@ Take the `/dev/nvme0n1` data disk as an example: ```bash mkdir /data1 && \ + systemctl daemon-reload && \ mount -a ``` @@ -112,21 +119,25 @@ Take the `/dev/nvme0n1` data disk as an example: ## Check and disable system swap -TiDB needs sufficient memory space for operation. When memory is insufficient, using swap as a buffer might degrade performance. Therefore, it is recommended to disable the system swap permanently by executing the following commands: +TiDB needs a sufficient amount of memory for operation. If the memory that TiDB uses gets swapped out and later gets swapped back in, this can cause latency spikes. If you want to maintain stable performance, it is recommended that you permanently disable the system swap, but it might trigger OOM issues when there is insufficient memory. If you want to avoid such OOM issues, you can just decrease the swap priority, instead of permanently disabling it. -{{< copyable "shell-regular" >}} +- Enabling and using swap might introduce performance jitter issues. It is recommended that you permanently disable the operating system tier swap for low-latency and stability-critical database services. To permanently disable swap, you can use the following method: -```bash -echo "vm.swappiness = 0">> /etc/sysctl.conf -swapoff -a && swapon -a -sysctl -p -``` + - During the initialization phase of the operating system, do not partition the swap partition disk separately. + - If you have already partitioned a separate swap partition disk during the initialization phase of the operating system and enabled swap, run the following command to disable it: -> **Note:** -> -> - Executing `swapoff -a` and then `swapon -a` is to refresh swap by dumping data to memory and cleaning up swap. If you drop the swappiness change and execute only `swapoff -a`, swap will be enabled again after you restart the system. -> -> - `sysctl -p` is to make the configuration effective without restarting the system. + ```bash + echo "vm.swappiness = 0">> /etc/sysctl.conf + sysctl -p + swapoff -a && swapon -a + ``` + +- If the host memory is insufficient, disabling the system swap might be more likely to trigger OOM issues. You can run the following command to decrease the swap priority instead of disabling it permanently: + + ```bash + echo "vm.swappiness = 0">> /etc/sysctl.conf + sysctl -p + ``` ## Set temporary spaces for TiDB instances (Recommended) @@ -138,33 +149,35 @@ Some operations in TiDB require writing temporary files to the server, so it is - `Fast Online DDL` work area - When the variable [`tidb_ddl_enable_fast_reorg`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) is set to `ON` (the default value in v6.5.0 and later versions), `Fast Online DDL` is enabled, and some DDL operations need to read and write temporary files in filesystems. The location is defined by the configuration item [`temp-dir`](/tidb-configuration-file.md#temp-dir-new-in-v630). You need to ensure that the user that runs TiDB has read and write permissions for that directory of the operating system. Taking the default directory `/tmp/tidb` as an example: + When the variable [`tidb_ddl_enable_fast_reorg`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) is set to `ON` (the default value in v6.5.0 and later versions), `Fast Online DDL` is enabled, and some DDL operations need to read and write temporary files in filesystems. The location is defined by the configuration item [`temp-dir`](/tidb-configuration-file.md#temp-dir-new-in-v630). You need to ensure that the user that runs TiDB has read and write permissions for that directory of the operating system. The default directory `/tmp/tidb` uses tmpfs (temporary file system). It is recommended to explicitly specify a disk directory. The following uses `/data/tidb-deploy/tempdir` as an example: > **Note:** > > If DDL operations on large objects exist in your application, it is highly recommended to configure an independent large file system for [`temp-dir`](/tidb-configuration-file.md#temp-dir-new-in-v630). ```shell - sudo mkdir /tmp/tidb + sudo mkdir -p /data/tidb-deploy/tempdir ``` - If the `/tmp/tidb` directory already exists, make sure the write permission is granted. + If the `/data/tidb-deploy/tempdir` directory already exists, make sure the write permission is granted. ```shell - sudo chmod -R 777 /tmp/tidb + sudo chmod -R 777 /data/tidb-deploy/tempdir ``` > **Note:** > - > If the directory does not exist, TiDB will automatically create it upon startup. If the directory creation fails or TiDB does not have the read and write permissions for that directory, [`Fast Online DDL`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) might experience unpredictable issues during runtime. + > If the directory does not exist, TiDB will automatically create it upon startup. If the directory creation fails or TiDB does not have the read and write permissions for that directory, [`Fast Online DDL`](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) will be disabled during runtime. -## Check and stop the firewall service of target machines +## Check the firewall service of target machines In TiDB clusters, the access ports between nodes must be open to ensure the transmission of information such as read and write requests and data heartbeats. In common online scenarios, the data interaction between the database and the application service and between the database nodes are all made within a secure network. Therefore, if there are no special security requirements, it is recommended to stop the firewall of the target machine. Otherwise, refer to [the port usage](/hardware-and-software-requirements.md#network-requirements) and add the needed port information to the allowlist of the firewall service. -The rest of this section describes how to stop the firewall service of a target machine. +### Stop and disable firewalld + +This section describes how to stop and disable the firewall service of a target machine. -1. Check the firewall status. Take CentOS Linux release 7.7.1908 (Core) as an example. +1. Check the firewall status. The following example uses CentOS Linux release 7.7.1908 (Core): {{< copyable "shell-regular" >}} @@ -173,7 +186,7 @@ The rest of this section describes how to stop the firewall service of a target sudo systemctl status firewalld.service ``` -2. Stop the firewall service. +2. Stop the firewall service: {{< copyable "shell-regular" >}} @@ -181,7 +194,7 @@ The rest of this section describes how to stop the firewall service of a target sudo systemctl stop firewalld.service ``` -3. Disable automatic start of the firewall service. +3. Disable automatic startup of the firewall service: {{< copyable "shell-regular" >}} @@ -189,7 +202,7 @@ The rest of this section describes how to stop the firewall service of a target sudo systemctl disable firewalld.service ``` -4. Check the firewall status. +4. Check the firewall status: {{< copyable "shell-regular" >}} @@ -197,6 +210,93 @@ The rest of this section describes how to stop the firewall service of a target sudo systemctl status firewalld.service ``` +### Change the firewall zone + +Instead of disabling the firewall completely, you can use a less restrictive zone. The default `public` zone allows only specific services and ports, while the `trusted` zone allows all traffic by default. + +To set the default zone to `trusted`: + +```bash +firewall-cmd --set-default-zone trusted +``` + +To verify the default zone: + +```bash +firewall-cmd --get-default-zone +# trusted +``` + +To list the policy for a zone: + +```bash +firewall-cmd --zone=trusted --list-all +# trusted +# target: ACCEPT +# icmp-block-inversion: no +# interfaces: +# sources: +# services: +# ports: +# protocols: +# forward: yes +# masquerade: no +# forward-ports: +# source-ports: +# icmp-blocks: +# rich rules: +``` + +### Configure the firewall + +To configure the firewall for TiDB cluster components, use the following commands. These examples are for reference only. Adjust the zone names, ports, and services based on your specific environment. + +Configure the firewall for the TiDB component: + +```bash +firewall-cmd --permanent --new-service tidb +firewall-cmd --permanent --service tidb --set-description="TiDB Server" +firewall-cmd --permanent --service tidb --set-short="TiDB" +firewall-cmd --permanent --service tidb --add-port=4000/tcp +firewall-cmd --permanent --service tidb --add-port=10080/tcp +firewall-cmd --permanent --zone=public --add-service=tidb +``` + +Configure the firewall for the TiKV component: + +```bash +firewall-cmd --permanent --new-service tikv +firewall-cmd --permanent --service tikv --set-description="TiKV Server" +firewall-cmd --permanent --service tikv --set-short="TiKV" +firewall-cmd --permanent --service tikv --add-port=20160/tcp +firewall-cmd --permanent --service tikv --add-port=20180/tcp +firewall-cmd --permanent --zone=public --add-service=tikv +``` + +Configure the firewall for the PD component: + +```bash +firewall-cmd --permanent --new-service pd +firewall-cmd --permanent --service pd --set-description="PD Server" +firewall-cmd --permanent --service pd --set-short="PD" +firewall-cmd --permanent --service pd --add-port=2379/tcp +firewall-cmd --permanent --service pd --add-port=2380/tcp +firewall-cmd --permanent --zone=public --add-service=pd +``` + +Configure the firewall for Prometheus: + +```bash +firewall-cmd --permanent --zone=public --add-service=prometheus +firewall-cmd --permanent --service=prometheus --add-port=12020/tcp +``` + +Configure the firewall for Grafana: + +```bash +firewall-cmd --permanent --zone=public --add-service=grafana +``` + ## Check and install the NTP service TiDB is a distributed database system that requires clock synchronization between nodes to guarantee linear consistency of transactions in the ACID model. @@ -311,6 +411,8 @@ To check whether the NTP service is installed and whether it synchronizes with t 506 Cannot talk to daemon ``` + - If the offset appears to be too high, you can run the `chronyc makestep` command to immediately correct the time offset. Otherwise, `chronyd` will gradually correct the time offset. + To make the NTP service start synchronizing as soon as possible, run the following command. Replace `pool.ntp.org` with your NTP server. {{< copyable "shell-regular" >}} @@ -336,7 +438,11 @@ sudo systemctl enable ntpd.service For TiDB in the production environment, it is recommended to optimize the operating system configuration in the following ways: 1. Disable THP (Transparent Huge Pages). The memory access pattern of databases tends to be sparse rather than consecutive. If the high-level memory fragmentation is serious, higher latency will occur when THP pages are allocated. -2. Set the I/O Scheduler of the storage media to `noop`. For the high-speed SSD storage media, the kernel's I/O scheduling operations can cause performance loss. After the Scheduler is set to `noop`, the performance is better because the kernel directly sends I/O requests to the hardware without other operations. Also, the noop Scheduler is better applicable. +2. Set the I/O Scheduler of the storage media. + + - For the high-speed SSD storage, the kernel's default I/O scheduling operations might cause performance loss. It is recommended to set the I/O Scheduler to first-in-first-out (FIFO), such as `noop` or `none`. This configuration allows the kernel to pass I/O requests directly to hardware without scheduling, thus improving performance. + - For NVMe storage, the default I/O Scheduler is `none`, so no adjustment is needed. + 3. Choose the `performance` mode for the cpufrequ module which controls the CPU frequency. The performance is maximized when the CPU frequency is fixed at its highest supported operating frequency without dynamic adjustment. Take the following steps to check the current operating system configuration and configure optimal parameters: @@ -357,9 +463,9 @@ Take the following steps to check the current operating system configuration and > > If `[always] madvise never` is output, THP is enabled. You need to disable it. -2. Execute the following command to see the I/O Scheduler of the disk where the data directory is located. Assume that you create data directories on both sdb and sdc disks: +2. Execute the following command to see the I/O Scheduler of the disk where the data directory is located. - {{< copyable "shell-regular" >}} + If your data directory uses an SD or VD device, run the following command to check the I/O Scheduler: ```bash cat /sys/block/sd[bc]/queue/scheduler @@ -374,6 +480,21 @@ Take the following steps to check the current operating system configuration and > > If `noop [deadline] cfq` is output, the I/O Scheduler for the disk is in the `deadline` mode. You need to change it to `noop`. + If your data directory uses an NVMe device, run the following command to check the I/O Scheduler: + + ```bash + cat /sys/block/nvme[01]*/queue/scheduler + ``` + + ``` + [none] mq-deadline kyber bfq + [none] mq-deadline kyber bfq + ``` + + > **Note:** + > + > `[none] mq-deadline kyber bfq` indicates that the NVMe device uses the `none` I/O Scheduler, and no changes are needed. + 3. Execute the following command to see the `ID_SERIAL` of the disk: {{< copyable "shell-regular" >}} @@ -389,7 +510,8 @@ Take the following steps to check the current operating system configuration and > **Note:** > - > If multiple disks are allocated with data directories, you need to execute the above command several times to record the `ID_SERIAL` of each disk. + > - If multiple disks are allocated with data directories, you need to execute the above command for each disk to record the `ID_SERIAL` of each disk. + > - If your device uses the `noop` or `none` Scheduler, you do not need to record the `ID_SERIAL` or configure udev rules or the tuned profile. 4. Execute the following command to see the power policy of the cpufreq module: @@ -466,6 +588,10 @@ Take the following steps to check the current operating system configuration and 3. Apply the new tuned profile: + > **Note:** + > + > If your device uses the `noop` or `none` I/O Scheduler, skip this step. No Scheduler configuration is needed in the tuned profile. + {{< copyable "shell-regular" >}} ```bash @@ -495,12 +621,12 @@ Take the following steps to check the current operating system configuration and {{< copyable "shell-regular" >}} ```bash - grubby --args="transparent_hugepage=never" --update-kernel /boot/vmlinuz-3.10.0-957.el7.x86_64 + grubby --args="transparent_hugepage=never" --update-kernel `grubby --default-kernel` ``` > **Note:** > - > `--update-kernel` is followed by the actual default kernel version. + > You can also specify the actual version number after `--update-kernel`, for example, `--update-kernel /boot/vmlinuz-3.10.0-957.el7.x86_64`. 3. Execute `grubby --info` to see the modified default kernel configuration: @@ -548,6 +674,10 @@ Take the following steps to check the current operating system configuration and 6. Apply the udev script: + > **Note:** + > + > If your device uses the `noop` or `none` I/O Scheduler, skip this step. No udev rules configuration is needed. + {{< copyable "shell-regular" >}} ```bash @@ -627,19 +757,22 @@ Take the following steps to check the current operating system configuration and ```bash echo "fs.file-max = 1000000">> /etc/sysctl.conf echo "net.core.somaxconn = 32768">> /etc/sysctl.conf - echo "net.ipv4.tcp_tw_recycle = 0">> /etc/sysctl.conf echo "net.ipv4.tcp_syncookies = 0">> /etc/sysctl.conf echo "vm.overcommit_memory = 1">> /etc/sysctl.conf echo "vm.min_free_kbytes = 1048576">> /etc/sysctl.conf sysctl -p ``` + > **Warning:** + > + > It is not recommended to increase the value of `vm.min_free_kbytes` on systems with less than 16 GiB of memory, because it might cause instability and boot failures. + > **Note:** > > - `vm.min_free_kbytes` is a Linux kernel parameter that controls the minimum amount of free memory reserved by the system, measured in KiB. > - The setting of `vm.min_free_kbytes` affects the memory reclaim mechanism. Setting it too large reduces the available memory, while setting it too small might cause memory request speeds to exceed background reclaim speeds, leading to memory reclamation and consequent delays in memory allocation. > - It is recommended to set `vm.min_free_kbytes` to `1048576` KiB (1 GiB) at least. If [NUMA is installed](/check-before-deployment.md#install-the-numactl-tool), it is recommended to set it to `number of NUMA nodes * 1048576` KiB. - > - For servers with memory sizes less than 16 GiB, it is recommended to keep the default value of `vm.min_free_kbytes` unchanged. + > - For systems running Linux kernel 4.11 or earlier, it is recommended to set `net.ipv4.tcp_tw_recycle = 0`. 10. Execute the following command to configure the user's `limits.conf` file: @@ -744,3 +877,11 @@ sudo yum -y install numactl ``` To get help information of the `tiup cluster exec` command, run the `tiup cluster exec --help` command. + +## Disable SELinux + +SELinux must be disabled or set to permissive mode. To check the current status, use the [getenforce(8)](https://linux.die.net/man/8/getenforce) utility. + +If SELinux is not disabled, open the `/etc/selinux/config` file, locate the line starting with `SELINUX=`, and change it to `SELINUX=disabled`. After making this change, you need to reboot the system because switching from `enforcing` or `permissive` to `disabled` does not take effect without a reboot. + +On some systems (such as Ubuntu), the `/etc/selinux/config` file might not exist, and the getenforce utility might not be installed. In that case, you can skip this step. diff --git a/choose-index.md b/choose-index.md index 3f8a4860eda67..19bf0f2f49d53 100644 --- a/choose-index.md +++ b/choose-index.md @@ -123,15 +123,15 @@ According to these factors and the cost model, the optimizer selects an index wi 1. The estimated number of rows is not accurate? - This is usually due to stale or inaccurate statistics. You can re-execute the `analyze table` statement or modify the parameters of the `analyze table` statement. + This is usually due to stale or inaccurate statistics. You can re-execute the `ANALYZE TABLE` statement or modify the parameters of the `ANALYZE TABLE` statement. 2. Statistics are accurate, and reading from TiFlash is faster, but why does the optimizer choose to read from TiKV? - At present, the cost model of distinguishing TiFlash from TiKV is still rough. You can decrease the value of `tidb_opt_seek_factor` parameter, then the optimizer prefers to choose TiFlash. + At present, the cost model of distinguishing TiFlash from TiKV is still rough. You can decrease the value of [`tidb_opt_seek_factor`](/system-variables.md#tidb_opt_seek_factor) parameter, then the optimizer prefers to choose TiFlash. 3. The statistics are accurate. Index A needs to retrieve rows from tables, but it actually executes faster than Index B that does not retrieve rows from tables. Why does the optimizer choose Index B? - In this case, the cost estimation may be too large for retrieving rows from tables. You can decrease the value of `tidb_opt_network_factor` parameter to reduce the cost of retrieving rows from tables. + In this case, the cost estimation may be too large for retrieving rows from tables. You can decrease the value of [`tidb_opt_network_factor`](/system-variables.md#tidb_opt_network_factor) parameter to reduce the cost of retrieving rows from tables. ## Control index selection @@ -143,7 +143,7 @@ The index selection can be controlled by a single query through [Optimizer Hints ## Use multi-valued indexes -[Multi-valued indexes](/sql-statements/sql-statement-create-index.md#multi-valued-indexes) are different from normal indexes. TiDB currently only uses [IndexMerge](/explain-index-merge.md) to access multi-valued indexes. Therefore, to use multi-valued indexes for data access, make sure that the value of the system variable `tidb_enable_index_merge` is set to `ON`. +[Multi-valued indexes](/sql-statements/sql-statement-create-index.md#multi-valued-indexes) are different from normal indexes. TiDB currently only uses [IndexMerge](/explain-index-merge.md) to access multi-valued indexes. Therefore, to use multi-valued indexes for data access, make sure that the value of the system variable [`tidb_enable_index_merge`](/system-variables.md#tidb_enable_index_merge-new-in-v40) is set to `ON`. For the limitations of multi-valued indexes, refer to [`CREATE INDEX`](/sql-statements/sql-statement-create-index.md#limitations). diff --git a/clinic/clinic-data-instruction-for-tiup.md b/clinic/clinic-data-instruction-for-tiup.md index 2285fa0a6a157..2fb3bb8407c09 100644 --- a/clinic/clinic-data-instruction-for-tiup.md +++ b/clinic/clinic-data-instruction-for-tiup.md @@ -112,7 +112,7 @@ This section lists the types of diagnostic data that can be collected by Diag fr | Data type | Exported file | Parameter for data collection by PingCAP Clinic | | :------ | :------ |:-------- | -| Log | `m-master.log` | `--include=log` | +| Log | `dm-master.log` | `--include=log` | | Error log | `dm-master_stderr.log` | `--include=log` | | Configuration file | `dm-master.toml` | `--include=config` | diff --git a/clinic/clinic-introduction.md b/clinic/clinic-introduction.md index 216c450ab404e..48b3b00f26b4f 100644 --- a/clinic/clinic-introduction.md +++ b/clinic/clinic-introduction.md @@ -1,6 +1,6 @@ --- title: PingCAP Clinic Overview -summary: PingCAP Clinic is a diagnostic service for TiDB clusters deployed using TiUP or TiDB Operator. It helps troubleshoot cluster problems remotely, ensures stable operation, and provides quick cluster status checks. The service includes Diag client for data collection and Clinic Server for online diagnostic reports. Users can troubleshoot problems remotely and quickly check cluster status. Diag collects diagnostic data through various methods, and Clinic Server has limitations on clusters, storage, and data size. The service is free until July 14, 2024. Next steps include using PingCAP Clinic in different environments. +summary: PingCAP Clinic is a diagnostic service for TiDB clusters deployed using TiUP or TiDB Operator. It helps troubleshoot cluster problems remotely, ensures stable operation, and provides quick cluster status checks. The service includes Diag client for data collection and Clinic Server for online diagnostic reports. Users can troubleshoot problems remotely and quickly check cluster status. Diag collects diagnostic data through various methods, and Clinic Server has limitations on clusters, storage, and data size. The service is free until April 15, 2025. Next steps include using PingCAP Clinic in different environments. --- # PingCAP Clinic Overview @@ -61,7 +61,7 @@ First, Diag gets cluster topology information from the deployment tool TiUP (tiu > **Note:** > -> - Clinic Server is free from July 15, 2022 to July 14, 2024. You will be notified through email before July 14, 2024 if the service starts charging fee afterwards. +> - Clinic Server is free from July 15, 2022 to April 15, 2025. You will be notified through email before April 15, 2025 if the service starts charging fee afterwards. > - If you want to adjust the usage limitations, [get support](/support.md) from PingCAP. | Service Type| Limitation | diff --git a/clinic/clinic-user-guide-for-tiup.md b/clinic/clinic-user-guide-for-tiup.md index cf9a972c79ffb..a743fa60c0220 100644 --- a/clinic/clinic-user-guide-for-tiup.md +++ b/clinic/clinic-user-guide-for-tiup.md @@ -30,7 +30,7 @@ Before using PingCAP Clinic, you need to install Diag (a component to collect da 1. Install Diag. - - If you have installed TiUP on your control machine, run the following command to install Diag: + - If you have installed TiUP on your control machine, run the following command to install Diag: ```bash tiup install diag @@ -58,7 +58,7 @@ Before using PingCAP Clinic, you need to install Diag (a component to collect da
- [Clinic Server for international users](https://clinic.pingcap.com): Data is stored in AWS in US. + [Clinic Server for international users](https://clinic.pingcap.com): Data is stored in AWS in US regions.
@@ -323,7 +323,7 @@ You can have a quick check on the cluster status locally using Diag. Even if you The following is the details of the abnormalities. ### Diagnostic result summary - The configuration rules are all derived from PingCAP’s OnCall Service. + The configuration rules are all derived from PingCAP's OnCall Service. If the results of the configuration rules are found to be abnormal, they may cause the cluster to fail. diff --git a/clustered-indexes.md b/clustered-indexes.md index 31a74827c58fa..4aae65449138a 100644 --- a/clustered-indexes.md +++ b/clustered-indexes.md @@ -7,7 +7,7 @@ summary: Learn the concept, user scenarios, usages, limitations, and compatibili TiDB supports the clustered index feature since v5.0. This feature controls how data is stored in tables containing primary keys. It provides TiDB the ability to organize tables in a way that can improve the performance of certain queries. -The term _clustered_ in this context refers to the _organization of how data is stored_ and not _a group of database servers working together_. Some database management systems refer to clustered indexes as _index-organized tables_ (IOT). +The term _clustered_ in this context refers to the _organization of how data is stored_ and not _a group of database servers working together_. Some database management systems refer to clustered index tables as _index-organized tables_ (IOT). Currently, tables containing primary keys in TiDB are divided into the following two categories: @@ -37,7 +37,7 @@ On the other hand, tables with clustered indexes have certain disadvantages. See ## Usages -## Create a table with clustered indexes +### Create a table with clustered indexes Since TiDB v5.0, you can add non-reserved keywords `CLUSTERED` or `NONCLUSTERED` after `PRIMARY KEY` in a `CREATE TABLE` statement to specify whether the table's primary key is a clustered index. For example: @@ -143,7 +143,7 @@ mysql> SELECT TIDB_PK_TYPE FROM information_schema.tables WHERE table_schema = ' Currently, there are several different types of limitations for the clustered index feature. See the following: - Situations that are not supported and not in the support plan: - - Using clustered indexes together with the attribute [`SHARD_ROW_ID_BITS`](/shard-row-id-bits.md) is not supported. Also, the attribute [`PRE_SPLIT_REGIONS`](/sql-statements/sql-statement-split-region.md#pre_split_regions) does not take effect on tables with clustered indexes. + - Using clustered indexes together with the attribute [`SHARD_ROW_ID_BITS`](/shard-row-id-bits.md) is not supported. Also, the attribute [`PRE_SPLIT_REGIONS`](/sql-statements/sql-statement-split-region.md#pre_split_regions) does not take effect on tables with clustered indexes that are not [`AUTO_RANDOM`](/auto-random.md). - Downgrading tables with clustered indexes is not supported. If you need to downgrade such tables, use logical backup tools to migrate data instead. - Situations that are not supported yet but in the support plan: - Adding, dropping, and altering clustered indexes using `ALTER TABLE` statements are not supported. diff --git a/command-line-flags-for-pd-configuration.md b/command-line-flags-for-pd-configuration.md index c12a03b8a3bf2..d3a8cb62d460b 100644 --- a/command-line-flags-for-pd-configuration.md +++ b/command-line-flags-for-pd-configuration.md @@ -1,7 +1,6 @@ --- title: PD Configuration Flags summary: Learn some configuration flags of PD. -aliases: ['/docs/dev/command-line-flags-for-pd-configuration/','/docs/dev/reference/configuration/pd-server/configuration/'] --- # PD Configuration Flags @@ -83,7 +82,7 @@ PD is configurable using command-line flags and environment variables. ## `--name` - The human-readable unique name for this PD member -- Default: `"pd"` +- Default: `"pd-${hostname}"` - If you want to start multiply PDs, you must use different name for each one. ## `--cacert` diff --git a/command-line-flags-for-tidb-configuration.md b/command-line-flags-for-tidb-configuration.md index c56c6b41c7575..739ebdb681e17 100644 --- a/command-line-flags-for-tidb-configuration.md +++ b/command-line-flags-for-tidb-configuration.md @@ -1,7 +1,6 @@ --- title: Configuration Options summary: Learn the configuration options in TiDB. -aliases: ['/docs/dev/command-line-flags-for-tidb-configuration/','/docs/dev/reference/configuration/tidb-server/configuration/','/docs/dev/reference/configuration/tidb-server/server-command-option/'] --- # Configuration Options diff --git a/command-line-flags-for-tikv-configuration.md b/command-line-flags-for-tikv-configuration.md index 86c5e8e810e70..18f2fea7b4a47 100644 --- a/command-line-flags-for-tikv-configuration.md +++ b/command-line-flags-for-tikv-configuration.md @@ -1,7 +1,6 @@ --- title: TiKV Configuration Flags summary: Learn some configuration flags of TiKV. -aliases: ['/docs/dev/command-line-flags-for-tikv-configuration/','/docs/dev/reference/configuration/tikv-server/configuration/'] --- # TiKV Configuration Flags diff --git a/comment-syntax.md b/comment-syntax.md index ac9530cb8091b..0a47026f5b80d 100644 --- a/comment-syntax.md +++ b/comment-syntax.md @@ -1,7 +1,6 @@ --- title: Comment Syntax summary: This document introduces the comment syntax supported by TiDB. -aliases: ['/docs/dev/comment-syntax/','/docs/dev/reference/sql/language-structure/comment-syntax/'] --- # Comment Syntax diff --git a/configure-load-base-split.md b/configure-load-base-split.md index 63abac9ff2194..7958c14d8583e 100644 --- a/configure-load-base-split.md +++ b/configure-load-base-split.md @@ -1,7 +1,6 @@ --- title: Load Base Split summary: Learn the feature of Load Base Split. -aliases: ['/docs/dev/configure-load-base-split/'] --- # Load Base Split diff --git a/configure-memory-usage.md b/configure-memory-usage.md index 71e03b7661184..4d38ff386b9ee 100644 --- a/configure-memory-usage.md +++ b/configure-memory-usage.md @@ -1,7 +1,6 @@ --- title: TiDB Memory Control summary: Learn how to configure the memory quota of a query and avoid OOM (out of memory). -aliases: ['/docs/dev/configure-memory-usage/','/docs/dev/how-to/configure/memory-control/'] --- # TiDB Memory Control @@ -149,7 +148,7 @@ TiDB supports disk spill for execution operators. When the memory usage of a SQL - The disk spill behavior is jointly controlled by the following parameters: [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query), [`tidb_enable_tmp_storage_on_oom`](/system-variables.md#tidb_enable_tmp_storage_on_oom), [`tmp-storage-path`](/tidb-configuration-file.md#tmp-storage-path), and [`tmp-storage-quota`](/tidb-configuration-file.md#tmp-storage-quota). - When the disk spill is triggered, TiDB outputs a log containing the keywords `memory exceeds quota, spill to disk now` or `memory exceeds quota, set aggregate mode to spill-mode`. -- Disk spill for the Sort, MergeJoin, and HashJoin operators is introduced in v4.0.0; disk spill for the non-concurrent algorithm of the HashAgg operator is introduced in v5.2.0; disk spill for the concurrent algorithm of the HashAgg operator is introduced in v8.0.0. +- Disk spill for the Sort, MergeJoin, and HashJoin operators is introduced in v4.0.0; disk spill for the non-parallel algorithm of the HashAgg operator is introduced in v5.2.0; disk spill for the parallel algorithm of the HashAgg operator is introduced in v8.0.0. - When the SQL executions containing Sort, MergeJoin, HashJoin, or HashAgg cause OOM, TiDB triggers disk spill by default. > **Note:** diff --git a/configure-placement-rules.md b/configure-placement-rules.md index dd77b8ac39746..790000969f309 100644 --- a/configure-placement-rules.md +++ b/configure-placement-rules.md @@ -1,7 +1,6 @@ --- title: Placement Rules summary: Learn how to configure Placement Rules. -aliases: ['/docs/dev/configure-placement-rules/','/docs/dev/how-to/configure/placement-rules/'] --- # Placement Rules @@ -189,7 +188,9 @@ cat > rules.json < rules.json < **Warning:** +> +> Store limit v2 is an experimental feature. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. + When [`store-limit-version`](/pd-configuration-file.md#store-limit-version-new-in-v710) is set to `v2`, store limit v2 takes effect. In v2 mode, the limit of operators are dynamically adjusted based on the capability of TiKV snapshots. When TiKV has fewer pending tasks, PD increases its scheduling tasks. Otherwise, PD reduces the scheduling tasks for the node. Therefore, you do not need to manually set `store limit` to speed up the scheduling process. In v2 mode, the execution speed of TiKV becomes the main bottleneck during migration. You can check whether the current scheduling speed has reached the upper limit through the **TiKV Details** > **Snapshot** > **Snapshot Speed** panel. To increase or decrease the scheduling speed of a node, you can adjust the TiKV snapshot limit ([`snap-io-max-bytes-per-sec`](/tikv-configuration-file.md#snap-io-max-bytes-per-sec)). diff --git a/configure-time-zone.md b/configure-time-zone.md index 246e226ca16cc..2c4c5028fff96 100644 --- a/configure-time-zone.md +++ b/configure-time-zone.md @@ -1,7 +1,6 @@ --- title: Time Zone Support summary: Learn how to set the time zone and its format. -aliases: ['/docs/dev/configure-time-zone/','/docs/dev/how-to/configure/time-zone/'] --- # Time Zone Support diff --git a/constraints.md b/constraints.md index 097a0d7e277ee..7599f51e083c2 100644 --- a/constraints.md +++ b/constraints.md @@ -1,7 +1,6 @@ --- title: Constraints summary: Learn how SQL Constraints apply to TiDB. -aliases: ['/docs/dev/constraints/','/docs/dev/reference/sql/constraints/'] --- # Constraints diff --git a/coprocessor-cache.md b/coprocessor-cache.md index 8c995ed0648c1..c386105fc863a 100644 --- a/coprocessor-cache.md +++ b/coprocessor-cache.md @@ -1,7 +1,6 @@ --- title: Coprocessor Cache summary: Learn the features of Coprocessor Cache. -aliases: ['/docs/dev/coprocessor-cache/'] --- # Coprocessor Cache diff --git a/daily-check.md b/daily-check.md index beda3f86b57ea..956d02a08ccb4 100644 --- a/daily-check.md +++ b/daily-check.md @@ -1,7 +1,6 @@ --- title: Daily Check summary: Learn about performance indicators of the TiDB cluster. -aliases: ['/docs/dev/daily-check/'] --- # Daily Check diff --git a/dashboard/dashboard-access.md b/dashboard/dashboard-access.md index 5a84e4fd76ce8..5ba01e8b3e449 100644 --- a/dashboard/dashboard-access.md +++ b/dashboard/dashboard-access.md @@ -1,7 +1,6 @@ --- title: Access TiDB Dashboard summary: To access TiDB Dashboard, visit the specified URL in your browser. For multiple PD instances, replace the address with any PD instance address and port. Use Chrome, Firefox, or Edge browsers of newer versions. Sign in with the TiDB root account or a user-defined SQL user. The session remains valid for 24 hours. Switch between English and Chinese languages. To log out, click the user name and then the Logout button. -aliases: ['/docs/dev/dashboard/dashboard-access/'] --- # Access TiDB Dashboard @@ -14,7 +13,7 @@ To access TiDB Dashboard, visit via your brows ## Access TiDB Dashboard when multiple PD instances are deployed -When multiple multiple PD instances are deployed in your cluster and you can directly access **every** PD instance and port, you can simply replace `127.0.0.1:2379` in the address with **any** PD instance address and port. +When multiple PD instances are deployed in your cluster and you can directly access **every** PD instance and port, you can simply replace `127.0.0.1:2379` in the address with **any** PD instance address and port. > **Note:** > @@ -34,13 +33,12 @@ You can use TiDB Dashboard in the following common desktop browsers of a relativ ## Sign in -After accessing TiDB Dashboard, you will be directed to the user login interface, as shown in the image below. +After accessing TiDB Dashboard, you will be directed to the user login interface. - You can sign in to TiDB Dashboard using the TiDB `root` account. +- You can also sign in to TiDB Dashboard via Single Sign-On (SSO). For more information, see [Configure SSO for TiDB Dashboard](/dashboard/dashboard-session-sso.md). - If you have created a [User-defined SQL User](/dashboard/dashboard-user.md), you can sign in using this account and the corresponding password. -![Login interface](/media/dashboard/dashboard-access-login.png) - If one of the following situations exists, the login might fail: - TiDB `root` user does not exist. diff --git a/dashboard/dashboard-cluster-info.md b/dashboard/dashboard-cluster-info.md index 0862560f06488..968f47b7acd27 100644 --- a/dashboard/dashboard-cluster-info.md +++ b/dashboard/dashboard-cluster-info.md @@ -1,7 +1,6 @@ --- title: TiDB Dashboard Cluster Information Page summary: The TiDB Dashboard Cluster Information Page allows users to view the running status of TiDB, TiKV, PD, and TiFlash components in the entire cluster, as well as the running status of the host on which these components are located. Users can access the page by logging in to TiDB Dashboard and clicking on Cluster Info in the left navigation menu, or by visiting a specific URL in their browser. The page provides instance, host, and disk lists, showing detailed information about each component and its running status. -aliases: ['/docs/dev/dashboard/dashboard-cluster-info/'] --- # TiDB Dashboard Cluster Information Page diff --git a/dashboard/dashboard-diagnostics-access.md b/dashboard/dashboard-diagnostics-access.md index 42996efe71f12..b7ed53b4e6089 100644 --- a/dashboard/dashboard-diagnostics-access.md +++ b/dashboard/dashboard-diagnostics-access.md @@ -1,7 +1,6 @@ --- title: TiDB Dashboard Cluster Diagnostic Page summary: TiDB Dashboard Cluster Diagnostics diagnoses cluster problems and summarizes results into a web page. Access the page through the dashboard or browser. Generate diagnostic and comparison reports for specified time ranges. Historical reports are also available. -aliases: ['/docs/dev/dashboard/dashboard-diagnostics-access/'] --- # TiDB Dashboard Cluster Diagnostics Page diff --git a/dashboard/dashboard-diagnostics-report.md b/dashboard/dashboard-diagnostics-report.md index defe049f2682c..28fa63dbae1be 100644 --- a/dashboard/dashboard-diagnostics-report.md +++ b/dashboard/dashboard-diagnostics-report.md @@ -1,7 +1,6 @@ --- title: TiDB Dashboard Diagnostic Report summary: TiDB Dashboard Diagnostic Report introduces diagnostic report content, including basic, diagnostic, load, overview, monitoring, and configuration information. It also includes comparison report details, DIFF_RATIO explanation, and Maximum Different Item table. -aliases: ['/docs/dev/dashboard/dashboard-diagnostics-report/'] --- # TiDB Dashboard Diagnostic Report @@ -33,7 +32,7 @@ In this report, some small buttons are described as follows: All monitoring metrics basically correspond to those on the TiDB Grafana monitoring dashboard. After a module is found to be abnormal, you can view more monitoring information on the TiDB Grafana. -In addition, the `TOTAL_TIME` and `TOTAL_COUNT` metrics in this report are monitoring data read from Prometheus, so calculation inaccuracy might exits in their statistics. +In addition, the `TOTAL_TIME` and `TOTAL_COUNT` metrics in this report are monitoring data read from Prometheus, so calculation inaccuracy might exist in their statistics. Each part of this report is introduced as follows. diff --git a/dashboard/dashboard-diagnostics-usage.md b/dashboard/dashboard-diagnostics-usage.md index 50d5c5215f5e7..bdb139d15fdc5 100644 --- a/dashboard/dashboard-diagnostics-usage.md +++ b/dashboard/dashboard-diagnostics-usage.md @@ -1,7 +1,6 @@ --- title: Locate Problems Using Diagnostic Report of TiDB Dashboard summary: TiDB Dashboard's diagnostic report helps locate problems by comparing system performance at different time ranges. It identifies issues like QPS decrease, latency increase, and slow queries, providing detailed analysis and SQL statements for further investigation. This comparison report is essential for quickly identifying and addressing performance issues. -aliases: ['/docs/dev/dashboard/dashboard-diagnostics-usage/'] --- # Locate Problems Using Diagnostic Report of TiDB Dashboard diff --git a/dashboard/dashboard-faq.md b/dashboard/dashboard-faq.md index 361451200ac17..b42e512ca2589 100644 --- a/dashboard/dashboard-faq.md +++ b/dashboard/dashboard-faq.md @@ -1,7 +1,6 @@ --- title: TiDB Dashboard FAQs summary: This document summarizes FAQs about TiDB Dashboard. It covers access-related, UI-related, and deployment issues, providing solutions for each problem. If further assistance is needed, support can be obtained from PingCAP or the community. -aliases: ['/docs/dev/dashboard/dashboard-faq/'] --- # TiDB Dashboard FAQs @@ -30,7 +29,7 @@ If you have deployed TiDB using the `tiup cluster` or `tiup playground` command, The **QPS** and **Latency** sections on the **Overview** page require a cluster with Prometheus deployed. Otherwise, the error is shown. You can solve this problem by deploying a Prometheus instance in the cluster. -If you still encounter this problem when the Prometheus instance has been deployed, the possible reason is that your deployment tool is out of date (TiUP or TiDB Operator), and your tool does not automatically report metrics addresses, which makes TiDB Dashboard unable to query metrics. You can upgrade you deployment tool to the latest version and try again. +If you still encounter this problem when the Prometheus instance has been deployed, the possible reason is that your deployment tool is out of date (TiUP or TiDB Operator), and your tool does not automatically report metrics addresses, which makes TiDB Dashboard unable to query metrics. You can upgrade your deployment tool to the latest version and try again. If your deployment tool is TiUP, take the following steps to solve this problem. For other deployment tools, refer to the corresponding documents of those tools. diff --git a/dashboard/dashboard-intro.md b/dashboard/dashboard-intro.md index e67034328b6f3..87a23253be3b9 100644 --- a/dashboard/dashboard-intro.md +++ b/dashboard/dashboard-intro.md @@ -1,7 +1,6 @@ --- title: TiDB Dashboard Introduction summary: TiDB Dashboard is a Web UI for monitoring, diagnosing, and managing the TiDB cluster. It shows overall running status, component and host status, traffic distribution, SQL statement execution information, slow queries, cluster diagnostics, log search, resource control, and profiling data collection. -aliases: ['/docs/dev/dashboard/dashboard-intro/'] --- # TiDB Dashboard Introduction diff --git a/dashboard/dashboard-key-visualizer.md b/dashboard/dashboard-key-visualizer.md index ea7d712adfe81..e31c884a29059 100644 --- a/dashboard/dashboard-key-visualizer.md +++ b/dashboard/dashboard-key-visualizer.md @@ -1,7 +1,6 @@ --- title: Key Visualizer Page summary: TiDB Dashboard's Key Visualizer page analyzes and troubleshoots traffic hotspots in the TiDB cluster. It visually shows traffic changes over time, and allows users to zoom in on specific time periods or region ranges. The page also provides settings to adjust brightness, select metrics, and refresh the heatmap. It identifies common heatmap types and offers solutions to address hotspot issues. -aliases: ['/docs/dev/dashboard/dashboard-key-visualizer/','/docs/dev/key-visualizer-monitoring-tool/'] --- # Key Visualizer Page @@ -38,7 +37,7 @@ This section introduces the basic concepts that relate to Key Visualizer. In a TiDB cluster, the stored data is distributed among TiKV instances. Logically, TiKV is a huge and orderly key-value map. The whole key-value space is divided into many segments and each segment consists of a series of adjacent keys. Such segment is called a `Region`. -For detailed introduction of Region, refer to [TiDB Internal (I) - Data Storage](https://en.pingcap.com/blog/tidb-internal-data-storage/). +For detailed introduction of Region, refer to [TiDB Internal (I) - Data Storage](https://www.pingcap.com/blog/tidb-internal-data-storage/). ### Hotspot diff --git a/dashboard/dashboard-log-search.md b/dashboard/dashboard-log-search.md index 85b2b27baf35a..4a32139f7a423 100644 --- a/dashboard/dashboard-log-search.md +++ b/dashboard/dashboard-log-search.md @@ -1,7 +1,6 @@ --- title: TiDB Dashboard Log Search Page summary: TiDB Dashboard log search page allows users to search logs, preview results, and download logs. Users can access the page after logging in, and specify time range, log level, keywords, and components for the search. The search result page displays parameter options, search progress, and search results. Users can download selected logs, cancel running tasks, and retry failed tasks. The search history list shows details of past searches and allows users to delete unnecessary history. -aliases: ['/docs/dev/dashboard/dashboard-log-search/'] --- # TiDB Dashboard Log Search Page diff --git a/dashboard/dashboard-monitoring.md b/dashboard/dashboard-monitoring.md index a2acc46f96909..eb9e0595a808d 100644 --- a/dashboard/dashboard-monitoring.md +++ b/dashboard/dashboard-monitoring.md @@ -69,11 +69,11 @@ Number of queries using plan cache per second in all TiDB instances ### KV/TSO Request OPS - kv request total: Total number of KV requests per second in all TiDB instances -- kv request by type: Number of KV requests per second in all TiDB instances based on such types as `Get`, `Prewrite`, and `Commit`. -- tso - cmd: Number of `tso cmd` requests per second in all TiDB instances -- tso - request: Number of `tso request` requests per second in all TiDB instances +- kv request by type: Number of KV requests per second in all TiDB instances based on such types as `Get`, `Prewrite`, and `Commit` +- tso - cmd: Number of gRPC requests per second that TiDB sends to PD in all TiDB instances; each gRPC request contains a batch of TSO requests +- tso - request: Number of TSO requests per second in all TiDB instances -Generally, dividing `tso - cmd` by `tso - request` yields the average batch size of requests per second. +Generally, `tso - request` divided by `tso - cmd` is the average size of TSO request batches per second. ### Connection Count diff --git a/dashboard/dashboard-ops-deploy.md b/dashboard/dashboard-ops-deploy.md index ac0653fee1b61..625d3976eddd5 100644 --- a/dashboard/dashboard-ops-deploy.md +++ b/dashboard/dashboard-ops-deploy.md @@ -1,7 +1,6 @@ --- title: Deploy TiDB Dashboard summary: TiDB Dashboard is built into PD for v4.0 or higher. No additional deployment is needed. It can also be deployed independently on Kubernetes. When multiple PD instances are deployed, only one serves the Dashboard. Use `tiup cluster display` to check the serving instance. You can disable and re-enable the Dashboard using `tiup ctl`. -aliases: ['/docs/dev/dashboard/dashboard-ops-deploy/'] --- # Deploy TiDB Dashboard diff --git a/dashboard/dashboard-ops-reverse-proxy.md b/dashboard/dashboard-ops-reverse-proxy.md index bbfab03cf007a..bc12148875168 100644 --- a/dashboard/dashboard-ops-reverse-proxy.md +++ b/dashboard/dashboard-ops-reverse-proxy.md @@ -1,6 +1,5 @@ --- title: Use TiDB Dashboard behind a Reverse Proxy -aliases: ['/docs/dev/dashboard/dashboard-ops-reverse-proxy/'] summary: TiDB Dashboard can be safely exposed using a reverse proxy. To do this, get the actual TiDB Dashboard address and configure the reverse proxy using either HAProxy or NGINX. You can also customize the path prefix for the TiDB Dashboard service. To enhance security, consider configuring a firewall. --- diff --git a/dashboard/dashboard-ops-security.md b/dashboard/dashboard-ops-security.md index 72e44afab5a55..7dcf49af8a152 100644 --- a/dashboard/dashboard-ops-security.md +++ b/dashboard/dashboard-ops-security.md @@ -1,7 +1,6 @@ --- title: Secure TiDB Dashboard summary: TiDB Dashboard requires enhanced security measures, including setting a strong password for the root user, creating a least-privileged user, and using a firewall to block untrusted access. It is also recommended to use a reverse proxy and enable TLS for further security. -aliases: ['/docs/dev/dashboard/dashboard-ops-security/'] --- # Secure TiDB Dashboard diff --git a/dashboard/dashboard-overview.md b/dashboard/dashboard-overview.md index 7239cdbf03f89..fd7c07a534cf1 100644 --- a/dashboard/dashboard-overview.md +++ b/dashboard/dashboard-overview.md @@ -1,7 +1,6 @@ --- title: Overview Page summary: The TiDB overview page displays cluster QPS, latency, top SQL statements, recent slow queries, instance status, and monitor/alert links. Access it via TiDB Dashboard or left navigation menu. QPS and latency require Prometheus monitoring. Top SQL and slow queries need SQL Statements and slow query logs enabled. Instance status shows total and abnormal instances. Monitor and alert links lead to Grafana dashboard, AlertManager, and cluster diagnostics. -aliases: ['/docs/dev/dashboard/dashboard-overview/'] --- # Overview Page diff --git a/dashboard/dashboard-profiling.md b/dashboard/dashboard-profiling.md index 7e00f7719ca2e..20f9e9514ff28 100644 --- a/dashboard/dashboard-profiling.md +++ b/dashboard/dashboard-profiling.md @@ -1,7 +1,6 @@ --- title: TiDB Dashboard Instance Profiling - Manual Profiling summary: Manual Profiling allows users to collect current performance data on demand for TiDB, TiKV, PD, and TiFlash instances. Experts can analyze resource consumption details like CPU and memory to pinpoint ongoing performance problems. Access the page through TiDB Dashboard or a browser. Start profiling by choosing target instances and modify the duration if needed. View real-time progress and download performance data after profiling is completed. View profiling history for detailed operations. -aliases: ['/docs/dev/dashboard/dashboard-profiling/'] --- # TiDB Dashboard Instance Profiling - Manual Profiling diff --git a/dashboard/dashboard-resource-manager.md b/dashboard/dashboard-resource-manager.md index a2cb2c65b9119..cbf0658ebe766 100644 --- a/dashboard/dashboard-resource-manager.md +++ b/dashboard/dashboard-resource-manager.md @@ -63,6 +63,10 @@ Before resource planning, you need to know the overall capacity of the cluster. You can select an appropriate time range using **CPU Usage** in the [Metrics](#metrics) section. +> **Note:** +> +> To use the capacity estimation feature, the current login user must have the `SUPER` or `RESOURCE_GROUP_ADMIN` privilege and the `SELECT` privilege for some system tables. Before using this feature, ensure the current user has these privileges. Otherwise, some features might not work properly. For more information, see [`CALIBRATE RESOURCE`](/sql-statements/sql-statement-calibrate-resource.md#privileges). + ## Metrics By observing the metrics on the panels, you can understand the current overall resource consumption status of the cluster. The monitoring metrics and their meanings are as follows: diff --git a/dashboard/dashboard-slow-query.md b/dashboard/dashboard-slow-query.md index 24f419d7c8fd3..40fb09c82adb4 100644 --- a/dashboard/dashboard-slow-query.md +++ b/dashboard/dashboard-slow-query.md @@ -1,7 +1,6 @@ --- title: Slow Queries Page of TiDB Dashboard summary: TiDB Dashboard's Slow Queries page allows users to search and view slow queries in the cluster. Queries with an execution time over 300 milliseconds are considered slow. Users can adjust the threshold and access the page through the dashboard or a browser. They can also change filters, display more columns, export queries, and view execution details. -aliases: ['/docs/dev/dashboard/dashboard-slow-query/'] --- # Slow Queries Page of TiDB Dashboard diff --git a/dashboard/dashboard-statement-details.md b/dashboard/dashboard-statement-details.md index 7c7ecd058da2a..7bd8040836d31 100644 --- a/dashboard/dashboard-statement-details.md +++ b/dashboard/dashboard-statement-details.md @@ -1,7 +1,6 @@ --- title: Statement Execution Details of TiDB Dashboard summary: TiDB Dashboard provides detailed information on SQL statement execution, including SQL template overview, execution plan list, and plan binding feature. Starting from v6.6.0, fast plan binding allows quick binding and dropping of execution plans. However, it has limitations and requires SUPER privilege. The execution detail of plans includes SQL sample, complete execution plan information, and basic execution details. Visual representations of execution plans are available in table, text, and graph formats. Additional tabs provide information on execution time, Coprocessor read, transaction, and slow queries. -aliases: ['/docs/dev/dashboard/dashboard-statement-details/'] --- # Statement Execution Details of TiDB Dashboard diff --git a/dashboard/dashboard-statement-list.md b/dashboard/dashboard-statement-list.md index c8de12141d099..14b612978b0db 100644 --- a/dashboard/dashboard-statement-list.md +++ b/dashboard/dashboard-statement-list.md @@ -1,7 +1,6 @@ --- title: SQL Statements Page of TiDB Dashboard summary: The SQL statements page in TiDB Dashboard shows the execution status of all SQL statements in the cluster. It allows users to analyze long-running SQL statements and provides options to access, filter, display more columns, sort, and change settings. The page also includes a feature to limit the number of stored SQL statements. For more details, visit the TiDB Dashboard documentation. -aliases: ['/docs/dev/dashboard/dashboard-statement-list/'] --- # SQL Statements Page of TiDB Dashboard diff --git a/dashboard/dashboard-user.md b/dashboard/dashboard-user.md index 0d06fc0c034d7..bc1c74c18a3e7 100644 --- a/dashboard/dashboard-user.md +++ b/dashboard/dashboard-user.md @@ -1,7 +1,6 @@ --- title: TiDB Dashboard User Management summary: TiDB Dashboard uses the same user privilege system as TiDB. SQL users need specific privileges to access the dashboard, including PROCESS, SHOW DATABASES, CONFIG, DASHBOARD_CLIENT, and more. It's recommended to create users with only the required privileges to prevent unintended operations. Users with high privileges can also sign in. To create a least-privileged SQL user, grant the necessary privileges and use role-based access control (RBAC) if needed. -aliases: ['/docs/dev/dashboard/dashboard-user/'] --- # TiDB Dashboard User Management diff --git a/data-type-date-and-time.md b/data-type-date-and-time.md index f754a1bb42f3a..b4b45f96c2da9 100644 --- a/data-type-date-and-time.md +++ b/data-type-date-and-time.md @@ -1,7 +1,6 @@ --- title: Date and Time Types summary: Learn about the supported date and time types. -aliases: ['/docs/dev/data-type-date-and-time/','/docs/dev/reference/sql/data-types/date-and-time/'] --- # Date and Time Types diff --git a/data-type-default-values.md b/data-type-default-values.md index 7dbdaf2b35a09..4fd348b1ff1ef 100644 --- a/data-type-default-values.md +++ b/data-type-default-values.md @@ -1,7 +1,6 @@ --- title: TiDB Data Type summary: Learn about default values for data types in TiDB. -aliases: ['/docs/dev/data-type-default-values/','/docs/dev/reference/sql/data-types/default-values/'] --- # Default Values @@ -14,7 +13,7 @@ You can set default values for all data types. Typically, default values must be - For integer types, you can use the `NEXT VALUE FOR` function to set the next value of a sequence as the default value for a column, and use the [`RAND()`](/functions-and-operators/numeric-functions-and-operators.md) function to generate a random floating-point value as the default value for a column. - For string types, you can use the [`UUID()`](/functions-and-operators/miscellaneous-functions.md) function to generate a [universally unique identifier (UUID)](/best-practices/uuid.md) as the default value for a column. - For binary types, you can use the [`UUID_TO_BIN()`](/functions-and-operators/miscellaneous-functions.md) function to convert a UUID to the binary format and set the converted value as the default value for a column. -- Starting from v8.0.0, TiDB additionally supports [specifying the default values](#specify-expressions-as-default-values) for [`BLOB`](/data-type-string.md#blob-type), [`TEXT`](/data-type-string.md#text-type), and [`JSON`](/data-type-json.md#json-type) data types, but you can only use expressions to set the [default values](#default-values) for them. +- Starting from v8.0.0, TiDB additionally supports [specifying the default values](#specify-expressions-as-default-values) for [`BLOB`](/data-type-string.md#blob-type), [`TEXT`](/data-type-string.md#text-type), and [`JSON`](/data-type-json.md#json-data-type) data types, but you can only use expressions to set the [default values](#default-values) for them. If a column definition includes no explicit `DEFAULT` value, TiDB determines the default value as follows: diff --git a/data-type-json.md b/data-type-json.md index fa043bc000a8c..e938ab5f2eedb 100644 --- a/data-type-json.md +++ b/data-type-json.md @@ -1,10 +1,9 @@ --- title: TiDB Data Type summary: Learn about the JSON data type in TiDB. -aliases: ['/docs/dev/data-type-json/','/docs/dev/reference/sql/data-types/json/'] --- -# JSON Type +# JSON Data Type TiDB supports the `JSON` (JavaScript Object Notation) data type, which is useful for storing semi-structured data. The `JSON` data type provides the following advantages over storing `JSON`-format strings in a string column: @@ -26,6 +25,27 @@ SELECT id FROM city WHERE population >= 100; For more information, see [JSON Functions](/functions-and-operators/json-functions.md) and [Generated Columns](/generated-columns.md). +## JSON value types + +The values inside a JSON document have types. This is visible in the output of [`JSON_TYPE`()](/functions-and-operators/json-functions/json-functions-return.md#json_type). + +| Type | Example | +|------------------|--------------------------------| +| ARRAY | `[]` | +| BIT | | +| BLOB | `0x616263` | +| BOOLEAN | `true` | +| DATE | `"2025-06-14"` | +| DATETIME | `"2025-06-14 09:05:10.000000"` | +| DOUBLE | `1.14` | +| INTEGER | `5` | +| NULL | `null` | +| OBJECT | `{}` | +| OPAQUE | | +| STRING | `"foobar"` | +| TIME | `"09:10:00.000000"` | +| UNSIGNED INTEGER | `9223372036854776000` | + ## Restrictions - Currently, TiDB only supports pushing down limited `JSON` functions to TiFlash. For more information, see [Push-down expressions](/tiflash/tiflash-supported-pushdown-calculations.md#push-down-expressions). @@ -102,4 +122,4 @@ For more information, see [JSON Functions](/functions-and-operators/json-functio INSERT INTO t VALUES (3); ``` -For more information about the `JSON` data type, see [JSON functions](/functions-and-operators/json-functions.md) and [Generated Columns](/generated-columns.md). \ No newline at end of file +For more information about the `JSON` data type, see [JSON functions](/functions-and-operators/json-functions.md) and [Generated Columns](/generated-columns.md). diff --git a/data-type-numeric.md b/data-type-numeric.md index 4406cc06f707a..1b7d69b19b74a 100644 --- a/data-type-numeric.md +++ b/data-type-numeric.md @@ -1,7 +1,6 @@ --- title: Numeric Types summary: Learn about numeric data types supported in TiDB. -aliases: ['/docs/dev/data-type-numeric/','/docs/dev/reference/sql/data-types/numeric/'] --- # Numeric Types diff --git a/data-type-overview.md b/data-type-overview.md index d0fe1b8c59586..07260fd94a579 100644 --- a/data-type-overview.md +++ b/data-type-overview.md @@ -1,7 +1,6 @@ --- title: Data Types summary: Learn about the data types supported in TiDB. -aliases: ['/docs/dev/data-type-overview/','/docs/dev/reference/sql/data-types/overview/'] --- # Data Types diff --git a/data-type-string.md b/data-type-string.md index 65dfae1666587..ce3b54dceba79 100644 --- a/data-type-string.md +++ b/data-type-string.md @@ -1,7 +1,6 @@ --- title: String types summary: Learn about the string types supported in TiDB. -aliases: ['/docs/dev/data-type-string/','/docs/dev/reference/sql/data-types/string/'] --- # String Types @@ -56,12 +55,12 @@ TINYTEXT [CHARACTER SET charset_name] [COLLATE collation_name] -The `MEDIUMTEXT` type is similar to the [`TEXT` type](#text-type). The difference is that the maximum column length of `MEDIUMTEXT` is 16,777,215. But due to the limitation of [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. +The `MEDIUMTEXT` type is similar to the [`TEXT` type](#text-type). The difference is that the maximum column length of `MEDIUMTEXT` is 16,777,215. But due to the limitation of [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. -The `MEDIUMTEXT` type is similar to the [`TEXT` type](#text-type). The difference is that the maximum column length of `MEDIUMTEXT` is 16,777,215. But due to the limitation of [`txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v50), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. +The `MEDIUMTEXT` type is similar to the [`TEXT` type](#text-type). The difference is that the maximum column length of `MEDIUMTEXT` is 16,777,215. But due to the limitation of [`txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v4010-and-v500), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. @@ -73,12 +72,12 @@ MEDIUMTEXT [CHARACTER SET charset_name] [COLLATE collation_name] -The `LONGTEXT` type is similar to the [`TEXT` type](#text-type). The difference is that the maximum column length of `LONGTEXT` is 4,294,967,295. But due to the limitation of [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. +The `LONGTEXT` type is similar to the [`TEXT` type](#text-type). The difference is that the maximum column length of `LONGTEXT` is 4,294,967,295. But due to the limitation of [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. -The `LONGTEXT` type is similar to the [`TEXT` type](#text-type). The difference is that the maximum column length of `LONGTEXT` is 4,294,967,295. But due to the limitation of [`txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v50), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. +The `LONGTEXT` type is similar to the [`TEXT` type](#text-type). The difference is that the maximum column length of `LONGTEXT` is 4,294,967,295. But due to the limitation of [`txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v4010-and-v500), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. @@ -122,12 +121,12 @@ TINYBLOB -The `MEDIUMBLOB` type is similar to the [`BLOB` type](#blob-type). The difference is that the maximum column length of `MEDIUMBLOB` is 16,777,215. But due to the limitation of [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. +The `MEDIUMBLOB` type is similar to the [`BLOB` type](#blob-type). The difference is that the maximum column length of `MEDIUMBLOB` is 16,777,215. But due to the limitation of [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. -The `MEDIUMBLOB` type is similar to the [`BLOB` type](#blob-type). The difference is that the maximum column length of `MEDIUMBLOB` is 16,777,215. But due to the limitation of [`txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v50), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. +The `MEDIUMBLOB` type is similar to the [`BLOB` type](#blob-type). The difference is that the maximum column length of `MEDIUMBLOB` is 16,777,215. But due to the limitation of [`txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v4010-and-v500), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. @@ -139,12 +138,12 @@ MEDIUMBLOB -The `LONGBLOB` type is similar to the [`BLOB` type](#blob-type). The difference is that the maximum column length of `LONGBLOB` is 4,294,967,295. But due to the limitation of [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. +The `LONGBLOB` type is similar to the [`BLOB` type](#blob-type). The difference is that the maximum column length of `LONGBLOB` is 4,294,967,295. But due to the limitation of [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. -The `LONGBLOB` type is similar to the [`BLOB` type](#blob-type). The difference is that the maximum column length of `LONGBLOB` is 4,294,967,295. But due to the limitation of [`txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v50), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. +The `LONGBLOB` type is similar to the [`BLOB` type](#blob-type). The difference is that the maximum column length of `LONGBLOB` is 4,294,967,295. But due to the limitation of [`txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v4010-and-v500), the maximum storage size of a single row in TiDB is 6 MiB by default and can be increased to 120 MiB by changing the configuration. diff --git a/ddl-introduction.md b/ddl-introduction.md index 7658ccfae57e7..69066f27da3d0 100644 --- a/ddl-introduction.md +++ b/ddl-introduction.md @@ -77,6 +77,31 @@ absent -> delete only -> write only -> write reorg -> public For users, the newly created index is unavailable before the `public` state. +
+ +Before TiDB v6.2.0, because the Owner can only execute one DDL task of the same type (either logical or physical) at a time, which is relatively strict, and affects the user experience. + +If there is no dependency between DDL tasks, parallel execution does not affect data correctness and consistency. For example, user A adds an index to the `T1` table, while user B deletes a column from the `T2` table. These two DDL statements can be executed in parallel. + +To improve the user experience of DDL execution, starting from v6.2.0, TiDB enables the Owner to determine the relevance of DDL tasks. The logic is as follows: + ++ DDL statements to be performed on the same table are mutually blocked. ++ `DROP DATABASE` and DDL statements that affect all objects in the database are mutually blocked. ++ Adding indexes and column type changes on different tables can be executed concurrently. ++ A logical DDL statement must wait for the previous logical DDL statement to be executed before it can be executed. ++ In other cases, DDL can be executed based on the level of availability for concurrent DDL execution. + +Specifically, TiDB 6.2.0 has enhanced the DDL execution framework in the following aspects: + ++ The DDL Owner can execute DDL tasks in parallel based on the preceding logic. ++ The first-in-first-out issue in the DDL Job queue has been addressed. The DDL Owner no longer selects the first job in the queue, but instead selects the job that can be executed at the current time. ++ The number of workers that handle physical DDL statements has been increased, enabling multiple physical DDL statements to be executed in parallel. + + Because all DDL tasks in TiDB are implemented using an online change approach, TiDB can determine the relevance of new DDL jobs through the Owner, and schedule DDL tasks based on this information. This approach enables the distributed database to achieve the same level of DDL concurrency as traditional databases. + +The concurrent DDL framework enhances the execution capability of DDL statements in TiDB, making it more compatible with the usage patterns of commercial databases. + +
Before v6.2.0, the process of handling asynchronous schema changes in the TiDB SQL layer is as follows: @@ -103,31 +128,6 @@ Before TiDB v6.2.0, the DDL execution framework had the following limitations: These limitations might lead to some "unintended" DDL blocking behavior. For more details, see [SQL FAQ - DDL Execution](https://docs.pingcap.com/tidb/stable/sql-faq#ddl-execution). -
-
- -Before TiDB v6.2.0, because the Owner can only execute one DDL task of the same type (either logical or physical) at a time, which is relatively strict, and affects the user experience. - -If there is no dependency between DDL tasks, parallel execution does not affect data correctness and consistency. For example, user A adds an index to the `T1` table, while user B deletes a column from the `T2` table. These two DDL statements can be executed in parallel. - -To improve the user experience of DDL execution, starting from v6.2.0, TiDB enables the Owner to determine the relevance of DDL tasks. The logic is as follows: - -+ DDL statements to be performed on the same table are mutually blocked. -+ `DROP DATABASE` and DDL statements that affect all objects in the database are mutually blocked. -+ Adding indexes and column type changes on different tables can be executed concurrently. -+ A logical DDL statement must wait for the previous logical DDL statement to be executed before it can be executed. -+ In other cases, DDL can be executed based on the level of availability for concurrent DDL execution. - -In specific, TiDB has upgraded the DDL execution framework in v6.2.0 in the following aspects: - -+ The DDL Owner can execute DDL tasks in parallel based on the preceding logic. -+ The first-in-first-out issue in the DDL Job queue has been addressed. The DDL Owner no longer selects the first job in the queue, but instead selects the job that can be executed at the current time. -+ The number of workers that handle physical DDL statements has been increased, enabling multiple physical DDL statements to be executed in parallel. - - Because all DDL tasks in TiDB are implemented using an online change approach, TiDB can determine the relevance of new DDL jobs through the Owner, and schedule DDL tasks based on this information. This approach enables the distributed database to achieve the same level of DDL concurrency as traditional databases. - -The concurrent DDL framework enhances the execution capability of DDL statements in TiDB, making it more compatible with the usage patterns of commercial databases. -
diff --git a/deploy-monitoring-services.md b/deploy-monitoring-services.md index 6c91115edd658..ebb3e3dae2825 100644 --- a/deploy-monitoring-services.md +++ b/deploy-monitoring-services.md @@ -1,14 +1,11 @@ --- title: Deploy Monitoring Services for the TiDB Cluster summary: Learn how to deploy monitoring services for the TiDB cluster. -aliases: ['/docs/dev/deploy-monitoring-services/','/docs/dev/how-to/monitor/monitor-a-cluster/','/docs/dev/monitor-a-tidb-cluster/'] --- # Deploy Monitoring Services for the TiDB Cluster -This document is intended for users who want to manually deploy TiDB monitoring and alert services. - -If you deploy the TiDB cluster using TiUP, the monitoring and alert services are automatically deployed, and no manual deployment is needed. +This document is intended for users who want to manually deploy TiDB monitoring and alert services. If you deploy the TiDB cluster using TiUP, the monitoring and alert services are automatically deployed, and no manual deployment is needed. [TiDB Dashboard](/dashboard/dashboard-intro.md) is built into the PD component and does not require an independent deployment. ## Deploy Prometheus and Grafana @@ -29,18 +26,18 @@ Assume that the TiDB cluster topology is as follows: ```bash # Downloads the package. -wget https://download.pingcap.org/prometheus-2.27.1.linux-amd64.tar.gz +wget https://github.com/prometheus/prometheus/releases/download/v2.49.1/prometheus-2.49.1.linux-amd64.tar.gz wget https://download.pingcap.org/node_exporter-v1.3.1-linux-amd64.tar.gz -wget https://download.pingcap.org/grafana-7.5.11.linux-amd64.tar.gz +wget https://download.pingcap.org/grafana-7.5.17.linux-amd64.tar.gz ``` {{< copyable "shell-regular" >}} ```bash # Extracts the package. -tar -xzf prometheus-2.27.1.linux-amd64.tar.gz +tar -xzf prometheus-2.49.1.linux-amd64.tar.gz tar -xzf node_exporter-v1.3.1-linux-amd64.tar.gz -tar -xzf grafana-7.5.11.linux-amd64.tar.gz +tar -xzf grafana-7.5.17.linux-amd64.tar.gz ``` ### Step 2: Start `node_exporter` on Node1, Node2, Node3, and Node4 @@ -62,7 +59,7 @@ Edit the Prometheus configuration file: {{< copyable "shell-regular" >}} ```bash -cd prometheus-2.27.1.linux-amd64 && +cd prometheus-2.49.1.linux-amd64 && vi prometheus.yml ``` @@ -112,7 +109,25 @@ scrape_configs: - '192.168.199.118:20180' ... +``` + +To enable alarm rules for components such as TiDB, PD, and TiKV, download the alarm rule files of the corresponding components separately, and then add the configurations of alarm rule files to the Prometheus configuration file. +- TiDB: [`tidb.rules.yml`](https://github.com/pingcap/tidb/blob/master/pkg/metrics/alertmanager/tidb.rules.yml) +- PD: [`pd.rules.yml`](https://github.com/tikv/pd/blob/master/metrics/alertmanager/pd.rules.yml) +- TiKV: [`tikv.rules.yml`](https://github.com/tikv/tikv/blob/master/metrics/alertmanager/tikv.rules.yml) +- TiFlash: [`tiflash.rules.yml`](https://github.com/pingcap/tiflash/blob/master/metrics/alertmanager/tiflash.rules.yml) +- TiCDC: [`ticdc.rules.yml`](https://github.com/pingcap/tiflow/blob/master/metrics/alertmanager/ticdc.rules.yml) +- TiDB Lightning: [`lightning.rules.yml`](https://github.com/pingcap/tidb/blob/master/br/metrics/alertmanager/lightning.rules.yml) + +```ini +rule_files: + - 'tidb.rules.yml' + - 'pd.rules.yml' + - 'tikv.rules.yml' + - 'tiflash.rules.yml' + - 'ticdc.rules.yml' + - 'lightning.rules.yml' ``` Start the Prometheus service: @@ -135,7 +150,7 @@ Edit the Grafana configuration file: {{< copyable "shell-regular" >}} ```ini -cd grafana-7.5.11 && +cd grafana-7.5.17 && vi conf/grafana.ini ... @@ -226,7 +241,7 @@ To import a Grafana dashboard for the PD server, the TiKV server, and the TiDB s 2. In the sidebar menu, click **Dashboards** -> **Import** to open the **Import Dashboard** window. -3. Click **Upload .json File** to upload a JSON file (Download TiDB Grafana configuration files from [pingcap/tidb](https://github.com/pingcap/tidb/tree/master/pkg/metrics/grafana), [tikv/tikv](https://github.com/tikv/tikv/tree/master/metrics/grafana), and [tikv/pd](https://github.com/tikv/pd/tree/master/metrics/grafana)). +3. Click **Upload .json File** to upload a JSON file (Download TiDB Grafana configuration files from [pingcap/tidb](https://github.com/pingcap/tidb/tree/release-8.1/pkg/metrics/grafana), [tikv/tikv](https://github.com/tikv/tikv/tree/release-8.1/metrics/grafana), and [tikv/pd](https://github.com/tikv/pd/tree/release-8.1/metrics/grafana)). > **Note:** > diff --git a/develop/dev-guide-aws-appflow-integration.md b/develop/dev-guide-aws-appflow-integration.md index 1305234a01a60..5e30e06ef874f 100644 --- a/develop/dev-guide-aws-appflow-integration.md +++ b/develop/dev-guide-aws-appflow-integration.md @@ -7,9 +7,9 @@ summary: Introduce how to integrate TiDB with Amazon AppFlow step by step. [Amazon AppFlow](https://aws.amazon.com/appflow/) is a fully managed API integration service that you use to connect your software as a service (SaaS) applications to AWS services, and securely transfer data. With Amazon AppFlow, you can import and export data from and to TiDB into many types of data providers, such as Salesforce, Amazon S3, LinkedIn, and GitHub. For more information, see [Supported source and destination applications](https://docs.aws.amazon.com/appflow/latest/userguide/app-specific.html) in AWS documentation. -This document describes how to integrate TiDB with Amazon AppFlow and takes integrating a TiDB Serverless cluster as an example. +This document describes how to integrate TiDB with Amazon AppFlow and takes integrating a TiDB Cloud Serverless cluster as an example. -If you do not have a TiDB cluster, you can create a [TiDB Serverless](https://tidbcloud.com/console/clusters) cluster, which is free and can be created in approximately 30 seconds. +If you do not have a TiDB cluster, you can create a [TiDB Cloud Serverless](https://tidbcloud.com/console/clusters) cluster, which is free and can be created in approximately 30 seconds. ## Prerequisites @@ -66,7 +66,7 @@ git clone https://github.com/pingcap-inc/tidb-appflow-integration > > - The `--guided` option uses prompts to guide you through the deployment. Your input will be stored in a configuration file, which is `samconfig.toml` by default. > - `stack_name` specifies the name of AWS Lambda that you are deploying. - > - This prompted guide uses AWS as the cloud provider of TiDB Serverless. To use Amazon S3 as the source or destination, you need to set the `region` of AWS Lambda as the same as that of Amazon S3. + > - This prompted guide uses AWS as the cloud provider of TiDB Cloud Serverless. To use Amazon S3 as the source or destination, you need to set the `region` of AWS Lambda as the same as that of Amazon S3. > - If you have already run `sam deploy --guided` before, you can just run `sam deploy` instead, and SAM CLI will use the configuration file `samconfig.toml` to simplify the interaction. If you see a similar output as follows, this Lambda is successfully deployed. @@ -148,7 +148,7 @@ Choose the **Source details** and **Destination details**. TiDB connector can be ``` 5. After the `sf_account` table is created, click **Connect**. A connection dialog is displayed. -6. In the **Connect to TiDB-Connector** dialog, enter the connection properties of the TiDB cluster. If you use a TiDB Serverless cluster, you need to set the **TLS** option to `Yes`, which lets the TiDB connector use the TLS connection. Then, click **Connect**. +6. In the **Connect to TiDB-Connector** dialog, enter the connection properties of the TiDB cluster. If you use a TiDB Cloud Serverless cluster, you need to set the **TLS** option to `Yes`, which lets the TiDB connector use the TLS connection. Then, click **Connect**. ![tidb connection message](/media/develop/aws-appflow-step-tidb-connection-message.png) @@ -244,5 +244,19 @@ test> SELECT * FROM sf_account; - If anything goes wrong, you can navigate to the [CloudWatch](https://console.aws.amazon.com/cloudwatch/home) page on the AWS Management Console to get logs. - The steps in this document are based on [Building custom connectors using the Amazon AppFlow Custom Connector SDK](https://aws.amazon.com/blogs/compute/building-custom-connectors-using-the-amazon-appflow-custom-connector-sdk/). -- [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) is **NOT** a production environment. -- To prevent excessive length, the examples in this document only show the `Insert` strategy, but `Update` and `Upsert` strategies are also tested and can be used. \ No newline at end of file +- [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) is **NOT** a production environment. +- To prevent excessive length, the examples in this document only show the `Insert` strategy, but `Update` and `Upsert` strategies are also tested and can be used. + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-bookshop-schema-design.md b/develop/dev-guide-bookshop-schema-design.md index 6add6879aa412..fc49cefd3d782 100644 --- a/develop/dev-guide-bookshop-schema-design.md +++ b/develop/dev-guide-bookshop-schema-design.md @@ -1,6 +1,6 @@ --- title: Bookshop Example Application -summary: Bookshop is an online bookstore app for buying and rating books. You can import table structures and data via TiUP or TiDB Cloud. Method 1 uses TiUP to quickly generate and import sample data, while Method 2 imports data from AWS S3 to TiDB Cloud. The database tables include books, authors, users, ratings, book_authors, and orders. The database initialization script `dbinit.sql` creates the table structures for the Bookshop application. +summary: Bookshop is an online bookstore app for buying and rating books. You can import table structures and data via TiUP or TiDB Cloud. Method 1 uses TiUP to quickly generate and import sample data, while Method 2 imports data from Amazon S3 to TiDB Cloud. The database tables include books, authors, users, ratings, book_authors, and orders. The database initialization script `dbinit.sql` creates the table structures for the Bookshop application. --- # Bookshop Example Application @@ -89,26 +89,28 @@ You can delete the original table structure through the `--drop-tables` paramete ### Method 2: Via TiDB Cloud Import -On the cluster detail page of TiDB Cloud, click **Import Data** in the **Import** area to enter the **Data Import** page. On this page, perform the following steps to import the Bookshop sample data from AWS S3 to TiDB Cloud. +1. Open the **Import** page for your target cluster. -1. Select **SQL File** for **Data Format**. -2. Copy the following **Bucket URI** and **Role ARN** to the corresponding input boxes: + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. - **Bucket URI**: + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. - ``` - s3://developer.pingcap.com/bookshop/ - ``` + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. - **Role ARN**: +2. Select **Import data from S3**. - ``` - arn:aws:iam::494090988690:role/s3-tidb-cloud-developer-access - ``` + If this is your first time using TiDB Cloud Import, select **Import From Amazon S3**. -3. Click **Next** to go to the **File and filter** step to confirm the information of the files to be imported. +3. On the **Import Data from Amazon S3** page, configure the following source data information: -4. Click **Next** again to go to the **Preview** step to confirm the preview of the data to be imported. + - **Import File Count**: select **Multiple files**. + - **Included Schema Files**: select **Yes**. + - **Data Format**: select **SQL**. + - **Folder URI**: enter `s3://developer.pingcap.com/bookshop/`. + - **Bucket Access**: select **AWS Role ARN**. + - **Role ARN**: enter `arn:aws:iam::494090988690:role/s3-tidb-cloud-developer-access`. In this example, the following data is generated in advance: @@ -118,7 +120,7 @@ On the cluster detail page of TiDB Cloud, click **Import Data** in the **Import* - 1,000,000 rows of rating records - 1,000,000 rows of order records -5. Click **Start Import** to start the import process and wait for TiDB Cloud to complete the import. +4. Click **Connect** > **Start Import** to start the import process and wait for TiDB Cloud to complete the import. For more information about how to import or migrate data to TiDB Cloud, see [TiDB Cloud Migration Overview](https://docs.pingcap.com/tidbcloud/tidb-cloud-migration-overview). @@ -290,3 +292,17 @@ CREATE TABLE `bookshop`.`orders` ( KEY `orders_book_id_idx` (`book_id`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ``` + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + diff --git a/develop/dev-guide-build-cluster-in-cloud.md b/develop/dev-guide-build-cluster-in-cloud.md index 9de57eb3a82be..0a92ecfd3ada7 100644 --- a/develop/dev-guide-build-cluster-in-cloud.md +++ b/develop/dev-guide-build-cluster-in-cloud.md @@ -1,15 +1,15 @@ --- -title: Build a TiDB Serverless Cluster -summary: Learn how to build a TiDB Serverless cluster in TiDB Cloud and connect to it. +title: Build a TiDB Cloud Serverless Cluster +summary: Learn how to build a TiDB Cloud Serverless cluster in TiDB Cloud and connect to it. --- -# Build a TiDB Serverless Cluster +# Build a TiDB Cloud Serverless Cluster -This document walks you through the quickest way to get started with TiDB. You will use [TiDB Cloud](https://en.pingcap.com/tidb-cloud) to create a TiDB Serverless cluster, connect to it, and run a sample application on it. +This document walks you through the quickest way to get started with TiDB. You will use [TiDB Cloud](https://www.pingcap.com/tidb-cloud) to create a TiDB Cloud Serverless cluster, connect to it, and run a sample application on it. If you need to run TiDB on your local machine, see [Starting TiDB Locally](/quick-start-with-tidb.md). @@ -21,7 +21,7 @@ This document walks you through the quickest way to get started with TiDB Cloud. -## Step 1. Create a TiDB Serverless cluster +## Step 1. Create a TiDB Cloud Serverless cluster 1. If you do not have a TiDB Cloud account, click [here](https://tidbcloud.com/free-trial) to sign up for an account. @@ -31,7 +31,7 @@ This document walks you through the quickest way to get started with TiDB Cloud. 4. On the **Create Cluster** page, **Serverless** is selected by default. Update the default cluster name if necessary, and then select the region where you want to create your cluster. -5. Click **Create** to create a TiDB Serverless cluster. +5. Click **Create** to create a TiDB Cloud Serverless cluster. Your TiDB Cloud cluster will be created in approximately 30 seconds. @@ -45,7 +45,7 @@ This document walks you through the quickest way to get started with TiDB Cloud. > **Note:** > -> For [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters, when you connect to your cluster, you must include the prefix for your cluster in the user name and wrap the name with quotation marks. For more information, see [User name prefix](https://docs.pingcap.com/tidbcloud/select-cluster-tier#user-name-prefix). +> For [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters, when you connect to your cluster, you must include the prefix for your cluster in the user name and wrap the name with quotation marks. For more information, see [User name prefix](https://docs.pingcap.com/tidbcloud/select-cluster-tier#user-name-prefix). @@ -53,7 +53,7 @@ This document walks you through the quickest way to get started with TiDB Cloud. > **Note:** > -> For [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters, when you connect to your cluster, you must include the prefix for your cluster in the user name and wrap the name with quotation marks. For more information, see [User name prefix](/tidb-cloud/select-cluster-tier.md#user-name-prefix). +> For [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters, when you connect to your cluster, you must include the prefix for your cluster in the user name and wrap the name with quotation marks. For more information, see [User name prefix](/tidb-cloud/select-cluster-tier.md#user-name-prefix). @@ -130,7 +130,7 @@ mysql Ver 15.1 Distrib 5.5.68-MariaDB, for Linux (x86_64) using readline 5.1 -2. Run the connection string obtained in [Step 1](#step-1-create-a-tidb-serverless-cluster). +2. Run the connection string obtained in [Step 1](#step-1-create-a-tidb-cloud-serverless-cluster). {{< copyable "shell-regular" >}} @@ -142,8 +142,8 @@ mysql Ver 15.1 Distrib 5.5.68-MariaDB, for Linux (x86_64) using readline 5.1 > **Note:** > -> - When you connect to a TiDB Serverless cluster, you must [use the TLS connection](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters). -> - If you encounter problems when connecting to a TiDB Serverless cluster, you can read [Secure Connections to TiDB Serverless Clusters](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters) for more information. +> - When you connect to a TiDB Cloud Serverless cluster, you must [use the TLS connection](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters). +> - If you encounter problems when connecting to a TiDB Cloud Serverless cluster, you can read [Secure Connections to TiDB Cloud Serverless Clusters](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters) for more information. @@ -151,8 +151,8 @@ mysql Ver 15.1 Distrib 5.5.68-MariaDB, for Linux (x86_64) using readline 5.1 > **Note:** > -> - When you connect to a TiDB Serverless cluster, you must [use the TLS connection](/tidb-cloud/secure-connections-to-serverless-clusters.md). -> - If you encounter problems when connecting to a TiDB Serverless cluster, you can read [Secure Connections to TiDB Serverless Clusters](/tidb-cloud/secure-connections-to-serverless-clusters.md) for more information. +> - When you connect to a TiDB Cloud Serverless cluster, you must [use the TLS connection](/tidb-cloud/secure-connections-to-serverless-clusters.md). +> - If you encounter problems when connecting to a TiDB Cloud Serverless cluster, you can read [Secure Connections to TiDB Cloud Serverless Clusters](/tidb-cloud/secure-connections-to-serverless-clusters.md) for more information. @@ -176,4 +176,18 @@ Expected output: +-------------------+ ``` -If your actual output is similar to the expected output, congratulations, you have successfully execute a SQL statement on TiDB Cloud. +If your actual output is similar to the expected output, congratulations, you have successfully executed a SQL statement on TiDB Cloud. + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + diff --git a/develop/dev-guide-choose-driver-or-orm.md b/develop/dev-guide-choose-driver-or-orm.md index f2a06af728a23..bbe5c011a1dbe 100644 --- a/develop/dev-guide-choose-driver-or-orm.md +++ b/develop/dev-guide-choose-driver-or-orm.md @@ -27,11 +27,11 @@ This section describes how to use drivers and ORM frameworks in Java. Support level: **Full** -You can follow the [MySQL documentation](https://dev.mysql.com/doc/connector-j/en/) to download and configure a Java JDBC driver. It is recommended to use MySQL Connector/J 8.0.33 or later with TiDB v6.3.0 and newer. +You can follow the [MySQL documentation](https://dev.mysql.com/doc/connector-j/en/) to download and configure a Java JDBC driver. It is recommended to use the latest GA version of MySQL Connector/J with TiDB v6.3.0 or later. -> **Tip:** +> **Warning:** > -> There is a [bug](https://bugs.mysql.com/bug.php?id=106252) in the Connector/J 8.0 versions before 8.0.32, which might cause threads to hang when using TiDB versions earlier than v6.3.0. To avoid this issue, it is recommended that you use either MySQL Connector/J 8.0.32 or a later version, or the TiDB JDBC (see the *TiDB-JDBC* tab). +> There is a [bug](https://bugs.mysql.com/bug.php?id=106252) in the MySQL Connector/J 8.0 versions before 8.0.31 (see [MySQL JDBC bugs](/develop/dev-guide-third-party-tools-compatibility.md#mysql-jdbc-bugs) for details), which might cause threads to hang when using TiDB versions earlier than v6.3.0. To avoid this issue, do **NOT** use MySQL Connector/J 8.0.31 or an earlier version. For an example of how to build a complete application, see [Build a simple CRUD app with TiDB and JDBC](/develop/dev-guide-sample-application-java-jdbc.md). @@ -87,15 +87,15 @@ implementation group: 'org.bouncycastle', name: 'bcpkix-jdk15on', version: '1.67 ### Java ORM frameworks + +
+ > **Note:** > > - Currently, Hibernate does [not support nested transactions](https://stackoverflow.com/questions/37927208/nested-transaction-in-spring-app-with-jpa-postgres). > > - Since v6.2.0, TiDB supports [savepoint](/sql-statements/sql-statement-savepoint.md). To use the `Propagation.NESTED` transaction propagation option in `@Transactional`, that is, to set `@Transactional(propagation = Propagation.NESTED)`, make sure that your TiDB is v6.2.0 or later. - -
- Support level: **Full** To avoid manually managing complex relationships between different dependencies of an application, you can use [Gradle](https://gradle.org/install) or [Maven](https://maven.apache.org/install.html) to get all dependencies of your application, including those indirect ones. Note that only Hibernate `6.0.0.Beta2` or above supports the TiDB dialect. @@ -106,21 +106,21 @@ If you are using Maven, add the following to your ` org.hibernate.orm hibernate-core - 6.0.0.CR2 + 6.2.3.Final mysql mysql-connector-java - 5.1.49 + 8.0.33 ``` If you are using Gradle, add the following to your `dependencies`: ```gradle -implementation 'org.hibernate:hibernate-core:6.0.0.CR2' -implementation 'mysql:mysql-connector-java:5.1.49' +implementation 'org.hibernate:hibernate-core:6.2.3.Final' +implementation 'mysql:mysql-connector-java:8.0.33' ``` - For an example of using Hibernate to build a TiDB application by native Java, see [Build a simple CRUD app with TiDB and Hibernate](/develop/dev-guide-sample-application-java-hibernate.md). @@ -146,21 +146,21 @@ If you are using Maven, add the following to your ` org.mybatis mybatis - 3.5.9 + 3.5.13 mysql mysql-connector-java - 5.1.49 + 8.0.33 ``` If you are using Gradle, add the following to your `dependencies`: ```gradle -implementation 'org.mybatis:mybatis:3.5.9' -implementation 'mysql:mysql-connector-java:5.1.49' +implementation 'org.mybatis:mybatis:3.5.13' +implementation 'mysql:mysql-connector-java:8.0.33' ``` For an example of using MyBatis to build a TiDB application, see [Build a simple CRUD app with TiDB and MyBatis](/develop/dev-guide-sample-application-java-mybatis.md). @@ -307,3 +307,17 @@ For an example of using peewee to build a TiDB application, see [Connect to TiDB After you have determined the driver or ORM, you can [connect to your TiDB cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-connect-to-tidb.md b/develop/dev-guide-connect-to-tidb.md index f9ed05e7f5e1e..6d31c6a93ccb5 100644 --- a/develop/dev-guide-connect-to-tidb.md +++ b/develop/dev-guide-connect-to-tidb.md @@ -133,3 +133,7 @@ For more information about TiDB SQL users, see [TiDB User Account Management](/u For more information about TiDB SQL users, see [TiDB User Account Management](https://docs.pingcap.com/tidb/stable/user-account-management). + +## Need help? + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). diff --git a/develop/dev-guide-connection-parameters.md b/develop/dev-guide-connection-parameters.md index efb4ba38101a6..f31890837bbec 100644 --- a/develop/dev-guide-connection-parameters.md +++ b/develop/dev-guide-connection-parameters.md @@ -276,3 +276,17 @@ TiDB provides two MySQL-compatible parameters to control the timeout: [`wait_tim The default value of [`wait_timeout`](/system-variables.md#wait_timeout) is relatively large. In scenarios where a transaction starts but is neither committed nor rolled back, you might need a finer-grained control and a shorter timeout to prevent prolonged lock holding. In this case, you can use [`tidb_idle_transaction_timeout`](/system-variables.md#tidb_idle_transaction_timeout-new-in-v760) (introduced in TiDB v7.6.0) to control the idle timeout for transactions in a user session. However, in an actual production environment, idle connections and SQL statements with excessively long execution time negatively affect databases and applications. To avoid idle connections and SQL statements that are executed for too long, you can configure these two parameters in your application's connection string. For example, set `sessionVariables=wait_timeout=3600` (1 hour) and `sessionVariables=max_execution_time=300000` (5 minutes). + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-create-database.md b/develop/dev-guide-create-database.md index c9ef93d081af3..934336aaa3dd8 100644 --- a/develop/dev-guide-create-database.md +++ b/develop/dev-guide-create-database.md @@ -11,7 +11,7 @@ This document describes how to create a database using SQL and various programmi Before creating a database, do the following: -- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). +- [Build a TiDB Cloud Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md). ## What is database @@ -80,3 +80,17 @@ The following is an example output: ## Next step After creating a database, you can add **tables** to it. For more information, see [Create a Table](/develop/dev-guide-create-table.md). + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-create-secondary-indexes.md b/develop/dev-guide-create-secondary-indexes.md index 0e57fd3c6f4a9..95f18c9fb8944 100644 --- a/develop/dev-guide-create-secondary-indexes.md +++ b/develop/dev-guide-create-secondary-indexes.md @@ -11,7 +11,7 @@ This document describes how to create a secondary index using SQL and various pr Before creating a secondary index, do the following: -- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). +- [Build a TiDB Cloud Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md). - [Create a Database](/develop/dev-guide-create-database.md). - [Create a Table](/develop/dev-guide-create-table.md). @@ -183,3 +183,17 @@ The following is an example output: ## Next step After creating a database and adding tables and secondary indexes to it, you can start adding the data [write](/develop/dev-guide-insert-data.md) and [read](/develop/dev-guide-get-data-from-single-table.md) features to your application. + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-create-table.md b/develop/dev-guide-create-table.md index 058828e5a82f5..545404365e2d9 100644 --- a/develop/dev-guide-create-table.md +++ b/develop/dev-guide-create-table.md @@ -5,13 +5,13 @@ summary: Learn the definitions, rules, and guidelines in table creation. # Create a Table -This document introduces how to create tables using the SQL statement and the related best practices. An example of the TiDB-based [Bookshop](/develop/dev-guide-bookshop-schema-design.md) application) is provided to illustrate the best practices. +This document introduces how to create tables using the SQL statement and the related best practices. An example of the TiDB-based [Bookshop](/develop/dev-guide-bookshop-schema-design.md) application is provided to illustrate the best practices. ## Before you start Before reading this document, make sure that the following tasks are completed: -- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). +- [Build a TiDB Cloud Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md). - [Create a Database](/develop/dev-guide-create-database.md). @@ -106,7 +106,7 @@ A [primary key](/constraints.md#primary-key) is a column or a set of columns in > > - In **InnoDB**: A **primary key** is unique, not null, and **index clustered**. > -> - In TiDB: A **primary key** is unique and is not null. But the primary key is not guaranteed to be a **clustered index**. Instead, another set of keywords `CLUSTERED` / `NONCLUSTERED` additionally controls whether the **primary key** is a **clustered index**. If the keyword is not specified, it is controlled by the system variable `@@global.tidb_enable_clustered_index`, as described in [clustered indexes](https://docs.pingcap.com/zh/tidb/stable/clustered-indexes). +> - In TiDB: A **primary key** is unique and is not null. But the primary key is not guaranteed to be a **clustered index**. Instead, another set of keywords `CLUSTERED` / `NONCLUSTERED` additionally controls whether the **primary key** is a **clustered index**. If the keyword is not specified, it is controlled by the system variable `@@global.tidb_enable_clustered_index`, as described in [clustered indexes](https://docs.pingcap.com/tidb/stable/clustered-indexes). The **primary key** is defined in the `CREATE TABLE` statement. The [primary key constraint](/constraints.md#primary-key) requires that all constrained columns contain only non-NULL values. @@ -135,7 +135,7 @@ CREATE TABLE `bookshop`.`users` ( TiDB supports the [clustered index](/clustered-indexes.md) feature since v5.0. This feature controls how data is stored in tables containing primary keys. It provides TiDB the ability to organize tables in a way that can improve the performance of certain queries. -The term clustered in this context refers to the organization of how data is stored and not a group of database servers working together. Some database management systems refer to clustered indexes as index-organized tables (IOT). +The term clustered in this context refers to the organization of how data is stored and not a group of database servers working together. Some database management systems refer to clustered index tables as index-organized tables (IOT). Currently, tables **_containing primary_** keys in TiDB are divided into the following two categories: @@ -183,7 +183,7 @@ CREATE TABLE `bookshop`.`ratings` ( ); ``` -In addition, if the current time is also filled in by default when the data is being updated, the following statements can be used (but only the [current time related statements](https://pingcap.github.io/sqlgram/#NowSymOptionFraction) can be filled in after `ON UPDATE`, and [more options](https://pingcap.github.io/sqlgram/#DefaultValueExpr) are supported after `DEFAULT`): +In addition, if the current time is also filled in by default when the data is being updated, the following statements can be used (but only the current-time related expressions can be filled in after `ON UPDATE`): ```sql CREATE TABLE `bookshop`.`ratings` ( @@ -195,6 +195,8 @@ CREATE TABLE `bookshop`.`ratings` ( ); ``` +For more information on default values of different data types, see [default values](/data-type-default-values.md). + ### Prevent duplicate values If you need to prevent duplicate values in a column, you can use the `UNIQUE` constraint. @@ -290,7 +292,7 @@ ALTER TABLE `bookshop`.`ratings` SET TIFLASH REPLICA 1; > **Note:** > -> If your cluster does not contain **TiFlash** nodes, this SQL statement will report an error: `1105 - the tiflash replica count: 1 should be less than the total tiflash server count: 0`. You can use [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster) to create a TiDB Serverless cluster that includes **TiFlash**. +> If your cluster does not contain **TiFlash** nodes, this SQL statement will report an error: `1105 - the tiflash replica count: 1 should be less than the total tiflash server count: 0`. You can use [Build a TiDB Cloud Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-cloud-serverless-cluster) to create a TiDB Cloud Serverless cluster that includes **TiFlash**. Then you can go on to perform the following query: @@ -407,3 +409,17 @@ This section provides guidelines you need to follow when creating a table. ## One more step Note that all the tables that have been created in this document do not contain secondary indexes. For a guide to add secondary indexes, refer to [Creating Secondary Indexes](/develop/dev-guide-create-secondary-indexes.md). + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-delete-data.md b/develop/dev-guide-delete-data.md index 96278a73313a5..b4cfb8984f42d 100644 --- a/develop/dev-guide-delete-data.md +++ b/develop/dev-guide-delete-data.md @@ -11,7 +11,7 @@ This document describes how to use the [DELETE](/sql-statements/sql-statement-de Before reading this document, you need to prepare the following: -- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md) +- [Build a TiDB Cloud Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md) - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md), [Create a Database](/develop/dev-guide-create-database.md), [Create a Table](/develop/dev-guide-create-table.md), and [Create Secondary Indexes](/develop/dev-guide-create-secondary-indexes.md) - [Insert Data](/develop/dev-guide-insert-data.md) @@ -411,3 +411,17 @@ In the same scenario as the [Bulk-delete example](#bulk-delete-example), the fol ```sql BATCH ON `rated_at` LIMIT 1000 DELETE FROM `ratings` WHERE `rated_at` >= "2022-04-15 00:00:00" AND `rated_at` <= "2022-04-15 00:15:00"; ``` + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-get-data-from-single-table.md b/develop/dev-guide-get-data-from-single-table.md index 2f3f6bf6f3e57..e74e5547de7a4 100644 --- a/develop/dev-guide-get-data-from-single-table.md +++ b/develop/dev-guide-get-data-from-single-table.md @@ -395,4 +395,18 @@ The result is as follows: 71 rows in set (0.00 sec) ``` -In addition to the `COUNT` function, TiDB also supports other aggregate functions. For more information, see [Aggregate (GROUP BY) Functions](/functions-and-operators/aggregate-group-by-functions.md). \ No newline at end of file +In addition to the `COUNT` function, TiDB also supports other aggregate functions. For more information, see [Aggregate (GROUP BY) Functions](/functions-and-operators/aggregate-group-by-functions.md). + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-gui-datagrip.md b/develop/dev-guide-gui-datagrip.md index 30e56a9f613bc..f3089d4e05921 100644 --- a/develop/dev-guide-gui-datagrip.md +++ b/develop/dev-guide-gui-datagrip.md @@ -9,7 +9,7 @@ TiDB is a MySQL-compatible database, and [JetBrains DataGrip](https://www.jetbra > **Note:** > -> This tutorial is compatible with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial is compatible with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. You can use DataGrip in two ways: @@ -29,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -37,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -47,7 +47,7 @@ To complete this tutorial, you need: Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -55,7 +55,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` - **Branch** is set to `main` - **Connect With** is set to `DataGrip` - **Operating System** matches your environment. @@ -76,26 +76,28 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 7. Copy the connection string from the TiDB Cloud connection dialog. Then, paste it into the **URL** field, and the remaining parameters will be auto-populated. An example result is as follows: - ![Configure the URL field for TiDB Serverless](/media/develop/datagrip-url-paste.jpg) + ![Configure the URL field for TiDB Cloud Serverless](/media/develop/datagrip-url-paste.jpg) If a **Download missing driver files** warning displays, click **Download** to acquire the driver files. -8. Click **Test Connection** to validate the connection to the TiDB Serverless cluster. +8. Click **Test Connection** to validate the connection to the TiDB Cloud Serverless cluster. - ![Test the connection to a TiDB Serverless clustser](/media/develop/datagrip-test-connection.jpg) + ![Test the connection to a TiDB Cloud Serverless cluster](/media/develop/datagrip-test-connection.jpg) 9. Click **OK** to save the connection configuration.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Launch DataGrip and create a project to manage your connections. @@ -105,9 +107,9 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele ![Select a data source in DataGrip](/media/develop/datagrip-data-source-select.jpg) -6. Copy and paste the appropriate connection string into the **Data Source and Drivers** window in DataGrip. The mappings between DataGrip fields and TiDB Dedicated connection string are as follows: +6. Copy and paste the appropriate connection string into the **Data Source and Drivers** window in DataGrip. The mappings between DataGrip fields and TiDB Cloud Dedicated connection string are as follows: - | DataGrip field | TiDB Dedicated connection string | + | DataGrip field | TiDB Cloud Dedicated connection string | | -------------- | ------------------------------- | | Host | `{host}` | | Port | `{port}` | @@ -116,26 +118,26 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele An example is as follows: - ![Configure the connection parameters for TiDB Dedicated](/media/develop/datagrip-dedicated-connect.jpg) + ![Configure the connection parameters for TiDB Cloud Dedicated](/media/develop/datagrip-dedicated-connect.jpg) 7. Click the **SSH/SSL** tab, select the **Use SSL** checkbox, and input the CA certificate path into the **CA file** field. - ![Configure the CA for TiDB Dedicated](/media/develop/datagrip-dedicated-ssl.jpg) + ![Configure the CA for TiDB Cloud Dedicated](/media/develop/datagrip-dedicated-ssl.jpg) If a **Download missing driver files** warning displays, click **Download** to acquire the driver files. 8. Click the **Advanced** tab, scroll to find the **enabledTLSProtocols** parameter, and set its value to `TLSv1.2,TLSv1.3`. - ![Configure the TLS for TiDB Dedicated](/media/develop/datagrip-dedicated-advanced.jpg) + ![Configure the TLS for TiDB Cloud Dedicated](/media/develop/datagrip-dedicated-advanced.jpg) -9. Click **Test Connection** to validate the connection to the TiDB Dedicated cluster. +9. Click **Test Connection** to validate the connection to the TiDB Cloud Dedicated cluster. - ![Test the connection to a TiDB Dedicated cluster](/media/develop/datagrip-dedicated-test-connection.jpg) + ![Test the connection to a TiDB Cloud Dedicated cluster](/media/develop/datagrip-dedicated-test-connection.jpg) 10. Click **OK** to save the connection configuration.
-
+
1. Launch DataGrip and create a project to manage your connections. @@ -147,20 +149,20 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Configure the following connection parameters: - - **Host**: The IP address or domain name of your TiDB Self-Hosted cluster. - - **Port**: The port number of your TiDB Self-Hosted cluster. - - **User**: The username to use to connect to your TiDB Self-Hosted cluster. + - **Host**: The IP address or domain name of your TiDB Self-Managed cluster. + - **Port**: The port number of your TiDB Self-Managed cluster. + - **User**: The username to use to connect to your TiDB Self-Managed cluster. - **Password**: The password of the username. An example is as follows: - ![Configure the connection parameters for TiDB Self-Hosted](/media/develop/datagrip-self-hosted-connect.jpg) + ![Configure the connection parameters for TiDB Self-Managed](/media/develop/datagrip-self-hosted-connect.jpg) If a **Download missing driver files** warning displays, click **Download** to acquire the driver files. -4. Click **Test Connection** to validate the connection to the TiDB Self-Hosted cluster. +4. Click **Test Connection** to validate the connection to the TiDB Self-Managed cluster. - ![Test the connection to a TiDB Self-Hosted cluster](/media/develop/datagrip-self-hosted-test-connection.jpg) + ![Test the connection to a TiDB Self-Managed cluster](/media/develop/datagrip-self-hosted-test-connection.jpg) 5. Click **OK** to save the connection configuration. @@ -177,12 +179,12 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-gui-dbeaver.md b/develop/dev-guide-gui-dbeaver.md index 2348a23325f56..aa83a5ba66c57 100644 --- a/develop/dev-guide-gui-dbeaver.md +++ b/develop/dev-guide-gui-dbeaver.md @@ -11,7 +11,7 @@ In this tutorial, you can learn how to connect to your TiDB cluster using DBeave > **Note:** > -> This tutorial is compatible with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial is compatible with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -24,7 +24,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -32,7 +32,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -42,7 +42,7 @@ To complete this tutorial, you need: Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -50,7 +50,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` - **Branch** is set to `main` - **Connect With** is set to `DBeaver` - **Operating System** matches your environment. @@ -69,9 +69,9 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 7. In the **Authentication (Database Native)** section, enter your **Username** and **Password**. An example is as follows: - ![Configure connection settings for TiDB Serverless](/media/develop/dbeaver-connection-settings-serverless.jpg) + ![Configure connection settings for TiDB Cloud Serverless](/media/develop/dbeaver-connection-settings-serverless.jpg) -8. Click **Test Connection** to validate the connection to the TiDB Serverless cluster. +8. Click **Test Connection** to validate the connection to the TiDB Cloud Serverless cluster. If the **Download driver files** dialog is displayed, click **Download** to get the driver files. @@ -84,23 +84,25 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 9. Click **Finish** to save the connection configuration.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere**, and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Launch DBeaver and click **New Database Connection** in the upper-left corner. In the **Connect to a database** dialog, select **TiDB** from the list, and then click **Next**. ![Select TiDB as the database in DBeaver](/media/develop/dbeaver-select-database.jpg) -5. Copy and paste the appropriate connection string into the DBeaver connection panel. The mappings between DBeaver fields and TiDB Dedicated connection string are as follows: +5. Copy and paste the appropriate connection string into the DBeaver connection panel. The mappings between DBeaver fields and TiDB Cloud Dedicated connection string are as follows: - | DBeaver field | TiDB Dedicated connection string | + | DBeaver field | TiDB Cloud Dedicated connection string | |---------------| ------------------------------- | | Server Host | `{host}` | | Port | `{port}` | @@ -109,9 +111,9 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele An example is as follows: - ![Configure connection settings for TiDB Dedicated](/media/develop/dbeaver-connection-settings-dedicated.jpg) + ![Configure connection settings for TiDB Cloud Dedicated](/media/develop/dbeaver-connection-settings-dedicated.jpg) -6. Click **Test Connection** to validate the connection to the TiDB Dedicated cluster. +6. Click **Test Connection** to validate the connection to the TiDB Cloud Dedicated cluster. If the **Download driver files** dialog is displayed, click **Download** to get the driver files. @@ -124,7 +126,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 7. Click **Finish** to save the connection configuration.
-
+
1. Launch DBeaver and click **New Database Connection** in the upper-left corner. In the **Connect to a database** dialog, select **TiDB** from the list, and then click **Next**. @@ -132,16 +134,16 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 2. Configure the following connection parameters: - - **Server Host**: The IP address or domain name of your TiDB Self-Hosted cluster. - - **Port**: The port number of your TiDB Self-Hosted cluster. - - **Username**: The username to use to connect to your TiDB Self-Hosted cluster. + - **Server Host**: The IP address or domain name of your TiDB Self-Managed cluster. + - **Port**: The port number of your TiDB Self-Managed cluster. + - **Username**: The username to use to connect to your TiDB Self-Managed cluster. - **Password**: The password of the username. An example is as follows: - ![Configure connection settings for TiDB Self-Hosted](/media/develop/dbeaver-connection-settings-self-hosted.jpg) + ![Configure connection settings for TiDB Self-Managed](/media/develop/dbeaver-connection-settings-self-hosted.jpg) -3. Click **Test Connection** to validate the connection to the TiDB Self-Hosted cluster. +3. Click **Test Connection** to validate the connection to the TiDB Self-Managed cluster. If the **Download driver files** dialog is displayed, click **Download** to get the driver files. @@ -166,12 +168,12 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-gui-mysql-workbench.md b/develop/dev-guide-gui-mysql-workbench.md index fb9b3ac832766..0aa8a63ccc3f2 100644 --- a/develop/dev-guide-gui-mysql-workbench.md +++ b/develop/dev-guide-gui-mysql-workbench.md @@ -16,7 +16,7 @@ In this tutorial, you can learn how to connect to your TiDB cluster using MySQL > **Note:** > -> This tutorial is compatible with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial is compatible with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -29,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -37,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -47,7 +47,7 @@ To complete this tutorial, you need: Connect to your TiDB cluster depending on the TiDB deployment option you have selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -55,7 +55,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public`. + - **Connection Type** is set to `Public`. - **Branch** is set to `main`. - **Connect With** is set to `MySQL Workbench`. - **Operating System** matches your environment. @@ -76,28 +76,30 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se - **Hostname**: enter the `HOST` parameter from the TiDB Cloud connection dialog. - **Port**: enter the `PORT` parameter from the TiDB Cloud connection dialog. - **Username**: enter the `USERNAME` parameter from the TiDB Cloud connection dialog. - - **Password**: click **Store in Keychain ...** or **Store in Vault**, enter the password of the TiDB Serverless cluster, and then click **OK** to store the password. + - **Password**: click **Store in Keychain ...** or **Store in Vault**, enter the password of the TiDB Cloud Serverless cluster, and then click **OK** to store the password. - ![MySQL Workbench: store the password of TiDB Serverless in keychain](/media/develop/mysql-workbench-store-password-in-keychain.png) + ![MySQL Workbench: store the password of TiDB Cloud Serverless in keychain](/media/develop/mysql-workbench-store-password-in-keychain.png) The following figure shows an example of the connection parameters: - ![MySQL Workbench: configure connection settings for TiDB Serverless](/media/develop/mysql-workbench-connection-config-serverless-parameters.png) + ![MySQL Workbench: configure connection settings for TiDB Cloud Serverless](/media/develop/mysql-workbench-connection-config-serverless-parameters.png) -7. Click **Test Connection** to validate the connection to the TiDB Serverless cluster. +7. Click **Test Connection** to validate the connection to the TiDB Cloud Serverless cluster. 8. If the connection test is successful, you can see the **Successfully made the MySQL connection** message. Click **OK** to save the connection configuration.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere**. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Launch MySQL Workbench and click **+** near the **MySQL Connections** title. @@ -109,20 +111,20 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se - **Hostname**: enter the `HOST` parameter from the TiDB Cloud connection dialog. - **Port**: enter the `PORT` parameter from the TiDB Cloud connection dialog. - **Username**: enter the `USERNAME` parameter from the TiDB Cloud connection dialog. - - **Password**: click **Store in Keychain ...**, enter the password of the TiDB Dedicated cluster, and then click **OK** to store the password. + - **Password**: click **Store in Keychain ...**, enter the password of the TiDB Cloud Dedicated cluster, and then click **OK** to store the password. - ![MySQL Workbench: store the password of TiDB Dedicated in keychain](/media/develop/mysql-workbench-store-dedicated-password-in-keychain.png) + ![MySQL Workbench: store the password of TiDB Cloud Dedicated in keychain](/media/develop/mysql-workbench-store-dedicated-password-in-keychain.png) The following figure shows an example of the connection parameters: - ![MySQL Workbench: configure connection settings for TiDB Dedicated](/media/develop/mysql-workbench-connection-config-dedicated-parameters.png) + ![MySQL Workbench: configure connection settings for TiDB Cloud Dedicated](/media/develop/mysql-workbench-connection-config-dedicated-parameters.png) -6. Click **Test Connection** to validate the connection to the TiDB Dedicated cluster. +6. Click **Test Connection** to validate the connection to the TiDB Cloud Dedicated cluster. 7. If the connection test is successful, you can see the **Successfully made the MySQL connection** message. Click **OK** to save the connection configuration.
-
+
1. Launch MySQL Workbench and click **+** near the **MySQL Connections** title. @@ -131,18 +133,18 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se 2. In the **Setup New Connection** dialog, configure the following connection parameters: - **Connection Name**: give this connection a meaningful name. - - **Hostname**: enter the IP address or domain name of your TiDB Self-Hosted cluster. - - **Port**: enter the port number of your TiDB Self-Hosted cluster. + - **Hostname**: enter the IP address or domain name of your TiDB Self-Managed cluster. + - **Port**: enter the port number of your TiDB Self-Managed cluster. - **Username**: enter the username to use to connect to your TiDB. - **Password**: click **Store in Keychain ...**, enter the password to use to connect to your TiDB cluster, and then click **OK** to store the password. - ![MySQL Workbench: store the password of TiDB Self-Hosted in keychain](/media/develop/mysql-workbench-store-self-hosted-password-in-keychain.png) + ![MySQL Workbench: store the password of TiDB Self-Managed in keychain](/media/develop/mysql-workbench-store-self-hosted-password-in-keychain.png) The following figure shows an example of the connection parameters: - ![MySQL Workbench: configure connection settings for TiDB Self-Hosted](/media/develop/mysql-workbench-connection-config-self-hosted-parameters.png) + ![MySQL Workbench: configure connection settings for TiDB Self-Managed](/media/develop/mysql-workbench-connection-config-self-hosted-parameters.png) -3. Click **Test Connection** to validate the connection to the TiDB Self-Hosted cluster. +3. Click **Test Connection** to validate the connection to the TiDB Self-Managed cluster. 4. If the connection test is successful, you can see the **Successfully made the MySQL connection** message. Click **OK** to save the connection configuration. @@ -172,12 +174,12 @@ For more information, see [MySQL Workbench frequently asked questions](https://d -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/tidb-cloud/tidb-cloud-support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-gui-navicat.md b/develop/dev-guide-gui-navicat.md index a8eb3be1b0bc2..a6f0e2295ef8b 100644 --- a/develop/dev-guide-gui-navicat.md +++ b/develop/dev-guide-gui-navicat.md @@ -5,32 +5,27 @@ summary: Learn how to connect to TiDB using Navicat. # Connect to TiDB with Navicat -TiDB is a MySQL-compatible database, and [Navicat](https://www.navicat.com) is a GUI tool set for database users. This tutorial uses the [Navicat for MySQL](https://www.navicat.com/en/products/navicat-for-mysql) tool to connect to TiDB. - -> **Warning:** -> -> - Although you can use Navicat to connect to TiDB due to its MySQL compatibility, Navicat does not fully support TiDB. You might encounter some issues during usage as it treats TiDB as MySQL. There is a known issue about [Navicat user management compatibility](https://github.com/pingcap/tidb/issues/45154). For more compatibility issues between Navicat and TiDB, see the [TiDB GitHub issue page](https://github.com/pingcap/tidb/issues?q=is%3Aissue+navicat+is%3Aopen). -> - It is recommended to use other GUI tools that officially support TiDB, such as [DataGrip](/develop/dev-guide-gui-datagrip.md), [DBeaver](/develop/dev-guide-gui-dbeaver.md), and [VS Code SQLTools](/develop/dev-guide-gui-vscode-sqltools.md). For a complete list of GUI tools that fully supported by TiDB, see [Third-party tools supported by TiDB](/develop/dev-guide-third-party-support.md#gui). +TiDB is a MySQL-compatible database, and [Navicat](https://www.navicat.com) is a GUI tool set for database users. This tutorial uses the [Navicat Premium](https://www.navicat.com/en/products/navicat-premium) tool to connect to TiDB. In this tutorial, you can learn how to connect to your TiDB cluster using Navicat. > **Note:** > -> This tutorial is compatible with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial is compatible with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites To complete this tutorial, you need: -- [Navicat for MySQL](https://www.navicat.com/en/download/navicat-for-mysql) **16.3.2** or later versions. -- A paid account for Navicat for MySQL. +- [Navicat Premium](https://www.navicat.com) **17.1.6** or later versions. +- A paid account for Navicat Premium. - A TiDB cluster. **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -38,7 +33,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -48,7 +43,7 @@ To complete this tutorial, you need: Connect to your TiDB cluster depending on the TiDB deployment option you have selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -56,7 +51,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public`. + - **Connection Type** is set to `Public`. - **Branch** is set to `main`. - **Connect With** is set to `Navicat`. - **Operating System** matches your environment. @@ -67,83 +62,85 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se > > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. -5. Launch Navicat for MySQL, click **Connection** in the upper-left corner, and select **MySQL** from the drop-down list. +5. Launch Navicat Premium, click **Connection** in the upper-left corner, select **PingCAP** from the **Vendor Filter** list, and double-click **TiDB** in the right panel. - ![Navicat: add new connection](/media/develop/navicat-add-new-connection.jpg) + ![Navicat: add new connection](/media/develop/navicat-premium-add-new-connection.png) -6. In the **New Connection (MySQL)** dialog, configure the following connection parameters: +6. In the **New Connection (TiDB)** dialog, configure the following connection parameters: - **Connection Name**: give this connection a meaningful name. - **Host**: enter the `HOST` parameter from the TiDB Cloud connection dialog. - **Port**: enter the `PORT` parameter from the TiDB Cloud connection dialog. - **User Name**: enter the `USERNAME` parameter from the TiDB Cloud connection dialog. - - **Password**: enter the password of the TiDB Serverless cluster. + - **Password**: enter the password of the TiDB Cloud Serverless cluster. - ![Navicat: configure connection general panel for TiDB Serverless](/media/develop/navicat-connection-config-serverless-general.png) + ![Navicat: configure connection general panel for TiDB Cloud Serverless](/media/develop/navicat-premium-connection-config-serverless-general.png) 7. Click the **SSL** tab and select **Use SSL**, **Use authentication**, and **Verify server certificate against CA** checkboxes. Then, select the `CA` file from the TiDB Cloud connection dialog into the **CA Certificate** field. - ![Navicat: configure connection SSL panel for TiDB Serverless](/media/develop/navicat-connection-config-serverless-ssl.png) + ![Navicat: configure connection SSL panel for TiDB Cloud Serverless](/media/develop/navicat-premium-connection-config-serverless-ssl.png) -8. Click **Test Connection** to validate the connection to the TiDB Serverless cluster. +8. Click **Test Connection** to validate the connection to the TiDB Cloud Serverless cluster. -9. If the connection test is successful, you can see the **Connection Successful** message. Click **Save** to finish the connection configuration. +9. If the connection test is successful, you can see the **Connection Successful** message. Click **OK** to finish the connection configuration.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere**. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list. + + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). -4. Click **Download CA cert** to download the CA file. +4. Click **CA cert** to download the CA certificate. -5. Launch Navicat for MySQL, click **Connection** in the upper-left corner, and select **MySQL** from the drop-down list. +5. Launch Navicat Premium, click **Connection** in the upper-left corner, select **PingCAP** from the **Vendor Filter** list, and double-click **TiDB** in the right panel. - ![Navicat: add new connection](/media/develop/navicat-add-new-connection.jpg) + ![Navicat: add new connection](/media/develop/navicat-premium-add-new-connection.png) -6. In the **New Connection (MySQL)** dialog, configure the following connection parameters: +6. In the **New Connection (TiDB)** dialog, configure the following connection parameters: - **Connection Name**: give this connection a meaningful name. - **Host**: enter the `HOST` parameter from the TiDB Cloud connection dialog. - **Port**: enter the `PORT` parameter from the TiDB Cloud connection dialog. - **User Name**: enter the `USERNAME` parameter from the TiDB Cloud connection dialog. - - **Password**: enter the password of the TiDB Dedicated cluster. + - **Password**: enter the password of the TiDB Cloud Dedicated cluster. - ![Navicat: configure connection general panel for TiDB Dedicated](/media/develop/navicat-connection-config-dedicated-general.png) + ![Navicat: configure connection general panel for TiDB Cloud Dedicated](/media/develop/navicat-premium-connection-config-dedicated-general.png) 7. Click the **SSL** tab and select **Use SSL**, **Use authentication**, and **Verify server certificate against CA** checkboxes. Then, select the CA file downloaded in step 4 into the **CA Certificate** field. - ![Navicat: configure connection SSL panel for TiDB Dedicated](/media/develop/navicat-connection-config-dedicated-ssl.jpg) + ![Navicat: configure connection SSL panel for TiDB Cloud Dedicated](/media/develop/navicat-premium-connection-config-dedicated-ssl.png) -8. **Test Connection** to validate the connection to the TiDB Dedicated cluster. +8. **Test Connection** to validate the connection to the TiDB Cloud Dedicated cluster. -9. If the connection test is successful, you can see the **Connection Successful** message. Click **Save** to finish the connection configuration. +9. If the connection test is successful, you can see the **Connection Successful** message. Click **OK** to finish the connection configuration.
-
+
-1. Launch Navicat for MySQL, click **Connection** in the upper-left corner, and select **MySQL** from the drop-down list. +1. Launch Navicat Premium, click **Connection** in the upper-left corner, select **PingCAP** from the **Vendor Filter** list, and double-click **TiDB** in the right panel. - ![Navicat: add new connection](/media/develop/navicat-add-new-connection.jpg) + ![Navicat: add new connection](/media/develop/navicat-premium-add-new-connection.png) -2. In the **New Connection (MySQL)** dialog, configure the following connection parameters: +2. In the **New Connection (TiDB)** dialog, configure the following connection parameters: - **Connection Name**: give this connection a meaningful name. - - **Host**: enter the IP address or domain name of your TiDB Self-Hosted cluster. - - **Port**: enter the port number of your TiDB Self-Hosted cluster. + - **Host**: enter the IP address or domain name of your TiDB Self-Managed cluster. + - **Port**: enter the port number of your TiDB Self-Managed cluster. - **User Name**: enter the username to use to connect to your TiDB. - **Password**: enter the password to use to connect to your TiDB. - ![Navicat: configure connection general panel for self-hosted TiDB](/media/develop/navicat-connection-config-self-hosted-general.png) + ![Navicat: configure connection general panel for self-hosted TiDB](/media/develop/navicat-premium-connection-config-self-hosted-general.png) -3. Click **Test Connection** to validate the connection to the TiDB Self-Hosted cluster. +3. Click **Test Connection** to validate the connection to the TiDB Self-Managed cluster. -4. If the connection test is successful, you can see the **Connection Successful** message. Click **Save** to finish the connection configuration. +4. If the connection test is successful, you can see the **Connection Successful** message. Click **OK** to finish the connection configuration.
@@ -157,12 +154,12 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/tidb-cloud/tidb-cloud-support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-gui-vscode-sqltools.md b/develop/dev-guide-gui-vscode-sqltools.md index 8931a3879ab65..f8230e47d4774 100644 --- a/develop/dev-guide-gui-vscode-sqltools.md +++ b/develop/dev-guide-gui-vscode-sqltools.md @@ -11,7 +11,7 @@ In this tutorial, you can learn how to connect to your TiDB cluster using Visual > **Note:** > -> - This tutorial is compatible with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> - This tutorial is compatible with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. > - This tutorial also works with Visual Studio Code Remote Development environments, such as [GitHub Codespaces](https://github.com/features/codespaces), [Visual Studio Code Dev Containers](https://code.visualstudio.com/docs/devcontainers/containers), and [Visual Studio Code WSL](https://code.visualstudio.com/docs/remote/wsl). ## Prerequisites @@ -29,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -37,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -47,7 +47,7 @@ To complete this tutorial, you need: Connect to your TiDB cluster depending on the TiDB deployment option you have selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -55,7 +55,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public`. + - **Connection Type** is set to `Public`. - **Branch** is set to `main`. - **Connect With** is set to `VS Code`. - **Operating System** matches your environment. @@ -87,33 +87,35 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se - In the **MySQL driver specific options** area, configure the following parameters: - **Authentication Protocol**: select **default**. - - **SSL**: select **Enabled**. TiDB Serverless requires a secure connection. In the **SSL Options (node.TLSSocket)** area, configure the **Certificate Authority (CA) Certificate File** field as the `CA` parameter from the TiDB Cloud connection dialog. + - **SSL**: select **Enabled**. TiDB Cloud Serverless requires a secure connection. In the **SSL Options (node.TLSSocket)** area, configure the **Certificate Authority (CA) Certificate File** field as the `CA` parameter from the TiDB Cloud connection dialog. > **Note:** > - > If you are running on Windows or GitHub Codespaces, you can leave **SSL** blank. By default SQLTools trusts well-known CAs curated by Let's Encrypt. For more information, see [TiDB Serverless root certificate management](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters#root-certificate-management). + > If you are running on Windows or GitHub Codespaces, you can leave **SSL** blank. By default SQLTools trusts well-known CAs curated by Let's Encrypt. For more information, see [TiDB Cloud Serverless root certificate management](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters#root-certificate-management). - ![VS Code SQLTools: configure connection settings for TiDB Serverless](/media/develop/vsc-sqltools-connection-config-serverless.jpg) + ![VS Code SQLTools: configure connection settings for TiDB Cloud Serverless](/media/develop/vsc-sqltools-connection-config-serverless.jpg) -7. Click **TEST CONNECTION** to validate the connection to the TiDB Serverless cluster. +7. Click **TEST CONNECTION** to validate the connection to the TiDB Cloud Serverless cluster. 1. In the pop-up window, click **Allow**. 2. In the **SQLTools Driver Credentials** dialog, enter the password you created in step 4. - ![VS Code SQLTools: enter password to connect to TiDB Serverless](/media/develop/vsc-sqltools-password.jpg) + ![VS Code SQLTools: enter password to connect to TiDB Cloud Serverless](/media/develop/vsc-sqltools-password.jpg) 8. If the connection test is successful, you can see the **Successfully connected!** message. Click **SAVE CONNECTION** to save the connection configuration.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere**. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Launch VS Code and select the **SQLTools** extension on the navigation pane. Under the **CONNECTIONS** section, click **Add New Connection** and select **TiDB** as the database driver. @@ -134,19 +136,19 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se - **Authentication Protocol**: select **default**. - **SSL**: select **Disabled**. - ![VS Code SQLTools: configure connection settings for TiDB Dedicated](/media/develop/vsc-sqltools-connection-config-dedicated.jpg) + ![VS Code SQLTools: configure connection settings for TiDB Cloud Dedicated](/media/develop/vsc-sqltools-connection-config-dedicated.jpg) -6. Click **TEST CONNECTION** to validate the connection to the TiDB Dedicated cluster. +6. Click **TEST CONNECTION** to validate the connection to the TiDB Cloud Dedicated cluster. 1. In the pop-up window, click **Allow**. - 2. In the **SQLTools Driver Credentials** dialog, enter the password of the TiDB Dedicated cluster. + 2. In the **SQLTools Driver Credentials** dialog, enter the password of the TiDB Cloud Dedicated cluster. - ![VS Code SQLTools: enter password to connect to TiDB Dedicated](/media/develop/vsc-sqltools-password.jpg) + ![VS Code SQLTools: enter password to connect to TiDB Cloud Dedicated](/media/develop/vsc-sqltools-password.jpg) 7. If the connection test is successful, you can see the **Successfully connected!** message. Click **SAVE CONNECTION** to save the connection configuration.
-
+
1. Launch VS Code and select the **SQLTools** extension on the navigation pane. Under the **CONNECTIONS** section, click **Add New Connection** and select **TiDB** as the database driver. @@ -157,10 +159,10 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se - **Connection name**: give this connection a meaningful name. - **Connection group**: (optional) give this group of connections a meaningful name. Connections with the same group name will be grouped together. - **Connect using**: select **Server and Port**. - - **Server Address**: enter the IP address or domain name of your TiDB Self-Hosted cluster. - - **Port**: enter the port number of your TiDB Self-Hosted cluster. + - **Server Address**: enter the IP address or domain name of your TiDB Self-Managed cluster. + - **Port**: enter the port number of your TiDB Self-Managed cluster. - **Database**: enter the database that you want to connect to. - - **Username**: enter the username to use to connect to your TiDB Self-Hosted cluster. + - **Username**: enter the username to use to connect to your TiDB Self-Managed cluster. - **Password mode**: - If the password is empty, select **Use empty password**. @@ -171,13 +173,13 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se - **Authentication Protocol**: select **default**. - **SSL**: select **Disabled**. - ![VS Code SQLTools: configure connection settings for TiDB Self-Hosted](/media/develop/vsc-sqltools-connection-config-self-hosted.jpg) + ![VS Code SQLTools: configure connection settings for TiDB Self-Managed](/media/develop/vsc-sqltools-connection-config-self-hosted.jpg) -3. Click **TEST CONNECTION** to validate the connection to the TiDB Self-Hosted cluster. +3. Click **TEST CONNECTION** to validate the connection to the TiDB Self-Managed cluster. - If the password is not empty, click **Allow** in the pop-up window, and then enter the password of the TiDB Self-Hosted cluster. + If the password is not empty, click **Allow** in the pop-up window, and then enter the password of the TiDB Self-Managed cluster. - ![VS Code SQLTools: enter password to connect to TiDB Self-Hosted](/media/develop/vsc-sqltools-password.jpg) + ![VS Code SQLTools: enter password to connect to TiDB Self-Managed](/media/develop/vsc-sqltools-password.jpg) 4. If the connection test is successful, you can see the **Successfully connected!** message. Click **SAVE CONNECTION** to save the connection configuration. @@ -195,12 +197,12 @@ Connect to your TiDB cluster depending on the TiDB deployment option you have se -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). - + \ No newline at end of file diff --git a/develop/dev-guide-hybrid-oltp-and-olap-queries.md b/develop/dev-guide-hybrid-oltp-and-olap-queries.md index 7e6a31c0b0f79..0b531acde0ab0 100644 --- a/develop/dev-guide-hybrid-oltp-and-olap-queries.md +++ b/develop/dev-guide-hybrid-oltp-and-olap-queries.md @@ -261,3 +261,17 @@ For more information about how TiDB chooses to use TiFlash, see [Use TiDB to rea - [Window Functions](/functions-and-operators/window-functions.md) - [Use TiFlash](/tiflash/tiflash-overview.md#use-tiflash) + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-implicit-type-conversion.md b/develop/dev-guide-implicit-type-conversion.md index d801fb2147af6..13711c29e7317 100644 --- a/develop/dev-guide-implicit-type-conversion.md +++ b/develop/dev-guide-implicit-type-conversion.md @@ -76,3 +76,17 @@ SELECT * FROM `t1` WHERE `a` BETWEEN '12123123' AND '1111222211111111200000'; ``` **Brief description of run results**: The above execution gives a wrong result. + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-index-best-practice.md b/develop/dev-guide-index-best-practice.md index 1445cde3b2778..c365c56e5187e 100644 --- a/develop/dev-guide-index-best-practice.md +++ b/develop/dev-guide-index-best-practice.md @@ -32,7 +32,7 @@ CREATE TABLE `books` ( - Create an appropriate index based on your application. In principle, create indexes only on the columns to be used in queries to improve performance. The following cases are suitable for creating an index: - Columns with a high distinction degree can significantly reduce the number of filtered rows. For example, it is recommended to create an index on the personal ID number, but not on the gender. - - Use combined indexes when querying with multiple conditions. Note that columns with equivalent conditions need to be placed in the front of the combined index. Here is an example: if the `select* from t where c1 = 10 and c2 = 100 and c3 > 10` query is frequently used, consider creating a combined index `Index cidx (c1, c2, c3)`, so that a index prefix can be constructed to scan by query conditions. + - Use combined indexes when querying with multiple conditions. Note that columns with equivalent conditions need to be placed in the front of the combined index. Here is an example: if the `select* from t where c1 = 10 and c2 = 100 and c3 > 10` query is frequently used, consider creating a combined index `Index cidx (c1, c2, c3)`, so that an index prefix can be constructed to scan by query conditions. - Name your secondary index meaningfully, and it is recommended to follow the table naming conventions of your company or organization. If such naming conventions do not exist, follow the rules in [Index Naming Specification](/develop/dev-guide-object-naming-guidelines.md). @@ -150,3 +150,17 @@ CREATE TABLE `books` ( ``` - When using the `IN` expression in a query condition, it is recommended that the number of value matched after it does not exceed 300, otherwise the execution efficiency will be poor. + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-insert-data.md b/develop/dev-guide-insert-data.md index 628911670e11f..da0096e85e057 100644 --- a/develop/dev-guide-insert-data.md +++ b/develop/dev-guide-insert-data.md @@ -13,7 +13,7 @@ This document describes how to insert data into TiDB by using the SQL language w Before reading this document, you need to prepare the following: -- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). +- [Build a TiDB Cloud Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md), [Create a Database](/develop/dev-guide-create-database.md), [Create a Table](/develop/dev-guide-create-table.md), and [Create Secondary Indexes](/develop/dev-guide-create-secondary-indexes.md) ## Insert rows @@ -290,7 +290,7 @@ There are two solutions to handle this error: INSERT INTO `bookshop`.`users` (`balance`, `nickname`) VALUES (0.00, 'nicky'); ``` -- If you are sure that you **_must_** specify this column, then you can use the [`SET` statement](https://docs.pingcap.com/zh/tidb/stable/sql-statement-set-variable) to allow the column of `AUTO_RANDOM` to be specified during insertion time by changing the user variable. +- If you are sure that you **_must_** specify this column, then you can use the [`SET` statement](https://docs.pingcap.com/tidb/stable/sql-statement-set-variable) to allow the column of `AUTO_RANDOM` to be specified during insertion time by changing the user variable. {{< copyable "sql" >}} @@ -302,3 +302,17 @@ There are two solutions to handle this error: ## Use HTAP In TiDB, HTAP capabilities save you from performing additional operations when inserting data. There is no additional insertion logic. TiDB automatically guarantees data consistency. All you need to do is [turn on column-oriented replica synchronization](/develop/dev-guide-create-table.md#use-htap-capabilities) after creating the table, and use the column-oriented replica to speed up your queries directly. + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-join-tables.md b/develop/dev-guide-join-tables.md index e22b59bb4ccb5..237dc9a8b4841 100644 --- a/develop/dev-guide-join-tables.md +++ b/develop/dev-guide-join-tables.md @@ -253,3 +253,17 @@ For more information about the implementation details and limitations of this Jo - [Explain Statements That Use Joins](/explain-joins.md) - [Introduction to Join Reorder](/join-reorder.md) + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-object-naming-guidelines.md b/develop/dev-guide-object-naming-guidelines.md index 04e03b8202b99..8939b7b8fd48f 100644 --- a/develop/dev-guide-object-naming-guidelines.md +++ b/develop/dev-guide-object-naming-guidelines.md @@ -43,4 +43,18 @@ It is recommended to differentiate database names by business, product, or other - Primary key index: `pk_{table_name_abbreviation}_{field_name_abbreviation}` - Unique index: `uk_{table_name_abbreviation}_{field_name_abbreviation}` - Common index: `idx_{table_name_abbreviation}_{field_name_abbreviation}` -- Column name with multiple words: use meaningful abbreviations \ No newline at end of file +- Column name with multiple words: use meaningful abbreviations + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-optimistic-and-pessimistic-transaction.md b/develop/dev-guide-optimistic-and-pessimistic-transaction.md index b20a2be9e3f7c..7f441d257a54c 100644 --- a/develop/dev-guide-optimistic-and-pessimistic-transaction.md +++ b/develop/dev-guide-optimistic-and-pessimistic-transaction.md @@ -1365,4 +1365,18 @@ mysql> SELECT * FROM users; | 2 | 9600.00 | Alice | +----+----------+----------+ 2 rows in set (0.00 sec) -``` \ No newline at end of file +``` + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-optimize-sql-best-practices.md b/develop/dev-guide-optimize-sql-best-practices.md index 6463e68195477..cdd8733c6dba9 100644 --- a/develop/dev-guide-optimize-sql-best-practices.md +++ b/develop/dev-guide-optimize-sql-best-practices.md @@ -190,3 +190,17 @@ See [Best Practices for Developing Java Applications with TiDB](https://docs.pin - [Highly Concurrent Write Best Practices](https://docs.pingcap.com/tidb/stable/high-concurrency-best-practices) + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-optimize-sql-overview.md b/develop/dev-guide-optimize-sql-overview.md index b3468ba51a505..9b6e30e4a0caf 100644 --- a/develop/dev-guide-optimize-sql-overview.md +++ b/develop/dev-guide-optimize-sql-overview.md @@ -50,3 +50,17 @@ After [tuning SQL performance](#sql-performance-tuning), if your application sti * [SQL Performance Tuning](/tidb-cloud/tidb-cloud-sql-tuning-overview.md) + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-optimize-sql.md b/develop/dev-guide-optimize-sql.md index 8651eff3452df..750b6c12e5d38 100644 --- a/develop/dev-guide-optimize-sql.md +++ b/develop/dev-guide-optimize-sql.md @@ -244,3 +244,17 @@ See [JOIN Execution Plan](/explain-joins.md). * [EXPLAIN Walkthrough](/explain-walkthrough.md) * [Explain Statements That Use Indexes](/explain-indexes.md) + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-overview.md b/develop/dev-guide-overview.md index 631d238aa2a9e..521f7693968ab 100644 --- a/develop/dev-guide-overview.md +++ b/develop/dev-guide-overview.md @@ -1,27 +1,193 @@ --- title: Developer Guide Overview -summary: Introduce the overview of the developer guide. -aliases: ['/tidb/dev/connectors-and-apis/','/appdev/dev/','/tidb/dev/dev-guide-outdated-for-laravel'] +summary: Introduce the overview of the developer guide for TiDB Cloud and TiDB Self-Managed. --- # Developer Guide Overview -This guide is written for application developers, but if you are interested in the inner workings of TiDB or want to get involved in TiDB development, read the [TiDB Kernel Development Guide](https://pingcap.github.io/tidb-dev-guide/) for more information about TiDB. + - + + + -This tutorial shows how to quickly build an application using TiDB, the possible use cases of TiDB and how to handle common problems. +## Guides by language and framework -Before reading this page, it is recommended that you read the [Quick Start Guide for the TiDB Database Platform](/quick-start-with-tidb.md). +Build your application with the language you use by following the guides with sample codes. - + + - +Connect to TiDB Cloud over HTTPS in Edge Function. + + + + +Connect Next.js with mysql2 to TiDB Cloud. + + + + +Connect to TiDB Cloud with Prisma ORM. + + + + +Connect to TiDB Cloud with TypeORM. + + + + +Connect to TiDB Cloud with Sequelize ORM. + + + + +Connect Node.js with mysql.js module to TiDB Cloud. + + + + +Connect Node.js with node-mysql2 module to TiDB Cloud. + + + + +Connect AWS Lambda Function with mysql2 to TiDB Cloud. + + + + + + + +Connect Django application with django-tidb to TiDB Cloud. + + + + +Connect to TiDB Cloud with MySQL official package. + + + + +Connect to TiDB Cloud with PyMySQL package. + + + + +Connect to TiDB Cloud with mysqlclient package. + + + + +Connect to TiDB Cloud with SQLAlchemy ORM. + + + + +Connect to TiDB Cloud with Peewee ORM. + + + + + + + +Connect to TiDB Cloud with JDBC (MySQL Connector/J). + + + + +Connect to TiDB Cloud with MyBatis ORM. + + + + +Connect to TiDB Cloud with Hibernate ORM. + + + + +Connect Spring based application with Spring Data JPA to TiDB Cloud. + + + + + + + +Connect to TiDB Cloud with MySQL driver for Go. + + + + +Connect to TiDB Cloud with GORM. + + + + + + + +Connect Ruby on Rails application with Active Record ORM to TiDB Cloud. + + + + +Connect to TiDB Cloud with mysql2 driver. -This tutorial shows how to quickly build an application using TiDB Cloud, the possible use cases of TiDB Cloud and how to handle common problems. + + + +In addition to these guides, PingCAP works with the community to [support the third-party MySQL driver, ORMs and tools](/develop/dev-guide-third-party-support.md). + +## Use MySQL client software + +As TiDB is a MySQL-compatible database, you can use many client software tools to connect to TiDB Cloud and manage the databases just like you did before. Or, use our command line tool to connect and manage your databases. + + + + +Connect and manage TiDB Cloud databases with MySQL Workbench. + + + + +Connect and manage TiDB Cloud databases with SQLTools extension in VSCode. + + + + +Connect and manage TiDB Cloud databases with DBeaver. + + + + +Connect and manage TiDB Cloud databases with DataGrip by JetBrains. + + + + +## Additional resources + +Learn other topics about developing with TiDB Cloud. + +- Use TiDB Cloud CLI to develop, manage and deploy your applications. +- Explore popular services integrations with TiDB Cloud. +- Use [TiDB database development reference](/develop/dev-guide-schema-design-overview.md) to design, interact, optimize and troubleshoot with your data and schema. +- Follow the free online course [Introduction to TiDB](https://eng.edu.pingcap.com/catalog/info/id:203/?utm_source=docs-dev-guide). + + +This guide is written for application developers, but if you are interested in the inner workings of TiDB or want to get involved in TiDB development, read the [TiDB Kernel Development Guide](https://pingcap.github.io/tidb-dev-guide/) for more information about TiDB. + +This tutorial shows how to quickly build an application using TiDB, the possible use cases of TiDB and how to handle common problems. + +Before reading this page, it is recommended that you read the [Quick Start Guide for the TiDB Database Platform](/quick-start-with-tidb.md). + ## TiDB basics Before you start working with TiDB, you need to understand some important mechanisms of how TiDB works: @@ -38,18 +204,8 @@ You can start a transaction using [`BEGIN`](/sql-statements/sql-statement-begin. TiDB guarantees atomicity for all statements between the start of `BEGIN` and the end of `COMMIT` or `ROLLBACK`, that is, all statements that are executed during this period either succeed or fail as a whole. This is used to ensure data consistency you need for application development. - - If you are not sure what an **optimistic transaction** is, do **_NOT_** use it yet. Because **optimistic transactions** require that the application can correctly handle [all errors](/error-codes.md) returned by the `COMMIT` statement. If you are not sure how your application handles them, use a **pessimistic transaction** instead. - - - - -If you are not sure what an **optimistic transaction** is, do **_NOT_** use it yet. Because **optimistic transactions** require that the application can correctly handle [all errors](https://docs.pingcap.com/tidb/stable/error-codes) returned by the `COMMIT` statement. If you are not sure how your application handles them, use a **pessimistic transaction** instead. - - - ## The way applications interact with TiDB TiDB is highly compatible with the MySQL protocol and supports [most MySQL syntax and features](/mysql-compatibility.md), so most MySQL connection libraries are compatible with TiDB. If your application framework or language does not have an official adaptation from PingCAP, it is recommended that you use MySQL's client libraries. More and more third-party libraries are actively supporting TiDB's different features. @@ -58,8 +214,6 @@ Since TiDB is compatible with the MySQL protocol and MySQL syntax, most of the O ## Read more - - - [Quick Start](/develop/dev-guide-build-cluster-in-cloud.md) - [Choose Driver or ORM](/develop/dev-guide-choose-driver-or-orm.md) - [Connect to TiDB](/develop/dev-guide-connect-to-tidb.md) @@ -70,34 +224,8 @@ Since TiDB is compatible with the MySQL protocol and MySQL syntax, most of the O - [Optimize](/develop/dev-guide-optimize-sql-overview.md) - [Example Applications](/develop/dev-guide-sample-application-java-spring-boot.md) - - - - -Here you can find additional resources to connect, manage and develop with TiDB Cloud. - -**To explore your data** +## Need help? -- [Quick Start](/develop/dev-guide-build-cluster-in-cloud.md) -- [Use AI-powered SQL Editor beta](/tidb-cloud/explore-data-with-chat2query.md) -- Connect with client tools such as [VSCode](/develop/dev-guide-gui-vscode-sqltools.md), [DBeaver](/develop/dev-guide-gui-dbeaver.md) or [DataGrip](/develop/dev-guide-gui-datagrip.md) - -**To build your application** - -- [Choose Driver or ORM](/develop/dev-guide-choose-driver-or-orm.md) -- [Use TiDB Cloud Data API beta](/tidb-cloud/data-service-overview.md) - -**To manage your cluster** - -- [TiDB Cloud Command Line Tools](/tidb-cloud/get-started-with-cli.md) -- [TiDB Cloud Administration API](https://docs.pingcap.com/tidbcloud/api/v1beta1) - -**To learn more about TiDB** - -- [Database Schema Design](/develop/dev-guide-schema-design-overview.md) -- [Write Data](/develop/dev-guide-insert-data.md) -- [Read Data](/develop/dev-guide-get-data-from-single-table.md) -- [Transaction](/develop/dev-guide-transaction-overview.md) -- [Optimize](/develop/dev-guide-optimize-sql-overview.md) +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). diff --git a/develop/dev-guide-paginate-results.md b/develop/dev-guide-paginate-results.md index f10f432033256..a61805e01575b 100644 --- a/develop/dev-guide-paginate-results.md +++ b/develop/dev-guide-paginate-results.md @@ -317,3 +317,17 @@ WHERE ) ORDER BY book_id, user_id; ``` + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-playground-gitpod.md b/develop/dev-guide-playground-gitpod.md index 7d1701617d7b8..1a1cec665bc87 100644 --- a/develop/dev-guide-playground-gitpod.md +++ b/develop/dev-guide-playground-gitpod.md @@ -33,7 +33,7 @@ After that, you will see a page similar to the following: ![playground gitpod workspace init](/media/develop/playground-gitpod-workspace-init.png) -This scenario in the page uses [TiUP](https://docs.pingcap.com/zh/tidb/stable/tiup-overview) to build a TiDB Playground. You can check the progress on the left side of the terminal area. +This scenario in the page uses [TiUP](https://docs.pingcap.com/tidb/stable/tiup-overview) to build a TiDB Playground. You can check the progress on the left side of the terminal area. Once the TiDB Playground is ready, another `Spring JPA Hibernate` task will run. You can check the progress on the right side of the terminal area. @@ -166,3 +166,17 @@ Visit `https://gitpod.io/workspaces` for all established workspaces. Gitpod provides a complete, automated, and pre-configured cloud-native development environment. You can develop, run, and test code directly in the browser without any local configurations. ![playground gitpod summary](/media/develop/playground-gitpod-summary.png) + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-prepared-statement.md b/develop/dev-guide-prepared-statement.md index 1177cb78c0cad..1bc3c510ef640 100644 --- a/develop/dev-guide-prepared-statement.md +++ b/develop/dev-guide-prepared-statement.md @@ -224,3 +224,17 @@ For a complete example in Java, see:
+ +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-proxysql-integration.md b/develop/dev-guide-proxysql-integration.md index e0465d329a0b4..dc57c8cae232d 100644 --- a/develop/dev-guide-proxysql-integration.md +++ b/develop/dev-guide-proxysql-integration.md @@ -119,11 +119,11 @@ systemctl start docker ### Option 1: Integrate TiDB Cloud with ProxySQL -For this integration, you will be using the [ProxySQL Docker image](https://hub.docker.com/r/proxysql/proxysql) along with a TiDB Serverless cluster. The following steps will set up ProxySQL on port `16033`, so make sure this port is available. +For this integration, you will be using the [ProxySQL Docker image](https://hub.docker.com/r/proxysql/proxysql) along with a TiDB Cloud Serverless cluster. The following steps will set up ProxySQL on port `16033`, so make sure this port is available. -#### Step 1. Create a TiDB Serverless cluster +#### Step 1. Create a TiDB Cloud Serverless cluster -1. [Create a free TiDB Serverless cluster](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart#step-1-create-a-tidb-cluster). Remember the root password that you set for your cluster. +1. [Create a free TiDB Cloud Serverless cluster](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart#step-1-create-a-tidb-cluster). Remember the root password that you set for your cluster. 2. Get your cluster hostname, port, and username for later use. 1. On the [Clusters](https://tidbcloud.com/console/clusters) page, click your cluster name to go to the cluster overview page. @@ -327,12 +327,12 @@ For this integration, you will be using the [ProxySQL Docker image](https://hub. > > 1. Adds a user using the username and password of your cluster. > 2. Assigns the user to the monitoring account. - > 3. Adds your TiDB Serverless cluster to the list of hosts. - > 4. Enables a secure connection between ProxySQL and the TiDB Serverless cluster. + > 3. Adds your TiDB Cloud Serverless cluster to the list of hosts. + > 4. Enables a secure connection between ProxySQL and the TiDB Cloud Serverless cluster. > > To have a better understanding, it is strongly recommended that you check the `proxysql-prepare.sql` file. To learn more about ProxySQL configuration, see [ProxySQL documentation](https://proxysql.com/documentation/proxysql-configuration/). - The following is an example output. You will see that the hostname of your cluster is shown in the output, which means that the connectivity between ProxySQL and the TiDB Serverless cluster is established. + The following is an example output. You will see that the hostname of your cluster is shown in the output, which means that the connectivity between ProxySQL and the TiDB Cloud Serverless cluster is established. ``` *************************** 1. row *************************** @@ -388,7 +388,7 @@ For this integration, you will be using the [ProxySQL Docker image](https://hub. SELECT VERSION(); ``` - If the TiDB version is displayed, you are successfully connected to your TiDB Serverless cluster through ProxySQL. To exit from the MySQL client anytime, enter `quit` and press enter. + If the TiDB version is displayed, you are successfully connected to your TiDB Cloud Serverless cluster through ProxySQL. To exit from the MySQL client anytime, enter `quit` and press enter. > **Note:** > @@ -624,7 +624,7 @@ The following steps will set up ProxySQL and TiDB on ports `6033` and `4000` res ## Production environment -For a production environment, it is recommended that you use [TiDB Cloud](https://en.pingcap.com/tidb-cloud/) directly for a fully-managed experience. +For a production environment, it is recommended that you use [TiDB Cloud Dedicated](https://www.pingcap.com/tidb-cloud-dedicated/) directly for a fully-managed experience. ### Prerequisite @@ -636,7 +636,7 @@ ProxySQL can be installed on many different platforms. The following takes CentO For a full list of supported platforms and the corresponding version requirements, see [ProxySQL documentation](https://proxysql.com/documentation/installing-proxysql/). -#### Step 1. Create a TiDB Dedicated cluster +#### Step 1. Create a TiDB Cloud Dedicated cluster For detailed steps, see [Create a TiDB Cluster](https://docs.pingcap.com/tidbcloud/create-tidb-cluster). @@ -679,7 +679,7 @@ To use ProxySQL as a proxy for TiDB, you need to configure ProxySQL. To do so, y ##### Option 1: Configure ProxySQL using the Admin Interface -1. Reconfigure ProxySQL’s internals using the standard ProxySQL Admin interface, accessible via any MySQL command line client (available by default on port `6032`): +1. Reconfigure ProxySQL's internals using the standard ProxySQL Admin interface, accessible via any MySQL command line client (available by default on port `6032`): ```bash mysql -u admin -padmin -h 127.0.0.1 -P6032 --prompt 'ProxySQL Admin> ' @@ -687,7 +687,7 @@ To use ProxySQL as a proxy for TiDB, you need to configure ProxySQL. To do so, y The above step will take you to the ProxySQL admin prompt. -2. Configure the TiDB clusters to be used, where you can add one or multiple TiDB clusters to ProxySQL. The following statement will add one TiDB Dedicated cluster for example. You need to replace `` and `` with your TiDB Cloud endpoint and port (the default port is `4000`). +2. Configure the TiDB clusters to be used, where you can add one or multiple TiDB clusters to ProxySQL. The following statement will add one TiDB Cloud Dedicated cluster for example. You need to replace `` and `` with your TiDB Cloud endpoint and port (the default port is `4000`). ```sql INSERT INTO mysql_servers(hostgroup_id, hostname, port) @@ -1124,4 +1124,18 @@ Databases can be overloaded by high traffic, faulty code, or malicious spam. Wit
- \ No newline at end of file + + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-sample-application-aws-lambda.md b/develop/dev-guide-sample-application-aws-lambda.md index df4d3d7f2c3bb..1452651bb392e 100644 --- a/develop/dev-guide-sample-application-aws-lambda.md +++ b/develop/dev-guide-sample-application-aws-lambda.md @@ -16,7 +16,7 @@ In this tutorial, you can learn how to use TiDB and mysql2 in AWS Lambda Functio > **Note** > -> This tutorial works with TiDB Serverless and TiDB Self-Hosted. +> This tutorial works with TiDB Cloud Serverless and TiDB Self-Managed. ## Prerequisites @@ -33,7 +33,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -41,7 +41,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -79,7 +79,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -87,7 +87,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -119,7 +119,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele
-
+
Copy and paste the corresponding connection string into `env.json`. The following is an example: @@ -355,7 +355,7 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). - To avoid SQL injection, it is recommended to use [prepared statements](https://github.com/sidorares/node-mysql2#using-prepared-statements). - In scenarios where there are not many complex SQL statements involved, using ORM frameworks like [Sequelize](https://sequelize.org/), [TypeORM](https://typeorm.io/), or [Prisma](https://www.prisma.io/) can greatly improve development efficiency. - For building a RESTful API for your application, it is recommended to [use AWS Lambda with API Gateway](https://docs.aws.amazon.com/lambda/latest/dg/services-apigateway.html). -- For designing high-performance applications using TiDB Serverless and AWS Lambda, refer to [this blog](https://aws.amazon.com/blogs/apn/designing-high-performance-applications-using-serverless-tidb-cloud-and-aws-lambda/). +- For designing high-performance applications using TiDB Cloud Serverless and AWS Lambda, refer to [this blog](https://aws.amazon.com/blogs/apn/designing-high-performance-applications-using-serverless-tidb-cloud-and-aws-lambda/). ## Next steps @@ -369,12 +369,12 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-golang-gorm.md b/develop/dev-guide-sample-application-golang-gorm.md index 2e5f15a4e98dc..fc3efc690f31b 100644 --- a/develop/dev-guide-sample-application-golang-gorm.md +++ b/develop/dev-guide-sample-application-golang-gorm.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and GORM to accomplish the follo > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -29,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -37,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -60,7 +60,7 @@ cd tidb-golang-gorm-quickstart Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -68,7 +68,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -102,20 +102,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. - TiDB Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. + TiDB Cloud Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -139,7 +141,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -199,7 +201,7 @@ func createDB() *gorm.DB { } ``` -When using this function, you need to replace `${tidb_host}`, `${tidb_port}`, `${tidb_user}`, `${tidb_password}`, and `${tidb_db_name}` with the actual values of your TiDB cluster. TiDB Serverless requires a secure connection. Therefore, you need to set the value of `${use_ssl}` to `true`. +When using this function, you need to replace `${tidb_host}`, `${tidb_port}`, `${tidb_user}`, `${tidb_password}`, and `${tidb_db_name}` with the actual values of your TiDB cluster. TiDB Cloud Serverless requires a secure connection. Therefore, you need to set the value of `${use_ssl}` to `true`. ### Insert data @@ -244,12 +246,12 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-golang-sql-driver.md b/develop/dev-guide-sample-application-golang-sql-driver.md index b59c38451c4fe..3ef0a3933c898 100644 --- a/develop/dev-guide-sample-application-golang-sql-driver.md +++ b/develop/dev-guide-sample-application-golang-sql-driver.md @@ -1,7 +1,6 @@ --- title: Connect to TiDB with Go-MySQL-Driver summary: Learn how to connect to TiDB using Go-MySQL-Driver. This tutorial gives Golang sample code snippets that work with TiDB using Go-MySQL-Driver. -aliases: ['/tidb/dev/dev-guide-outdated-for-go-sql-driver-mysql','/tidb/dev/dev-guide-outdated-for-gorm','/tidb/dev/dev-guide-sample-application-golang'] --- # Connect to TiDB with Go-MySQL-Driver @@ -16,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and Go-MySQL-Driver to accomplis > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -30,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -38,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -61,7 +60,7 @@ cd tidb-golang-sql-driver-quickstart Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -69,7 +68,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -103,20 +102,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. - TiDB Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. + TiDB Cloud Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -140,7 +141,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -198,7 +199,7 @@ func openDB(driverName string, runnable func(db *sql.DB)) { } ``` -When using this function, you need to replace `${tidb_host}`, `${tidb_port}`, `${tidb_user}`, `${tidb_password}`, and `${tidb_db_name}` with the actual values of your TiDB cluster. TiDB Serverless requires a secure connection. Therefore, you need to set the value of `${use_ssl}` to `true`. +When using this function, you need to replace `${tidb_host}`, `${tidb_port}`, `${tidb_user}`, `${tidb_password}`, and `${tidb_db_name}` with the actual values of your TiDB cluster. TiDB Cloud Serverless requires a secure connection. Therefore, you need to set the value of `${use_ssl}` to `true`. ### Insert data @@ -295,12 +296,12 @@ Unless you need to write complex SQL statements, it is recommended to use [ORM]( -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-java-hibernate.md b/develop/dev-guide-sample-application-java-hibernate.md index 40958421dcbbe..c493c84e4bdb3 100644 --- a/develop/dev-guide-sample-application-java-hibernate.md +++ b/develop/dev-guide-sample-application-java-hibernate.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and Hibernate to accomplish the > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -30,7 +30,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -38,7 +38,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -61,7 +61,7 @@ cd tidb-java-hibernate-quickstart Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -69,7 +69,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -103,20 +103,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. - TiDB Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. + TiDB Cloud Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. 7. Save the `env.sh` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -140,7 +142,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `env.sh` file.
-
+
1. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -267,12 +269,12 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-java-jdbc.md b/develop/dev-guide-sample-application-java-jdbc.md index bd3b7373e6557..d780b00bb3bd1 100644 --- a/develop/dev-guide-sample-application-java-jdbc.md +++ b/develop/dev-guide-sample-application-java-jdbc.md @@ -1,7 +1,6 @@ --- title: Connect to TiDB with JDBC summary: Learn how to connect to TiDB using JDBC. This tutorial gives Java sample code snippets that work with TiDB using JDBC. -aliases: ['/tidb/dev/sample-application-java','/tidb/dev/dev-guide-sample-application-java'] --- # Connect to TiDB with JDBC @@ -14,9 +13,23 @@ In this tutorial, you can learn how to use TiDB and JDBC to accomplish the follo - Connect to your TiDB cluster using JDBC. - Build and run your application. Optionally, you can find [sample code snippets](#sample-code-snippets) for basic CRUD operations. + + +> **Note:** +> +> - This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. +> - Starting from TiDB v7.4, if `connectionCollation` is not configured, and `characterEncoding` is either not configured or set to `UTF-8` in the JDBC URL, the collation used in a JDBC connection depends on the JDBC driver version. For more information, see [Collation used in JDBC connections](/faq/sql-faq.md#collation-used-in-jdbc-connections). + + + + + > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> - This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. +> - Starting from TiDB v7.4, if `connectionCollation` is not configured, and `characterEncoding` is either not configured or set to `UTF-8` in the JDBC URL, the collation used in a JDBC connection depends on the JDBC driver version. For more information, see [Collation used in JDBC connections](https://docs.pingcap.com/tidb/stable/sql-faq#collation-used-in-jdbc-connections). + + ## Prerequisites @@ -31,7 +44,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -39,11 +52,11 @@ To complete this tutorial, you need: > **Note:** > -> For security considerations, it is recommended that you use `VERIFY_IDENTITY` to establish TLS connections to TiDB clusters when connecting over the internet. Both TiDB Serverless and TiDB Dedicated use Subject Alternative Name (SAN) certificates, which require MySQL Connector/J version to be greater than or equal to [8.0.22](https://dev.mysql.com/doc/relnotes/connector-j/8.0/en/news-8-0-22.html). +> For security considerations, it is recommended that you use `VERIFY_IDENTITY` to establish TLS connections to TiDB clusters when connecting over the internet. Both TiDB Cloud Serverless and TiDB Cloud Dedicated use Subject Alternative Name (SAN) certificates, which require MySQL Connector/J version to be greater than or equal to [8.0.22](https://dev.mysql.com/doc/relnotes/connector-j/en/news-8-0-22.html). **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -66,7 +79,7 @@ cd tidb-java-jdbc-quickstart Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -74,7 +87,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -108,20 +121,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. - TiDB Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. + TiDB Cloud Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. 7. Save the `env.sh` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. + + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -145,7 +160,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `env.sh` file.
-
+
1. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -306,12 +321,12 @@ Unless you need to write complex SQL statements, it is recommended to use [ORM]( -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-java-mybatis.md b/develop/dev-guide-sample-application-java-mybatis.md index 60eeb2d8ecc40..e9a49b3efec71 100644 --- a/develop/dev-guide-sample-application-java-mybatis.md +++ b/develop/dev-guide-sample-application-java-mybatis.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and MyBatis to accomplish the fo > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -30,7 +30,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -38,7 +38,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -61,7 +61,7 @@ cd tidb-java-mybatis-quickstart Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -69,7 +69,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -103,20 +103,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. - TiDB Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. + TiDB Cloud Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. 7. Save the `env.sh` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -140,7 +142,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `env.sh` file.
-
+
1. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -317,12 +319,12 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-java-spring-boot.md b/develop/dev-guide-sample-application-java-spring-boot.md index 24dfe5678b7bc..33c832577e3e5 100644 --- a/develop/dev-guide-sample-application-java-spring-boot.md +++ b/develop/dev-guide-sample-application-java-spring-boot.md @@ -1,7 +1,6 @@ --- title: Connect to TiDB with Spring Boot summary: Learn how to connect to TiDB using Spring Boot. This tutorial gives Java sample code snippets that work with TiDB using Spring Boot. -aliases: ['/tidbcloud/dev-guide-sample-application-spring-boot','/tidb/dev/dev-guide-sample-application-spring-boot'] --- # Connect to TiDB with Spring Boot @@ -16,7 +15,7 @@ In this tutorial, you can learn how to use TiDB along with [Spring Data JPA](htt > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -31,7 +30,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -39,7 +38,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -62,7 +61,7 @@ cd tidb-java-springboot-jpa-quickstart Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -70,7 +69,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -104,20 +103,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. - TiDB Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. + TiDB Cloud Serverless requires a secure connection. Therefore, you need to set the value of `USE_SSL` to `true`. 7. Save the `env.sh` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -141,7 +142,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `env.sh` file.
-
+
1. Run the following command to copy `env.sh.example` and rename it to `env.sh`: @@ -271,12 +272,12 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-nextjs.md b/develop/dev-guide-sample-application-nextjs.md index 6ef9504a7321a..6d2f39b3bd6ce 100644 --- a/develop/dev-guide-sample-application-nextjs.md +++ b/develop/dev-guide-sample-application-nextjs.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and mysql2 in Next.js to accompl > **Note** > -> This tutorial works with TiDB Serverless and TiDB Self-Hosted. +> This tutorial works with TiDB Cloud Serverless and TiDB Self-Managed. ## Prerequisites @@ -29,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -37,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -73,7 +73,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele -
+
1. Navigate to the [**Clusters** page](https://tidbcloud.com/console/clusters), and then click the name of your target cluster to go to its overview page. @@ -81,7 +81,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -124,7 +124,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -285,12 +285,12 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-nodejs-mysql2.md b/develop/dev-guide-sample-application-nodejs-mysql2.md index 612e04f72211d..12e882abdf0b7 100644 --- a/develop/dev-guide-sample-application-nodejs-mysql2.md +++ b/develop/dev-guide-sample-application-nodejs-mysql2.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and node-mysql2 to accomplish th > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -29,13 +29,13 @@ To complete this tutorial, you need: -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -77,7 +77,7 @@ npm install mysql2 dotenv --save Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -85,7 +85,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public`. + - **Connection Type** is set to `Public`. - **Branch** is set to `main`. - **Connect With** is set to `General`. - **Operating System** matches the operating system where you run the application. @@ -111,20 +111,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > For TiDB Serverless, TLS connection **MUST** be enabled via `TIDB_ENABLE_SSL` when using public endpoint. + > For TiDB Cloud Serverless, TLS connection **MUST** be enabled via `TIDB_ENABLE_SSL` when using public endpoint. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -146,14 +148,14 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > It is recommended to enable TLS connection when using the public endpoint to connect to TiDB Dedicated. + > It is recommended to enable TLS connection when using the public endpoint to connect to TiDB Cloud Dedicated. > > To enable TLS connection, modify `TIDB_ENABLE_SSL` to `true` and using `TIDB_CA_PATH` to specify the file path of CA certificate downloaded from the connection dialog. 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -189,7 +191,7 @@ npm start If the connection is successful, the console will output the version of the TiDB cluster as follows: ``` -🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.0.0) +🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.1.2) ⏳ Loading sample game data... ✅ Loaded sample game data. @@ -244,7 +246,7 @@ void main(); > **Note** > -> For TiDB Serverless, you **MUST** enable TLS connection via `TIDB_ENABLE_SSL` when using public endpoint. However, you **don't** have to specify an SSL CA certificate via `TIDB_CA_PATH`, because Node.js uses the built-in [Mozilla CA certificate](https://wiki.mozilla.org/CA/Included_Certificates) by default, which is trusted by TiDB Serverless. +> For TiDB Cloud Serverless, you **MUST** enable TLS connection via `TIDB_ENABLE_SSL` when using public endpoint. However, you **don't** have to specify an SSL CA certificate via `TIDB_CA_PATH`, because Node.js uses the built-in [Mozilla CA certificate](https://wiki.mozilla.org/CA/Included_Certificates) by default, which is trusted by TiDB Cloud Serverless. ### Insert data @@ -311,12 +313,12 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-nodejs-mysqljs.md b/develop/dev-guide-sample-application-nodejs-mysqljs.md index e050831f0900c..347786fabcfed 100644 --- a/develop/dev-guide-sample-application-nodejs-mysqljs.md +++ b/develop/dev-guide-sample-application-nodejs-mysqljs.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and mysql.js driver to accomplis > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -29,13 +29,13 @@ To complete this tutorial, you need: -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -77,7 +77,7 @@ npm install mysql dotenv --save Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -85,7 +85,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public`. + - **Connection Type** is set to `Public`. - **Branch** is set to `main`. - **Connect With** is set to `General`. - **Operating System** matches the operating system where you run the application. @@ -111,20 +111,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > For TiDB Serverless, TLS connection **MUST** be enabled via `TIDB_ENABLE_SSL` when using public endpoint. + > For TiDB Cloud Serverless, TLS connection **MUST** be enabled via `TIDB_ENABLE_SSL` when using public endpoint. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -146,14 +148,14 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > It is recommended to enable TLS connection when using the public endpoint to connect to TiDB Dedicated. + > It is recommended to enable TLS connection when using the public endpoint to connect to TiDB Cloud Dedicated. > > To enable TLS connection, modify `TIDB_ENABLE_SSL` to `true` and using `TIDB_CA_PATH` to specify the file path of CA certificate downloaded from the connection dialog. 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -189,7 +191,7 @@ npm start If the connection is successful, the console will output the version of the TiDB cluster as follows: ``` -🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.0.0) +🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.1.2) ⏳ Loading sample game data... ✅ Loaded sample game data. @@ -240,7 +242,7 @@ conn.end(); > **Note** > -> For TiDB Serverless, you **MUST** enable TLS connection via `TIDB_ENABLE_SSL` when using public endpoint. However, you **don't** have to specify an SSL CA certificate via `TIDB_CA_PATH`, because Node.js uses the built-in [Mozilla CA certificate](https://wiki.mozilla.org/CA/Included_Certificates) by default, which is trusted by TiDB Serverless. +> For TiDB Cloud Serverless, you **MUST** enable TLS connection via `TIDB_ENABLE_SSL` when using public endpoint. However, you **don't** have to specify an SSL CA certificate via `TIDB_CA_PATH`, because Node.js uses the built-in [Mozilla CA certificate](https://wiki.mozilla.org/CA/Included_Certificates) by default, which is trusted by TiDB Cloud Serverless. ### Insert data @@ -334,12 +336,12 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-nodejs-prisma.md b/develop/dev-guide-sample-application-nodejs-prisma.md index e3fe11a0416f7..9994dc0e96e8e 100644 --- a/develop/dev-guide-sample-application-nodejs-prisma.md +++ b/develop/dev-guide-sample-application-nodejs-prisma.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and Prisma to accomplish the fol > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -29,13 +29,13 @@ To complete this tutorial, you need: -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -77,7 +77,7 @@ npm install prisma typescript ts-node @types/node --save-dev Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -85,7 +85,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public`. + - **Connection Type** is set to `Public`. - **Branch** is set to `main`. - **Connect With** is set to `Prisma`. - **Operating System** matches the operating system where you run the application. @@ -106,7 +106,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > For TiDB Serverless, you **MUST** enable TLS connection by setting `sslaccept=strict` when using public endpoint. + > For TiDB Cloud Serverless, you **MUST** enable TLS connection by setting `sslaccept=strict` when using public endpoint. 7. Save the `.env` file. 8. In the `prisma/schema.prisma`, set up `mysql` as the connection provider and `env("DATABASE_URL")` as the connection URL: @@ -119,15 +119,17 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele ```
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -143,7 +145,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > For TiDB Serverless, It is **RECOMMENDED** to enable TLS connection by setting `sslaccept=strict` when using public endpoint. When you set up `sslaccept=strict` to enable TLS connection, you **MUST** specify the file path of the CA certificate downloaded from connection dialog via `sslcert=/path/to/ca.pem`. + > For TiDB Cloud Serverless, It is **RECOMMENDED** to enable TLS connection by setting `sslaccept=strict` when using public endpoint. When you set up `sslaccept=strict` to enable TLS connection, you **MUST** specify the file path of the CA certificate downloaded from connection dialog via `sslcert=/path/to/ca.pem`. 6. Save the `.env` file. 7. In the `prisma/schema.prisma`, set up `mysql` as the connection provider and `env("DATABASE_URL")` as the connection URL: @@ -156,7 +158,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele ```
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -268,7 +270,7 @@ void main(); If the connection is successful, the terminal will output the version of the TiDB cluster as follows: ``` -🔌 Connected to TiDB cluster! (TiDB version: 5.7.25-TiDB-v6.6.0-serverless) +🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.1.2) 🆕 Created a new player with ID 1. ℹ️ Got Player 1: Player { id: 1, coins: 100, goods: 100 } 🔢 Added 50 coins and 50 goods to player 1, now player 1 has 150 coins and 150 goods. @@ -372,12 +374,12 @@ To check [referential integrity](https://en.wikipedia.org/wiki/Referential_integ -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-nodejs-sequelize.md b/develop/dev-guide-sample-application-nodejs-sequelize.md index e9648b2d921a0..8648f8b7511cf 100644 --- a/develop/dev-guide-sample-application-nodejs-sequelize.md +++ b/develop/dev-guide-sample-application-nodejs-sequelize.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and Sequelize to accomplish the > **Note** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -29,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -37,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -73,7 +73,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -81,7 +81,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -117,15 +117,17 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -149,7 +151,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -325,12 +327,12 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-nodejs-typeorm.md b/develop/dev-guide-sample-application-nodejs-typeorm.md index 681da4620d895..c05e24799f553 100644 --- a/develop/dev-guide-sample-application-nodejs-typeorm.md +++ b/develop/dev-guide-sample-application-nodejs-typeorm.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and TypeORM to accomplish the fo > **Note** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -29,13 +29,13 @@ To complete this tutorial, you need: -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -85,7 +85,7 @@ npm install @types/node ts-node typescript --save-dev Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -93,7 +93,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public`. + - **Connection Type** is set to `Public`. - **Branch** is set to `main`. - **Connect With** is set to `General`. - **Operating System** matches the operating system where you run the application. @@ -119,20 +119,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > For TiDB Serverless, you **MUST** enable TLS connection via `TIDB_ENABLE_SSL` when using public endpoint. + > For TiDB Cloud Serverless, you **MUST** enable TLS connection via `TIDB_ENABLE_SSL` when using public endpoint. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -154,12 +156,12 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > For TiDB Dedicated, it is **RECOMMENDED** to enable TLS connection via `TIDB_ENABLE_SSL` when using public endpoint. When you set up `TIDB_ENABLE_SSL=true`, you **MUST** specify the path of the CA certificate downloaded from connection dialog via `TIDB_CA_PATH=/path/to/ca.pem`. + > For TiDB Cloud Dedicated, it is **RECOMMENDED** to enable TLS connection via `TIDB_ENABLE_SSL` when using public endpoint. When you set up `TIDB_ENABLE_SSL=true`, you **MUST** specify the path of the CA certificate downloaded from connection dialog via `TIDB_CA_PATH=/path/to/ca.pem`. 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -231,7 +233,7 @@ npm start If the connection is successful, the terminal will output the version of the TiDB cluster as follows: ``` -🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.0.0) +🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.1.2) 🆕 Created a new player with ID 2. ℹ️ Got Player 2: Player { id: 2, coins: 100, goods: 100 } 🔢 Added 50 coins and 50 goods to player 2, now player 2 has 100 coins and 150 goods. @@ -274,9 +276,9 @@ export const AppDataSource = new DataSource({ > **Note** > -> For TiDB Serverless, you MUST enable TLS connection when using public endpoint. In this sample code, please set up the environment variable `TIDB_ENABLE_SSL` in the `.env` file to `true`. +> For TiDB Cloud Serverless, you MUST enable TLS connection when using public endpoint. In this sample code, please set up the environment variable `TIDB_ENABLE_SSL` in the `.env` file to `true`. > -> However, you **don't** have to specify an SSL CA certificate via `TIDB_CA_PATH`, because Node.js uses the built-in [Mozilla CA certificate](https://wiki.mozilla.org/CA/Included_Certificates) by default, which is trusted by TiDB Serverless. +> However, you **don't** have to specify an SSL CA certificate via `TIDB_CA_PATH`, because Node.js uses the built-in [Mozilla CA certificate](https://wiki.mozilla.org/CA/Included_Certificates) by default, which is trusted by TiDB Cloud Serverless. ### Insert data @@ -371,12 +373,12 @@ For more information, refer to the [TypeORM FAQ](https://typeorm.io/relations-fa -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-python-django.md b/develop/dev-guide-sample-application-python-django.md index 6d603709dd5ba..ae9e0f494f8ed 100644 --- a/develop/dev-guide-sample-application-python-django.md +++ b/develop/dev-guide-sample-application-python-django.md @@ -1,7 +1,6 @@ --- title: Connect to TiDB with Django summary: Learn how to connect to TiDB using Django. This tutorial gives Python sample code snippets that work with TiDB using Django. -aliases: ['/tidb/dev/dev-guide-outdated-for-django'] --- # Connect to TiDB with Django @@ -16,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and Django to accomplish the fol > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted clusters. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed clusters. ## Prerequisites @@ -30,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -38,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -79,7 +78,7 @@ For more information, refer to [django-tidb repository](https://github.com/pingc Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -87,7 +86,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -121,20 +120,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. - TiDB Serverless requires a secure connection. Since the `ssl_mode` of mysqlclient defaults to `PREFERRED`, you don't need to manually specify `CA_PATH`. Just leave it empty. But if you have a special reason to specify `CA_PATH` manually, you can refer to the [TLS connections to TiDB Serverless](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters) to get the certificate paths for different operating systems. + TiDB Cloud Serverless requires a secure connection. Since the `ssl_mode` of mysqlclient defaults to `PREFERRED`, you don't need to manually specify `CA_PATH`. Just leave it empty. But if you have a special reason to specify `CA_PATH` manually, you can refer to the [TLS connections to TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters) to get the certificate paths for different operating systems. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -158,7 +159,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -332,12 +333,12 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-python-mysql-connector.md b/develop/dev-guide-sample-application-python-mysql-connector.md index 2debeb43cf2c9..230ce6cd90b60 100644 --- a/develop/dev-guide-sample-application-python-mysql-connector.md +++ b/develop/dev-guide-sample-application-python-mysql-connector.md @@ -1,7 +1,6 @@ --- title: Connect to TiDB with MySQL Connector/Python summary: Learn how to connect to TiDB using MySQL Connector/Python. This tutorial gives Python sample code snippets that work with TiDB using MySQL Connector/Python. -aliases: ['/tidb/dev/dev-guide-sample-application-python','/tidb/dev/dev-guide-outdated-for-python-mysql-connector'] --- # Connect to TiDB with MySQL Connector/Python @@ -16,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and MySQL Connector/Python to ac > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted clusters. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed clusters. ## Prerequisites @@ -30,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -38,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -69,7 +68,7 @@ pip install -r requirements.txt Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -77,7 +76,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -114,15 +113,17 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -146,7 +147,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -283,12 +284,12 @@ Unless you need to write complex SQL statements, it is recommended to use [ORM]( -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-python-mysqlclient.md b/develop/dev-guide-sample-application-python-mysqlclient.md index a90d0bfa88e5d..de6da0da9c77d 100644 --- a/develop/dev-guide-sample-application-python-mysqlclient.md +++ b/develop/dev-guide-sample-application-python-mysqlclient.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and mysqlclient to accomplish th > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -29,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -37,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -70,7 +70,7 @@ If you encounter installation issues, refer to the [mysqlclient official documen Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -78,7 +78,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -112,20 +112,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. - TiDB Serverless requires a secure connection. Since the `ssl_mode` of mysqlclient defaults to `PREFERRED`, you don't need to manually specify `CA_PATH`. Just leave it empty. But if you have a special reason to specify `CA_PATH` manually, you can refer to the [TLS connections to TiDB Serverless](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters) to get the certificate paths for different operating systems. + TiDB Cloud Serverless requires a secure connection. Since the `ssl_mode` of mysqlclient defaults to `PREFERRED`, you don't need to manually specify `CA_PATH`. Just leave it empty. But if you have a special reason to specify `CA_PATH` manually, you can refer to the [TLS connections to TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters) to get the certificate paths for different operating systems. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -149,7 +151,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -284,12 +286,12 @@ Unless you need to write complex SQL statements, it is recommended to use [ORM]( -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-python-peewee.md b/develop/dev-guide-sample-application-python-peewee.md index 9b9868dae14e8..f7e00d744dc7c 100644 --- a/develop/dev-guide-sample-application-python-peewee.md +++ b/develop/dev-guide-sample-application-python-peewee.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and peewee to accomplish the fol > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted clusters. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed clusters. ## Prerequisites @@ -29,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -37,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -72,7 +72,7 @@ peewee is an ORM library that works with multiple databases. It provides a high- Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -80,7 +80,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -117,15 +117,17 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -149,7 +151,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -307,12 +309,12 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-python-pymysql.md b/develop/dev-guide-sample-application-python-pymysql.md index 500a0b18075a7..5ccab47a4a503 100644 --- a/develop/dev-guide-sample-application-python-pymysql.md +++ b/develop/dev-guide-sample-application-python-pymysql.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and PyMySQL to accomplish the fo > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted clusters. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed clusters. ## Prerequisites @@ -29,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -37,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -68,7 +68,7 @@ pip install -r requirements.txt Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -76,7 +76,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -113,15 +113,17 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -145,7 +147,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -287,12 +289,12 @@ Unless you need to write complex SQL statements, it is recommended to use [ORM]( -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-python-sqlalchemy.md b/develop/dev-guide-sample-application-python-sqlalchemy.md index f41c7a90f4672..8b398419e1fd9 100644 --- a/develop/dev-guide-sample-application-python-sqlalchemy.md +++ b/develop/dev-guide-sample-application-python-sqlalchemy.md @@ -1,7 +1,6 @@ --- title: Connect to TiDB with SQLAlchemy summary: Learn how to connect to TiDB using SQLAlchemy. This tutorial gives Python sample code snippets that work with TiDB using SQLAlchemy. -aliases: ['/tidb/dev/dev-guide-outdated-for-sqlalchemy'] --- # Connect to TiDB with SQLAlchemy @@ -16,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and SQLAlchemy to accomplish the > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted clusters. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed clusters. ## Prerequisites @@ -30,7 +29,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. @@ -38,7 +37,7 @@ To complete this tutorial, you need: **If you don't have a TiDB cluster, you can create one as follows:** -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -75,11 +74,11 @@ You can also use other database drivers, such as [mysqlclient](https://github.co Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
> **Note:** > -> Currently, TiDB Serverless clusters have a limitation: if there are no active connections for 5 minutes, they will shut down, which closes all connections. Therefore, when using SQLAlchemy with TiDB Serverless clusters, pooled connections might encounter `OperationalError` such as `Lost connection to MySQL server during query` or `MySQL Connection not available`. To avoid this error, you can set the `pool_recycle` parameter to `300`. For more information, see [Dealing with Disconnects](https://docs.sqlalchemy.org/en/20/core/pooling.html#dealing-with-disconnects) in SQLAlchemy documentation. +> Currently, TiDB Cloud Serverless clusters have a limitation: if there are no active connections for 5 minutes, they will shut down, which closes all connections. Therefore, when using SQLAlchemy with TiDB Cloud Serverless clusters, pooled connections might encounter `OperationalError` such as `Lost connection to MySQL server during query` or `MySQL Connection not available`. To avoid this error, you can set the `pool_recycle` parameter to `300`. For more information, see [Dealing with Disconnects](https://docs.sqlalchemy.org/en/20/core/pooling.html#dealing-with-disconnects) in SQLAlchemy documentation. 1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -87,7 +86,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public` + - **Connection Type** is set to `Public` - **Branch** is set to `main` - **Connect With** is set to `General` - **Operating System** matches your environment. @@ -124,15 +123,17 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -156,7 +157,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -301,12 +302,12 @@ For more information, refer to [Delete data](/develop/dev-guide-delete-data.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](/support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). -Ask questions on the [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc), or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/develop/dev-guide-sample-application-ruby-mysql2.md b/develop/dev-guide-sample-application-ruby-mysql2.md index 7c60da4a442d4..7494442faf7e6 100644 --- a/develop/dev-guide-sample-application-ruby-mysql2.md +++ b/develop/dev-guide-sample-application-ruby-mysql2.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and mysql2 to accomplish the fol > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -30,13 +30,13 @@ To complete this tutorial, you need: -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -78,7 +78,7 @@ bundle add mysql2 dotenv Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -86,7 +86,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public`. + - **Connection Type** is set to `Public`. - **Branch** is set to `main`. - **Connect With** is set to `General`. - **Operating System** matches the operating system where you run the application. @@ -112,20 +112,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > For TiDB Serverless, TLS connection **MUST** be enabled via `DATABASE_ENABLE_SSL` when using public endpoint. + > For TiDB Cloud Serverless, TLS connection **MUST** be enabled via `DATABASE_ENABLE_SSL` when using public endpoint. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -147,14 +149,14 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > It is recommended to enable TLS connection when using the public endpoint to connect to a TiDB Dedicated cluster. + > It is recommended to enable TLS connection when using the public endpoint to connect to a TiDB Cloud Dedicated cluster. > > To enable TLS connection, modify `DATABASE_ENABLE_SSL` to `true` and use `DATABASE_SSL_CA` to specify the file path of CA certificate downloaded from the connection dialog. 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -190,7 +192,7 @@ ruby app.rb If the connection is successful, the console will output the version of the TiDB cluster as follows: ``` -🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.0.0) +🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.1.2) ⏳ Loading sample game data... ✅ Loaded sample game data. @@ -229,7 +231,7 @@ client = Mysql2::Client.new(options) > **Note** > -> For TiDB Serverless, TLS connection **MUST** be enabled via `DATABASE_ENABLE_SSL` when using public endpoint, but you **don't** have to specify an SSL CA certificate via `DATABASE_SSL_CA`, because mysql2 gem will search for existing CA certificates in a particular order until a file is discovered. +> For TiDB Cloud Serverless, TLS connection **MUST** be enabled via `DATABASE_ENABLE_SSL` when using public endpoint, but you **don't** have to specify an SSL CA certificate via `DATABASE_SSL_CA`, because mysql2 gem will search for existing CA certificates in a particular order until a file is discovered. ### Insert data @@ -310,4 +312,14 @@ While it is possible to specify the CA certificate path manually, doing so might ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX) channel. \ No newline at end of file + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-sample-application-ruby-rails.md b/develop/dev-guide-sample-application-ruby-rails.md index 142bac8ec1988..44ccb35d72eb2 100644 --- a/develop/dev-guide-sample-application-ruby-rails.md +++ b/develop/dev-guide-sample-application-ruby-rails.md @@ -15,7 +15,7 @@ In this tutorial, you can learn how to use TiDB and Rails to accomplish the foll > **Note:** > -> This tutorial works with TiDB Serverless, TiDB Dedicated, and TiDB Self-Hosted. +> This tutorial works with TiDB Cloud Serverless, TiDB Cloud Dedicated, and TiDB Self-Managed. ## Prerequisites @@ -30,13 +30,13 @@ To complete this tutorial, you need: -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](/production-deployment-using-tiup.md) to create a local cluster. -- (Recommended) Follow [Creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. +- (Recommended) Follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster. - Follow [Deploy a local test TiDB cluster](https://docs.pingcap.com/tidb/stable/quick-start-with-tidb#deploy-a-local-test-cluster) or [Deploy a production TiDB cluster](https://docs.pingcap.com/tidb/stable/production-deployment-using-tiup) to create a local cluster. @@ -78,13 +78,13 @@ bundle add mysql2 dotenv Connect to your TiDB cluster depending on the TiDB deployment option you've selected. -
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. In the connection dialog, select `Rails` from the **Connect With** drop-down list and keep the default setting of the **Endpoint Type** as `Public`. +3. In the connection dialog, select `Rails` from the **Connect With** drop-down list and keep the default setting of the **Connection Type** as `Public`. 4. If you have not set a password yet, click **Generate Password** to generate a random password. @@ -102,20 +102,22 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > For TiDB Serverless, TLS connection **MUST** be enabled with the `ssl_mode=verify_identity` query parameter when using public endpoint. + > For TiDB Cloud Serverless, TLS connection **MUST** be enabled with the `ssl_mode=verify_identity` query parameter when using public endpoint. 7. Save the `.env` file.
-
+
1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Click **Allow Access from Anywhere** and then click **Download TiDB cluster CA** to download the CA certificate. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list, and then click **CA cert** to download the CA certificate. - For more details about how to obtain the connection string, refer to [TiDB Dedicated standard connection](https://docs.pingcap.com/tidbcloud/connect-via-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + + In addition to the **Public** connection type, TiDB Cloud Dedicated supports **Private Endpoint** and **VPC Peering** connection types. For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](https://docs.pingcap.com/tidbcloud/connect-to-tidb-cluster). 4. Run the following command to copy `.env.example` and rename it to `.env`: @@ -131,14 +133,14 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele > **Note** > - > It is recommended to enable TLS connection when using the public endpoint to connect to TiDB Dedicated. + > It is recommended to enable TLS connection when using the public endpoint to connect to TiDB Cloud Dedicated. > > To enable TLS connection, modify the value of the `ssl_mode` query parameter to `verify_identity` and the value of `sslca` to the file path of CA certificate downloaded from the connection dialog. 6. Save the `.env` file.
-
+
1. Run the following command to copy `.env.example` and rename it to `.env`: @@ -183,7 +185,7 @@ Connect to your TiDB cluster depending on the TiDB deployment option you've sele If the connection is successful, the console will output the version of the TiDB cluster as follows: ``` -🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.0.0) +🔌 Connected to TiDB cluster! (TiDB version: 8.0.11-TiDB-v8.1.2) ⏳ Loading sample game data... ✅ Loaded sample game data. @@ -223,7 +225,7 @@ production: > **Note** > -> For TiDB Serverless, TLS connection **MUST** be enabled via setting the `ssl_mode` query parameter to `verify_identity` in `DATABASE_URL` when using public endpoint, but you **don't** have to specify an SSL CA certificate via `DATABASE_URL`, because mysql2 gem will search for existing CA certificates in a particular order until a file is discovered. +> For TiDB Cloud Serverless, TLS connection **MUST** be enabled via setting the `ssl_mode` query parameter to `verify_identity` in `DATABASE_URL` when using public endpoint, but you **don't** have to specify an SSL CA certificate via `DATABASE_URL`, because mysql2 gem will search for existing CA certificates in a particular order until a file is discovered. ### Insert data @@ -284,4 +286,14 @@ While it is possible to specify the CA certificate path manually, this approach ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX) channel. \ No newline at end of file + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-schema-design-overview.md b/develop/dev-guide-schema-design-overview.md index cd2c47df16e56..b83fb6d735f4c 100644 --- a/develop/dev-guide-schema-design-overview.md +++ b/develop/dev-guide-schema-design-overview.md @@ -88,3 +88,17 @@ As a best practice, it is recommended that you use a [MySQL client](https://dev. ## Object limitations For more information, see [TiDB Limitations](/tidb-limitations.md). + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-sql-development-specification.md b/develop/dev-guide-sql-development-specification.md index d0254495eb946..c19f431c97150 100644 --- a/develop/dev-guide-sql-development-specification.md +++ b/develop/dev-guide-sql-development-specification.md @@ -48,7 +48,21 @@ This document introduces some general development specifications for using SQL. - Replace `OR` with `IN` or `UNION`. The number of `IN` must be less than `300`. - Avoid using the `%` prefix for fuzzy prefix queries. - If the application uses **Multi Statements** to execute SQL, that is, multiple SQLs are joined with semicolons and sent to the client for execution at once, TiDB only returns the result of the first SQL execution. -- When you use expressions, check if the expressions support computing push-down to the storage layer (TiKV or TiFlash). If not, you should expect more memory consumption and even OOM at the TiDB layer. Computing that can be pushe down the storage layer is as follows: +- When you use expressions, check if the expressions support computing push-down to the storage layer (TiKV or TiFlash). If not, you should expect more memory consumption and even OOM at the TiDB layer. Computing that can be pushed down to the storage layer is as follows: - [TiFlash supported push-down calculations](/tiflash/tiflash-supported-pushdown-calculations.md). - [TiKV - List of Expressions for Pushdown](/functions-and-operators/expressions-pushed-down.md). - [Predicate push down](/predicate-push-down.md). + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-third-party-support.md b/develop/dev-guide-third-party-support.md index 7e0a9637767a0..c8a92eea70365 100644 --- a/develop/dev-guide-third-party-support.md +++ b/develop/dev-guide-third-party-support.md @@ -44,7 +44,7 @@ If you encounter problems when connecting to TiDB using the tools listed in this v1.6.0 Full N/A - Connect to TiDB with Go-MySQL-Driver + Connect to TiDB with Go-MySQL-Driver Java @@ -53,11 +53,11 @@ If you encounter problems when connecting to TiDB using the tools listed in this Full - Connect to TiDB with JDBC + Connect to TiDB with JDBC @@ -82,7 +82,7 @@ If you encounter problems when connecting to TiDB using the tools listed in this v1.23.5 Full N/A - Connect to TiDB with GORM + Connect to TiDB with GORM beego @@ -111,21 +111,21 @@ If you encounter problems when connecting to TiDB using the tools listed in this 6.1.0.Final Full N/A - Connect to TiDB with Hibernate + Connect to TiDB with Hibernate MyBatis v3.5.10 Full N/A - Connect to TiDB with MyBatis + Connect to TiDB with MyBatis Spring Data JPA 2.7.2 Full N/A - Connect to TiDB with Spring Boot + Connect to TiDB with Spring Boot jOOQ @@ -140,7 +140,7 @@ If you encounter problems when connecting to TiDB using the tools listed in this v7.0 Full N/A - Connect to TiDB with Rails Framework and ActiveRecord ORM + Connect to TiDB with Rails Framework and ActiveRecord ORM JavaScript / TypeScript @@ -148,21 +148,21 @@ If you encounter problems when connecting to TiDB using the tools listed in this v6.20.1 Full N/A - Connect to TiDB with Sequelize + Connect to TiDB with Sequelize Prisma 4.16.2 Full N/A - Connect to TiDB with Prisma + Connect to TiDB with Prisma TypeORM v0.3.17 Full N/A - Connect to TiDB with TypeORM + Connect to TiDB with TypeORM Python @@ -170,22 +170,37 @@ If you encounter problems when connecting to TiDB using the tools listed in this v4.2 Full django-tidb - Connect to TiDB with Django + Connect to TiDB with Django SQLAlchemy v1.4.37 Full N/A - Connect to TiDB with SQLAlchemy + Connect to TiDB with SQLAlchemy ## GUI -| GUI | Latest tested version | Support level | Tutorial | -|-----------------------------------------------------------|-----------------------|---------------|-------------------------------------------------------------------------------| -| [JetBrains DataGrip](https://www.jetbrains.com/datagrip/) | 2023.2.1 | Full | [Connect to TiDB with JetBrains DataGrip](/develop/dev-guide-gui-datagrip.md) | -| [DBeaver](https://dbeaver.io/) | 23.0.3 | Full | [Connect to TiDB with DBeaver](/develop/dev-guide-gui-dbeaver.md) | -| [Visual Studio Code](https://code.visualstudio.com/) | 1.72.0 | Full | [Connect to TiDB with Visual Studio Code](/develop/dev-guide-gui-vscode-sqltools.md) | +| GUI | Latest tested version | Support level | Tutorial | +|-----------------------------------------------------------|-----------------------|---------------|--------------------------------------------------------------------------------------| +| [Beekeeper Studio](https://www.beekeeperstudio.io/) | 4.3.0 | Full | N/A | +| [JetBrains DataGrip](https://www.jetbrains.com/datagrip/) | 2023.2.1 | Full | [Connect to TiDB with JetBrains DataGrip](/develop/dev-guide-gui-datagrip.md) | +| [DBeaver](https://dbeaver.io/) | 23.0.3 | Full | [Connect to TiDB with DBeaver](/develop/dev-guide-gui-dbeaver.md) | +| [Visual Studio Code](https://code.visualstudio.com/) | 1.72.0 | Full | [Connect to TiDB with Visual Studio Code](/develop/dev-guide-gui-vscode-sqltools.md) | + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-third-party-tools-compatibility.md b/develop/dev-guide-third-party-tools-compatibility.md index 32dc4a0a48749..8e6c5f0bbf1ad 100644 --- a/develop/dev-guide-third-party-tools-compatibility.md +++ b/develop/dev-guide-third-party-tools-compatibility.md @@ -231,3 +231,17 @@ To allow the removal of the `AUTO_INCREMENT` attribute, set `@@tidb_allow_remove **Description** `FULLTEXT`, `HASH`, and `SPATIAL` indexes are not supported. + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-tidb-crud-sql.md b/develop/dev-guide-tidb-crud-sql.md index 9fa4493e245f6..f0315f1f4540b 100644 --- a/develop/dev-guide-tidb-crud-sql.md +++ b/develop/dev-guide-tidb-crud-sql.md @@ -1,15 +1,15 @@ --- title: CRUD SQL in TiDB -summary: A brief introduction to TiDB's CURD SQL. +summary: A brief introduction to TiDB's CRUD SQL. --- # CRUD SQL in TiDB -This document briefly introduces how to use TiDB's CURD SQL. +This document briefly introduces how to use TiDB's CRUD SQL. ## Before you start -Please make sure you are connected to a TiDB cluster. If not, refer to [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster) to create a TiDB Serverless cluster. +Please make sure you are connected to a TiDB cluster. If not, refer to [Build a TiDB Cloud Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-cloud-serverless-cluster) to create a TiDB Cloud Serverless cluster. ## Explore SQL with TiDB @@ -21,7 +21,7 @@ TiDB is compatible with MySQL, you can use MySQL statements directly in most cas To experiment with SQL and test out TiDB compatibility with MySQL queries, you can try [TiDB Playground](https://play.tidbcloud.com/?utm_source=docs&utm_medium=basic-sql-operations). You can also first deploy a TiDB cluster and then run SQL statements in it. -This page walks you through the basic TiDB SQL statements such as DDL, DML, and CRUD operations. For a complete list of TiDB statements, see [TiDB SQL Syntax Diagram](https://pingcap.github.io/sqlgram/). +This page walks you through the basic TiDB SQL statements such as DDL, DML, and CRUD operations. For a complete list of TiDB statements, see [SQL Statement Overview](/sql-statements/sql-statement-overview.md). ## Category @@ -35,7 +35,7 @@ SQL is divided into the following 4 types according to their functions: - **DCL (Data Control Language)**: It is used to define access privileges and security levels. -The following mainly introduces DML and DQL. For more information about DDL and DCL, see [Explore SQL with TiDB](/basic-sql-operations.md) or [TiDB SQL syntax detailed explanation](https://pingcap.github.io/sqlgram/). +The following mainly introduces DML and DQL. For more information about DDL and DCL, see [Explore SQL with TiDB](/basic-sql-operations.md) or [SQL Statement Overview](/sql-statements/sql-statement-overview.md). ## Data Manipulation Language @@ -101,3 +101,17 @@ Use the `WHERE` clause to filter all records that match the conditions and then ```sql SELECT * FROM person WHERE id < 5; ``` + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-timeouts-in-tidb.md b/develop/dev-guide-timeouts-in-tidb.md index 632391c9bf836..395cd564a1303 100644 --- a/develop/dev-guide-timeouts-in-tidb.md +++ b/develop/dev-guide-timeouts-in-tidb.md @@ -30,7 +30,7 @@ Note that the system variable configuration takes effect globally and immediatel > **Tip:** > -> Specifically, when Dumpling is exporting data from TiDB (less than 1 TB), if the TiDB version is later than or equal to v4.0.0 and Dumpling can access the PD address of the TiDB cluster, Dumpling automatically extends the GC time without affecting the original cluster. +> Specifically, when Dumpling is exporting data from TiDB (less than 1 TB), if the TiDB version is v4.0.0 or later and Dumpling can access the PD address and the [`INFORMATION_SCHEMA.CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) table of the TiDB cluster, Dumpling automatically adjusts the GC safe point to block GC without affecting the original cluster. > > However, in either of the following scenarios, Dumpling cannot automatically adjust the GC time: > @@ -88,3 +88,17 @@ However, in a real production environment, idle connections and indefinitely exe - `sessionVariables=wait_timeout=3600` (1 hour) - `sessionVariables=max_execution_time=300000` (5 minutes) + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-transaction-overview.md b/develop/dev-guide-transaction-overview.md index 4e9358707726f..aa32c042aa499 100644 --- a/develop/dev-guide-transaction-overview.md +++ b/develop/dev-guide-transaction-overview.md @@ -158,4 +158,18 @@ mysql> SET TRANSACTION ISOLATION LEVEL SERIALIZABLE; ERROR 8048 (HY000): The isolation level 'SERIALIZABLE' is not supported. Set tidb_skip_isolation_level_check=1 to skip this error ``` -TiDB implements Snapshot Isolation (SI) level consistency, also known as "repeatable read" for consistency with MySQL. This isolation level is different from [ANSI Repeatable Read Isolation Level](/transaction-isolation-levels.md#difference-between-tidb-and-ansi-repeatable-read) and [MySQL Repeatable Read Isolation Level](/transaction-isolation-levels.md#difference-between-tidb-and-mysql-repeatable-read). For more details, see [TiDB Transaction Isolation Levels](/transaction-isolation-levels.md). \ No newline at end of file +TiDB implements Snapshot Isolation (SI) level consistency, also known as "repeatable read" for consistency with MySQL. This isolation level is different from [ANSI Repeatable Read Isolation Level](/transaction-isolation-levels.md#difference-between-tidb-and-ansi-repeatable-read) and [MySQL Repeatable Read Isolation Level](/transaction-isolation-levels.md#difference-between-tidb-and-mysql-repeatable-read). For more details, see [TiDB Transaction Isolation Levels](/transaction-isolation-levels.md). + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-transaction-restraints.md b/develop/dev-guide-transaction-restraints.md index 6c180e6d29bf6..c6fdc32923243 100644 --- a/develop/dev-guide-transaction-restraints.md +++ b/develop/dev-guide-transaction-restraints.md @@ -681,6 +681,10 @@ mysql> SELECT * FROM doctors; ## Support for `savepoint` and nested transactions +> **Note:** +> +> Starting from v6.2.0, TiDB supports the [`savepoint`](/sql-statements/sql-statement-savepoint.md) feature. If your TiDB cluster is earlier than v6.2.0, your TiDB cluster does not support the `PROPAGATION_NESTED` behavior. It is recommended to upgrade to v6.2.0 or a later version. If upgrading TiDB is not possible, and your applications are based on the **Java Spring** framework that uses the `PROPAGATION_NESTED` propagation behavior, you need to adapt it on the application side to remove the logic for nested transactions. + The `PROPAGATION_NESTED` propagation behavior supported by **Spring** triggers a nested transaction, which is a child transaction that is started independently of the current transaction. A `savepoint` is recorded when the nested transaction starts. If the nested transaction fails, the transaction will roll back to the `savepoint` state. The nested transaction is part of the outer transaction and will be committed together with the outer transaction. The following example demonstrates the `savepoint` mechanism: @@ -701,16 +705,19 @@ mysql> SELECT * FROM T2; +------+ ``` -> **Note:** -> -> Since v6.2.0, TiDB supports the `savepoint` feature. If your TiDB cluster is earlier than v6.2.0, your TiDB cluster does not support the `PROPAGATION_NESTED` behavior. If your applications are based on the **Java Spring** framework that use the `PROPAGATION_NESTED` propagation behavior, you need to adapt it on the application side to remove the logic for nested transactions. - ## Large transaction restrictions The basic principle is to limit the size of the transaction. At the KV level, TiDB has a restriction on the size of a single transaction. At the SQL level, one row of data is mapped to one KV entry, and each additional index will add one KV entry. The restriction is as follows at the SQL level: -- The maximum single row record size is `120 MB`. You can configure it by `performance.txn-entry-size-limit` for TiDB v5.0 and later versions. The value is `6 MB` for earlier versions. -- The maximum single transaction size supported is `10 GB`. You can configure it by `performance.txn-total-size-limit` for TiDB v4.0 and later versions. The value is `100 MB` for earlier versions. +- The maximum single row record size is 120 MiB. + + - You can adjust it by using the [`performance.txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v4010-and-v500) configuration parameter of tidb-server for TiDB v4.0.10 and later v4.0.x versions, TiDB v5.0.0 and later versions. The value is `6 MB` for versions earlier than v4.0.10. + - Starting from v7.6.0, you can use the [`tidb_txn_entry_size_limit`](/system-variables.md#tidb_txn_entry_size_limit-new-in-v760) system variable to dynamically modify the value of this configuration item. + +- The maximum single transaction size supported is 1 TiB. + + - For TiDB v4.0 and later versions, You can configure it by [`performance.txn-total-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-total-size-limit)). The value is `100 MB` for earlier versions. + - For TiDB v6.5.0 and later versions, this configuration is no longer recommended. For more information, see [`performance.txn-total-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-total-size-limit)). Note that for both the size restrictions and row restrictions, you should also consider the overhead of encoding and additional keys for the transaction during the transaction execution. To achieve optimal performance, it is recommended to write one transaction every 100 ~ 500 rows. @@ -721,3 +728,17 @@ Currently locks are not added to auto-committed `SELECT FOR UPDATE` statements. ![The situation in TiDB](/media/develop/autocommit_selectforupdate_nowaitlock.png) This is a known incompatibility issue with MySQL. You can solve this issue by using the explicit `BEGIN;COMMIT;` statements. + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + diff --git a/develop/dev-guide-transaction-troubleshoot.md b/develop/dev-guide-transaction-troubleshoot.md index 03d37ceb22c67..000703d0c2679 100644 --- a/develop/dev-guide-transaction-troubleshoot.md +++ b/develop/dev-guide-transaction-troubleshoot.md @@ -153,4 +153,18 @@ For information about how to troubleshoot and resolve transaction conflicts, see - [Troubleshoot Write Conflicts in Optimistic Transactions](https://docs.pingcap.com/tidb/stable/troubleshoot-write-conflicts) + + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + \ No newline at end of file diff --git a/develop/dev-guide-troubleshoot-overview.md b/develop/dev-guide-troubleshoot-overview.md index acc046c208d33..77428c7aaea03 100644 --- a/develop/dev-guide-troubleshoot-overview.md +++ b/develop/dev-guide-troubleshoot-overview.md @@ -42,3 +42,17 @@ See [Handle transaction errors](/develop/dev-guide-transaction-troubleshoot.md). - [TiDB FAQs](/faq/tidb-faq.md) + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-unique-serial-number-generation.md b/develop/dev-guide-unique-serial-number-generation.md index 0af1adb2917f9..ee608ac39fa52 100644 --- a/develop/dev-guide-unique-serial-number-generation.md +++ b/develop/dev-guide-unique-serial-number-generation.md @@ -49,3 +49,17 @@ The number allocation solution can be understood as bulk acquisition of auto-inc Every time, the application gets a segment of sequence numbers at the configured step. It updates the database at the same time to persist the maximum value of the current sequence that has been allocated. The processing and allocation of sequence numbers are completed in the application's memory. After a segment of sequence numbers is used up, the application gets a new segment of sequence numbers, which effectively alleviates the pressure on the database write. In practice, you can also adjust the step to control the frequency of database updates. Finally, note that the IDs generated by the above two solutions are not random enough to be directly used as **primary keys** for TiDB tables. In practice, you can perform bit-reverse on the generated IDs to get more random new IDs. For example, after performing bit-reverse, the ID `00000010100101000001111010011100` becomes `00111001011110000010100101000000`, and `11111111111111111111111111111101` becomes `10111111111111111111111111111111`. + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-unstable-result-set.md b/develop/dev-guide-unstable-result-set.md index dacc60b3d332c..2b166f8b75d37 100644 --- a/develop/dev-guide-unstable-result-set.md +++ b/develop/dev-guide-unstable-result-set.md @@ -234,3 +234,17 @@ To let `GROUP_CONCAT()` get the result set output in order, you need to add the ## Unstable results in `SELECT * FROM T LIMIT N` The returned result is related to the distribution of data on the storage node (TiKV). If multiple queries are performed, different storage units (Regions) of the storage nodes (TiKV) return results at different speeds, which can cause unstable results. + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-update-data.md b/develop/dev-guide-update-data.md index 2dcb69a80ccbb..bc5f207f627b0 100644 --- a/develop/dev-guide-update-data.md +++ b/develop/dev-guide-update-data.md @@ -14,7 +14,7 @@ This document describes how to use the following SQL statements to update the da Before reading this document, you need to prepare the following: -- [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). +- [Build a TiDB Cloud Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). - Read [Schema Design Overview](/develop/dev-guide-schema-design-overview.md), [Create a Database](/develop/dev-guide-create-database.md), [Create a Table](/develop/dev-guide-create-table.md), and [Create Secondary Indexes](/develop/dev-guide-create-secondary-indexes.md). - If you want to `UPDATE` data, you need to [insert data](/develop/dev-guide-insert-data.md) first. @@ -447,3 +447,17 @@ In each iteration, `SELECT` queries in order of the primary key. It selects prim
+ +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-use-common-table-expression.md b/develop/dev-guide-use-common-table-expression.md index 71854c515692f..5ba101523e59e 100644 --- a/develop/dev-guide-use-common-table-expression.md +++ b/develop/dev-guide-use-common-table-expression.md @@ -214,3 +214,17 @@ The result is as follows: ## Read more - [WITH](/sql-statements/sql-statement-with.md) + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-use-follower-read.md b/develop/dev-guide-use-follower-read.md index 4571756bb3337..611730161374e 100644 --- a/develop/dev-guide-use-follower-read.md +++ b/develop/dev-guide-use-follower-read.md @@ -160,3 +160,17 @@ public static class AuthorDAO { - [TiDB Cloud Key Visualizer Page](/tidb-cloud/tune-performance.md#key-visualizer) + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-use-stale-read.md b/develop/dev-guide-use-stale-read.md index fbdb8654cd33f..9f374c5ac1a7a 100644 --- a/develop/dev-guide-use-stale-read.md +++ b/develop/dev-guide-use-stale-read.md @@ -497,3 +497,17 @@ public static class StaleReadHelper{ - [Usage Scenarios of Stale Read](/stale-read.md) - [Read Historical Data Using the `AS OF TIMESTAMP` Clause](/as-of-timestamp.md) - [Read Historical Data Using the `tidb_read_staleness` System Variable](/tidb-read-staleness.md) + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-use-subqueries.md b/develop/dev-guide-use-subqueries.md index 1eb078c5ea9a2..9e6fe2c86b23d 100644 --- a/develop/dev-guide-use-subqueries.md +++ b/develop/dev-guide-use-subqueries.md @@ -9,7 +9,7 @@ This document introduces subquery statements and categories in TiDB. ## Overview -An subquery is a query within another SQL query. With subquery, the query result can be used in another query. +A subquery is a query within another SQL query. With subquery, the query result can be used in another query. The following takes the [Bookshop](/develop/dev-guide-bookshop-schema-design.md) application as an example to introduce subquery. @@ -127,4 +127,18 @@ As a best practice, in actual development, it is recommended to avoid querying t - [Subquery Related Optimizations](/subquery-optimization.md) - [Decorrelation of Correlated Subquery](/correlated-subquery-optimization.md) -- [Subquery Optimization in TiDB](https://en.pingcap.com/blog/subquery-optimization-in-tidb/) +- [Subquery Optimization in TiDB](https://www.pingcap.com/blog/subquery-optimization-in-tidb/) + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-use-temporary-tables.md b/develop/dev-guide-use-temporary-tables.md index 279a3dbab8e27..f74168012decc 100644 --- a/develop/dev-guide-use-temporary-tables.md +++ b/develop/dev-guide-use-temporary-tables.md @@ -257,3 +257,17 @@ For limitations of temporary tables in TiDB, see [Compatibility restrictions wit ## Read more - [Temporary Tables](/temporary-tables.md) + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/develop/dev-guide-use-views.md b/develop/dev-guide-use-views.md index 01143079f5f00..83f30d2141cbc 100644 --- a/develop/dev-guide-use-views.md +++ b/develop/dev-guide-use-views.md @@ -122,3 +122,17 @@ For limitations of views in TiDB, see [Limitations of Views](/views.md#limitatio - [DROP VIEW Statement](/sql-statements/sql-statement-drop-view.md) - [EXPLAIN Statements Using Views](/explain-views.md) - [TiFlink: Strongly Consistent Materialized Views Using TiKV and Flink](https://github.com/tiflink/tiflink) + +## Need help? + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](/support.md). + + + + + +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). + + \ No newline at end of file diff --git a/dm/deploy-a-dm-cluster-using-binary.md b/dm/deploy-a-dm-cluster-using-binary.md index 17e9b26f55fce..8e94e724aeee3 100644 --- a/dm/deploy-a-dm-cluster-using-binary.md +++ b/dm/deploy-a-dm-cluster-using-binary.md @@ -1,7 +1,6 @@ --- title: Deploy Data Migration Using DM Binary summary: Learn how to deploy a Data Migration cluster using DM binary. -aliases: ['/docs/tidb-data-migration/dev/deploy-a-dm-cluster-using-binary/'] --- # Deploy Data Migration Using DM Binary diff --git a/dm/deploy-a-dm-cluster-using-tiup-offline.md b/dm/deploy-a-dm-cluster-using-tiup-offline.md index 16a4f464f1784..007cf6ff7c0a5 100644 --- a/dm/deploy-a-dm-cluster-using-tiup-offline.md +++ b/dm/deploy-a-dm-cluster-using-tiup-offline.md @@ -127,7 +127,7 @@ alertmanager_servers: > > - Use `.` to indicate the subcategory of the configuration, such as `log.slow-threshold`. For more formats, see [TiUP configuration template](https://github.com/pingcap/tiup/blob/master/embed/examples/dm/topology.example.yaml). > -> - For more parameter description, see [master `config.toml.example`](https://github.com/pingcap/tiflow/blob/master/dm/master/dm-master.toml) and [worker `config.toml.example`](https://github.com/pingcap/tiflow/blob/master/dm/worker/dm-worker.toml). +> - For more parameter description, see [master `config.toml.example`](https://github.com/pingcap/tiflow/blob/release-8.1/dm/master/dm-master.toml) and [worker `config.toml.example`](https://github.com/pingcap/tiflow/blob/release-8.1/dm/worker/dm-worker.toml). > > - Make sure that the ports among the following components are interconnected: > - The `peer_port` (`8291` by default) among the DM-master nodes are interconnected. diff --git a/dm/deploy-a-dm-cluster-using-tiup.md b/dm/deploy-a-dm-cluster-using-tiup.md index 26b6cdb054b24..c0ffabe3c3b97 100644 --- a/dm/deploy-a-dm-cluster-using-tiup.md +++ b/dm/deploy-a-dm-cluster-using-tiup.md @@ -1,7 +1,6 @@ --- title: Deploy a DM Cluster Using TiUP summary: Learn how to deploy TiDB Data Migration using TiUP DM. -aliases: ['/docs/tidb-data-migration/dev/deploy-a-dm-cluster-using-ansible/','/docs/tools/dm/deployment/'] --- # Deploy a DM Cluster Using TiUP @@ -16,9 +15,11 @@ TiUP supports deploying DM v2.0 or later DM versions. This document introduces h ## Prerequisites -When DM performs a full data replication task, the DM-worker is bound with only one upstream database. The DM-worker first exports the full amount of data locally, and then imports the data into the downstream database. Therefore, the worker's host space must be large enough to store all upstream tables to be exported. The storage path is specified later when you create the task. +- When DM performs a full data replication task, the DM-worker is bound with only one upstream database. The DM-worker first exports the full amount of data locally, and then imports the data into the downstream database. Therefore, the worker's host space must be large enough to store all upstream tables to be exported. The storage path is specified later when you create the task. -In addition, you need to meet the [hardware and software requirements](/dm/dm-hardware-and-software-requirements.md) when deploying a DM cluster. +- You need to meet the [hardware and software requirements](/dm/dm-hardware-and-software-requirements.md) when deploying a DM cluster. + +- Starting from v8.0.0, if you need to [encrypt database passwords](/dm/dm-manage-source.md#encrypt-the-database-password), you must store [a key file used for encrypting and decrypting database passwords](/dm/dm-customized-secret-key.md) in the DM-master in advance, and configure [`secret-key-path`](/dm/dm-master-configuration-file.md) for DM-master before using the `dmctl encrypt` command. ## Step 1: Install TiUP on the control machine @@ -144,7 +145,7 @@ alertmanager_servers: > - The TiUP nodes can connect to the `port` of all DM-master nodes (`8261` by default). > - The TiUP nodes can connect to the `port` of all DM-worker nodes (`8262` by default). -For more `master_servers.host.config` parameter description, refer to [master parameter](https://github.com/pingcap/tiflow/blob/master/dm/master/dm-master.toml). For more `worker_servers.host.config` parameter description, refer to [worker parameter](https://github.com/pingcap/tiflow/blob/master/dm/worker/dm-worker.toml). +For more `master_servers.host.config` parameter description, refer to [master parameter](https://github.com/pingcap/tiflow/blob/release-8.1/dm/master/dm-master.toml). For more `worker_servers.host.config` parameter description, refer to [worker parameter](https://github.com/pingcap/tiflow/blob/release-8.1/dm/worker/dm-worker.toml). ## Step 3: Execute the deployment command @@ -230,3 +231,7 @@ If the `Status` is `Up` in the output, the cluster status is normal. dmctl is a command-line tool used to control DM clusters. You are recommended to [use dmctl via TiUP](/dm/maintain-dm-using-tiup.md#dmctl). dmctl supports both the command mode and the interactive mode. For details, see [Maintain DM Clusters Using dmctl](/dm/dmctl-introduction.md#maintain-dm-clusters-using-dmctl). + +## Step 9: Encrypt the database password + +After deployment, you can configure DM tasks to encrypt the database password. For more information, see [Encrypt the database password](/dm/dm-manage-source.md#encrypt-the-database-password). diff --git a/dm/dm-best-practices.md b/dm/dm-best-practices.md index dddfb81835afe..dfeff1a1c2c01 100644 --- a/dm/dm-best-practices.md +++ b/dm/dm-best-practices.md @@ -5,7 +5,7 @@ summary: Learn about best practices when you use TiDB Data Migration (DM) to mig # TiDB Data Migration (DM) Best Practices -[TiDB Data Migration (DM)](https://github.com/pingcap/tiflow/tree/master/dm) is a data migration tool developed by PingCAP. It supports full and incremental data migration from MySQL-compatible databases such as MySQL, Percona MySQL, MariaDB, Amazon RDS for MySQL, and Amazon Aurora into TiDB. +[TiDB Data Migration (DM)](https://github.com/pingcap/tiflow/tree/release-8.1/dm) is a data migration tool developed by PingCAP. It supports full and incremental data migration from MySQL-compatible databases such as MySQL, Percona MySQL, MariaDB, Amazon RDS for MySQL, and Amazon Aurora into TiDB. You can use DM in the following scenarios: @@ -41,7 +41,7 @@ To distribute the workload evenly on multiple nodes, the design for the distribu `AUTO_INCREMENT` in TiDB is compatible with `AUTO_INCREMENT` in MySQL. However, as a distributed database, TiDB usually has multiple computing nodes (entries for the client end). When the application data is written, the workload is evenly distributed. This leads to the result that when there is an `AUTO_INCREMENT` column in the table, the auto-increment IDs of the column might be inconsecutive. For more details, see [AUTO_INCREMENT](/auto-increment.md#implementation-principles). -If your business has a strong dependence on auto-increment IDs, consider using the [SEQUENCE function](/sql-statements/sql-statement-create-sequence.md#sequence-function). +If your business has a strong dependence on auto-increment IDs, consider using the [MySQL-compatible `AUTO_INCREMENT` mode](/auto-increment.md#mysql-compatibility-mode) or the [`SEQUENCE` function](/sql-statements/sql-statement-create-sequence.md#sequence-function). #### Usage of clustered indexes @@ -99,7 +99,7 @@ The following table summarizes the pros and cons of optimistic mode and pessimis | Scenario | Pros | Cons | | :--- | :--- | :--- | -| Pessimistic mode (Default) | It can ensure that the data migrated to the downstream will not go wrong. | If there are a large number of shards, the migration task will be blocked for a long time, or even stop if the upstream binlogs have been cleaned up. You can enable the relay log to avoid this problem. For more information, see [Use the relay log](#use-the relay-log). | +| Pessimistic mode (Default) | It can ensure that the data migrated to the downstream will not go wrong. | If there are a large number of shards, the migration task will be blocked for a long time, or even stop if the upstream binlogs have been cleaned up. You can enable the relay log to avoid this problem. For more information, see [Use the relay log](#use-the-relay-log). | | Optimistic mode| Upstream schema changes will not cause data migration latency. | In this mode, ensure that schema changes are compatible (check whether the incremental column has a default value). It is possible that the inconsistent data can be overlooked. For more information, see [Merge and Migrate Data from Sharded Tables in Optimistic Mode](/dm/feature-shard-merge-optimistic.md#restrictions).| ### Other restrictions and impact diff --git a/dm/dm-compatibility-catalog.md b/dm/dm-compatibility-catalog.md index c3fbe60afc57f..d6bd26c0184ea 100644 --- a/dm/dm-compatibility-catalog.md +++ b/dm/dm-compatibility-catalog.md @@ -32,7 +32,9 @@ DM supports migrating data from different sources to TiDB clusters. Based on the |Target database|Compatibility level|DM version| |-|-|-| -|TiDB 6.0|GA|≥ 5.3.1| +|TiDB 8.x|GA|≥ 5.3.1| +|TiDB 7.x|GA|≥ 5.3.1| +|TiDB 6.x|GA|≥ 5.3.1| |TiDB 5.4|GA|≥ 5.3.1| |TiDB 5.3|GA|≥ 5.3.1| |TiDB 5.2|GA|≥ 2.0.7, recommended: 5.4| @@ -41,4 +43,4 @@ DM supports migrating data from different sources to TiDB clusters. Based on the |TiDB 4.x|GA|≥ 2.0.1, recommended: 2.0.7| |TiDB 3.x|GA|≥ 2.0.1, recommended: 2.0.7| |MySQL|Experimental|| -|MariaDB|Experimental|| \ No newline at end of file +|MariaDB|Experimental|| diff --git a/dm/dm-config-overview.md b/dm/dm-config-overview.md index 449a836fee8c8..118067df24373 100644 --- a/dm/dm-config-overview.md +++ b/dm/dm-config-overview.md @@ -1,7 +1,6 @@ --- title: Data Migration Configuration File Overview summary: This document gives an overview of Data Migration configuration files. -aliases: ['/docs/tidb-data-migration/dev/config-overview/'] --- # Data Migration Configuration File Overview diff --git a/dm/dm-continuous-data-validation.md b/dm/dm-continuous-data-validation.md index 1507761b1e76f..d42748d50ecd4 100644 --- a/dm/dm-continuous-data-validation.md +++ b/dm/dm-continuous-data-validation.md @@ -282,7 +282,7 @@ The lifecycle of continuous data validation is as follows: The detailed implementation of continuous data validation is as follows: 1. The validator pulls a binlog event from the upstream and gets the changed rows: - - The validator only checks a event that has been incrementally migrated by the syncer. If the event has not been processed by the syncer, the validator pauses and waits for the syncer to complete processing. + - The validator only checks an event that has been incrementally migrated by the syncer. If the event has not been processed by the syncer, the validator pauses and waits for the syncer to complete processing. - If the event has been processed by the syncer, the validator moves on to the following steps. 2. The validator parses the binlog event and filters out the rows based on the block and allow lists, the table filters, and table routing. After that, the validator submits the changed rows to the validation worker that runs in the background. 3. The validation worker merges the changed rows that affect the same table and the same primary key to avoid validating "expired" data. The changed rows are cached in memory. diff --git a/dm/dm-customized-secret-key.md b/dm/dm-customized-secret-key.md index af71bb0bdf4bc..1b0dd51f11c97 100644 --- a/dm/dm-customized-secret-key.md +++ b/dm/dm-customized-secret-key.md @@ -9,7 +9,7 @@ Before v8.0.0, [DM](/dm/dm-overview.md) uses a [fixed AES-256 secret key](https: ## Usage -1. Create a custom key file, which must contain a 64-character hexadecimal AES-256 secret key. +1. Create a custom key file, which must contain a 64-character hexadecimal AES-256 secret key. One way to generate this key is by calculating SHA256 checksum of random data, such as `head -n 256 /dev/urandom | sha256sum`. 2. In the DM-master [command-line flags](/dm/dm-command-line-flags.md) or [configuration file](/dm/dm-master-configuration-file.md), specify `secret-key-path` as the path of your custom key file. ## Upgrade from a version earlier than v8.0.0 diff --git a/dm/dm-daily-check.md b/dm/dm-daily-check.md index c410c5dd99f7b..0376d91544d62 100644 --- a/dm/dm-daily-check.md +++ b/dm/dm-daily-check.md @@ -1,7 +1,6 @@ --- title: Daily Check for TiDB Data Migration summary: Learn about the daily check of TiDB Data Migration (DM). -aliases: ['/docs/tidb-data-migration/dev/daily-check/'] --- # Daily Check for TiDB Data Migration diff --git a/dm/dm-error-handling.md b/dm/dm-error-handling.md index 2ee6a5045c587..9c11d2fef3ea8 100644 --- a/dm/dm-error-handling.md +++ b/dm/dm-error-handling.md @@ -1,7 +1,6 @@ --- title: Handle Errors in TiDB Data Migration summary: Learn about the error system and how to handle common errors when you use DM. -aliases: ['/docs/tidb-data-migration/dev/error-handling/','/docs/tidb-data-migration/dev/troubleshoot-dm/','/docs/tidb-data-migration/dev/error-system/'] --- # Handle Errors in TiDB Data Migration @@ -70,7 +69,7 @@ In the error system, usually, the information of a specific error is as follows: Whether DM outputs the error stack information depends on the error severity and the necessity. The error stack records the complete stack call information when the error occurs. If you cannot figure out the error cause based on the basic information and the error message, you can trace the execution path of the code when the error occurs using the error stack. -For the complete list of error codes, refer to the [error code lists](https://github.com/pingcap/tiflow/blob/master/dm/_utils/terror_gen/errors_release.txt). +For the complete list of error codes, refer to the [error code lists](https://github.com/pingcap/tiflow/blob/release-8.1/dm/_utils/terror_gen/errors_release.txt). ## Troubleshooting diff --git a/dm/dm-faq.md b/dm/dm-faq.md index 5ee968c2986e6..ca4ee4d42f135 100644 --- a/dm/dm-faq.md +++ b/dm/dm-faq.md @@ -1,7 +1,6 @@ --- title: TiDB Data Migration FAQs summary: Learn about frequently asked questions (FAQs) about TiDB Data Migration (DM). -aliases: ['/docs/tidb-data-migration/dev/faq/'] --- # TiDB Data Migration FAQs diff --git a/dm/dm-glossary.md b/dm/dm-glossary.md index a92a834e43a4d..5427cd4e47bc6 100644 --- a/dm/dm-glossary.md +++ b/dm/dm-glossary.md @@ -1,7 +1,6 @@ --- title: TiDB Data Migration Glossary summary: Learn the terms used in TiDB Data Migration. -aliases: ['/docs/tidb-data-migration/dev/glossary/'] --- # TiDB Data Migration Glossary diff --git a/dm/dm-hardware-and-software-requirements.md b/dm/dm-hardware-and-software-requirements.md index 8a64e5f5ff86a..e4f6077305e14 100644 --- a/dm/dm-hardware-and-software-requirements.md +++ b/dm/dm-hardware-and-software-requirements.md @@ -1,7 +1,6 @@ --- title: Software and Hardware Requirements for TiDB Data Migration summary: Learn the software and hardware requirements for DM cluster. -aliases: ['/docs/tidb-data-migration/dev/hardware-and-software-requirements/'] --- # Software and Hardware Requirements for TiDB Data Migration diff --git a/dm/dm-master-configuration-file.md b/dm/dm-master-configuration-file.md index ec58677c0ab18..d275ffab278f0 100644 --- a/dm/dm-master-configuration-file.md +++ b/dm/dm-master-configuration-file.md @@ -1,7 +1,6 @@ --- title: DM-master Configuration File summary: Learn the configuration file of DM-master. -aliases: ['/docs/tidb-data-migration/dev/dm-master-configuration-file/'] --- # DM-master Configuration File @@ -60,4 +59,4 @@ This section introduces the configuration parameters of DM-master. | `ssl-cert` | The path of the file that contains X509 certificate in PEM format for DM-master to connect with other components. | | `ssl-key` | The path of the file that contains X509 key in PEM format for DM-master to connect with other components. | | `cert-allowed-cn` | Common Name list. | -| `secret-key-path` | The file path of the secret key, which is used to encrypt and decrypt upstream and downstream passwords. The file must contain a 64-character hexadecimal AES-256 secret key. | \ No newline at end of file +| `secret-key-path` | The file path of the secret key, which is used to encrypt and decrypt upstream and downstream passwords. The file must contain a 64-character hexadecimal AES-256 secret key. One way to generate this key is by calculating SHA256 checksum of random data, such as head -n 256 /dev/urandom \| sha256sum. For more information, see [Customize a secret key for DM encryption and decryption](/dm/dm-customized-secret-key.md). | \ No newline at end of file diff --git a/dm/dm-open-api.md b/dm/dm-open-api.md index ada86e1cbf718..1dda9c2df95f2 100644 --- a/dm/dm-open-api.md +++ b/dm/dm-open-api.md @@ -25,7 +25,7 @@ To enable OpenAPI, perform one of the following operations: > **Note:** > -> - DM provides the [specification document](https://github.com/pingcap/tiflow/blob/master/dm/openapi/spec/dm.yaml) that meets the OpenAPI 3.0.0 standard. This document contains all the request parameters and returned values. You can copy the document yaml and preview it in [Swagger Editor](https://editor.swagger.io/). +> - DM provides the [specification document](https://github.com/pingcap/tiflow/blob/release-8.1/dm/openapi/spec/dm.yaml) that meets the OpenAPI 3.0.0 standard. This document contains all the request parameters and returned values. You can copy the document yaml and preview it in [Swagger Editor](https://editor.swagger.io/). > > - After you deploy the DM-master nodes, you can access `http://{master-addr}/api/v1/docs` to preview the documentation online. > diff --git a/dm/dm-overview.md b/dm/dm-overview.md index a11d048cd5a51..f8b95f5ef43fa 100644 --- a/dm/dm-overview.md +++ b/dm/dm-overview.md @@ -1,7 +1,6 @@ --- title: TiDB Data Migration Overview summary: Learn about the Data Migration tool, the architecture, the key components, and features. -aliases: ['/docs/tidb-data-migration/dev/overview/','/docs/tidb-data-migration/dev/feature-overview/','/tidb/dev/dm-key-features'] --- @@ -12,7 +11,7 @@ aliases: ['/docs/tidb-data-migration/dev/overview/','/docs/tidb-data-migration/d ![star](https://img.shields.io/github/stars/pingcap/tiflow?style=for-the-badge&logo=github) ![license](https://img.shields.io/github/license/pingcap/tiflow?style=for-the-badge) ![forks](https://img.shields.io/github/forks/pingcap/tiflow?style=for-the-badge) --> -[TiDB Data Migration](https://github.com/pingcap/tiflow/tree/master/dm) (DM) is an integrated data migration task management platform, which supports the full data migration and the incremental data replication from MySQL-compatible databases (such as MySQL, MariaDB, and Aurora MySQL) into TiDB. It can help to reduce the operation cost of data migration and simplify the troubleshooting process. +[TiDB Data Migration](https://github.com/pingcap/tiflow/tree/release-8.1/dm) (DM) is an integrated data migration task management platform, which supports the full data migration and the incremental data replication from MySQL-compatible databases (such as MySQL, MariaDB, and Aurora MySQL) into TiDB. It can help to reduce the operation cost of data migration and simplify the troubleshooting process. ## Basic features @@ -69,15 +68,15 @@ Before using the DM tool, note the following restrictions: ## Contributing -You are welcome to participate in the DM open sourcing project. Your contribution would be highly appreciated. For more details, see [CONTRIBUTING.md](https://github.com/pingcap/tiflow/blob/master/dm/CONTRIBUTING.md). +You are welcome to participate in the DM open sourcing project. Your contribution would be highly appreciated. For more details, see [CONTRIBUTING.md](https://github.com/pingcap/tiflow/blob/release-8.1/dm/CONTRIBUTING.md). ## Community support -You can learn about DM through the online documentation. If you have any questions, contact us on [GitHub](https://github.com/pingcap/tiflow/tree/master/dm). +You can learn about DM through the online documentation. If you have any questions, contact us on [GitHub](https://github.com/pingcap/tiflow/tree/release-8.1/dm). ## License -DM complies with the Apache 2.0 license. For more details, see [LICENSE](https://github.com/pingcap/tiflow/blob/master/LICENSE). +DM complies with the Apache 2.0 license. For more details, see [LICENSE](https://github.com/pingcap/tiflow/blob/release-8.1/LICENSE). ## DM versions @@ -89,5 +88,5 @@ Before v5.4, the DM documentation is independent of the TiDB documentation. To a > **Note:** > -> - Since October 2021, DM's GitHub repository has been moved to [pingcap/tiflow](https://github.com/pingcap/tiflow/tree/master/dm). If you see any issues with DM, submit your issue to the `pingcap/tiflow` repository for feedback. +> - Since October 2021, DM's GitHub repository has been moved to [pingcap/tiflow](https://github.com/pingcap/tiflow/tree/release-8.1/dm). If you see any issues with DM, submit your issue to the `pingcap/tiflow` repository for feedback. > - In earlier versions (v1.0 and v2.0), DM uses version numbers that are independent of TiDB. Since v5.3, DM uses the same version number as TiDB. The next version of DM v2.0 is DM v5.3. There are no compatibility changes from DM v2.0 to v5.3, and the upgrade process is the same as a normal upgrade, only an increase in version number. diff --git a/dm/dm-precheck.md b/dm/dm-precheck.md index 387111155d534..46ac64e0c2d19 100644 --- a/dm/dm-precheck.md +++ b/dm/dm-precheck.md @@ -1,7 +1,6 @@ --- title: Migration Task Precheck summary: Learn the precheck that DM performs before starting a migration task. -aliases: ['/docs/tidb-data-migration/dev/precheck/'] --- # Migration Task Precheck @@ -82,7 +81,7 @@ For the full data migration mode (`task-mode: full`), in addition to the [common - Primary key - Unique index - - In the optimistic mode, check whether the schemas of all sharded tables meet the [optimistic compatibility](https://github.com/pingcap/tiflow/blob/master/dm/docs/RFCS/20191209_optimistic_ddl.md#modifying-column-types). + - In the optimistic mode, check whether the schemas of all sharded tables meet the [optimistic compatibility](https://github.com/pingcap/tiflow/blob/release-8.1/dm/docs/RFCS/20191209_optimistic_ddl.md#modifying-column-types). - If a migration task was started successfully by the `start-task` command, the precheck of this task skips the consistency check. diff --git a/dm/dm-query-status.md b/dm/dm-query-status.md index eccbaa92912b3..895e163aaa277 100644 --- a/dm/dm-query-status.md +++ b/dm/dm-query-status.md @@ -1,7 +1,6 @@ --- title: Query Task Status in TiDB Data Migration summary: Learn how to query the status of a data replication task. -aliases: ['/docs/tidb-data-migration/dev/query-status/'] --- # Query Task Status in TiDB Data Migration diff --git a/dm/dm-source-configuration-file.md b/dm/dm-source-configuration-file.md index d540958e1e0d3..a38bebfef82a2 100644 --- a/dm/dm-source-configuration-file.md +++ b/dm/dm-source-configuration-file.md @@ -1,7 +1,6 @@ --- title: Upstream Database Configuration File of TiDB Data Migration summary: Learn the configuration file of the upstream database -aliases: ['/docs/tidb-data-migration/dev/source-configuration-file/'] --- # Upstream Database Configuration File of TiDB Data Migration @@ -19,7 +18,7 @@ source-id: "mysql-replica-01" enable-gtid: false # Whether to enable relay log. -enable-relay: false # Since DM v2.0.2, this configuration item is deprecated. To enable the relay log feature, use the `start-relay` command instead. +enable-relay: false relay-binlog-name: "" # The file name from which DM-worker starts to pull the binlog. relay-binlog-gtid: "" # The GTID from which DM-worker starts to pull the binlog. # relay-dir: "relay-dir" # The directory used to store relay log. The default value is "relay-dir". This configuration item is marked as deprecated since v6.1 and replaced by a parameter of the same name in the dm-worker configuration. @@ -70,7 +69,7 @@ This section describes each configuration parameter in the configuration file. | :------------ | :--------------------------------------- | | `source-id` | Represents a MySQL instance ID. | | `enable-gtid` | Determines whether to pull binlog from the upstream using GTID. The default value is `false`. In general, you do not need to configure `enable-gtid` manually. However, if GTID is enabled in the upstream database, and the primary/secondary switch is required, you need to set `enable-gtid` to `true`. | -| `enable-relay` | Determines whether to enable the relay log feature. The default value is `false`. Since DM v2.0.2, this configuration item is deprecated. To [enable the relay log feature](/dm/relay-log.md#enable-and-disable-relay-log), use the `start-relay` command instead. | +| `enable-relay` | Determines whether to enable the relay log feature. The default value is `false`. This parameter takes effect from v5.4. Additionally, you can [enable relay log dynamically](/dm/relay-log.md#enable-and-disable-relay-log) using the `start-relay` command. | | `relay-binlog-name` | Specifies the file name from which DM-worker starts to pull the binlog. For example, `"mysql-bin.000002"`. It only works when `enable_gtid` is `false`. If this parameter is not specified, DM-worker will start pulling from the earliest binlog file being replicated. Manual configuration is generally not required. | | `relay-binlog-gtid` | Specifies the GTID from which DM-worker starts to pull the binlog. For example, `"e9a1fc22-ec08-11e9-b2ac-0242ac110003:1-7849"`. It only works when `enable_gtid` is `true`. If this parameter is not specified, DM-worker will start pulling from the latest GTID being replicated. Manual configuration is generally not required. | | `relay-dir` | Specifies the relay log directory. | diff --git a/dm/dm-stop-task.md b/dm/dm-stop-task.md index 680c6bd7ab6bb..c5c9c5ba212d7 100644 --- a/dm/dm-stop-task.md +++ b/dm/dm-stop-task.md @@ -61,4 +61,11 @@ stop-task test } ] } -``` \ No newline at end of file +``` + +> **Note:** +> +> After you stop a migration task with the `stop-task` command, running [`query-status`](/dm/dm-query-status.md) will no longer display the task. However, the checkpoint and other related information for this task is still retained in the `dm_meta` database. +> +> + To start over the migration task, add the `--remove-meta` option when running the [`start-task`](/dm/dm-create-task.md) command. +> + To completely remove the migration task, manually delete the four tables that use the task name as a prefix from the `dm_meta` database. \ No newline at end of file diff --git a/dm/dm-webui-guide.md b/dm/dm-webui-guide.md index 847d7dd40fbd8..fcf02fd95b3a4 100644 --- a/dm/dm-webui-guide.md +++ b/dm/dm-webui-guide.md @@ -39,7 +39,7 @@ When [OpenAPI](/dm/dm-open-api.md#maintain-dm-clusters-using-openapi) is enabled Before creating a migration task, you need to create the data source information of the upstream for the replication task. You can create the upstream configuration in the **Source** page. When creating sources, pay attention to the following items: -- If there is a auto failover between primary and secondary instance, enable GTID in the upstream MySQL and set GTID to `True` when creating the upstream configuration; otherwise, the migration task will be interrupted during the failover (except for AWS Aurora). +- If there is an auto failover between primary and secondary instance, enable GTID in the upstream MySQL and set GTID to `True` when creating the upstream configuration; otherwise, the migration task will be interrupted during the failover (except for AWS Aurora). - If a MySQL instance needs to be temporarily offline, you can disable the instance. However, when the MySQL instance is being disabled, other MySQL instances running migration tasks should not execute DDL operations; otherwise, the disabled instance cannot properly migrate data after it is enabled. - When multiple migration tasks use the same upstream, it might cause additional stress. Enabling relay log can reduce the impact on the upstream, so it is recommended to enable relay log. diff --git a/dm/dm-worker-configuration-file.md b/dm/dm-worker-configuration-file.md index 39c3614ccc50d..fb437e26eff5f 100644 --- a/dm/dm-worker-configuration-file.md +++ b/dm/dm-worker-configuration-file.md @@ -1,7 +1,6 @@ --- title: DM-worker Configuration File summary: Learn the configuration file of DM-worker. -aliases: ['/docs/tidb-data-migration/dev/dm-worker-configuration-file/','/docs/tidb-data-migration/dev/dm-worker-configuration-file-full/'] --- # DM-worker Configuration File diff --git a/dm/dm-worker-intro.md b/dm/dm-worker-intro.md index c5c2c2c8ed9c4..a6328065ee005 100644 --- a/dm/dm-worker-intro.md +++ b/dm/dm-worker-intro.md @@ -1,7 +1,6 @@ --- title: DM-worker Introduction summary: Learn the features of DM-worker. -aliases: ['/docs/tidb-data-migration/dev/dm-worker-intro/'] --- # DM-worker Introduction @@ -55,7 +54,7 @@ The upstream database (MySQL/MariaDB) user must have the following privileges: If you need to migrate the data from `db1` to TiDB, execute the following `GRANT` statement: ```sql -GRANT RELOAD,REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'your_user'@'your_wildcard_of_host' +GRANT RELOAD,REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'your_user'@'your_wildcard_of_host'; GRANT SELECT ON db1.* TO 'your_user'@'your_wildcard_of_host'; ``` diff --git a/dm/dmctl-introduction.md b/dm/dmctl-introduction.md index b3d479de9957d..6bf32516f9036 100644 --- a/dm/dmctl-introduction.md +++ b/dm/dmctl-introduction.md @@ -1,7 +1,6 @@ --- title: Maintain DM Clusters Using dmctl summary: Learn how to maintain a DM cluster using dmctl. -aliases: ['/docs/tidb-data-migration/dev/manage-replication-tasks/'] --- # Maintain DM Clusters Using dmctl diff --git a/dm/feature-expression-filter.md b/dm/feature-expression-filter.md index a34dc195522c0..e3ed93cee0085 100644 --- a/dm/feature-expression-filter.md +++ b/dm/feature-expression-filter.md @@ -1,6 +1,5 @@ --- title: Filter DMLs Using SQL Expressions -aliases: ['/tidb/dev/feature-expression-filter/'] summary: In incremental data migration, you can filter binlog events using SQL expressions. DM supports filtering data during migration using binlog value filter since v2.0.5. You can configure SQL expressions based on the values in binlog events to determine whether to migrate a row change downstream. For detailed operation and implementation, refer to "Filter DML Events Using SQL Expressions". --- diff --git a/dm/feature-online-ddl.md b/dm/feature-online-ddl.md index e853ef67ff688..121539fd62dd5 100644 --- a/dm/feature-online-ddl.md +++ b/dm/feature-online-ddl.md @@ -1,7 +1,6 @@ --- title: Migrate from Databases that Use GH-ost/PT-osc summary: This document introduces the `online-ddl/online-ddl-scheme` feature of DM. -aliases: ['/docs/tidb-data-migration/dev/online-ddl-scheme/','tidb-data-migration/dev/feature-online-ddl-scheme'] --- # Migrate from Databases that Use GH-ost/PT-osc diff --git a/dm/feature-shard-merge.md b/dm/feature-shard-merge.md index 7bc428738efab..2ebf9812dd19b 100644 --- a/dm/feature-shard-merge.md +++ b/dm/feature-shard-merge.md @@ -1,7 +1,6 @@ --- title: Merge and Migrate Data from Sharded Tables summary: Learn how DM merges and migrates data from sharded tables. -aliases: ['/docs/tidb-data-migration/dev/feature-shard-merge/'] --- # Merge and Migrate Data from Sharded Tables diff --git a/dm/handle-failed-ddl-statements.md b/dm/handle-failed-ddl-statements.md index 99415b02ee934..aef8aa90200f2 100644 --- a/dm/handle-failed-ddl-statements.md +++ b/dm/handle-failed-ddl-statements.md @@ -1,7 +1,6 @@ --- title: Handle Failed DDL Statements in TiDB Data Migration summary: Learn how to handle failed DDL statements when you're using the TiDB Data Migration tool to migrate data. -aliases: ['/docs/tidb-data-migration/dev/skip-or-replace-abnormal-sql-statements/'] --- # Handle Failed DDL Statements in TiDB Data Migration diff --git a/dm/maintain-dm-using-tiup.md b/dm/maintain-dm-using-tiup.md index 9ae7cf5842eaa..5e3217ff42a05 100644 --- a/dm/maintain-dm-using-tiup.md +++ b/dm/maintain-dm-using-tiup.md @@ -1,7 +1,6 @@ --- title: Maintain a DM Cluster Using TiUP summary: Learn how to maintain a DM cluster using TiUP. -aliases: ['/docs/tidb-data-migration/dev/cluster-operations/'] --- # Maintain a DM Cluster Using TiUP @@ -389,12 +388,12 @@ tiup dmctl --master-addr master1:8261 operate-source create /tmp/source1.yml All operations above performed on the cluster machine use the SSH client embedded in TiUP to connect to the cluster and execute commands. However, in some scenarios, you might also need to use the SSH client native to the control machine system to perform such cluster operations. For example: -- To use a SSH plug-in for authentication +- To use an SSH plug-in for authentication - To use a customized SSH client Then you can use the `--native-ssh` command-line flag to enable the system-native command-line tool: -- Deploy a cluster: `tiup dm deploy --native-ssh`. Fill in the name of your cluster for ``, the DM version to be deployed (such as `v8.0.0`) for `` , and the topology file name for ``. +- Deploy a cluster: `tiup dm deploy --native-ssh`. Fill in the name of your cluster for ``, the DM version to be deployed (such as `v8.1.2`) for `` , and the topology file name for ``. - Start a cluster: `tiup dm start --native-ssh`. - Upgrade a cluster: `tiup dm upgrade ... --native-ssh` diff --git a/dm/manually-handling-sharding-ddl-locks.md b/dm/manually-handling-sharding-ddl-locks.md index 663547ec5b3d1..ccae25b79b77a 100644 --- a/dm/manually-handling-sharding-ddl-locks.md +++ b/dm/manually-handling-sharding-ddl-locks.md @@ -1,7 +1,6 @@ --- title: Handle Sharding DDL Locks Manually in DM summary: Learn how to handle sharding DDL locks manually in DM. -aliases: ['/docs/tidb-data-migration/dev/feature-manually-handling-sharding-ddl-locks/'] --- # Handle Sharding DDL Locks Manually in DM diff --git a/dm/migrate-data-using-dm.md b/dm/migrate-data-using-dm.md index 395b03d2c3d3f..5cb316c28ee1a 100644 --- a/dm/migrate-data-using-dm.md +++ b/dm/migrate-data-using-dm.md @@ -1,7 +1,6 @@ --- title: Migrate Data Using Data Migration summary: Use the Data Migration tool to migrate the full data and the incremental data. -aliases: ['/docs/tidb-data-migration/dev/replicate-data-using-dm/'] --- # Migrate Data Using Data Migration diff --git a/dm/monitor-a-dm-cluster.md b/dm/monitor-a-dm-cluster.md index 2af6bb1ac852e..9c77d0e413194 100644 --- a/dm/monitor-a-dm-cluster.md +++ b/dm/monitor-a-dm-cluster.md @@ -1,7 +1,6 @@ --- title: Data Migration Monitoring Metrics summary: Learn about the monitoring metrics when you use Data Migration to migrate data. -aliases: ['/docs/tidb-data-migration/dev/monitor-a-dm-cluster/'] --- # Data Migration Monitoring Metrics diff --git a/dm/quick-start-create-task.md b/dm/quick-start-create-task.md index a42af559e0c2a..039a4520c1b8c 100644 --- a/dm/quick-start-create-task.md +++ b/dm/quick-start-create-task.md @@ -1,7 +1,6 @@ --- title: Create a Data Migration Task summary: Learn how to create a migration task after the DM cluster is deployed. -aliases: ['/docs/tidb-data-migration/dev/create-task-and-verify/'] --- # Create a Data Migration Task @@ -74,7 +73,7 @@ To run a TiDB server, use the following command: {{< copyable "shell-regular" >}} ```bash -wget https://download.pingcap.org/tidb-community-server-v8.0.0-linux-amd64.tar.gz +wget https://download.pingcap.org/tidb-community-server-v8.1.2-linux-amd64.tar.gz tar -xzvf tidb-latest-linux-amd64.tar.gz mv tidb-latest-linux-amd64/bin/tidb-server ./ ./tidb-server diff --git a/dm/quick-start-with-dm.md b/dm/quick-start-with-dm.md index 3386f01ffa96f..2df08c00dfb05 100644 --- a/dm/quick-start-with-dm.md +++ b/dm/quick-start-with-dm.md @@ -1,13 +1,16 @@ --- title: TiDB Data Migration Quick Start summary: Learn how to quickly deploy a DM cluster using binary packages. -aliases: ['/docs/tidb-data-migration/dev/get-started/'] --- # Quick Start Guide for TiDB Data Migration This document describes how to migrate data from MySQL to TiDB using [TiDB Data Migration (DM)](/dm/dm-overview.md). This guide is a quick demo of DM features and is not recommended for any production environment. +> **Note:** +> +> If your target machine's operating system supports SELinux, ensure that SELinux is **disabled**. + ## Step 1: Deploy a DM cluster 1. Install TiUP, and install [`dmctl`](/dm/dmctl-introduction.md) using TiUP: @@ -32,7 +35,7 @@ This document describes how to migrate data from MySQL to TiDB using [TiDB Data {{< copyable "shell-regular" >}} ```shell - tiup dm deploy dm-test 6.0.0 topology.yaml -p + tiup dm deploy dm-test 8.1.2 topology.yaml -p ``` ## Step 2: Prepare the data source diff --git a/dm/relay-log.md b/dm/relay-log.md index cafb2f51011d0..e630b7a26ada2 100644 --- a/dm/relay-log.md +++ b/dm/relay-log.md @@ -1,7 +1,6 @@ --- title: Data Migration Relay Log summary: Learn the directory structure, initial migration rules and data purge of DM relay logs. -aliases: ['/docs/tidb-data-migration/dev/relay-log/'] --- # Data Migration Relay Log diff --git a/dm/shard-merge-best-practices.md b/dm/shard-merge-best-practices.md index 637f97716cc06..7169d599e5358 100644 --- a/dm/shard-merge-best-practices.md +++ b/dm/shard-merge-best-practices.md @@ -1,7 +1,6 @@ --- title: Best Practices of Data Migration in the Shard Merge Scenario summary: Learn the best practices of data migration in the shard merge scenario. -aliases: ['/docs/tidb-data-migration/dev/shard-merge-best-practices/'] --- # Best Practices of Data Migration in the Shard Merge Scenario diff --git a/dm/table-selector.md b/dm/table-selector.md index ee05ace71b3d8..b3eef57182ff6 100644 --- a/dm/table-selector.md +++ b/dm/table-selector.md @@ -1,7 +1,6 @@ --- title: Table Selector of TiDB Data Migration summary: Learn about Table Selector used by the table routing, binlog event filtering, and column mapping rule of Data Migration. -aliases: ['/docs/tidb-data-migration/dev/table-selector/'] --- # Table Selector of TiDB Data Migration diff --git a/dm/task-configuration-file-full.md b/dm/task-configuration-file-full.md index dfdb69fbf72f1..ee35e9a4fd87a 100644 --- a/dm/task-configuration-file-full.md +++ b/dm/task-configuration-file-full.md @@ -1,6 +1,5 @@ --- title: DM Advanced Task Configuration File -aliases: ['/docs/tidb-data-migration/dev/task-configuration-file-full/','/docs/tidb-data-migration/dev/dm-portal/'] summary: This document introduces the advanced task configuration file of Data Migration (DM), covering global and instance configuration. The global configuration includes basic and feature settings, while the instance configuration defines subtasks for data migration from one or multiple MySQL instances in the upstream to the same instance in the downstream. --- @@ -126,7 +125,7 @@ loaders: # The import mode during the full import phase. The following modes are supported: # - "logical" (default). Uses TiDB Lightning's logical import mode to import data. Document: https://docs.pingcap.com/tidb/stable/tidb-lightning-logical-import-mode - # - "physical". Uses TiDB Lightning's physical import mode to import data. Document: https://docs.pingcap.com/zh/tidb/stable/tidb-lightning-physical-import-mode + # - "physical". Uses TiDB Lightning's physical import mode to import data. Document: https://docs.pingcap.com/tidb/stable/tidb-lightning-physical-import-mode # The "physical" mode is still an experimental feature and is not recommended in production. import-mode: "logical" # Methods to resolve conflicts in logical import. diff --git a/download-ecosystem-tools.md b/download-ecosystem-tools.md index fe65a624f7cc1..5c781d16f794b 100644 --- a/download-ecosystem-tools.md +++ b/download-ecosystem-tools.md @@ -1,7 +1,6 @@ --- title: Download TiDB Tools summary: Download the most officially maintained versions of TiDB tools. -aliases: ['/docs/dev/download-ecosystem-tools/','/docs/dev/reference/tools/download/'] --- # Download TiDB Tools @@ -28,7 +27,7 @@ You can download TiDB Toolkit from the following link: https://download.pingcap.org/tidb-community-toolkit-{version}-linux-{arch}.tar.gz ``` -`{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v6.2.0` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-toolkit-v6.2.0-linux-amd64.tar.gz`. +`{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v8.1.2` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-toolkit-v8.1.2-linux-amd64.tar.gz`. > **Note:** > diff --git a/dr-secondary-cluster.md b/dr-secondary-cluster.md index d8160cb1794df..37126f1e3e58d 100644 --- a/dr-secondary-cluster.md +++ b/dr-secondary-cluster.md @@ -299,13 +299,20 @@ It is important to conduct regular DR drills for critical business systems to te 2. After there are no more writes, query the latest TSO (`Position`) of the TiDB cluster: ```sql - mysql> show master status; - +-------------+--------------------+--------------+------------------+-------------------+ - | File | Position | Binlog_Do_DB | Binlog_Ignore_DB | Executed_Gtid_Set | - +-------------+--------------------+--------------+------------------+-------------------+ - | tidb-binlog | 438223974697009153 | | | | - +-------------+--------------------+--------------+------------------+-------------------+ - 1 row in set (0.33 sec) + BEGIN; SELECT TIDB_CURRENT_TSO(); ROLLBACK; + ``` + + ```sql + Query OK, 0 rows affected (0.00 sec) + + +--------------------+ + | TIDB_CURRENT_TSO() | + +--------------------+ + | 452654700157468673 | + +--------------------+ + 1 row in set (0.00 sec) + + Query OK, 0 rows affected (0.00 sec) ``` 3. Poll the changefeed `dr-primary-to-secondary` until it meets the condition `TSO >= Position`. diff --git a/dumpling-overview.md b/dumpling-overview.md index d376f157f9276..690628bca0be2 100644 --- a/dumpling-overview.md +++ b/dumpling-overview.md @@ -1,12 +1,11 @@ --- title: Dumpling Overview summary: Use the Dumpling tool to export data from TiDB. -aliases: ['/docs/dev/mydumper-overview/','/docs/dev/reference/tools/mydumper/','/tidb/dev/mydumper-overview/'] --- # Use Dumpling to Export Data -This document introduces the data export tool - [Dumpling](https://github.com/pingcap/tidb/tree/master/dumpling). Dumpling exports data stored in TiDB/MySQL as SQL or CSV data files and can be used to make a logical full backup or export. Dumpling also supports exporting data to Amazon S3. +This document introduces the data export tool - [Dumpling](https://github.com/pingcap/tidb/tree/release-8.1/dumpling). Dumpling exports data stored in TiDB/MySQL as SQL or CSV data files and can be used to make a logical full backup or export. Dumpling also supports exporting data to Amazon S3. @@ -55,7 +54,7 @@ Dumpling has the following advantages: - Support exporting data to Amazon S3 cloud storage. - More optimizations are made for TiDB: - Support configuring the memory limit of a single TiDB SQL statement. - - If Dumpling can connect directly to PD, Dumpling supports automatic adjustment of TiDB GC time for TiDB v4.0.0 and later versions. + - If Dumpling can access the PD address and the [`INFORMATION_SCHEMA.CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) table of the TiDB cluster, Dumpling supports automatically adjusting the [GC](/garbage-collection-overview.md) safe point time to block GC for TiDB v4.0.0 and later versions. - Use TiDB's hidden column `_tidb_rowid` to optimize the performance of concurrent data export from a single table. - For TiDB, you can set the value of [`tidb_snapshot`](/read-historical-data.md#how-tidb-reads-data-from-history-versions) to specify the time point of the data backup. This ensures the consistency of the backup, instead of using `FLUSH TABLES WITH READ LOCK` to ensure the consistency. @@ -95,7 +94,7 @@ In the command above: + The `-h`, `-P`, and `-u` option respectively mean the address, the port, and the user. If a password is required for authentication, you can use `-p $YOUR_SECRET_PASSWORD` to pass the password to Dumpling. + The `-o` (or `--output`) option specifies the export directory of the storage, which supports an absolute local file path or an [external storage URI](/external-storage-uri.md). + The `-t` option specifies the number of threads for the export. Increasing the number of threads improves the concurrency of Dumpling and the export speed, and also increases the database's memory consumption. Therefore, it is not recommended to set the number too large. Usually, it's less than 64. -+ The `-r` option enables the in-table concurrency to speed up the export. The default value is `0`, which means disabled. A value greater than 0 means it is enabled, and the value is of `INT` type. When the source database is TiDB, a `-r` value greater than 0 indicates that the TiDB region information is used for splitting, and reduces the memory usage. The specific `-r` value does not affect the split algorithm. When the source database is MySQL and the primary key is of the `INT` type, specifying `-r` can also enable the in-table concurrency. ++ The `-r` option enables the in-table concurrency to speed up the export. The default value is `0`, which means disabled. A value greater than 0 means it is enabled, and the value is of `INT` type. When the source database is TiDB, a `-r` value greater than 0 indicates that the TiDB region information is used for splitting, and reduces the memory usage. The specific `-r` value does not affect the split algorithm. When the source database is MySQL and the primary key or the first column of the composite primary key is of the `INT` type, specifying `-r` can also enable the in-table concurrency. + The `-F` option is used to specify the maximum size of a single file (the unit here is `MiB`; inputs like `5GiB` or `8KB` are also acceptable). It is recommended to keep its value to 256 MiB or less if you plan to use TiDB Lightning to load this file into a TiDB instance. > **Note:** @@ -282,7 +281,7 @@ Examples: The exported file is stored in the `./export-` directory by default. Commonly used options are as follows: - The `-t` option specifies the number of threads for the export. Increasing the number of threads improves the concurrency of Dumpling and the export speed, and also increases the database's memory consumption. Therefore, it is not recommended to set the number too large. -- The `-r` option enables the in-table concurrency to speed up the export. The default value is `0`, which means disabled. A value greater than 0 means it is enabled, and the value is of `INT` type. When the source database is TiDB, a `-r` value greater than 0 indicates that the TiDB region information is used for splitting, and reduces the memory usage. The specific `-r` value does not affect the split algorithm. When the source database is MySQL and the primary key is of the `INT` type, specifying `-r` can also enable the in-table concurrency. +- The `-r` option enables the in-table concurrency to speed up the export. The default value is `0`, which means disabled. A value greater than 0 means it is enabled, and the value is of `INT` type. When the source database is TiDB, a `-r` value greater than 0 indicates that the TiDB region information is used for splitting, and reduces the memory usage. The specific `-r` value does not affect the split algorithm. When the source database is MySQL and the primary key or the first column of the composite primary key is of the `INT` type, specifying `-r` can also enable the in-table concurrency. - The `--compress ` option specifies the compression format of the dump. It supports the following compression algorithms: `gzip`, `snappy`, and `zstd`. This option can speed up dumping of data if storage is the bottleneck or if storage capacity is a concern. The drawback is an increase in CPU usage. Each file is compressed individually. With the above options specified, Dumpling can have a quicker speed of data export. @@ -345,7 +344,7 @@ When Dumpling is exporting a large single table from TiDB, Out of Memory (OOM) m ### Manually set the TiDB GC time -When exporting data from TiDB (less than 1 TB), if the TiDB version is later than or equal to v4.0.0 and Dumpling can access the PD address of the TiDB cluster, Dumpling automatically extends the GC time without affecting the original cluster. +When exporting data from TiDB (less than 1 TB), if the TiDB version is v4.0.0 or later and Dumpling can access the PD address and the [`INFORMATION_SCHEMA.CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) table of the TiDB cluster, Dumpling automatically adjusts the GC safe point to block GC without affecting the original cluster. However, in either of the following scenarios, Dumpling cannot automatically adjust the GC time: @@ -377,7 +376,7 @@ SET GLOBAL tidb_gc_life_time = '10m'; | `--case-sensitive` | whether table-filter is case-sensitive | false (case-insensitive) | | `-h` or `--host` | The IP address of the connected database host | "127.0.0.1" | | `-t` or `--threads` | The number of concurrent backup threads | 4 | -| `-r` or `--rows` | Enable the in-table concurrency to speed up the export. The default value is `0`, which means disabled. A value greater than 0 means it is enabled, and the value is of `INT` type. When the source database is TiDB, a `-r` value greater than 0 indicates that the TiDB region information is used for splitting, and reduces the memory usage. The specific `-r` value does not affect the split algorithm. When the source database is MySQL and the primary key is of the `INT` type, specifying `-r` can also enable the in-table concurrency. | +| `-r` or `--rows` | Enable the in-table concurrency to speed up the export. The default value is `0`, which means disabled. A value greater than 0 means it is enabled, and the value is of `INT` type. When the source database is TiDB, a `-r` value greater than 0 indicates that the TiDB region information is used for splitting, and reduces the memory usage. The specific `-r` value does not affect the split algorithm. When the source database is MySQL and the primary key or the first column of the composite primary key is of the `INT` type, specifying `-r` can also enable the in-table concurrency. | | `-L` or `--logfile` | Log output address. If it is empty, the log will be output to the console | "" | | `--loglevel` | Log level {debug,info,warn,error,dpanic,panic,fatal} | "info" | | `--logfmt` | Log output format {text,json} | "text" | diff --git a/dynamic-config.md b/dynamic-config.md index 9c6c40a8db132..b02cb85cf0989 100644 --- a/dynamic-config.md +++ b/dynamic-config.md @@ -1,7 +1,6 @@ --- title: Modify Configuration Dynamically summary: Learn how to dynamically modify the cluster configuration. -aliases: ['/docs/dev/dynamic-config/'] --- # Modify Configuration Dynamically @@ -12,7 +11,7 @@ You can dynamically update the configuration of components (including TiDB, TiKV > **Note:** > -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). For TiDB Cloud, you need to contact [TiDB Cloud Support](https://docs.pingcap.com/tidbcloud/tidb-cloud-support) to modify the configurations. +> This feature is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). For TiDB Cloud, you need to contact [TiDB Cloud Support](https://docs.pingcap.com/tidbcloud/tidb-cloud-support) to modify the configurations. ## Common Operations @@ -135,6 +134,7 @@ The following TiKV configuration items can be modified dynamically: | `raftstore.raft-max-size-per-msg` | The soft limit on the size of a single message packet that is allowed to be generated | | `raftstore.raft-entry-max-size` | The hard limit on the maximum size of a single Raft log | | `raftstore.raft-entry-cache-life-time` | The maximum remaining time allowed for the log cache in memory | +| `raftstore.max-apply-unpersisted-log-limit` | The maximum number of committed but not persisted Raft logs that can be applied | | `raftstore.split-region-check-tick-interval` | The time interval at which to check whether the Region split is needed | | `raftstore.region-split-check-diff` | The maximum value by which the Region data is allowed to exceed before Region split | | `raftstore.region-compact-check-interval` | The time interval at which to check whether it is necessary to manually trigger RocksDB compaction | @@ -225,6 +225,7 @@ The following TiKV configuration items can be modified dynamically: | `server.simplify-metrics` | Controls whether to simplify the sampling monitoring metrics | | `storage.block-cache.capacity` | The size of shared block cache (supported since v4.0.3) | | `storage.scheduler-worker-pool-size` | The number of threads in the Scheduler thread pool | +| `import.num-threads` | The number of threads to process restore or import RPC requests (dynamic modification is supported starting from v8.1.2) | | `backup.num-threads` | The number of backup threads (supported since v4.0.3) | | `split.qps-threshold` | The threshold to execute `load-base-split` on a Region. If the QPS of read requests for a Region exceeds `qps-threshold` for 10 consecutive seconds, this Region should be split.| | `split.byte-threshold` | The threshold to execute `load-base-split` on a Region. If the traffic of read requests for a Region exceeds the `byte-threshold` for 10 consecutive seconds, this Region should be split. | diff --git a/ecosystem-tool-user-case.md b/ecosystem-tool-user-case.md index 8b3380d5f3f56..ef02e74a20ddc 100644 --- a/ecosystem-tool-user-case.md +++ b/ecosystem-tool-user-case.md @@ -1,7 +1,6 @@ --- title: TiDB Tools Use Cases summary: Learn the common use cases of TiDB tools and how to choose the tools. -aliases: ['/docs/dev/ecosystem-tool-user-case/'] --- # TiDB Tools Use Cases diff --git a/ecosystem-tool-user-guide.md b/ecosystem-tool-user-guide.md index 60b1f17d81512..80909d82c3320 100644 --- a/ecosystem-tool-user-guide.md +++ b/ecosystem-tool-user-guide.md @@ -1,7 +1,6 @@ --- title: TiDB Tools Overview summary: Learn the tools and applicable scenarios. -aliases: ['/docs/dev/ecosystem-tool-user-guide/','/docs/dev/reference/tools/user-guide/','/docs/dev/how-to/migrate/from-mysql/','/docs/dev/how-to/migrate/incrementally-from-mysql/','/docs/dev/how-to/migrate/overview/'] --- # TiDB Tools Overview @@ -75,7 +74,7 @@ The following are the basics of Dumpling: > **Note:** > -> PingCAP previously maintained a fork of the [mydumper project](https://github.com/maxbube/mydumper) with enhancements specific to TiDB. Starting from v7.5.0, [Mydumper](https://docs.pingcap.com/tidb/v4.0/mydumper-overview) is deprecated and most of its features have been replaced by [Dumpling](/dumpling-overview.md). It is strongly recommended that you use Dumpling instead of mydumper. +> PingCAP previously maintained a fork of the [mydumper project](https://github.com/maxbube/mydumper) with enhancements specific to TiDB. Starting from v7.5.0, [Mydumper](https://docs-archive.pingcap.com/tidb/v4.0/mydumper-overview/) is deprecated and most of its features have been replaced by [Dumpling](/dumpling-overview.md). It is strongly recommended that you use Dumpling instead of mydumper. ### Full data import - TiDB Lightning @@ -91,7 +90,7 @@ The following are the basics of TiDB Lightning: - Data source: - The output files of Dumpling - Other compatible CSV files - - Parquet files exported from Amazon Aurora or Apache Hive + - Parquet files exported from Amazon Aurora, Apache Hive, or Snowflake - Supported TiDB versions: v2.1 and later versions - Kubernetes support: Yes. See [Quickly restore data into a TiDB cluster on Kubernetes using TiDB Lightning](https://docs.pingcap.com/tidb-in-kubernetes/stable/restore-data-using-tidb-lightning) for details. @@ -146,4 +145,4 @@ The following are the basics of sync-diff-inspector: ## OLAP Query tool - TiSpark -[TiSpark](/tispark-overview.md) is a product developed by PingCAP to address the complexiy of OLAP queries. It combines strengths of Spark, and the features of distributed TiKV clusters and TiDB to provide a one-stop Hybrid Transactional and Analytical Processing (HTAP) solution. +[TiSpark](/tispark-overview.md) is a product developed by PingCAP to address the complexity of OLAP queries. It combines strengths of Spark, and the features of distributed TiKV clusters and TiDB to provide a one-stop Hybrid Transactional and Analytical Processing (HTAP) solution. diff --git a/enable-tls-between-clients-and-servers.md b/enable-tls-between-clients-and-servers.md index 3e47532a9bb88..8ee11daeb8b78 100644 --- a/enable-tls-between-clients-and-servers.md +++ b/enable-tls-between-clients-and-servers.md @@ -1,7 +1,6 @@ --- title: Enable TLS Between TiDB Clients and Servers summary: Use secure connections to ensure data security. -aliases: ['/docs/dev/enable-tls-between-clients-and-servers/','/docs/dev/how-to/secure/enable-tls-clients/','/docs/dev/encrypted-connections-with-tls-protocols/'] --- # Enable TLS between TiDB Clients and Servers diff --git a/enable-tls-between-components.md b/enable-tls-between-components.md index 3f48566c2da66..698045d1240e0 100644 --- a/enable-tls-between-components.md +++ b/enable-tls-between-components.md @@ -1,7 +1,6 @@ --- title: Enable TLS Between TiDB Components summary: Learn how to enable TLS authentication between TiDB components. -aliases: ['/docs/dev/enable-tls-between-components/','/docs/dev/how-to/secure/enable-tls-between-components/'] --- # Enable TLS Between TiDB Components @@ -84,7 +83,7 @@ Currently, it is not supported to only enable encrypted transmission of some spe - TiFlash (New in v4.0.5) - Configure in the `tiflash.toml` file, and change the `http_port` item to `https_port`: + Configure in the `tiflash.toml` file: ```toml [security] @@ -111,7 +110,7 @@ Currently, it is not supported to only enable encrypted transmission of some spe - TiCDC - Configure in the configuration file： + Configure in the configuration file: ```toml [security] diff --git a/encryption-at-rest.md b/encryption-at-rest.md index 9653ba186c16d..12cf636f60af6 100644 --- a/encryption-at-rest.md +++ b/encryption-at-rest.md @@ -1,7 +1,6 @@ --- title: Encryption at Rest summary: Learn how to enable encryption at rest to protect sensitive data. -aliases: ['/docs/dev/encryption at rest/'] --- # Encryption at Rest @@ -28,11 +27,11 @@ TiKV currently does not exclude encryption keys and user data from core dumps. I TiKV tracks encrypted data files using the absolute path of the files. As a result, once encryption is turned on for a TiKV node, the user should not change data file paths configuration such as `storage.data-dir`, `raftstore.raftdb-path`, `rocksdb.wal-dir` and `raftdb.wal-dir`. -SM4 encryption is only supported in v6.3.0 and later versions of TiKV. TiKV versions earlier than v6.3.0 only support AES encryption. SM4 encryption might lead to 50% to 80% degradation on throughput. +SM4 encryption is only supported in v6.3.0 and later versions of TiKV. TiKV versions earlier than v6.3.0 only support AES encryption. SM4 encryption affects performance. In the worst-case scenario, it might cause a ​50% to 80% throughput degradation. However, a sufficiently large [`storage.block-cache`](/tikv-configuration-file.md#storageblock-cache) can significantly mitigate this impact, reducing the throughput degradation to around 10%. ### TiFlash -TiFlash supports encryption at rest. Data keys are generated by TiFlash. All files (including data files, schema files, and temporary files) written into TiFlash (including TiFlash Proxy) are encrypted using the current data key. The encryption algorithms, the encryption configuration (in the [`tiflash-learner.toml` file](/tiflash/tiflash-configuration.md#configure-the-tiflashtoml-file) supported by TiFlash, and the meanings of monitoring metrics are consistent with those of TiKV. +TiFlash supports encryption at rest. Data keys are generated by TiFlash. All files (including data files, schema files, and temporary files) written into TiFlash (including TiFlash Proxy) are encrypted using the current data key. The encryption algorithms, the encryption configuration (in the [`tiflash-learner.toml` file](/tiflash/tiflash-configuration.md#configure-the-tiflashtoml-file) supported by TiFlash), and the meanings of monitoring metrics are consistent with those of TiKV. If you have deployed TiFlash with Grafana, you can check the **TiFlash-Proxy-Details** -> **Encryption** panel. diff --git a/error-codes.md b/error-codes.md index ff7dd32a95f9a..b795039472a3f 100644 --- a/error-codes.md +++ b/error-codes.md @@ -1,7 +1,6 @@ --- title: Error Codes and Troubleshooting summary: Learn about the error codes and solutions in TiDB. -aliases: ['/docs/dev/error-codes/','/docs/dev/reference/error-codes/'] --- # Error Codes and Troubleshooting @@ -90,7 +89,7 @@ TiDB is compatible with the error codes in MySQL, and in most cases returns the The single Key-Value pair being written is too large. The largest single Key-Value pair supported in TiDB is 6 MB by default. - If a pair exceeds this limit, you need to properly adjust the [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50) configuration value to relax the limit. + If a pair exceeds this limit, you need to properly adjust the [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500) configuration value to relax the limit. * Error Number: 8026 diff --git a/explain-joins.md b/explain-joins.md index ceef0558d5bd8..37b5b6e102b04 100644 --- a/explain-joins.md +++ b/explain-joins.md @@ -70,7 +70,7 @@ Index join is efficient in memory usage, but might be slower to execute than oth SELECT * FROM t1 INNER JOIN t2 ON t1.id=t2.t1_id WHERE t1.pad1 = 'value' and t2.pad1='value'; ``` -In an inner join operation, TiDB implements join reordering and might access either `t1` or `t2` first. Assume that TiDB selects `t1` as the first table to apply the `build` step, and then TiDB is able to filter on the predicate `t1.col = 'value'` before probing the table `t2`. The filter for the predicate `t2.col='value'` will be applied on each probe of table `t2`, which might be less efficient than other join methods. +In an inner join operation, TiDB implements join reordering and might access either `t1` or `t2` first. Assume that TiDB selects `t1` as the first table to apply the `build` step, and then TiDB is able to filter on the predicate `t1.pad1 = 'value'` before probing the table `t2`. The filter for the predicate `t2.pad1='value'` will be applied on each probe of table `t2`, which might be less efficient than other join methods. Index join is effective if the build side is small and the probe side is pre-indexed and large. Consider the following query where an index join performs worse than a hash join and is not chosen by the SQL Optimizer: diff --git a/explain-overview.md b/explain-overview.md index 03041376ac3dc..291b6254f17c0 100644 --- a/explain-overview.md +++ b/explain-overview.md @@ -1,7 +1,6 @@ --- title: TiDB Query Execution Plan Overview summary: Learn about the execution plan information returned by the `EXPLAIN` statement in TiDB. -aliases: ['/docs/dev/query-execution-plan/','/docs/dev/reference/performance/understanding-the-query-execution-plan/','/docs/dev/index-merge/','/docs/dev/reference/performance/index-merge/','/tidb/dev/index-merge','/tidb/dev/query-execution-plan'] --- # TiDB Query Execution Plan Overview @@ -64,7 +63,7 @@ The following describes the output of the `EXPLAIN` statement above: > Before v6.4.0, `estRows` means the number of estimated rows to be processed by the probe side operators for each row from the build side operators. Since v6.4.0, `estRows` means the **total number** of estimated rows to be processed by the probe side operators. The actual number of rows displayed (indicated by the `actRows` column) in the result of `EXPLAIN ANALYZE` means the total row count, so since v6.4.0 the meanings of `estRows` and `actRows` for the probe side child nodes of `IndexJoin` and `Apply` operators are consistent. > > -> For example： +> For example: > > ```sql > CREATE TABLE t1(a INT, b INT); diff --git a/explore-htap.md b/explore-htap.md index 01a49469a7dd2..56fd277d6db16 100644 --- a/explore-htap.md +++ b/explore-htap.md @@ -29,7 +29,7 @@ The following are the typical use cases of HTAP: When using TiDB as a data hub, TiDB can meet specific business needs by seamlessly connecting the data for the application and the data warehouse. -For more information about use cases of TiDB HTAP, see [blogs about HTAP on the PingCAP website](https://en.pingcap.com/blog/?tag=htap). +For more information about use cases of TiDB HTAP, see [blogs about HTAP on the PingCAP website](https://www.pingcap.com/blog/?tag=htap). To enhance the overall performance of TiDB, it is recommended to use HTAP in the following technical scenarios: diff --git a/expression-syntax.md b/expression-syntax.md index fe5fae943a5ba..a4dabf83b515f 100644 --- a/expression-syntax.md +++ b/expression-syntax.md @@ -1,7 +1,6 @@ --- title: Expression Syntax summary: Learn about the expression syntax in TiDB. -aliases: ['/docs/dev/expression-syntax/','/docs/dev/reference/sql/language-structure/expression-syntax/'] --- # Expression Syntax @@ -18,7 +17,7 @@ The expressions can be divided into the following types: - ParamMarker (`?`), system variables, user variables and CASE expressions. -The following rules are the expression syntax, which is based on the [`parser.y`](https://github.com/pingcap/tidb/blob/master/pkg/parser/parser.y) rules of TiDB parser. For the navigable version of the following syntax diagram, refer to [TiDB SQL Syntax Diagram](https://pingcap.github.io/sqlgram/#Expression). +The following rules are the expression syntax, which is based on the [`parser.y`](https://github.com/pingcap/tidb/blob/release-8.1/pkg/parser/parser.y) rules of TiDB parser. ```ebnf+diagram Expression ::= diff --git a/extended-statistics.md b/extended-statistics.md index b6a2fae9456c7..5cc526e7a9ff6 100644 --- a/extended-statistics.md +++ b/extended-statistics.md @@ -5,20 +5,18 @@ summary: Learn how to use extended statistics to guide the optimizer. # Introduction to Extended Statistics -TiDB can collect the following two types of statistics: +TiDB can collect the following two types of statistics. This document describes how to use extended statistics to guide the optimizer. Before reading this document, it is recommended that you read [Introduction to Statistics](/statistics.md) first. -- Basic statistics: statistics such as histograms and Count-Min Sketch. See [Introduction to Statistics](/statistics.md) for details. -- Extended statistics: statistics filtered by tables and columns. - -> **Tip:** -> -> Before reading this document, it is recommended that you read [Introduction to Statistics](/statistics.md) first. +- Basic statistics: statistics such as histograms and Count-Min Sketch, which primarily focus on individual columns. They are essential for the optimizer to estimate the query cost. See [Introduction to Statistics](/statistics.md) for details. +- Extended statistics: statistics that focus on data correlations between specified columns, which guide the optimizer to estimate the query cost more precisely when the queried columns are correlated. When the `ANALYZE` statement is executed manually or automatically, TiDB by default only collects the basic statistics and does not collect the extended statistics. This is because the extended statistics are only used for optimizer estimates in specific scenarios, and collecting them requires additional overhead. -Extended statistics are disabled by default. To collect extended statistics, you need to first enable the extended statistics, and then register each individual extended statistics object. +Extended statistics are disabled by default. To collect extended statistics, you need to first enable extended statistics, and then create your desired extended statistics objects one by one. After the object creation, the next time the `ANALYZE` statement is executed, TiDB collects both the basic statistics and the corresponding extended statistics of the created objects. -After the registration, the next time the `ANALYZE` statement is executed, TiDB collects both the basic statistics and the registered extended statistics. +> **Warning:** +> +> This feature is experimental. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. ## Limitations @@ -40,17 +38,17 @@ SET GLOBAL tidb_enable_extended_stats = ON; The default value of this variable is `OFF`. The setting of this system variable applies to all extended statistics objects. -### Register extended statistics +### Create extended statistics objects -The registration for extended statistics is not a one-time task, and you need repeat the registration for each extended statistics object. +The creation of extended statistics objects is not a one-time task. You need to repeat the creation for each extended statistics object. -To register extended statistics, use the SQL statement `ALTER TABLE ADD STATS_EXTENDED`. The syntax is as follows: +To create an extended statistics object, use the SQL statement `ALTER TABLE ADD STATS_EXTENDED`. The syntax is as follows: ```sql ALTER TABLE table_name ADD STATS_EXTENDED IF NOT EXISTS stats_name stats_type(column_name, column_name...); ``` -In the syntax, you can specify the table name, statistics name, statistics type, and column name of the extended statistics to be collected. +In the syntax, you can specify the table name, statistics name, statistics type, and column name of the extended statistics object to be collected. - `table_name` specifies the name of the table from which the extended statistics are collected. - `stats_name` specifies the name of the statistics object, which must be unique for each table. @@ -60,7 +58,7 @@ In the syntax, you can specify the table name, statistics name, statistics type,
How it works -To improve access performance, each TiDB node maintains a cache in the system table `mysql.stats_extended` for extended statistics. After you register the extended statistics, the next time the `ANALYZE` statement is executed, TiDB will collect the extended statistics if the system table `mysql.stats_extended` has the corresponding objects. +To improve access performance, each TiDB node maintains a cache in the system table `mysql.stats_extended` for extended statistics. After you create the extended statistics objects, the next time the `ANALYZE` statement is executed, TiDB will collect the extended statistics if the system table `mysql.stats_extended` has the corresponding objects. Each row in the `mysql.stats_extended` table has a `version` column. Once a row is updated, the value of `version` is increased. In this way, TiDB loads the table into memory incrementally, instead of fully. @@ -78,7 +76,7 @@ TiDB loads `mysql.stats_extended` periodically to ensure that the cache is kept
-### Delete extended statistics +### Delete extended statistics objects To delete an extended statistics object, use the following statement: @@ -107,7 +105,7 @@ Other TiDB nodes will read this change and delete the object in their memory cac ### Export and import extended statistics -The way of exporting or importing extended statistics is the same as exporting or importing basic statistics. See [Introduction to Statistics - Import and export statistics](/statistics.md#import-and-export-statistics) for details. +The way of exporting or importing extended statistics is the same as exporting or importing basic statistics. See [Introduction to Statistics - Import and export statistics](/statistics.md#export-and-import-statistics) for details. ## Usage examples for correlation-type extended statistics @@ -140,13 +138,13 @@ Without extended statistics, the TiDB optimizer only supposes that `col1` and `c ### Step 3. Enable extended statistics -Set `tidb_enable_extended_stats` to `ON`, and register the extended statistics object for `col1` and `col2`: +Set `tidb_enable_extended_stats` to `ON`, and create the extended statistics object for `col1` and `col2`: ```sql ALTER TABLE t ADD STATS_EXTENDED s1 correlation(col1, col2); ``` -When you execute `ANALYZE` after the registration, TiDB calculates the [Pearson correlation coefficient](https://en.wikipedia.org/wiki/Pearson_correlation_coefficient) of `col` and `col2` of table `t`, and writes the object into the `mysql.stats_extended` table. +When you execute `ANALYZE` after the object creation, TiDB calculates the [Pearson correlation coefficient](https://en.wikipedia.org/wiki/Pearson_correlation_coefficient) of `col1` and `col2` of table `t`, and writes the object into the `mysql.stats_extended` table. ### Step 4. See how extended statistics make a difference diff --git a/external-storage-uri.md b/external-storage-uri.md index 090629e331479..cb99bcb77018f 100644 --- a/external-storage-uri.md +++ b/external-storage-uri.md @@ -26,7 +26,7 @@ The basic format of the URI is as follows: - `endpoint`: Specifies the URL of custom endpoint for S3-compatible services (for example, ``). - `force-path-style`: Use path style access rather than virtual hosted style access (defaults to `true`). - `storage-class`: Specifies the storage class of the uploaded objects (for example, `STANDARD` or `STANDARD_IA`). - - `sse`: Specifies the server-side encryption algorithm used to encrypt the uploaded objects (value options: ``, `AES256`, or `aws:kms`). + - `sse`: Specifies the server-side encryption algorithm used to encrypt the uploaded objects (value options: empty, `AES256`, or `aws:kms`). - `sse-kms-key-id`: Specifies the KMS ID if `sse` is set to `aws:kms`. - `acl`: Specifies the canned ACL of the uploaded objects (for example, `private` or `authenticated-read`). - `role-arn`: When you need to access Amazon S3 data from a third party using a specified [IAM role](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html), you can specify the corresponding [Amazon Resource Name (ARN)](https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) of the IAM role with the `role-arn` URL query parameter, such as `arn:aws:iam::888888888888:role/my-role`. For more information about using an IAM role to access Amazon S3 data from a third party, see [AWS documentation](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_common-scenarios_third-party.html). BR supports this parameter starting from v7.6.0. diff --git a/faq/backup-and-restore-faq.md b/faq/backup-and-restore-faq.md index de4efdfc1cda6..81c43a15ded81 100644 --- a/faq/backup-and-restore-faq.md +++ b/faq/backup-and-restore-faq.md @@ -1,7 +1,6 @@ --- title: Backup & Restore FAQs summary: Learn about Frequently Asked Questions (FAQs) and the solutions of backup and restore. -aliases: ['/docs/dev/br/backup-and-restore-faq/','/tidb/dev/pitr-troubleshoot/','/tidb/dev/pitr-known-issues/'] --- # Backup & Restore FAQs @@ -151,7 +150,7 @@ To handle this issue, you can try to scale out the cluster resources, reduce the You can try to reduce the number of tables to be created in a batch by setting `--ddl-batch-size` to `128` or a smaller value. -When using BR to restore the backup data with the value of [`--ddl-batch-size`](/br/br-batch-create-table.md#use-batch-create-table) greater than `1`, TiDB writes a DDL job of table creation to the DDL jobs queue that is maintained by TiKV. At this time, the total size of all tables schema sent by TiDB at one time should not exceed 6 MB, because the maximum value of job messages is `6 MB` by default (it is **not recommended** to modify this value. For details, see [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50) and [`raft-entry-max-size`](/tikv-configuration-file.md#raft-entry-max-size)). Therefore, if you set `--ddl-batch-size` to an excessively large value, the schema size of the tables sent by TiDB in a batch at one time exceeds the specified value, which causes BR to report the `entry too large, the max entry size is 6291456, the size of data is 7690800` error. +When using BR to restore the backup data with the value of [`--ddl-batch-size`](/br/br-batch-create-table.md#use-batch-create-table) greater than `1`, TiDB writes a DDL job of table creation to the DDL jobs queue that is maintained by TiKV. At this time, the total size of all tables schema sent by TiDB at one time should not exceed 6 MB, because the maximum value of job messages is `6 MB` by default (it is **not recommended** to modify this value. For details, see [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500) and [`raft-entry-max-size`](/tikv-configuration-file.md#raft-entry-max-size)). Therefore, if you set `--ddl-batch-size` to an excessively large value, the schema size of the tables sent by TiDB in a batch at one time exceeds the specified value, which causes BR to report the `entry too large, the max entry size is 6291456, the size of data is 7690800` error. ### Where are the backed up files stored when I use `local` storage? @@ -276,7 +275,7 @@ Note that even if you configures [table filter](/table-filter.md#syntax), **BR d - Statistics tables (`mysql.stat_*`). But statistics can be restored. See [Back up statistics](/br/br-snapshot-manual.md#back-up-statistics). - System variable tables (`mysql.tidb`, `mysql.global_variables`) -- [Other system tables](https://github.com/pingcap/tidb/blob/master/br/pkg/restore/systable_restore.go#L31) +- [Other system tables](https://github.com/pingcap/tidb/blob/release-8.1/br/pkg/restore/systable_restore.go#L31) ### How to deal with the error of `cannot find rewrite rule` during restoration? diff --git a/faq/faq-overview.md b/faq/faq-overview.md index 242079f70da94..e9ea2d78e4c44 100644 --- a/faq/faq-overview.md +++ b/faq/faq-overview.md @@ -17,27 +17,27 @@ This document summarizes frequently asked questions (FAQs) about TiDB. TiDB architecture and principles - TiDB Architecture FAQs + TiDB Architecture FAQs Deployment - + Data migration @@ -45,31 +45,31 @@ This document summarizes frequently asked questions (FAQs) about TiDB. Data backup and restore - Backup & Restore FAQs + Backup & Restore FAQs SQL operations - SQL FAQs + SQL FAQs Cluster upgrade - TiDB Upgrade FAQs + TiDB Upgrade FAQs Cluster management - Cluster Management FAQs + Cluster Management FAQs Monitor and alert - + High availability and high reliability - + Common error codes - Error Codes and Troubleshooting + Error Codes and Troubleshooting \ No newline at end of file diff --git a/faq/manage-cluster-faq.md b/faq/manage-cluster-faq.md index 1a6955b1e1cf6..30e4120e9b4de 100644 --- a/faq/manage-cluster-faq.md +++ b/faq/manage-cluster-faq.md @@ -29,7 +29,7 @@ TiKV data is located in the [`--data-dir`](/command-line-flags-for-tikv-configur ### What are the system tables in TiDB? -Similar to MySQL, TiDB includes system tables as well, used to store the information required by the server when it runs. See [TiDB system table](/mysql-schema.md). +Similar to MySQL, TiDB includes system tables as well, used to store the information required by the server when it runs. See [TiDB system table](/mysql-schema/mysql-schema.md). ### Where are the TiDB/PD/TiKV logs? @@ -415,7 +415,10 @@ The memory usage of TiKV mainly comes from the block-cache of RocksDB, which is ### Can both TiDB data and RawKV data be stored in the same TiKV cluster? -No. TiDB (or data created from the transactional API) relies on a specific key format. It is not compatible with data created from RawKV API (or data from other RawKV-based services). +It depends on your TiDB version and whether TiKV API V2 is enabled ([`storage.api-version = 2`](/tikv-configuration-file.md#api-version-new-in-v610)). + +- If your TiDB version is v6.1.0 or later and TiKV API V2 is enabled, TiDB data and RawKV data can be stored in the same TiKV cluster. +- Otherwise, the answer is no because the key format of TiDB data (or data created using the transactional API) is incompatible with data created using the RawKV API (or data from other RawKV-based services). ## TiDB testing diff --git a/faq/migration-tidb-faq.md b/faq/migration-tidb-faq.md index 477c9559bc390..3d291fbe6569a 100644 --- a/faq/migration-tidb-faq.md +++ b/faq/migration-tidb-faq.md @@ -147,7 +147,7 @@ The total read capacity has no limit. You can increase the read capacity by addi ### The error message `transaction too large` is displayed -Due to the limitation of the underlying storage engine, each key-value entry (one row) in TiDB should be no more than 6MB. You can adjust the [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50) configuration value up to 120MB. +Due to the limitation of the underlying storage engine, each key-value entry (one row) in TiDB should be no more than 6MB. You can adjust the [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500) configuration value up to 120MB. Distributed transactions need two-phase commit and the bottom layer performs the Raft replication. If a transaction is very large, the commit process would be quite slow and the write conflict is more likely to occur. Moreover, the rollback of a failed transaction leads to an unnecessary performance penalty. To avoid these problems, we limit the total size of key-value entries to no more than 100MB in a transaction by default. If you need larger transactions, modify the value of `txn-total-size-limit` in the TiDB configuration file. The maximum value of this configuration item is up to 10G. The actual limitation is also affected by the physical memory of the machine. @@ -171,13 +171,13 @@ Yes. ### Why does the query speed getting slow after deleting data? -Deleting a large amount of data leaves a lot of useless keys, affecting the query efficiency. The [Region Merge feature](/best-practices/massive-regions-best-practices.md#method-3-enable-region-merge) can solve this problem. For more details, see the [deleting data section in TiDB Best Practices](https://en.pingcap.com/blog/tidb-best-practice/#write). +Deleting a large amount of data leaves a lot of useless keys, affecting the query efficiency. The [Region Merge feature](/best-practices/massive-regions-best-practices.md#method-3-enable-region-merge) can solve this problem. For more details, see the [deleting data section in TiDB Best Practices](https://www.pingcap.com/blog/tidb-best-practice/#write). ### What is the most efficient way of deleting data? When deleting a large amount of data, it is recommended to use `Delete from t where xx limit 5000;`. It deletes through the loop and uses `Affected Rows == 0` as a condition to end the loop, so as not to exceed the limit of transaction size. With the prerequisite of meeting business filtering logic, it is recommended to add a strong filter index column or directly use the primary key to select the range, such as `id >= 5000*n+m and id < 5000*(n+1)+m`. -If the amount of data that needs to be deleted at a time is very large, this loop method will get slower and slower because each deletion traverses backward. After deleting the previous data, lots of deleted flags remain for a short period (then all will be processed by Garbage Collection) and influence the following Delete statement. If possible, it is recommended to refine the Where condition. See [details in TiDB Best Practices](https://en.pingcap.com/blog/tidb-best-practice/#write). +If the amount of data that needs to be deleted at a time is very large, this loop method will get slower and slower because each deletion traverses backward. After deleting the previous data, lots of deleted flags remain for a short period (then all will be processed by Garbage Collection) and influence the following Delete statement. If possible, it is recommended to refine the Where condition. See [details in TiDB Best Practices](https://www.pingcap.com/blog/tidb-best-practice/#write). ### How to improve the data loading speed in TiDB? diff --git a/faq/sql-faq.md b/faq/sql-faq.md index f28100b19b2a8..b916866708405 100644 --- a/faq/sql-faq.md +++ b/faq/sql-faq.md @@ -32,7 +32,9 @@ In addition, you can also use the [SQL binding](/sql-plan-management.md#sql-bind ## How to prevent the execution of a particular SQL statement? -You can create [SQL bindings](/sql-plan-management.md#sql-binding) with the [`MAX_EXECUTION_TIME`](/optimizer-hints.md#max_execution_timen) hint to limit the execution time of a particular statement to a small value (for example, 1ms). In this way, the statement is terminated automatically by the threshold. +For TiDB v7.5.0 or later versions, you can use the [`QUERY WATCH`](/sql-statements/sql-statement-query-watch.md) statement to terminate specific SQL statements. For more details, see [Manage queries that consume more resources than expected (Runaway Queries)](/tidb-resource-control.md#query-watch-parameters). + +For versions earlier than TiDB v7.5.0, you can create [SQL bindings](/sql-plan-management.md#sql-binding) with the [`MAX_EXECUTION_TIME`](/optimizer-hints.md#max_execution_timen) hint to limit the execution time of a particular statement to a small value (for example, 1ms). In this way, the statement is terminated automatically by the threshold. For example, to prevent the execution of `SELECT * FROM t1, t2 WHERE t1.id = t2.id`, you can use the following SQL binding to limit the execution time of the statement to 1ms: @@ -132,7 +134,7 @@ For details, see [description of the `SELECT` syntax elements](/sql-statements/s ## Can the codec of TiDB guarantee that the UTF-8 string is memcomparable? Is there any coding suggestion if our key needs to support UTF-8? -TiDB uses the UTF-8 character set by default and currently only supports UTF-8. The string of TiDB uses the memcomparable format. +The default character set in TiDB is `utf8mb4`. The strings are in memcomparable format. For more information about the character set in TiDB, see [Character Set and Collation](/character-set-and-collation.md). ## What is the maximum number of statements in a transaction? @@ -184,7 +186,7 @@ None of the `DELETE`, `TRUNCATE` and `DROP` operations release data immediately. ## Why does the query speed get slow after data is deleted? -Deleting a large amount of data leaves a lot of useless keys, affecting the query efficiency. To solve the problem, you can use the [Region Merge](/best-practices/massive-regions-best-practices.md#method-3-enable-region-merge) feature. For details, see the [deleting data section in TiDB Best Practices](https://en.pingcap.com/blog/tidb-best-practice/#write). +Deleting a large amount of data leaves a lot of useless keys, affecting the query efficiency. To solve the problem, you can use the [Region Merge](/best-practices/massive-regions-best-practices.md#method-3-enable-region-merge) feature. For details, see the [deleting data section in TiDB Best Practices](https://www.pingcap.com/blog/tidb-best-practice/#write). ## What should I do if it is slow to reclaim storage space after deleting data? @@ -298,7 +300,7 @@ In the preceding causes, only Cause 1 is related to tables. Cause 1 and Cause 2 ### What are the causes of the "Information schema is out of date" error? -Before TiDB v6.5.0, when executing a DML statement, if TiDB fails to load the latest schema within a DDL lease (45s by default), the `Information schema is out of date` error might occur. Possible causes are: +When executing a DML statement, if TiDB fails to load the latest schema within a DDL lease (45s by default), the `Information schema is out of date` error might occur. Possible causes are as follows: - The TiDB instance that executed this DML was killed, and the transaction execution corresponding to this DML statement took longer than a DDL lease. When the transaction was committed, the error occurred. - TiDB failed to connect to PD or TiKV while executing this DML statement. As a result, TiDB failed to load schema within a DDL lease or disconnected from PD due to the keepalive setting. @@ -335,6 +337,73 @@ Whether your cluster is a new cluster or an upgraded cluster from an earlier ver - If the owner does not exist, try manually triggering owner election with: `curl -X POST http://{TiDBIP}:10080/ddl/owner/resign`. - If the owner exists, export the Goroutine stack and check for the possible stuck location. +## Collation used in JDBC connections + +This section lists questions related to collations used in JDBC connections. For information about character sets and collations supported by TiDB, see [Character Set and Collation](/character-set-and-collation.md). + +### What collation is used in a JDBC connection when `connectionCollation` is not configured in the JDBC URL? + +When `connectionCollation` is not configured in the JDBC URL, there are two scenarios: + +**Scenario 1**: Neither `connectionCollation` nor `characterEncoding` is configured in the JDBC URL + +- For Connector/J 8.0.25 and earlier versions, the JDBC driver attempts to use the server's default character set. Because the default character set of TiDB is `utf8mb4`, the driver uses `utf8mb4_bin` as the connection collation. +- For Connector/J 8.0.26 and later versions, the JDBC driver uses the `utf8mb4` character set and automatically selects the collation based on the return value of `SELECT VERSION()`. + + - When the return value is less than `8.0.1`, the driver uses `utf8mb4_general_ci` as the connection collation. TiDB follows the driver and uses `utf8mb4_general_ci` as the collation. + - When the return value is greater than or equal to `8.0.1`, the driver uses `utf8mb4_0900_ai_ci` as the connection collation. TiDB v7.4.0 and later versions follow the driver and use `utf8mb4_0900_ai_ci` as the collation, while TiDB versions earlier than v7.4.0 fall back to using the default collation `utf8mb4_bin` because the `utf8mb4_0900_ai_ci` collation is not supported in these versions. + +**Scenario 2**: `characterEncoding=utf8` is configured in the JDBC URL but `connectionCollation` is not configured. The JDBC driver uses the `utf8mb4` character set according to the mapping rules. The collation is determined according to the rules described in scenario 1. + +### How to handle collation changes after upgrading TiDB? + +In TiDB v7.4 and earlier versions, if `connectionCollation` is not configured, and `characterEncoding` is either not configured or set to `UTF-8` in the JDBC URL, the TiDB [`collation_connection`](/system-variables.md#collation_connection) variable defaults to the `utf8mb4_bin` collation. + +Starting from TiDB v7.4, if `connectionCollation` is not configured, and `characterEncoding` is either not configured or set to `UTF-8` in the JDBC URL, the value of the [`collation_connection`](/system-variables.md#collation_connection) variable depends on the JDBC driver version. For example, for Connector/J 8.0.26 and later versions, the JDBC driver defaults to the `utf8mb4` character set and uses `utf8mb4_general_ci` as the connection collation. TiDB follows the driver, and the [`collation_connection`](/system-variables.md#collation_connection) variable uses the `utf8mb4_0900_ai_ci` collation. For more information, see [Collation used in JDBC connections](#what-collation-is-used-in-a-jdbc-connection-when-connectioncollation-is-not-configured-in-the-jdbc-url). + +When upgrading from an earlier version to v7.4 or later (for example, from v6.5 to v7.5), if you need to maintain the `collation_connection` as `utf8mb4_bin` for JDBC connections, it is recommended to configure the `connectionCollation` parameter in the JDBC URL. + +The following is a common JDBC URL configuration in TiDB v6.5: + +``` +spring.datasource.url=JDBC:mysql://{TiDBIP}:{TiDBPort}/{DBName}?characterEncoding=UTF-8&useSSL=false&useServerPrepStmts=true&cachePrepStmts=true&prepStmtCacheSqlLimit=10000&prepStmtCacheSize=1000&useConfigs=maxPerformance&rewriteBatchedStatements=true&defaultFetchSize=-2147483648&allowMultiQueries=true +``` + +After upgrading to TiDB v7.5 or a later version, it is recommended to configure the `connectionCollation` parameter in the JDBC URL: + +``` +spring.datasource.url=JDBC:mysql://{TiDBIP}:{TiDBPort}/{DBName}?characterEncoding=UTF-8&connectionCollation=utf8mb4_bin&useSSL=false&useServerPrepStmts=true&cachePrepStmts=true&prepStmtCacheSqlLimit=10000&prepStmtCacheSize=1000&useConfigs=maxPerformance&rewriteBatchedStatements=true&defaultFetchSize=-2147483648&allowMultiQueries=true +``` + +### What are the differences between the `utf8mb4_bin` and `utf8mb4_0900_ai_ci` collations? + +| Collation | Case-sensitive | Ignore trailing spaces | Accent-sensitive | Comparison method | +|----------------------|----------------|------------------|--------------|------------------------| +| `utf8mb4_bin` | Yes | Yes | Yes | Compare binary values | +| `utf8mb4_0900_ai_ci` | No | No | No | Use Unicode sorting algorithm | + +For example: + +```sql +-- utf8mb4_bin is case-sensitive +SELECT 'apple' = 'Apple' COLLATE utf8mb4_bin; -- Returns 0 (FALSE) + +-- utf8mb4_0900_ai_ci is case-insensitive +SELECT 'apple' = 'Apple' COLLATE utf8mb4_0900_ai_ci; -- Returns 1 (TRUE) + +-- utf8mb4_bin ignores trailing spaces +SELECT 'Apple ' = 'Apple' COLLATE utf8mb4_bin; -- Returns 1 (TRUE) + +-- utf8mb4_0900_ai_ci does not ignore trailing spaces +SELECT 'Apple ' = 'Apple' COLLATE utf8mb4_0900_ai_ci; -- Returns 0 (FALSE) + +-- utf8mb4_bin is accent-sensitive +SELECT 'café' = 'cafe' COLLATE utf8mb4_bin; -- Returns 0 (FALSE) + +-- utf8mb4_0900_ai_ci is accent-insensitive +SELECT 'café' = 'cafe' COLLATE utf8mb4_0900_ai_ci; -- Returns 1 (TRUE) +``` + ## SQL optimization ### TiDB execution plan description diff --git a/faq/tidb-faq.md b/faq/tidb-faq.md index 7d30346996796..98f5ec6a3e594 100644 --- a/faq/tidb-faq.md +++ b/faq/tidb-faq.md @@ -1,7 +1,6 @@ --- title: TiDB Architecture FAQs summary: Learn about the most frequently asked questions (FAQs) relating to TiDB. -aliases: ['/docs/dev/faq/tidb-faq/','/docs/dev/faq/tidb/','/docs/dev/tiflash/tiflash-faq/','/docs/dev/reference/tiflash/faq/','/tidb/dev/tiflash-faq'] --- # TiDB Architecture FAQs diff --git a/faq/upgrade-faq.md b/faq/upgrade-faq.md index ce6be8527ed2a..d9c6a1589ac36 100644 --- a/faq/upgrade-faq.md +++ b/faq/upgrade-faq.md @@ -1,7 +1,6 @@ --- title: Upgrade and After Upgrade FAQs summary: Learn about some FAQs and the solutions during and after upgrading TiDB. -aliases: ['/docs/dev/faq/upgrade-faq/','/docs/dev/faq/upgrade/'] --- # Upgrade and After Upgrade FAQs @@ -36,6 +35,12 @@ It is not recommended to upgrade TiDB using the binary. Instead, it is recommend This section lists some FAQs and their solutions after you upgrade TiDB. +### The collation in JDBC connections changes after upgrading TiDB + +When upgrading from an earlier version to v7.4 or later, if the `connectionCollation` is not configured, and the `characterEncoding` is either not configured or configured as `UTF-8` in the JDBC URL, the default collation in your JDBC connections might change from `utf8mb4_bin` to `utf8mb4_0900_ai_ci` after upgrading. If you need to maintain the collation as `utf8mb4_bin`, configure `connectionCollation=utf8mb4_bin` in the JDBC URL. + +For more information, see [Collation used in JDBC connections](/faq/sql-faq.md#collation-used-in-jdbc-connections). + ### The character set (charset) errors when executing DDL operations In v2.1.0 and earlier versions (including all versions of v2.0), the character set of TiDB is UTF-8 by default. But starting from v2.1.1, the default character set has been changed into UTF8MB4. diff --git a/follower-read.md b/follower-read.md index b0146a8aab98d..6a8f413bea93e 100644 --- a/follower-read.md +++ b/follower-read.md @@ -1,7 +1,6 @@ --- title: Follower Read summary: This document describes the use and implementation of Follower Read. -aliases: ['/docs/dev/follower-read/','/docs/dev/reference/performance/follower-read/'] --- # Follower Read diff --git a/foreign-key.md b/foreign-key.md index 4ecb881ca4f19..377a39ce065c5 100644 --- a/foreign-key.md +++ b/foreign-key.md @@ -308,7 +308,7 @@ Create Table | CREATE TABLE `child` ( - [DM](/dm/dm-overview.md) does not support foreign keys. DM disables the [`foreign_key_checks`](/system-variables.md#foreign_key_checks) of the downstream TiDB when replicating data to TiDB. Therefore, the cascading operations caused by foreign keys are not replicated from the upstream to the downstream, which might cause data inconsistency. - [TiCDC](/ticdc/ticdc-overview.md) v6.6.0 is compatible with foreign keys. The previous versions of TiCDC might report an error when replicating tables with foreign keys. It is recommended to disable the `foreign_key_checks` of the downstream TiDB cluster when using a TiCDC version earlier than v6.6.0. - [BR](/br/backup-and-restore-overview.md) v6.6.0 is compatible with foreign keys. The previous versions of BR might report an error when restoring tables with foreign keys to a v6.6.0 or later cluster. It is recommended to disable the `foreign_key_checks` of the downstream TiDB cluster before restoring the cluster when using a BR earlier than v6.6.0. -- When you use [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md), it is recommended to disable the `foreign_key_checks` of the downstream TiDB cluster before importing data. +- When you use [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md), if the target table uses a foreign key, it is recommended to disable the `foreign_key_checks` of the downstream TiDB cluster before importing data. For versions earlier than v6.6.0, disabling this system variable does not take effect, and you need to grant the `REFERENCES` privilege for the downstream database user, or manually create the target table in the downstream database in advance to ensure smooth data import. diff --git a/functions-and-operators/aggregate-group-by-functions.md b/functions-and-operators/aggregate-group-by-functions.md index 2693c910f6ee9..f01c4139524d3 100644 --- a/functions-and-operators/aggregate-group-by-functions.md +++ b/functions-and-operators/aggregate-group-by-functions.md @@ -1,7 +1,6 @@ --- title: Aggregate (GROUP BY) Functions summary: Learn about the supported aggregate functions in TiDB. -aliases: ['/docs/dev/functions-and-operators/aggregate-group-by-functions/','/docs/dev/reference/sql/functions-and-operators/aggregate-group-by-functions/'] --- # Aggregate (GROUP BY) Functions @@ -12,20 +11,21 @@ This document describes details about the supported aggregate functions in TiDB. This section describes the supported MySQL `GROUP BY` aggregate functions in TiDB. -| Name | Description | -|:--------------------------------------------------------------------------------------------------------------|:--------------------------------------------------| +| Name | Description | +|:---------------------------------------------------------------------------------------------------------------|:--------------------------------------------------| | [`COUNT()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_count) | Return a count of the number of rows returned | | [`COUNT(DISTINCT)`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_count-distinct) | Return the count of a number of different values | | [`SUM()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_sum) | Return the sum | | [`AVG()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_avg) | Return the average value of the argument | | [`MAX()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_max) | Return the maximum value | | [`MIN()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_min) | Return the minimum value | -| [`GROUP_CONCAT()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_group-concat) | Return a concatenated string | -| [`VARIANCE()`, `VAR_POP()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_var-pop) | Return the population standard variance| -| [`STD()`, `STDDEV()`, `STDDEV_POP`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_std) | Return the population standard deviation | -| [`VAR_SAMP()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_var-samp) | Return the sample variance | -| [`STDDEV_SAMP()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_stddev-samp) | Return the sample standard deviation | -| [`JSON_OBJECTAGG(key, value)`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_json-objectagg) | Return the result set as a single JSON object containing key-value pairs | +| [`GROUP_CONCAT()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_group-concat) | Return a concatenated string | +| [`VARIANCE()`, `VAR_POP()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_var-pop) | Return the population standard variance | +| [`STD()`, `STDDEV()`, `STDDEV_POP`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_std) | Return the population standard deviation | +| [`VAR_SAMP()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_var-samp) | Return the sample variance | +| [`STDDEV_SAMP()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_stddev-samp) | Return the sample standard deviation | +| [`JSON_ARRAYAGG()`](/functions-and-operators/json-functions/json-functions-aggregate.md#json_arrayagg) | Return the result set as a single JSON array | +| [`JSON_OBJECTAGG()`](/functions-and-operators/json-functions/json-functions-aggregate.md#json_objectagg) | Return the result set as a single JSON object containing key-value pairs | - Unless otherwise stated, group functions ignore `NULL` values. - If you use a group function in a statement containing no `GROUP BY` clause, it is equivalent to grouping on all rows. @@ -40,30 +40,52 @@ In addition, TiDB also provides the following aggregate functions: The following example shows how to calculate the fiftieth percentile of a `INT` column: - {{< copyable "sql" >}} - ```sql - drop table if exists t; - create table t(a int); - insert into t values(1), (2), (3); + DROP TABLE IF EXISTS t; + CREATE TABLE t(a INT); + INSERT INTO t VALUES(1), (2), (3); ``` - {{< copyable "sql" >}} - ```sql - select approx_percentile(a, 50) from t; + SELECT APPROX_PERCENTILE(a, 50) FROM t; ``` ```sql +--------------------------+ - | approx_percentile(a, 50) | + | APPROX_PERCENTILE(a, 50) | +--------------------------+ | 2 | +--------------------------+ 1 row in set (0.00 sec) ``` -Except for the `GROUP_CONCAT()` and `APPROX_PERCENTILE()` functions, all the preceding functions can serve as [Window functions](/functions-and-operators/window-functions.md). ++ `APPROX_COUNT_DISTINCT(expr, [expr...])` + + This function is similar to `COUNT(DISTINCT)` in counting the number of distinct values but returns an approximate result. It uses the `BJKST` algorithm, significantly reducing memory consumption when processing large datasets with a power-law distribution. Moreover, for low-cardinality data, this function provides high accuracy while maintaining efficient CPU utilization. + + The following example shows how to use this function: + + ```sql + DROP TABLE IF EXISTS t; + CREATE TABLE t(a INT, b INT, c INT); + INSERT INTO t VALUES(1, 1, 1), (2, 1, 1), (2, 2, 1), (3, 1, 1), (5, 1, 2), (5, 1, 2), (6, 1, 2), (7, 1, 2); + ``` + + ```sql + SELECT APPROX_COUNT_DISTINCT(a, b) FROM t GROUP BY c; + ``` + + ``` + +-----------------------------+ + | approx_count_distinct(a, b) | + +-----------------------------+ + | 3 | + | 4 | + +-----------------------------+ + 2 rows in set (0.00 sec) + ``` + +Except for the `GROUP_CONCAT()`, `APPROX_PERCENTILE()`, and `APPROX_COUNT_DISTINCT` functions, all the preceding functions can serve as [Window functions](/functions-and-operators/window-functions.md). ## GROUP BY modifiers @@ -153,4 +175,4 @@ group by id, val; ## Related system variables -The `group_concat_max_len` variable sets the maximum number of items for the `GROUP_CONCAT()` function. \ No newline at end of file +The [`group_concat_max_len`](/system-variables.md#group_concat_max_len) variable sets the maximum number of items for the `GROUP_CONCAT()` function. \ No newline at end of file diff --git a/functions-and-operators/bit-functions-and-operators.md b/functions-and-operators/bit-functions-and-operators.md index 3760495c08e54..c5d6046a5369e 100644 --- a/functions-and-operators/bit-functions-and-operators.md +++ b/functions-and-operators/bit-functions-and-operators.md @@ -1,7 +1,6 @@ --- title: Bit Functions and Operators summary: Learn about the bit functions and operators. -aliases: ['/docs/dev/functions-and-operators/bit-functions-and-operators/','/docs/dev/reference/sql/functions-and-operators/bit-functions-and-operators/'] --- # Bit Functions and Operators @@ -12,13 +11,291 @@ TiDB supports all of the [bit functions and operators](https://dev.mysql.com/doc | Name | Description | | :------| :------------- | -| [`BIT_COUNT()`](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#function_bit-count) | Return the number of bits that are set as 1 | -| [&](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#operator_bitwise-and) | Bitwise AND | -| [~](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#operator_bitwise-invert) | Bitwise inversion | -| [\|](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#operator_bitwise-or) | Bitwise OR | -| [^](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#operator_bitwise-xor) | Bitwise XOR | -| [<<](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#operator_left-shift) | Left shift | -| [>>](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#operator_right-shift) | Right shift | +| [`BIT_COUNT()`](#bit_count) | Return the number of bits that are set as 1 | +| [&](#-bitwise-and) | Bitwise AND | +| [~](#-bitwise-inversion) | Bitwise inversion | +| [\|](#-bitwise-or) | Bitwise OR | +| [`^`](#-bitwise-xor) | Bitwise XOR | +| [`<<`](#-left-shift) | Left shift | +| [`>>`](#-right-shift) | Right shift | + +## [`BIT_COUNT()`](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#function_bit-count) + +The `BIT_COUNT(expr)` function returns the number of bits that are set as 1 in `expr`. + +```sql +SELECT BIT_COUNT(b'00101001'); +``` + +``` ++------------------------+ +| BIT_COUNT(b'00101001') | ++------------------------+ +| 3 | ++------------------------+ +1 row in set (0.00 sec) +``` + +> **Note:** +> +> If the argument `expr` is a binary number, you need to specify `b` explicitly before the number, such as `b'00101001'`. Otherwise, this function treats it as a string and returns a different result. For example, `BIT_COUNT('00101001')` returns `7` because it converts the string `'00101001'` as the decimal number `101001` and counts the number of `1` bits in its binary format `11000100001010001`. + +The following example is similar to the preceding one, but it uses a hex-literal instead of a bit-literal as the argument. The `CONV()` function converts `0x29` from hexadecimal (base 16) to binary (base 2), showing that it equals `00101001` in binary. + +```sql +SELECT BIT_COUNT(0x29), CONV(0x29,16,2); +``` + +``` ++-----------------+-----------------+ +| BIT_COUNT(0x29) | CONV(0x29,16,2) | ++-----------------+-----------------+ +| 3 | 101001 | ++-----------------+-----------------+ +1 row in set (0.01 sec) +``` + +A practical use of the `BIT_COUNT(expr)` function is to convert a netmask to a [CIDR](https://en.wikipedia.org/wiki/Classless_Inter-Domain_Routing) notation. In the following example, the netmask `255.255.255.0` is converted to its CIDR representation `24`. + +```sql +SELECT BIT_COUNT(INET_ATON('255.255.255.0')); +``` + +``` ++---------------------------------------+ +| BIT_COUNT(INET_ATON('255.255.255.0')) | ++---------------------------------------+ +| 24 | ++---------------------------------------+ +1 row in set (0.00 sec) +``` + +## [`&` (bitwise AND)](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#operator_bitwise-and) + +The `&` operator performs a bitwise AND operation. It compares the corresponding bits of two numbers: if both corresponding bits are 1, the corresponding bit of the result is 1; otherwise, it is 0. + +For example, a bitwise AND operation between `1010` and `1100` returns `1000`, because only the leftmost bit is set as 1 in both numbers. + +``` + 1010 +& 1100 + ---- + 1000 +``` + +In SQL, you can use the `&` operator as follows: + +```sql +SELECT CONV(b'1010' & b'1000',10,2); +``` + +``` ++------------------------------+ +| CONV(b'1010' & b'1000',10,2) | ++------------------------------+ +| 1000 | ++------------------------------+ +1 row in set (0.00 sec) +``` + +You can use the `&` operator with `INET_NTOA()` and `INET_ATON()` functions to perform a bitwise AND operation on an IP address and a network mask to get the network address. This is useful to determine whether multiple IP addresses belong to the same network or not. + +In the following two examples, the IP addresses `192.168.1.1` and `192.168.1.2` are in the same network `192.168.1.0/24` when masked with `255.255.255.0`. + +```sql +SELECT INET_NTOA(INET_ATON('192.168.1.1') & INET_ATON('255.255.255.0')); +``` + +``` ++------------------------------------------------------------------+ +| INET_NTOA(INET_ATON('192.168.1.1') & INET_ATON('255.255.255.0')) | ++------------------------------------------------------------------+ +| 192.168.1.0 | ++------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +```sql +SELECT INET_NTOA(INET_ATON('192.168.1.2') & INET_ATON('255.255.255.0')); +``` + +``` ++------------------------------------------------------------------+ +| INET_NTOA(INET_ATON('192.168.1.2') & INET_ATON('255.255.255.0')) | ++------------------------------------------------------------------+ +| 192.168.1.0 | ++------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## [`~` (bitwise inversion)](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#operator_bitwise-invert) + +The `~` operator performs a bitwise inversion (or bitwise NOT) operation on a given value. It inverts each bit of the given value: bits that are 0 become 1, and bits that are 1 become 0. + +Before the operation, the value is expanded to 64 bits. + +Take the binary number `1111000011110000` as an example. When expanded to 64 bits and inverted, it looks like this: + +``` +Original (16 bits): 1111000011110000 +Expanded and inverted (64 bits): 1111111111111111111111111111111111111111111111110000111100001111 +``` + +In SQL, you can use the `~` operator as follows: + +```sql +SELECT CONV(~ b'1111000011110000',10,2); ++------------------------------------------------------------------+ +| CONV(~ b'1111000011110000',10,2) | ++------------------------------------------------------------------+ +| 1111111111111111111111111111111111111111111111110000111100001111 | ++------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +You can reverse the inversion by applying the `~` operator again to the result: + +```sql +SELECT CONV(~ b'1111111111111111111111111111111111111111111111110000111100001111',10,2); +``` + +``` ++----------------------------------------------------------------------------------+ +| CONV(~ b'1111111111111111111111111111111111111111111111110000111100001111',10,2) | ++----------------------------------------------------------------------------------+ +| 1111000011110000 | ++----------------------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## [`|` (bitwise OR)](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#operator_bitwise-or) + +The `|` operator performs a bitwise OR operation. It compares the corresponding bits of two numbers: if at least one of the corresponding bits is 1, the corresponding bit in the result is 1. + +For example, a bitwise OR operation between `1010` and `1100` returns `1110`, because among the first three bits of the two numbers, at least one of the corresponding bits is set as 1. + +``` + 1010 +| 1100 + ---- + 1110 +``` + +In SQL, you can use the `|` operator as follows: + +```sql +SELECT CONV(b'1010' | b'1100',10,2); +``` + +``` ++------------------------------+ +| CONV(b'1010' | b'1100',10,2) | ++------------------------------+ +| 1110 | ++------------------------------+ +1 row in set (0.00 sec) +``` + +## [`^` (bitwise XOR)](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#operator_bitwise-xor) + +The `^` operator performs a bitwise XOR (exclusive OR) operation. It compares the corresponding bits of two numbers: if the corresponding bits are different, the corresponding bit in the result is 1. + +For example, a bitwise XOR operation between `1010` and `1100` returns `0110`, because the second and the third bits of the two numbers are different. + +``` + 1010 +^ 1100 + ---- + 0110 +``` + +In SQL, you can use the `^` operator as follows: + +```sql +SELECT CONV(b'1010' ^ b'1100',10,2); +``` + +``` ++------------------------------+ +| CONV(b'1010' ^ b'1100',10,2) | ++------------------------------+ +| 110 | ++------------------------------+ +1 row in set (0.00 sec) +``` + +Note that the result is shown as `110` instead of `0110` because the leading zero is removed. + +## [`<<` (left shift)](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#operator_left-shift) + +The `<<` operator performs a left shift operation, which shifts the bits of a number to the left by a specified number of positions, filling the vacated bits with zeros on the right. + +For example, the following statement uses `1<>` (right shift)](https://dev.mysql.com/doc/refman/8.0/en/bit-functions.html#operator_right-shift) + +The `>>` operator performs a right shift operation, which shifts the bits of a number to the right by a specified number of positions, filling the vacated bits with zeros on the left. + +For example, the following statement uses `1024>>n` to shift a value of `1024` (`10000000000` in binary) to the right by `n` positions: + +```sql +WITH RECURSIVE cte(n) AS ( + SELECT 0 AS n + UNION ALL + SELECT n+1 FROM cte WHERE n<11 +) +SELECT n,1024>>n,LPAD(CONV(1024>>n,10,2),11,0) FROM cte; +``` + +``` ++------+---------+-------------------------------+ +| n | 1024>>n | LPAD(CONV(1024>>n,10,2),11,0) | ++------+---------+-------------------------------+ +| 0 | 1024 | 10000000000 | +| 1 | 512 | 01000000000 | +| 2 | 256 | 00100000000 | +| 3 | 128 | 00010000000 | +| 4 | 64 | 00001000000 | +| 5 | 32 | 00000100000 | +| 6 | 16 | 00000010000 | +| 7 | 8 | 00000001000 | +| 8 | 4 | 00000000100 | +| 9 | 2 | 00000000010 | +| 10 | 1 | 00000000001 | +| 11 | 0 | 00000000000 | ++------+---------+-------------------------------+ +12 rows in set (0.00 sec) +``` + +The `>>` operator can also be useful for extracting specific parts of a larger number, such as extracting a UNIX timestamp from a TiDB [TSO](/tso.md) timestamp. ## MySQL compatibility diff --git a/functions-and-operators/cast-functions-and-operators.md b/functions-and-operators/cast-functions-and-operators.md index 9ac5cab6eb9d0..588ce5c3e9c05 100644 --- a/functions-and-operators/cast-functions-and-operators.md +++ b/functions-and-operators/cast-functions-and-operators.md @@ -1,7 +1,6 @@ --- title: Cast Functions and Operators summary: Learn about the cast functions and operators. -aliases: ['/docs/dev/functions-and-operators/cast-functions-and-operators/','/docs/dev/reference/sql/functions-and-operators/cast-functions-and-operators/'] --- # Cast Functions and Operators @@ -24,11 +23,31 @@ The [`BINARY`](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#opera ## CAST -The [`CAST()`](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#function_cast) function is used to cast an expression to a specific type. +The [`CAST( AS [ARRAY])`](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#function_cast) function is used to cast an expression to a specific type. This function is also used to create [Multi-valued indexes](/sql-statements/sql-statement-create-index.md#multi-valued-indexes). -Example: +The following types are supported: + +| Type | Description | Whether it can be used with multi-valued indexes | +|----------------------|------------------|------------------------------------------------------------| +| `BINARY(n)` | Binary string | No | +| `CHAR(n)` | Character string | Yes, but only if a length is specified | +| `DATE` | Date | Yes | +| `DATETIME(fsp)` | Date/time, where `fsp` is optional | Yes | +| `DECIMAL(n, m)` | Decimal number, where `n` and `m` are optional and are `10` and `0` if not specified | No | +| `DOUBLE` | Double precision floating-point number | No | +| `FLOAT(n)` | Floating-point number, where `n` is optional and should be between `0` and `53` | No | +| `JSON` | JSON | No | +| `REAL` | Floating-point number | Yes | +| `SIGNED [INTEGER]` | Signed integer | Yes | +| `TIME(fsp)` | Time | Yes | +| `UNSIGNED [INTEGER]` | Unsigned integer | Yes | +| `YEAR` | Year | No | + +Examples: + +The following statement converts a binary string from a HEX literal to a `CHAR`. ```sql SELECT CAST(0x54694442 AS CHAR); @@ -43,6 +62,43 @@ SELECT CAST(0x54694442 AS CHAR); 1 row in set (0.0002 sec) ``` +The following statement casts the values of the `a` attribute extracted from the JSON column to an unsigned array. Note that casting to an array is only supported as part of an index definition for multi-valued indexes. + +```sql +CREATE TABLE t ( + id INT PRIMARY KEY, + j JSON, + INDEX idx_a ((CAST(j->'$.a' AS UNSIGNED ARRAY))) +); +INSERT INTO t VALUES (1, JSON_OBJECT('a',JSON_ARRAY(1,2,3))); +INSERT INTO t VALUES (2, JSON_OBJECT('a',JSON_ARRAY(4,5,6))); +INSERT INTO t VALUES (3, JSON_OBJECT('a',JSON_ARRAY(7,8,9))); +ANALYZE TABLE t; +``` + +```sql + EXPLAIN SELECT * FROM t WHERE 1 MEMBER OF(j->'$.a')\G +*************************** 1. row *************************** + id: IndexMerge_10 + estRows: 2.00 + task: root +access object: +operator info: type: union +*************************** 2. row *************************** + id: ├─IndexRangeScan_8(Build) + estRows: 2.00 + task: cop[tikv] +access object: table:t, index:idx_a(cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array)) +operator info: range:[1,1], keep order:false, stats:partial[j:unInitialized] +*************************** 3. row *************************** + id: └─TableRowIDScan_9(Probe) + estRows: 2.00 + task: cop[tikv] +access object: table:t +operator info: keep order:false, stats:partial[j:unInitialized] +3 rows in set (0.00 sec) +``` + ## CONVERT The [`CONVERT()`](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#function_convert) function is used to convert between [character sets](/character-set-and-collation.md). diff --git a/functions-and-operators/control-flow-functions.md b/functions-and-operators/control-flow-functions.md index 0d2fa9ce88a12..e884fd347221e 100644 --- a/functions-and-operators/control-flow-functions.md +++ b/functions-and-operators/control-flow-functions.md @@ -1,7 +1,6 @@ --- title: Control Flow Functions summary: Learn about the Control Flow functions. -aliases: ['/docs/dev/functions-and-operators/control-flow-functions/','/docs/dev/reference/sql/functions-and-operators/control-flow-functions/'] --- # Control Flow Functions diff --git a/functions-and-operators/date-and-time-functions.md b/functions-and-operators/date-and-time-functions.md index 17ac1032a2e8c..848203957d608 100644 --- a/functions-and-operators/date-and-time-functions.md +++ b/functions-and-operators/date-and-time-functions.md @@ -1,7 +1,6 @@ --- title: Date and Time Functions summary: Learn how to use the data and time functions. -aliases: ['/docs/dev/functions-and-operators/date-and-time-functions/','/docs/dev/reference/sql/functions-and-operators/date-and-time-functions/'] --- # Date and Time Functions diff --git a/functions-and-operators/encryption-and-compression-functions.md b/functions-and-operators/encryption-and-compression-functions.md index 342a6f6ceff06..0e2234b6f1c7e 100644 --- a/functions-and-operators/encryption-and-compression-functions.md +++ b/functions-and-operators/encryption-and-compression-functions.md @@ -1,7 +1,6 @@ --- title: Encryption and Compression Functions summary: Learn about the encryption and compression functions. -aliases: ['/docs/dev/functions-and-operators/encryption-and-compression-functions/','/docs/dev/reference/sql/functions-and-operators/encryption-and-compression-functions/'] --- # Encryption and Compression Functions @@ -10,32 +9,420 @@ TiDB supports most of the [encryption and compression functions](https://dev.mys ## Supported functions -| Name | Description | -|:------------------------------------------------------------------------------------------------------------------------------------------------------|:--------------------------------------------------| -| [`MD5()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_md5) | Calculate MD5 checksum | -| [`PASSWORD()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_password) | Calculate and return a password string | -| [`RANDOM_BYTES()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_random-bytes) | Return a random byte vector | -| [`SHA1(), SHA()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_sha1) | Calculate an SHA-1 160-bit checksum | -| [`SHA2()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_sha2) | Calculate an SHA-2 checksum | -| [`SM3()`](https://en.wikipedia.org/wiki/SM3_(hash_function)) | Calculate an SM3 checksum (currently MySQL does not support this function) | -| [`AES_DECRYPT()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_aes-decrypt) | Decrypt using AES | -| [`AES_ENCRYPT()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_aes-encrypt) | Encrypt using AES | -| [`COMPRESS()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_compress) | Return result as a binary string | -| [`UNCOMPRESS()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_uncompress) | Uncompress a string compressed | -| [`UNCOMPRESSED_LENGTH()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_uncompressed-length) | Return the length of a string before compression | -| [`VALIDATE_PASSWORD_STRENGTH()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_validate-password-strength) | Validate the password strength | +| Name | Description | +|:--------------------------------------------------------------|:--------------------------------------------------| +| [`AES_DECRYPT()`](#aes_decrypt) | Decrypt using AES | +| [`AES_ENCRYPT()`](#aes_encrypt) | Encrypt using AES | +| [`COMPRESS()`](#compress) | Compress and return result as a binary string | +| [`MD5()`](#md5) | Calculate MD5 checksum | +| [`PASSWORD()`](#password) | Calculate and return a password string | +| [`RANDOM_BYTES()`](#random_bytes) | Return a random byte vector | +| [`SHA()`](#sha) | Calculate an SHA-1 160-bit checksum | +| [`SHA1()`](#sha1) | Calculate an SHA-1 160-bit checksum | +| [`SHA2()`](#sha2) | Calculate an SHA-2 checksum | +| [`SM3()`](#sm3) | Calculate an SM3 checksum | +| [`UNCOMPRESS()`](#uncompress) | Uncompress a compressed string | +| [`UNCOMPRESSED_LENGTH()`](#uncompressed_length) | Return the length of a string before compression | +| [`VALIDATE_PASSWORD_STRENGTH()`](#validate_password_strength) | Validate the password strength | -## Related system variables +### [`AES_DECRYPT()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_aes-decrypt) -The [`block_encryption_mode`](/system-variables.md#block_encryption_mode) variable sets the encryption mode that is used for `AES_ENCRYPT()` and `AES_DECRYPT()`. +The `AES_DECRYPT(data, key [,iv])` function decrypts `data` that was previously encrypted using the [`AES_ENCRYPT()`](#aes_encrypt) function with the same `key`. -The [`validate_password.*`](/system-variables.md) variables affect the `VALIDATE_PASSWORD_STRENGTH()` function. +You can use the [`block_encryption_mode`](/system-variables.md#block_encryption_mode) system variable to select the [Advanced Encryption Standard (AES)](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard) encryption mode. + +For encryption modes that require an initialization vector, set it with the `iv` argument. The default value is `NULL`. + +```sql +SELECT AES_DECRYPT(0x28409970815CD536428876175F1A4923, 'secret'); +``` + +``` ++----------------------------------------------------------------------------------------------------------------------+ +| AES_DECRYPT(0x28409970815CD536428876175F1A4923, 'secret') | ++----------------------------------------------------------------------------------------------------------------------+ +| 0x616263 | ++----------------------------------------------------------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +### [`AES_ENCRYPT()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_aes-encrypt) + +The `AES_ENCRYPT(data, key [,iv])` function encrypts `data` with `key` using the [Advanced Encryption Standard (AES)](https://en.wikipedia.org/wiki/Advanced_Encryption_Standard) algorithm. + +You can use the [`block_encryption_mode`](/system-variables.md#block_encryption_mode) system variable to select the AES encryption mode. + +For encryption modes that require an initialization vector, set it with the `iv` argument. The default value is `NULL`. + +```sql +SELECT AES_ENCRYPT(0x616263,'secret'); +``` + +``` ++----------------------------------------------------------------+ +| AES_ENCRYPT(0x616263,'secret') | ++----------------------------------------------------------------+ +| 0x28409970815CD536428876175F1A4923 | ++----------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +### [`COMPRESS()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_compress) + +The `COMPRESS(expr)` function returns a compressed version of the input data `expr`. + +- If the argument is `NULL`, the function returns `NULL`. +- If the argument is an empty string, the function returns a zero-length value. + +For non-zero length argument, the function returns a binary string with the following structure: + +- Bytes 0 to 3: the uncompressed length +- Bytes 4 to the end: the zlib compressed data + +```sql +SELECT COMPRESS(0x414243); +``` + +``` ++------------------------------------------+ +| COMPRESS(0x414243) | ++------------------------------------------+ +| 0x03000000789C72747206040000FFFF018D00C7 | ++------------------------------------------+ +1 row in set (0.00 sec) +``` + +In this output, `0x03000000` represents the uncompressed length (3) and `0x789C72747206040000FFFF018D00C7` is the zlib compressed data. + +An example of using Python to decode this outside of TiDB: + +```python +import codecs +import zlib + +data = codecs.decode('03000000789C72747206040000FFFF018D00C7','hex') +print(int.from_bytes(data[:4], byteorder='little')) # 3 +print(zlib.decompress(data[4:])) # b'ABC' +``` + +For short strings, `COMPRESS()` might return more bytes than the input. The following example shows that a string of 100 `a` characters compresses to 19 bytes. + +```sql +WITH x AS (SELECT REPEAT('a',100) 'a') +SELECT LENGTH(a),LENGTH(COMPRESS(a)) FROM x; +``` + +``` ++-----------+---------------------+ +| LENGTH(a) | LENGTH(COMPRESS(a)) | ++-----------+---------------------+ +| 100 | 19 | ++-----------+---------------------+ +1 row in set (0.00 sec) +``` + +### [`MD5()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_md5) + +The `MD5(expr)` function calculates a 128-bit [MD5](https://en.wikipedia.org/wiki/MD5) hash for the given argument `expr`. + +```sql +SELECT MD5('abc'); +``` + +``` ++----------------------------------+ +| MD5('abc') | ++----------------------------------+ +| 900150983cd24fb0d6963f7d28e17f72 | ++----------------------------------+ +1 row in set (0.00 sec) +``` + +### [`PASSWORD()`](https://dev.mysql.com/doc/refman/5.7/en/encryption-functions.html#function_password) + +> **Warning:** +> +> This function is deprecated in MySQL 5.7 and removed in MySQL 8.0. It is deprecated in TiDB. It is not recommended to use this function. + +The `PASSWORD(str)` function calculates a password hash that can be used with the `mysql_native_password` authentication method. + +```sql +SELECT PASSWORD('secret'); +``` + +``` ++-------------------------------------------+ +| PASSWORD('secret') | ++-------------------------------------------+ +| *14E65567ABDB5135D0CFD9A70B3032C179A49EE7 | ++-------------------------------------------+ +1 row in set, 1 warning (0.00 sec) + +Warning (Code 1681): PASSWORD is deprecated and will be removed in a future release. +``` + +### [`RANDOM_BYTES()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_random-bytes) + +The `RANDOM_BYTES(n)` function returns `n` random bytes. + +```sql +SELECT RANDOM_BYTES(3); +``` + +``` ++----------------------------------+ +| RANDOM_BYTES(3) | ++----------------------------------+ +| 0x1DBC0D | ++----------------------------------+ +1 row in set (0.00 sec) +``` + +### [`SHA()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_sha1) + +The `SHA()` function is an alias for [`SHA1`](#sha1). + +### [`SHA1()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_sha1) + +The `SHA1(expr)` function calculates a 160-bit [SHA-1](https://en.wikipedia.org/wiki/SHA-1) hash for the given argument `expr`. + +```sql +SELECT SHA1('abc'); +``` + +``` ++------------------------------------------+ +| SHA1('abc') | ++------------------------------------------+ +| a9993e364706816aba3e25717850c26c9cd0d89d | ++------------------------------------------+ +1 row in set (0.00 sec) +``` + +### [`SHA2()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_sha2) + +The `SHA2(str, n)` function calculates a hash using an algorithm from the [SHA-2](https://en.wikipedia.org/wiki/SHA-2) family. The `n` argument is used to select the algorithm. `SHA2()` returns `NULL` if any of the arguments are `NULL` or if the algorithm selected by `n` is unknown or unsupported. + +The following lists supported algorithms: + +| n | Algorithm | +|-----|-----------| +| 0 | SHA-256 | +| 224 | SHA-224 | +| 256 | SHA-256 | +| 384 | SHA-384 | +| 512 | SHA-512 | + +```sql +SELECT SHA2('abc',224); +``` + +``` ++----------------------------------------------------------+ +| SHA2('abc',224) | ++----------------------------------------------------------+ +| 23097d223405d8228642a477bda255b32aadbce4bda0b3f7e36c9da7 | ++----------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +### `SM3()` + +> **Note:** +> +> The `SM3()` function is a TiDB extension and is not implemented in MySQL. + +The `SM3(str)` function calculates a 256-bit [ShangMi 3 (SM3)](https://en.wikipedia.org/wiki/SM3_(hash_function)) hash for the given argument `str`. + +```sql +SELECT SM3('abc'); +``` + +``` ++------------------------------------------------------------------+ +| SM3('abc') | ++------------------------------------------------------------------+ +| 66c7f0f462eeedd9d1f2d46bdc10e4e24167c4875cf2f7a2297da02b8f4ba8e0 | ++------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +### [`UNCOMPRESS()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_uncompress) + +The `UNCOMPRESS(data)` function decompresses the data that was compressed with the [`COMPRESS()`](#compress) function. + +```sql +SELECT UNCOMPRESS(0x03000000789C72747206040000FFFF018D00C7); +``` + +``` ++------------------------------------------------------------------------------------------------------------+ +| UNCOMPRESS(0x03000000789C72747206040000FFFF018D00C7) | ++------------------------------------------------------------------------------------------------------------+ +| 0x414243 | ++------------------------------------------------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +### [`UNCOMPRESSED_LENGTH()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_uncompressed-length) + +The `UNCOMPRESSED_LENGTH(data)` function returns the first 4 bytes of the compressed data, which store the length that the compressed string had before being compressed with the [`COMPRESS()`](#compress) function. + +```sql +SELECT UNCOMPRESSED_LENGTH(0x03000000789C72747206040000FFFF018D00C7); +``` + +``` ++---------------------------------------------------------------+ +| UNCOMPRESSED_LENGTH(0x03000000789C72747206040000FFFF018D00C7) | ++---------------------------------------------------------------+ +| 3 | ++---------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +### [`VALIDATE_PASSWORD_STRENGTH()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_validate-password-strength) + + + +The `VALIDATE_PASSWORD_STRENGTH(str)` function is used as part of [password management](/password-management.md). It calculates the strength of a password and returns a value between 0 and 100. + + + + + +The `VALIDATE_PASSWORD_STRENGTH(str)` function is used as part of password management. It calculates the strength of a password and returns a value between 0 and 100. + + + +The [`validate_password.*`](/system-variables.md) system variables affect the behavior of the `VALIDATE_PASSWORD_STRENGTH()` function. + +Examples: + +- To enable the password complexity check, set the [`validate_password.enable`](/system-variables.md#validate_passwordenable-new-in-v650) system variable to `ON`: + + ```sql + SET GLOBAL validate_password.enable=ON; + ``` + +- View password validation-related system variables: + + ```sql + SHOW VARIABLES LIKE 'validate_password.%'; + ``` + + ``` + +--------------------------------------+--------+ + | Variable_name | Value | + +--------------------------------------+--------+ + | validate_password.check_user_name | ON | + | validate_password.dictionary | | + | validate_password.enable | ON | + | validate_password.length | 8 | + | validate_password.mixed_case_count | 1 | + | validate_password.number_count | 1 | + | validate_password.policy | MEDIUM | + | validate_password.special_char_count | 1 | + +--------------------------------------+--------+ + 8 rows in set (0.01 sec) + ``` + +- Check the password strength of an empty string, which returns `0`: + + ```sql + SELECT VALIDATE_PASSWORD_STRENGTH(''); + ``` + + ``` + +--------------------------------+ + | VALIDATE_PASSWORD_STRENGTH('') | + +--------------------------------+ + | 0 | + +--------------------------------+ + 1 row in set (0.00 sec) + ``` + +- Check the password strength of a short string `abcdef`, which returns `25`: + + ```sql + SELECT VALIDATE_PASSWORD_STRENGTH('abcdef'); + ``` + + ``` + +--------------------------------------+ + | VALIDATE_PASSWORD_STRENGTH('abcdef') | + +--------------------------------------+ + | 25 | + +--------------------------------------+ + 1 row in set (0.00 sec) + ``` + +- Check the password strength of a longer string `abcdefghi`, which returns `50`. This string is longer than the default value of [`validate_password.length`](/system-variables.md#validate_passwordlength-new-in-v650): + + ```sql + SELECT VALIDATE_PASSWORD_STRENGTH('abcdefghi'); + ``` + + ``` + +-----------------------------------------+ + | VALIDATE_PASSWORD_STRENGTH('abcdefghi') | + +-----------------------------------------+ + | 50 | + +-----------------------------------------+ + 1 row in set (0.00 sec) + ``` + +- Adding an upper-case character to the string does not improve the password strength: + + ```sql + SELECT VALIDATE_PASSWORD_STRENGTH('Abcdefghi'); + ``` + + ``` + +-----------------------------------------+ + | VALIDATE_PASSWORD_STRENGTH('Abcdefghi') | + +-----------------------------------------+ + | 50 | + +-----------------------------------------+ + 1 row in set (0.01 sec) + ``` + +- Adding numbers to the string also does not improve the password strength: + + ```sql + SELECT VALIDATE_PASSWORD_STRENGTH('Abcdefghi123'); + ``` + + ``` + +--------------------------------------------+ + | VALIDATE_PASSWORD_STRENGTH('Abcdefghi123') | + +--------------------------------------------+ + | 50 | + +--------------------------------------------+ + 1 row in set (0.00 sec) + ``` + +- Finally, adding special characters to the string brings the password strength to `100`, indicating a strong password: + + ```sql + SELECT VALIDATE_PASSWORD_STRENGTH('Abcdefghi123%$#'); + ``` + + ``` + +-----------------------------------------------+ + | VALIDATE_PASSWORD_STRENGTH('Abcdefghi123%$#') | + +-----------------------------------------------+ + | 100 | + +-----------------------------------------------+ + 1 row in set (0.00 sec) + ``` ## Unsupported functions -* `DES_DECRYPT()`, `DES_ENCRYPT()`, `OLD_PASSWORD()`, `ENCRYPT()`: these functions were deprecated in MySQL 5.7 and removed in 8.0. -* Functions only available in MySQL Enterprise [Issue #2632](https://github.com/pingcap/tidb/issues/2632). +* TiDB does not support the functions only available in MySQL Enterprise [Issue #2632](https://github.com/pingcap/tidb/issues/2632). ## MySQL compatibility * TiDB does not support the `STATEMENT_DIGEST()` and `STATEMENT_DIGEST_TEXT()` functions. +* TiDB does not support the `kdf_name`, `salt`, and `iterations` arguments for [`AES_ENCRYPT()`](#aes_encrypt) and [`AES_DECRYPT`](#aes_decrypt) that MySQL added in MySQL 8.0.30. +* MySQL does not implement the [`SM3()`](#sm3) function. \ No newline at end of file diff --git a/functions-and-operators/expressions-pushed-down.md b/functions-and-operators/expressions-pushed-down.md index 0d6f0a388cf3f..0dfe8433c8534 100644 --- a/functions-and-operators/expressions-pushed-down.md +++ b/functions-and-operators/expressions-pushed-down.md @@ -1,7 +1,6 @@ --- title: List of Expressions for Pushdown summary: Learn a list of expressions that can be pushed down to TiKV and the related operations. -aliases: ['/docs/dev/functions-and-operators/expressions-pushed-down/','/docs/dev/reference/sql/functions-and-operators/expressions-pushed-down/'] --- # List of Expressions for Pushdown @@ -10,6 +9,10 @@ When TiDB reads data from TiKV, TiDB tries to push down some expressions (includ TiFlash also supports pushdown for the functions and operators [listed on this page](/tiflash/tiflash-supported-pushdown-calculations.md). +> **Note:** +> +> The Aggregate functions do not support pushdown to TiKV when used as [window functions](/functions-and-operators/window-functions.md). + ## Supported expressions for pushdown to TiKV | Expression Type | Operations | @@ -19,13 +22,14 @@ TiFlash also supports pushdown for the functions and operators [listed on this p | [Comparison functions and operators](/functions-and-operators/operators.md#comparison-functions-and-operators) | [`<`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_less-than), [`<=`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_less-than-or-equal), [`=`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_equal), [`!= (<>)`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_not-equal), [`>`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_greater-than), [`>=`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_greater-than-or-equal), [`<=>`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_equal-to), [BETWEEN ... AND ...](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_between), [COALESCE()](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#function_coalesce), [IN()](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_in), [INTERVAL()](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#function_interval), [IS NOT NULL](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_is-not-null), [IS NOT](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_is-not), [IS NULL](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_is-null), [IS](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_is), [ISNULL()](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#function_isnull), [LIKE](https://dev.mysql.com/doc/refman/8.0/en/string-comparison-functions.html#operator_like), [NOT BETWEEN ... AND ...](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_not-between), [NOT IN()](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_not-in), [NOT LIKE](https://dev.mysql.com/doc/refman/8.0/en/string-comparison-functions.html#operator_not-like), [STRCMP()](https://dev.mysql.com/doc/refman/8.0/en/string-comparison-functions.html#function_strcmp) | | [Numeric functions and operators](/functions-and-operators/numeric-functions-and-operators.md) | [+](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_plus), [-](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_minus), [*](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_times), [/](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_divide), [DIV](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_div), [% (MOD)](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_mod), [-](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_unary-minus), [ABS()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_abs), [ACOS()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_acos), [ASIN()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_asin), [ATAN()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_atan), [ATAN2(), ATAN()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_atan2), [CEIL()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_ceil), [CEILING()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_ceiling), [CONV()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_conv), [COS()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_cos), [COT()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_cot), [CRC32()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_crc32), [DEGREES()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_degrees), [EXP()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_exp), [FLOOR()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_floor), [LN()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_ln), [LOG()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_log), [LOG10()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_log10), [LOG2()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_log2), [MOD()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_mod), [PI()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_pi), [POW()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_pow), [POWER()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_power), [RADIANS()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_radians), [RAND()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_rand), [ROUND()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_round), [SIGN()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_sign), [SIN()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_sin), [SQRT()](https://dev.mysql.com/doc/refman/8.0/en/mathematical-functions.html#function_sqrt) | | [Control flow functions](/functions-and-operators/control-flow-functions.md) | [CASE](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#operator_case), [IF()](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#function_if), [IFNULL()](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#function_ifnull) | -| [JSON functions](/functions-and-operators/json-functions.md) | [JSON_ARRAY([val[, val] ...])](https://dev.mysql.com/doc/refman/8.0/en/json-creation-functions.html#function_json-array),
[JSON_CONTAINS(target, candidate[, path])](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-contains),
[JSON_EXTRACT(json_doc, path[, path] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-extract),
[JSON_INSERT(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-insert),
[JSON_LENGTH(json_doc[, path])](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-length),
[JSON_MERGE(json_doc, json_doc[, json_doc] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-merge),
[JSON_OBJECT([key, val[, key, val] ...])](https://dev.mysql.com/doc/refman/8.0/en/json-creation-functions.html#function_json-object),
[JSON_REMOVE(json_doc, path[, path] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-remove),
[JSON_REPLACE(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-replace),
[JSON_SET(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-set),
[JSON_TYPE(json_val)](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-type),
[JSON_UNQUOTE(json_val)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-unquote),
[JSON_VALID(val)](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-valid),
[value MEMBER OF(json_array)](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#operator_member-of) | +| [JSON functions](/functions-and-operators/json-functions.md) | [JSON_ARRAY([val[, val] ...])](https://dev.mysql.com/doc/refman/8.0/en/json-creation-functions.html#function_json-array),
[JSON_CONTAINS(target, candidate[, path])](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-contains),
[JSON_EXTRACT(json_doc, path[, path] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-extract),
[JSON_INSERT(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-insert),
[JSON_LENGTH(json_doc[, path])](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-length),
[JSON_MERGE(json_doc, json_doc[, json_doc] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-merge),
[JSON_OBJECT([key, val[, key, val] ...])](https://dev.mysql.com/doc/refman/8.0/en/json-creation-functions.html#function_json-object),
[JSON_REMOVE(json_doc, path[, path] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-remove),
[JSON_SET(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-set),
[JSON_TYPE(json_val)](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-type),
[JSON_UNQUOTE(json_val)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-unquote),
[JSON_VALID(val)](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-valid),
[value MEMBER OF(json_array)](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#operator_member-of) | | [Date and time functions](/functions-and-operators/date-and-time-functions.md) | [DATE()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_date), [DATE_FORMAT()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_date-format), [DATEDIFF()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_datediff), [DAYOFMONTH()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_dayofmonth), [DAYOFWEEK()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_dayofweek), [DAYOFYEAR()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_dayofyear), [FROM_DAYS()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_from-days), [HOUR()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_hour), [MAKEDATE()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_makedate), [MAKETIME()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_maketime), [MICROSECOND()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_microsecond), [MINUTE()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_minute), [MONTH()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_month), [MONTHNAME()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_monthname), [PERIOD_ADD()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_period-add), [PERIOD_DIFF()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_period-diff), [SEC_TO_TIME()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_sec-to-time), [SECOND()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_second), [SYSDATE()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_sysdate), [TIME_TO_SEC()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_time-to-sec), [TIMEDIFF()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_timediff), [WEEK()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_week), [WEEKOFYEAR()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_weekofyear), [YEAR()](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_year) | | [String functions](/functions-and-operators/string-functions.md) | [ASCII()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_ascii), [BIT_LENGTH()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_bit-length), [CHAR()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_char), [CHAR_LENGTH()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_char-length), [CONCAT()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_concat), [CONCAT_WS()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_concat-ws), [ELT()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_elt), [FIELD()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_field), [HEX()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_hex), [LENGTH()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_length), [LIKE](https://dev.mysql.com/doc/refman/8.0/en/string-comparison-functions.html#operator_like), [LOWER()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_lower), [LTRIM()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_ltrim), [MID()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_mid), [NOT LIKE](https://dev.mysql.com/doc/refman/8.0/en/string-comparison-functions.html#operator_not-like), [NOT REGEXP](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#operator_not-regexp), [REGEXP](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#operator_regexp), [REGEXP_INSTR()](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#function_regexp-instr), [REGEXP_LIKE()](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#function_regexp-like), [REGEXP_REPLACE()](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#function_regexp-replace), [REGEXP_SUBSTR()](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#function_regexp-substr), [REPLACE()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_replace), [REVERSE()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_reverse), [RIGHT()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_right), [RLIKE](https://dev.mysql.com/doc/refman/8.0/en/regexp.html#operator_regexp), [RTRIM()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_rtrim), [SPACE()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_space), [STRCMP()](https://dev.mysql.com/doc/refman/8.0/en/string-comparison-functions.html#function_strcmp), [SUBSTR()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_substr), [SUBSTRING()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_substring), [UPPER()](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_upper) | | [Aggregation functions](/functions-and-operators/aggregate-group-by-functions.md#aggregate-group-by-functions) | [COUNT()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_count), [COUNT(DISTINCT)](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_count-distinct), [SUM()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_sum), [AVG()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_avg), [MAX()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_max), [MIN()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_min), [VARIANCE()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_variance), [VAR_POP()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_var-pop), [STD()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_std), [STDDEV()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_stddev), [STDDEV_POP](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_stddev-pop), [VAR_SAMP()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_var-samp), [STDDEV_SAMP()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_stddev-samp), [JSON_ARRAYAGG(key)](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_json-arrayagg), [JSON_OBJECTAGG(key, value)](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_json-objectagg) | -| [Encryption and compression functions](/functions-and-operators/encryption-and-compression-functions.md#encryption-and-compression-functions) | [MD5()](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_md5), [SHA1(), SHA()](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_sha1), [UNCOMPRESSED_LENGTH()](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_uncompressed-length) | +| [Encryption and compression functions](/functions-and-operators/encryption-and-compression-functions.md#encryption-and-compression-functions) | [MD5()](/functions-and-operators/encryption-and-compression-functions.md#md5), [SHA1(), SHA()](/functions-and-operators/encryption-and-compression-functions.md#sha1), [UNCOMPRESSED_LENGTH()](/functions-and-operators/encryption-and-compression-functions.md#uncompressed_length) | | [Cast functions and operators](/functions-and-operators/cast-functions-and-operators.md#cast-functions-and-operators) | [CAST()](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#function_cast), [CONVERT()](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#function_convert) | | [Miscellaneous functions](/functions-and-operators/miscellaneous-functions.md#supported-functions) | [UUID()](https://dev.mysql.com/doc/refman/8.0/en/miscellaneous-functions.html#function_uuid) | +| [Window Functions](/functions-and-operators/window-functions.md) | None | ## Blocklist specific expressions diff --git a/functions-and-operators/functions-and-operators-overview.md b/functions-and-operators/functions-and-operators-overview.md index ec1253087ea5e..6956801656523 100644 --- a/functions-and-operators/functions-and-operators-overview.md +++ b/functions-and-operators/functions-and-operators-overview.md @@ -1,13 +1,15 @@ --- title: Function and Operator Reference summary: Learn how to use the functions and operators. -aliases: ['/docs/dev/functions-and-operators/functions-and-operators-overview/','/docs/dev/reference/sql/functions-and-operators/reference/'] --- # Function and Operator Reference The usage of the functions and operators in TiDB is similar to MySQL. See [Functions and Operators in MySQL](https://dev.mysql.com/doc/refman/8.0/en/functions.html). -In SQL statements, expressions can be used on the `ORDER BY` and `HAVING` clauses of the `SELECT` statement, the `WHERE` clause of `SELECT`/`DELETE`/`UPDATE` statements, and `SET` statements. +In SQL statements, expressions can be used on the `ORDER BY` and `HAVING` clauses of the [`SELECT`](/sql-statements/sql-statement-select.md) statement, the `WHERE` clause of [`SELECT`](/sql-statements/sql-statement-select.md)/[`DELETE`](/sql-statements/sql-statement-delete.md)/[`UPDATE`](/sql-statements/sql-statement-update.md) statements, and [`SET`](/sql-statements/sql-statement-set-variable.md) statements. -You can write expressions using literals, column names, NULL, built-in functions, operators and so on. For expressions that TiDB supports pushing down to TiKV, see [List of Expressions for Pushdown](/functions-and-operators/expressions-pushed-down.md). +You can write expressions using literals, column names, `NULL`, built-in functions, operators, and so on. + +- For expressions that TiDB supports pushing down to TiKV, see [List of expressions for pushdown](/functions-and-operators/expressions-pushed-down.md). +- For expressions that TiDB supports pushing down to [TiFlash](/tiflash/tiflash-overview.md), see [Push-down expressions](/tiflash/tiflash-supported-pushdown-calculations.md#push-down-expressions). diff --git a/functions-and-operators/group-by-modifier.md b/functions-and-operators/group-by-modifier.md index f95e122f26e3c..710758e14387a 100644 --- a/functions-and-operators/group-by-modifier.md +++ b/functions-and-operators/group-by-modifier.md @@ -18,7 +18,7 @@ In the `GROUP BY` clause, you can specify one or more columns as a group list an With this grouping method, if there are `N` columns in the group list, TiDB aggregates the query results on `N+1` groups. -For example:： +For example: ```sql SELECT count(1) FROM t GROUP BY a,b,c WITH ROLLUP; @@ -217,4 +217,4 @@ After the `Expand` operator is applied, you can get the following three rows of +------------+------+-------+-----+ ``` -Note that the `SELECT` clause in the query uses the `GROUPING` function. When the `GROUPING` function is used in the `SELECT`, `HAVING`, or `ORDER BY` clauses, TiDB rewrites it during the logical optimization phase, transforms the relationship between the `GROUPING` function and the `GROUP BY` items into a `GID` related to the logic of dimension group (also known as `grouping set`), and fills this `GID` as metadata into the new `GROUPING` function. \ No newline at end of file +Note that the `SELECT` clause in the query uses the `GROUPING` function. When the `GROUPING` function is used in the `SELECT`, `HAVING`, or `ORDER BY` clauses, TiDB rewrites it during the logical optimization phase, transforms the relationship between the `GROUPING` function and the `GROUP BY` items into a `GID` related to the logic of dimension group (also known as `grouping set`), and fills this `GID` as metadata into the new `GROUPING` function. diff --git a/functions-and-operators/information-functions.md b/functions-and-operators/information-functions.md index eb9484cce28cc..ae5d04892daeb 100644 --- a/functions-and-operators/information-functions.md +++ b/functions-and-operators/information-functions.md @@ -1,7 +1,6 @@ --- title: Information Functions summary: Learn about the information functions. -aliases: ['/docs/dev/functions-and-operators/information-functions/','/docs/dev/reference/sql/functions-and-operators/information-functions/'] --- # Information Functions diff --git a/functions-and-operators/json-functions.md b/functions-and-operators/json-functions.md index 5f53cf940ef1e..ab36bb403cba2 100644 --- a/functions-and-operators/json-functions.md +++ b/functions-and-operators/json-functions.md @@ -1,78 +1,142 @@ --- title: JSON Functions summary: Learn about JSON functions. -aliases: ['/docs/dev/functions-and-operators/json-functions/','/docs/dev/reference/sql/functions-and-operators/json-functions/'] --- # JSON Functions -TiDB supports most of the [JSON functions](https://dev.mysql.com/doc/refman/8.0/en/json-functions.html) available in MySQL 8.0. +You can use JSON functions to work with data in the [JSON data type](/data-type-json.md). ## Functions that create JSON values | Function Name | Description | | --------------------------------- | ----------- | -| [JSON_ARRAY([val[, val] ...])](https://dev.mysql.com/doc/refman/8.0/en/json-creation-functions.html#function_json-array) | Evaluates a (possibly empty) list of values and returns a JSON array containing those values | -| [JSON_OBJECT(key, val[, key, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-creation-functions.html#function_json-object) | Evaluates a (possibly empty) list of key-value pairs and returns a JSON object containing those pairs | -| [JSON_QUOTE(string)](https://dev.mysql.com/doc/refman/8.0/en/json-creation-functions.html#function_json-quote) | Returns a string as a JSON value with quotes | +| [JSON_ARRAY()](/functions-and-operators/json-functions/json-functions-create.md#json_array) | Evaluates a (possibly empty) list of values and returns a JSON array containing those values | +| [JSON_OBJECT()](/functions-and-operators/json-functions/json-functions-create.md#json_object) | Evaluates a (possibly empty) list of key-value pairs and returns a JSON object containing those pairs | +| [JSON_QUOTE()](/functions-and-operators/json-functions/json-functions-create.md#json_quote) | Returns a string as a JSON value with quotes | ## Functions that search JSON values | Function Name | Description | | --------------------------------- | ----------- | -| [JSON_CONTAINS(target, candidate[, path])](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-contains) | Indicates by returning 1 or 0 whether a given candidate JSON document is contained within a target JSON document | -| [JSON_CONTAINS_PATH(json_doc, one_or_all, path[, path] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-contains-path) | Returns 0 or 1 to indicate whether a JSON document contains data at a given path or paths | -| [JSON_EXTRACT(json_doc, path[, path] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-extract)| Returns data from a JSON document, selected from the parts of the document matched by the `path` arguments | -| [->](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#operator_json-column-path) | Returns the value from a JSON column after the evaluating path; an alias for `JSON_EXTRACT(doc, path_literal)` | -| [->>](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#operator_json-inline-path) | Returns the value from a JSON column after the evaluating path and unquoting the result; an alias for `JSON_UNQUOTE(JSON_EXTRACT(doc, path_literal))` | -| [JSON_KEYS(json_doc[, path])](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-keys) | Returns the keys from the top-level value of a JSON object as a JSON array, or, if a path argument is given, the top-level keys from the selected path | -| [JSON_SEARCH(json_doc, one_or_all, search_str[, escape_char[, path] ...])](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-search) | Search a JSON document for one or all matches of a string | -| [value MEMBER OF(json_array)](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#operator_member-of) | If the passed value is an element of the JSON array, returns 1. Otherwise, returns 0. | -| [JSON_OVERLAPS(json_doc1, json_doc2)](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-overlaps) | Indicates whether two JSON documents have overlapping part. If yes, returns 1. If not, returns 0. | +| [JSON_CONTAINS()](/functions-and-operators/json-functions/json-functions-search.md#json_contains) | Indicates by returning 1 or 0 whether a given candidate JSON document is contained within a target JSON document | +| [JSON_CONTAINS_PATH()](/functions-and-operators/json-functions/json-functions-search.md#json_contains_path) | Returns 0 or 1 to indicate whether a JSON document contains data at a given path or paths | +| [JSON_EXTRACT()](/functions-and-operators/json-functions/json-functions-search.md#json_extract) | Returns data from a JSON document, selected from the parts of the document matched by the `path` arguments | +| [->](/functions-and-operators/json-functions/json-functions-search.md#-) | Returns the value from a JSON column after the evaluating path; an alias for `JSON_EXTRACT(doc, path_literal)` | +| [->>](/functions-and-operators/json-functions/json-functions-search.md#--1) | Returns the value from a JSON column after the evaluating path and unquoting the result; an alias for `JSON_UNQUOTE(JSON_EXTRACT(doc, path_literal))` | +| [JSON_KEYS()](/functions-and-operators/json-functions/json-functions-search.md#json_keys) | Returns the keys from the top-level value of a JSON object as a JSON array, or, if a path argument is given, the top-level keys from the selected path | +| [JSON_SEARCH()](/functions-and-operators/json-functions/json-functions-search.md#json_search) | Search a JSON document for one or all matches of a string | +| [MEMBER OF()](/functions-and-operators/json-functions/json-functions-search.md#member-of) | If the passed value is an element of the JSON array, returns 1. Otherwise, returns 0. | +| [JSON_OVERLAPS()](/functions-and-operators/json-functions/json-functions-search.md#json_overlaps) | Indicates whether two JSON documents have overlapping part. If yes, returns 1. If not, returns 0. | ## Functions that modify JSON values | Function Name | Description | | --------------------------------- | ----------- | -| [JSON_APPEND(json_doc, path, value)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-append) | An alias to `JSON_ARRAY_APPEND` | -| [JSON_ARRAY_APPEND(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-array-append) | Appends values to the end of the indicated arrays within a JSON document and returns the result | -| [JSON_ARRAY_INSERT(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-array-insert) | Insert values into the specified locations of a JSON document and returns the result | -| [JSON_INSERT(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-insert) | Inserts data into a JSON document and returns the result | -| [JSON_MERGE_PATCH(json_doc, json_doc[, json_doc] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-merge-patch) | Merge JSON documents | -| [JSON_MERGE_PRESERVE(json_doc, json_doc[, json_doc] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-merge-preserve) | Merges two or more JSON documents and returns the merged result | -| [JSON_MERGE(json_doc, json_doc[, json_doc] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-merge) | A deprecated alias for `JSON_MERGE_PRESERVE` | -| [JSON_REMOVE(json_doc, path[, path] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-remove) | Removes data from a JSON document and returns the result | -| [JSON_REPLACE(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-replace) | Replaces existing values in a JSON document and returns the result | -| [JSON_SET(json_doc, path, val[, path, val] ...)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-set) | Inserts or updates data in a JSON document and returns the result | -| [JSON_UNQUOTE(json_val)](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-unquote) | Unquotes a JSON value and returns the result as a string | +| [JSON_APPEND()](/functions-and-operators/json-functions/json-functions-modify.md#json_append) | An alias to `JSON_ARRAY_APPEND()` | +| [JSON_ARRAY_APPEND()](/functions-and-operators/json-functions/json-functions-modify.md#json_array_append) | Appends values to the end of the indicated arrays within a JSON document and returns the result | +| [JSON_ARRAY_INSERT()](/functions-and-operators/json-functions/json-functions-modify.md#json_array_insert) | Insert values into the specified locations of a JSON document and returns the result | +| [JSON_INSERT()](/functions-and-operators/json-functions/json-functions-modify.md#json_insert) | Inserts data into a JSON document and returns the result | +| [JSON_MERGE_PATCH()](/functions-and-operators/json-functions/json-functions-modify.md#json_merge_patch) | Merges two or more JSON documents, without preserving values of duplicate keys | +| [JSON_MERGE_PRESERVE()](/functions-and-operators/json-functions/json-functions-modify.md#json_merge_preserve) | Merges two or more JSON documents by preserving all values | +| [JSON_MERGE()](/functions-and-operators/json-functions/json-functions-modify.md#json_merge) | A deprecated alias for `JSON_MERGE_PRESERVE()` | +| [JSON_REMOVE()](/functions-and-operators/json-functions/json-functions-modify.md#json_remove) | Removes data from a JSON document and returns the result | +| [JSON_REPLACE()](/functions-and-operators/json-functions/json-functions-modify.md#json_replace) | Replaces existing values in a JSON document and returns the result | +| [JSON_SET()](/functions-and-operators/json-functions/json-functions-modify.md#json_set) | Inserts or updates data in a JSON document and returns the result | +| [JSON_UNQUOTE()](/functions-and-operators/json-functions/json-functions-modify.md#json_unquote) | Unquotes a JSON value and returns the result as a string | ## Functions that return JSON value attributes | Function Name | Description | | --------------------------------- | ----------- | -| [JSON_DEPTH(json_doc)](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-depth) | Returns the maximum depth of a JSON document | -| [JSON_LENGTH(json_doc[, path])](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-length) | Returns the length of a JSON document, or, if a path argument is given, the length of the value within the path | -| [JSON_TYPE(json_val)](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-type) | Returns a string indicating the type of a JSON value | -| [JSON_VALID(json_doc)](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-valid) | Checks if a json\_doc is valid JSON. Useful for checking a column before converting it to the json type. | +| [JSON_DEPTH()](/functions-and-operators/json-functions/json-functions-return.md#json_depth) | Returns the maximum depth of a JSON document | +| [JSON_LENGTH()](/functions-and-operators/json-functions/json-functions-return.md#json_length) | Returns the length of a JSON document, or, if a path argument is given, the length of the value within the path | +| [JSON_TYPE()](/functions-and-operators/json-functions/json-functions-return.md#json_type) | Returns a string indicating the type of a JSON value | +| [JSON_VALID()](/functions-and-operators/json-functions/json-functions-return.md#json_valid) | Checks if a json\_doc is valid JSON. | ## Utility functions | Function Name | Description | | --------------------------------- | ----------- | -| [JSON_PRETTY(json_doc)](https://dev.mysql.com/doc/refman/8.0/en/json-utility-functions.html#function_json-pretty) | Pretty formatting of a JSON document | -| [JSON_STORAGE_FREE(json_doc)](https://dev.mysql.com/doc/refman/8.0/en/json-utility-functions.html#function_json-storage-free) | Returns how much storage space was freed in the binary representation of the JSON value after it was updated in place. As TiDB has different storage architecture from MySQL, this function always returns 0 for a valid JSON value, and it is implemented for compatibility with MySQL 8.0. | -| [JSON_STORAGE_SIZE(json_doc)](https://dev.mysql.com/doc/refman/8.0/en/json-utility-functions.html#function_json-storage-size) | Returns an approximate size of bytes required to store the json value. As the size does not account for TiKV using compression, the output of this function is not strictly compatible with MySQL. | +| [JSON_PRETTY()](/functions-and-operators/json-functions/json-functions-utility.md#json_pretty) | Pretty formatting of a JSON document | +| [JSON_STORAGE_FREE()](/functions-and-operators/json-functions/json-functions-utility.md#json_storage_free) | Returns how much storage space was freed in the binary representation of the JSON value after it was updated in place. | +| [JSON_STORAGE_SIZE()](/functions-and-operators/json-functions/json-functions-utility.md#json_storage_size) | Returns an approximate size of bytes required to store the json value. As the size does not account for TiKV using compression, the output of this function is not strictly compatible with MySQL. | ## Aggregate functions | Function Name | Description | | --------------------------------- | ----------- | -| [JSON_ARRAYAGG(key)](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_json-arrayagg) | Provides an aggregation of keys. | -| [JSON_OBJECTAGG(key, value)](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_json-objectagg) | Provides an aggregation of values for a given key. | +| [JSON_ARRAYAGG()](/functions-and-operators/json-functions/json-functions-aggregate.md#json_arrayagg) | Provides an aggregation of keys. | +| [JSON_OBJECTAGG()](/functions-and-operators/json-functions/json-functions-aggregate.md#json_objectagg) | Provides an aggregation of values for a given key. | + +## JSONPath + +Many of JSON functions use [JSONPath](https://www.rfc-editor.org/rfc/rfc9535.html) to select parts of a JSON document. + +| Symbol | Description | +| -------------- | ---------------------------- | +| `$` | Document root | +| `.` | Member selection | +| `[]` | Array selection | +| `*` | Wildcard | +| `**` | Path wildcard | +| `[ to ]` | Array range selection | + +The subsequent content takes the following JSON document as an example to demonstrate how to use JSONPath: + +```json +{ + "database": { + "name": "TiDB", + "features": [ + "distributed", + "scalable", + "relational", + "cloud native" + ], + "license": "Apache-2.0 license", + "versions": [ + { + "version": "v8.1.2", + "type": "lts", + "release_date": "2024-12-26" + }, + { + "version": "v8.0.0", + "type": "dmr", + "release_date": "2024-03-29" + } + ] + }, + "migration_tool": { + "name": "TiDB Data Migration", + "features": [ + "MySQL compatible", + "Shard merging" + ], + "license": "Apache-2.0 license" + } +} +``` + +| JSONPath | Description | Example with [`JSON_EXTRACT()`](/functions-and-operators/json-functions/json-functions-search.md#json_extract) | +|-------------------------------------- |-----------------------------------------|-------------------------------| +| `$` | The root of the document | Returns the full document | +| `$.database` | The `database` object | Returns the full structure starting with `"database"`. It does not include `"migration_tool"` and the structure below that. | +| `$.database.name` | The name of the database. | `"TiDB"` | +| `$.database.features` | All database features | `["distributed", "scalable", "relational", "cloud native"]` | +| `$.database.features[0]` | The first database feature. | `"distributed"` | +| `$.database.features[2]` | The third database feature. | `"relational"` | +| `$.database.versions[0].type` | The type of the first database version. | `"lts"` | +| `$.database.versions[*].release_date` | The release date for all versions. | `["2024-12-26","2024-03-29"]` | +| `$.*.features` | Two array's of features | `[["distributed", "scalable", "relational", "cloud native"], ["MySQL compatible", "Shard merging"]]` | +| `$**.version` | All versions, with path wildcard | `["v8.1.2","v8.0.0"]` | +| `$.database.features[0 to 2]` | Range of database features from the first to the third. | `["scalable","relational"]` | + +For more information, see [the IETF draft for JSONPath](https://www.ietf.org/archive/id/draft-goessner-dispatch-jsonpath-00.html). ## See also -* [JSON Function Reference](https://dev.mysql.com/doc/refman/8.0/en/json-function-reference.html) * [JSON Data Type](/data-type-json.md) ## Unsupported functions @@ -82,4 +146,8 @@ TiDB supports most of the [JSON functions](https://dev.mysql.com/doc/refman/8.0/ - `JSON_TABLE()` - `JSON_VALUE()` -For more information, see [#14486](https://github.com/pingcap/tidb/issues/14486). \ No newline at end of file +For more information, see [#14486](https://github.com/pingcap/tidb/issues/14486). + +## MySQL compatibility + +- TiDB supports most of the [JSON functions](https://dev.mysql.com/doc/refman/8.0/en/json-functions.html) available in MySQL 8.0. diff --git a/functions-and-operators/json-functions/json-functions-aggregate.md b/functions-and-operators/json-functions/json-functions-aggregate.md new file mode 100644 index 0000000000000..e564119bc4feb --- /dev/null +++ b/functions-and-operators/json-functions/json-functions-aggregate.md @@ -0,0 +1,132 @@ +--- +title: JSON Functions That Aggregate JSON Values +summary: Learn about JSON functions that aggregate JSON values. +--- + +# JSON Functions That Aggregate JSON Values + +The functions listed on this page are part of the [aggregate functions](/functions-and-operators/aggregate-group-by-functions.md) that TiDB supports, but are specific to working with JSON. + +## [JSON_ARRAYAGG()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_json-arrayagg) + +The `JSON_ARRAYAGG(key)` function aggregates values of keys into a JSON array according to the given `key`. `key` is typically an expression or a column name. + +Example: + +Here the two rows in one column of a table get aggregated into a JSON array. + +```sql +SELECT JSON_ARRAYAGG(v) FROM (SELECT 1 'v' UNION SELECT 2); +``` + +``` ++------------------+ +| JSON_ARRAYAGG(v) | ++------------------+ +| [2, 1] | ++------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_OBJECTAGG()](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_json-objectagg) + +The `JSON_OBJECTAGG(key,value)` function aggregates keys and values of keys into a JSON object according to the given `key` and `value`. Both `key` or `value` are typically an expression or a column name. + +Example: + +First, create two tables and add a few rows to them. + +```sql +CREATE TABLE plants ( + id INT PRIMARY KEY, + name VARCHAR(255) +); + +CREATE TABLE plant_attributes ( + id INT PRIMARY KEY AUTO_INCREMENT, + plant_id INT, attribute VARCHAR(255), + value VARCHAR(255), + FOREIGN KEY (plant_id) REFERENCES plants(id) +); + +INSERT INTO plants +VALUES +(1,"rose"), +(2,"tulip"), +(3,"orchid"); + +INSERT INTO plant_attributes(plant_id,attribute,value) +VALUES +(1,"color","red"), +(1,"thorns","yes"), +(2,"color","orange"), +(2,"thorns","no"), +(2,"grows_from","bulb"), +(3,"color","white"), +(3, "thorns","no"); +``` + +Now you can check what the created tables look like. + +```sql +TABLE plants; +``` + +``` ++----+--------+ +| id | name | ++----+--------+ +| 1 | rose | +| 2 | tulip | +| 3 | orchid | ++----+--------+ +3 rows in set (0.00 sec) +``` + +```sql +TABLE plant_attributes; +``` + +``` ++----+----------+------------+--------+ +| id | plant_id | attribute | value | ++----+----------+------------+--------+ +| 1 | 1 | color | red | +| 2 | 1 | thorns | yes | +| 3 | 2 | color | orange | +| 4 | 2 | thorns | no | +| 5 | 2 | grows_from | bulb | +| 6 | 3 | color | white | +| 7 | 3 | thorns | no | ++----+----------+------------+--------+ +7 rows in set (0.00 sec) +``` + +You can use the `JSON_OBJECTAGG()` function with this data. Here you can see that for every group multiple key/value pairs are aggregated into a JSON object. + +```sql +SELECT + p.name, + JSON_OBJECTAGG(attribute,value) +FROM + plant_attributes pa + LEFT JOIN plants p ON pa.plant_id=p.id +GROUP BY + plant_id; +``` + +``` ++--------+-----------------------------------------------------------+ +| name | JSON_OBJECTAGG(attribute,value) | ++--------+-----------------------------------------------------------+ +| rose | {"color": "red", "thorns": "yes"} | +| orchid | {"color": "white", "thorns": "no"} | +| tulip | {"color": "orange", "grows_from": "bulb", "thorns": "no"} | ++--------+-----------------------------------------------------------+ +3 rows in set (0.00 sec) +``` + +## See also + +- [JSON Functions Overview](/functions-and-operators/json-functions.md) +- [JSON Data Type](/data-type-json.md) \ No newline at end of file diff --git a/functions-and-operators/json-functions/json-functions-create.md b/functions-and-operators/json-functions/json-functions-create.md new file mode 100644 index 0000000000000..cbe0eb79761ea --- /dev/null +++ b/functions-and-operators/json-functions/json-functions-create.md @@ -0,0 +1,64 @@ +--- +title: JSON Functions That Create JSON Values +summary: Learn about JSON functions that create JSON values. +--- + +# JSON Functions That Create JSON Values + +This document describes JSON functions that create JSON values. + +## [JSON_ARRAY()](https://dev.mysql.com/doc/refman/8.0/en/json-creation-functions.html#function_json-array) + +The `JSON_ARRAY([val[, val] ...])` function evaluates a (possibly empty) list of values and returns a JSON array containing those values. + +```sql +SELECT JSON_ARRAY(1,2,3,4,5), JSON_ARRAY("foo", "bar"); +``` + +``` ++-----------------------+--------------------------+ +| JSON_ARRAY(1,2,3,4,5) | JSON_ARRAY("foo", "bar") | ++-----------------------+--------------------------+ +| [1, 2, 3, 4, 5] | ["foo", "bar"] | ++-----------------------+--------------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_OBJECT()](https://dev.mysql.com/doc/refman/8.0/en/json-creation-functions.html#function_json-object) + +The `JSON_OBJECT([key, val[, key, val] ...])` function evaluates a (possibly empty) list of key-value pairs and returns a JSON object containing those pairs. + +```sql +SELECT JSON_OBJECT("database", "TiDB", "distributed", TRUE); +``` + +``` ++------------------------------------------------------+ +| JSON_OBJECT("database", "TiDB", "distributed", TRUE) | ++------------------------------------------------------+ +| {"database": "TiDB", "distributed": true} | ++------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_QUOTE()](https://dev.mysql.com/doc/refman/8.0/en/json-creation-functions.html#function_json-quote) + +The `JSON_QUOTE(str)` function returns a string as a JSON value with quotes. + +```sql +SELECT JSON_QUOTE('The name is "O\'Neil"'); +``` + +``` ++-------------------------------------+ +| JSON_QUOTE('The name is "O\'Neil"') | ++-------------------------------------+ +| "The name is \"O'Neil\"" | ++-------------------------------------+ +1 row in set (0.00 sec) +``` + +## See also + +- [JSON Functions Overview](/functions-and-operators/json-functions.md) +- [JSON Data Type](/data-type-json.md) \ No newline at end of file diff --git a/functions-and-operators/json-functions/json-functions-modify.md b/functions-and-operators/json-functions/json-functions-modify.md new file mode 100644 index 0000000000000..4d839a5efc7d3 --- /dev/null +++ b/functions-and-operators/json-functions/json-functions-modify.md @@ -0,0 +1,346 @@ +--- +title: JSON Functions That Modify JSON Values +summary: Learn about JSON functions that modify JSON values. +--- + +# JSON Functions That Modify JSON Values + +This document describes JSON functions that modify JSON values. + +## [JSON_APPEND()](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-append) + +An alias to [`JSON_ARRAY_APPEND()`](#json_array_append). + +## [JSON_ARRAY_APPEND()](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-array-append) + +The `JSON_ARRAY_APPEND(json_array, path, value [,path, value] ...)` function appends values to the end of the indicated arrays within a JSON document at the specified `path` and returns the result. + +This function takes arguments in pairs, where each pair is a `path` and a `value`. + +Examples: + +The following example adds an item to an array, which is the root of the JSON document. + +```sql +SELECT JSON_ARRAY_APPEND('["Car", "Boat", "Train"]', '$', "Airplane") AS "Transport options"; +``` + +``` ++--------------------------------------+ +| Transport options | ++--------------------------------------+ +| ["Car", "Boat", "Train", "Airplane"] | ++--------------------------------------+ +1 row in set (0.00 sec) +``` + +The following example adds an item to an array at the specified path. + +```sql +SELECT JSON_ARRAY_APPEND('{"transport_options": ["Car", "Boat", "Train"]}', '$.transport_options', "Airplane") AS "Transport options"; +``` + +``` ++-------------------------------------------------------------+ +| Transport options | ++-------------------------------------------------------------+ +| {"transport_options": ["Car", "Boat", "Train", "Airplane"]} | ++-------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_ARRAY_INSERT()](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-array-insert) + +The `JSON_ARRAY_INSERT(json_array, path, value [,path, value] ...)` function inserts a `value` into the specified position of the `json_array` in the `path` and returns the result. + +This function takes arguments in pairs, where each pair is a `path` and a `value`. + +Examples: + +The following example inserts a value at the position of index 0 in the array. + +```sql +SELECT JSON_ARRAY_INSERT('["Car", "Boat", "Train"]', '$[0]', "Airplane") AS "Transport options"; +``` + +``` ++--------------------------------------+ +| Transport options | ++--------------------------------------+ +| ["Airplane", "Car", "Boat", "Train"] | ++--------------------------------------+ +1 row in set (0.01 sec) +``` + +The following example inserts a value at the position of index 1 in the array. + +```sql +SELECT JSON_ARRAY_INSERT('["Car", "Boat", "Train"]', '$[1]', "Airplane") AS "Transport options"; +``` + +``` ++--------------------------------------+ +| Transport options | ++--------------------------------------+ +| ["Car", "Airplane", "Boat", "Train"] | ++--------------------------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_INSERT()](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-insert) + +The `JSON_INSERT(json_doc, path, value [,path, value] ...)` function inserts one or more values into a JSON document and returns the result. + +This function takes arguments in pairs, where each pair is a `path` and a `value`. + +```sql +SELECT JSON_INSERT( + '{"language": ["Go", "Rust", "C++"]}', + '$.architecture', 'riscv', + '$.os', JSON_ARRAY("linux","freebsd") +) AS "Demo"; +``` + +``` ++------------------------------------------------------------------------------------------+ +| Demo | ++------------------------------------------------------------------------------------------+ +| {"architecture": "riscv", "language": ["Go", "Rust", "C++"], "os": ["linux", "freebsd"]} | ++------------------------------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +Note that this function does not overwrite values of existing attributes. For example, the following statement appears to overwrite the `"a"` attribute, but it does not actually do so. + +```sql +SELECT JSON_INSERT('{"a": 61, "b": 62}', '$.a', 41, '$.c', 63); +``` + +``` ++---------------------------------------------------------+ +| JSON_INSERT('{"a": 61, "b": 62}', '$.a', 41, '$.c', 63) | ++---------------------------------------------------------+ +| {"a": 61, "b": 62, "c": 63} | ++---------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_MERGE_PATCH()](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-merge-patch) + +The `JSON_MERGE_PATCH(json_doc, json_doc [,json_doc] ...)` function merges two or more JSON documents into a single JSON document, without preserving values of duplicate keys. For `json_doc` arguments with duplicated keys, only the values from the later specified `json_doc` argument are preserved in the merged result. + +Examples: + +In the following example, you can see that the value of `a` gets overwritten by argument 2 and that `c` is added as a new attribute in the merged result. + +```sql +SELECT JSON_MERGE_PATCH( + '{"a": 1, "b": 2}', + '{"a": 100}', + '{"c": 300}' +); +``` + +``` ++-----------------------------------------------------------------+ +| JSON_MERGE_PATCH('{"a": 1, "b": 2}','{"a": 100}', '{"c": 300}') | ++-----------------------------------------------------------------+ +| {"a": 100, "b": 2, "c": 300} | ++-----------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_MERGE_PRESERVE()](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-merge-preserve) + +The `JSON_MERGE_PRESERVE(json_doc, json_doc [,json_doc] ...)` function merges two or more JSON documents while preserving all values associated with each key and returns the merged result. + +Examples: + +In the following example, you can see that the value of argument 2 is appended to `a` and that `c` is added as a new attribute. + +```sql +SELECT JSON_MERGE_PRESERVE('{"a": 1, "b": 2}','{"a": 100}', '{"c": 300}'); +``` + +``` ++--------------------------------------------------------------------+ +| JSON_MERGE_PRESERVE('{"a": 1, "b": 2}','{"a": 100}', '{"c": 300}') | ++--------------------------------------------------------------------+ +| {"a": [1, 100], "b": 2, "c": 300} | ++--------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_MERGE()](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-merge) + +> **Warning:** +> +> This function is deprecated. + +A deprecated alias for [`JSON_MERGE_PRESERVE()`](#json_merge_preserve). + +## [JSON_REMOVE()](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-remove) + +The `JSON_REMOVE(json_doc, path [,path] ...)` function removes data of the specified `path` from a JSON document and returns the result. + +Examples: + +This example removes the `b` attribute from the JSON document. + +```sql +SELECT JSON_REMOVE('{"a": 61, "b": 62, "c": 63}','$.b'); +``` + +``` ++--------------------------------------------------+ +| JSON_REMOVE('{"a": 61, "b": 62, "c": 63}','$.b') | ++--------------------------------------------------+ +| {"a": 61, "c": 63} | ++--------------------------------------------------+ +1 row in set (0.00 sec) +``` + +This example removes the `b` and `c` attributes from the JSON document. + +```sql +SELECT JSON_REMOVE('{"a": 61, "b": 62, "c": 63}','$.b','$.c'); +``` + +``` ++--------------------------------------------------------+ +| JSON_REMOVE('{"a": 61, "b": 62, "c": 63}','$.b','$.c') | ++--------------------------------------------------------+ +| {"a": 61} | ++--------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_REPLACE()](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-replace) + +The `JSON_REPLACE(json_doc, path, value [, path, value] ...)` function replaces values in specified paths of a JSON document and returns the result. If a specified path does not exist, the value corresponding to the path is not added to the result. + +This function takes arguments in pairs, where each pair is a `path` and a `value`. + +Examples: + +In the following example, you change the value at `$.b` from `62` to `42`. + +```sql +SELECT JSON_REPLACE('{"a": 41, "b": 62}','$.b',42); +``` + +``` ++---------------------------------------------+ +| JSON_REPLACE('{"a": 41, "b": 62}','$.b',42) | ++---------------------------------------------+ +| {"a": 41, "b": 42} | ++---------------------------------------------+ +1 row in set (0.00 sec) +``` + +In the following example, you can change the value at `$.b` from `62` to `42`. In addition, this statement tries to replace the value at `$.c` with `43`, but it does not work because the `$.c` path does not exist in `{"a": 41, "b": 62}`. + +```sql +SELECT JSON_REPLACE('{"a": 41, "b": 62}','$.b',42,'$.c',43); +``` + +``` ++------------------------------------------------------+ +| JSON_REPLACE('{"a": 41, "b": 62}','$.b',42,'$.c',43) | ++------------------------------------------------------+ +| {"a": 41, "b": 42} | ++------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_SET()](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-set) + +The `JSON_SET(json_doc, path, value [,path, value] ...)` function inserts or updates data in a JSON document and returns the result. + +This function takes arguments in pairs, where each pair is a `path` and a `value`. + +Examples: + +In the following example, you can update the `$.version` from `1.1` to `1.2`. + +```sql +SELECT JSON_SET('{"version": 1.1, "name": "example"}','$.version',1.2); +``` + +``` ++-----------------------------------------------------------------+ +| JSON_SET('{"version": 1.1, "name": "example"}','$.version',1.2) | ++-----------------------------------------------------------------+ +| {"name": "example", "version": 1.2} | ++-----------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +In the following example, you can update the `$.version` from `1.1` to `1.2`. And you can update `$.branch`, which does not exist before, to `main`. + +```sql +SELECT JSON_SET('{"version": 1.1, "name": "example"}','$.version',1.2,'$.branch', "main"); +``` + +``` ++------------------------------------------------------------------------------------+ +| JSON_SET('{"version": 1.1, "name": "example"}','$.version',1.2,'$.branch', "main") | ++------------------------------------------------------------------------------------+ +| {"branch": "main", "name": "example", "version": 1.2} | ++------------------------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_UNQUOTE()](https://dev.mysql.com/doc/refman/8.0/en/json-modification-functions.html#function_json-unquote) + +The `JSON_UNQUOTE(json)` function unquotes a JSON value and returns the result as a string. This is the opposite of the [`JSON_QUOTE()`](/functions-and-operators/json-functions/json-functions-create.md#json_quote) function. + +Examples: + +In the example, `"foo"` is unquoted to `foo`. + +```sql +SELECT JSON_UNQUOTE('"foo"'); +``` + +``` ++-----------------------+ +| JSON_UNQUOTE('"foo"') | ++-----------------------+ +| foo | ++-----------------------+ +1 row in set (0.00 sec) +``` + +This function is often used together with [`JSON_EXTRACT()`](/functions-and-operators/json-functions/json-functions-search.md#json_extract). For the following examples, you can extract a JSON value with quotes in the first example and then use the two functions together to unquote the value in the second example. Note that instead of `JSON_UNQUOTE(JSON_EXTRACT(...))`, you can use the [`->>`](/functions-and-operators/json-functions/json-functions-search.md#--1) operator. + +```sql +SELECT JSON_EXTRACT('{"database": "TiDB"}', '$.database'); +``` + +``` ++----------------------------------------------------+ +| JSON_EXTRACT('{"database": "TiDB"}', '$.database') | ++----------------------------------------------------+ +| "TiDB" | ++----------------------------------------------------+ +1 row in set (0.00 sec) +``` + +```sql +SELECT JSON_UNQUOTE(JSON_EXTRACT('{"database": "TiDB"}', '$.database')); +``` + +``` ++------------------------------------------------------------------+ +| JSON_UNQUOTE(JSON_EXTRACT('{"database": "TiDB"}', '$.database')) | ++------------------------------------------------------------------+ +| TiDB | ++------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## See also + +- [JSON Functions Overview](/functions-and-operators/json-functions.md) +- [JSON Data Type](/data-type-json.md) \ No newline at end of file diff --git a/functions-and-operators/json-functions/json-functions-return.md b/functions-and-operators/json-functions/json-functions-return.md new file mode 100644 index 0000000000000..cc4afd41a04fe --- /dev/null +++ b/functions-and-operators/json-functions/json-functions-return.md @@ -0,0 +1,168 @@ +--- +title: JSON Functions That Return JSON Values +summary: Learn about JSON functions that return JSON values. +--- + +# JSON Functions That Return JSON Values + +This document describes JSON functions that return JSON values. + +## [JSON_DEPTH()](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-depth) + +The `JSON_DEPTH(json_doc)` function returns the maximum depth of a JSON document. + +Examples: + +In the following example, `JSON_DEPTH()` returns `3` because there are three levels: + +- root (`$`) +- weather (`$.weather`) +- weather current (`$.weather.sunny`) + +```sql +SELECT JSON_DEPTH('{"weather": {"current": "sunny"}}'); +``` + +``` ++-------------------------------------------------+ +| JSON_DEPTH('{"weather": {"current": "sunny"}}') | ++-------------------------------------------------+ +| 3 | ++-------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_LENGTH()](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-length) + +The `JSON_LENGTH(json_doc [,path])` function returns the length of a JSON document. If a `path` argument is given, it returns the length of the value within the path. + +Examples: + +In the following example, the returned value is `1` because the only item at the root of the document is `weather`. + +```sql +SELECT JSON_LENGTH('{"weather": {"current": "sunny", "tomorrow": "cloudy"}}','$'); +``` + +``` ++----------------------------------------------------------------------------+ +| JSON_LENGTH('{"weather": {"current": "sunny", "tomorrow": "cloudy"}}','$') | ++----------------------------------------------------------------------------+ +| 1 | ++----------------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +In the following example, the returned value is `2` because there are two items at `$.weather`: `current` and `tomorrow`. + +```sql +SELECT JSON_LENGTH('{"weather": {"current": "sunny", "tomorrow": "cloudy"}}','$.weather'); +``` + +``` ++------------------------------------------------------------------------------------+ +| JSON_LENGTH('{"weather": {"current": "sunny", "tomorrow": "cloudy"}}','$.weather') | ++------------------------------------------------------------------------------------+ +| 2 | ++------------------------------------------------------------------------------------+ +1 row in set (0.01 sec) +``` + +## [JSON_TYPE()](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-type) + +The `JSON_TYPE(json_val)` function returns a string indicating [the type of a JSON value](/data-type-json.md#json-value-types). + +Example: + +```sql +WITH demo AS ( + SELECT 'null' AS 'v' + UNION SELECT '"foobar"' + UNION SELECT 'true' + UNION SELECT '5' + UNION SELECT '1.14' + UNION SELECT '[]' + UNION SELECT '{}' + UNION SELECT POW(2,63) +) +SELECT v, JSON_TYPE(v) FROM demo ORDER BY 2; +``` + +``` ++----------------------+--------------+ +| v | JSON_TYPE(v) | ++----------------------+--------------+ +| [] | ARRAY | +| true | BOOLEAN | +| 1.14 | DOUBLE | +| 9.223372036854776e18 | DOUBLE | +| 5 | INTEGER | +| null | NULL | +| {} | OBJECT | +| "foobar" | STRING | ++----------------------+--------------+ +8 rows in set (0.00 sec) +``` + +Note that values that look the same might not have the same type, as demonstrated in the following example. + +```sql +SELECT '"2025-06-14"',CAST(CAST('2025-06-14' AS date) AS json); +``` + +``` ++--------------+------------------------------------------+ +| "2025-06-14" | CAST(CAST('2025-06-14' AS date) AS json) | ++--------------+------------------------------------------+ +| "2025-06-14" | "2025-06-14" | ++--------------+------------------------------------------+ +1 row in set (0.00 sec) +``` + +```sql +SELECT JSON_TYPE('"2025-06-14"'),JSON_TYPE(CAST(CAST('2025-06-14' AS date) AS json)); +``` + +``` ++---------------------------+-----------------------------------------------------+ +| JSON_TYPE('"2025-06-14"') | JSON_TYPE(CAST(CAST('2025-06-14' AS date) AS json)) | ++---------------------------+-----------------------------------------------------+ +| STRING | DATE | ++---------------------------+-----------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_VALID()](https://dev.mysql.com/doc/refman/8.0/en/json-attribute-functions.html#function_json-valid) + +The `JSON_VALID(str)` function checks if the argument is valid JSON. This can be useful for checking a column before converting it to the `JSON` type. + +```sql +SELECT JSON_VALID('{"foo"="bar"}'); +``` + +``` ++-----------------------------+ +| JSON_VALID('{"foo"="bar"}') | ++-----------------------------+ +| 0 | ++-----------------------------+ +1 row in set (0.01 sec) +``` + +```sql +SELECT JSON_VALID('{"foo": "bar"}'); +``` + +``` ++------------------------------+ +| JSON_VALID('{"foo": "bar"}') | ++------------------------------+ +| 1 | ++------------------------------+ +1 row in set (0.01 sec) +``` + +## See also + +- [JSON Functions Overview](/functions-and-operators/json-functions.md) +- [JSON Data Type](/data-type-json.md) diff --git a/functions-and-operators/json-functions/json-functions-search.md b/functions-and-operators/json-functions/json-functions-search.md new file mode 100644 index 0000000000000..89e0d13877dfe --- /dev/null +++ b/functions-and-operators/json-functions/json-functions-search.md @@ -0,0 +1,360 @@ +--- +title: JSON Functions That Search JSON Values +summary: Learn about JSON functions that search JSON values. +--- + +# JSON Functions That Search JSON Values + +This document describes JSON functions that search JSON values. + +## [JSON_CONTAINS()](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-contains) + +By returning `1` or `0`, the `JSON_CONTAINS(json_doc, candidate [,path])` function indicates whether a given `candidate` JSON document is contained within a target JSON document. + +Examples: + +Here `a` is contained in the target document. + +```sql +SELECT JSON_CONTAINS('["a","b","c"]','"a"'); +``` + +``` ++--------------------------------------+ +| JSON_CONTAINS('["a","b","c"]','"a"') | ++--------------------------------------+ +| 1 | ++--------------------------------------+ +1 row in set (0.00 sec) +``` + +Here `e` is not contained in the target document. + +```sql +SELECT JSON_CONTAINS('["a","b","c"]','"e"'); +``` + +``` ++--------------------------------------+ +| JSON_CONTAINS('["a","b","c"]','"e"') | ++--------------------------------------+ +| 0 | ++--------------------------------------+ +1 row in set (0.00 sec) +``` + +Here `{"foo": "bar"}` is contained in the target document. + +```sql +SELECT JSON_CONTAINS('{"foo": "bar", "aaa": 5}','{"foo": "bar"}'); +``` + +``` ++------------------------------------------------------------+ +| JSON_CONTAINS('{"foo": "bar", "aaa": 5}','{"foo": "bar"}') | ++------------------------------------------------------------+ +| 1 | ++------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +Here `"bar"` is not contained in the root of the target document. + +```sql +SELECT JSON_CONTAINS('{"foo": "bar", "aaa": 5}','"bar"'); +``` + +``` ++---------------------------------------------------+ +| JSON_CONTAINS('{"foo": "bar", "aaa": 5}','"bar"') | ++---------------------------------------------------+ +| 0 | ++---------------------------------------------------+ +1 row in set (0.00 sec) +``` + +Here `"bar"` is contained in the `$.foo` attribute of the target document. + +```sql +SELECT JSON_CONTAINS('{"foo": "bar", "aaa": 5}','"bar"', '$.foo'); +``` + +``` ++------------------------------------------------------------+ +| JSON_CONTAINS('{"foo": "bar", "aaa": 5}','"bar"', '$.foo') | ++------------------------------------------------------------+ +| 1 | ++------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_CONTAINS_PATH()](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-contains-path) + +The `JSON_CONTAINS_PATH(json_doc, all_or_one, path [,path, ...])` function returns `0` or `1` to indicate whether a JSON document contains data at a given path or paths. + +Examples: + +Here the document contains `$.foo`. + +```sql +SELECT JSON_CONTAINS_PATH('{"foo": "bar", "aaa": 5}','all','$.foo'); +``` + +``` ++--------------------------------------------------------------+ +| JSON_CONTAINS_PATH('{"foo": "bar", "aaa": 5}','all','$.foo') | ++--------------------------------------------------------------+ +| 1 | ++--------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +Here the document does not contain `$.bar`. + +```sql +SELECT JSON_CONTAINS_PATH('{"foo": "bar", "aaa": 5}','all','$.bar'); +``` + +``` ++--------------------------------------------------------------+ +| JSON_CONTAINS_PATH('{"foo": "bar", "aaa": 5}','all','$.bar') | ++--------------------------------------------------------------+ +| 0 | ++--------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +Here the document contains both `$.foo` and `$.aaa`. + +```sql +SELECT JSON_CONTAINS_PATH('{"foo": "bar", "aaa": 5}','all','$.foo', '$.aaa'); +``` + +``` ++-----------------------------------------------------------------------+ +| JSON_CONTAINS_PATH('{"foo": "bar", "aaa": 5}','all','$.foo', '$.aaa') | ++-----------------------------------------------------------------------+ +| 1 | ++-----------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_EXTRACT()](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-extract) + +The `JSON_EXTRACT(json_doc, path[, path] ...)` function extracts data from a JSON document, selected from the parts of the document matched by the `path` arguments. + +```sql +SELECT JSON_EXTRACT('{"foo": "bar", "aaa": 5}', '$.foo'); +``` + +``` ++---------------------------------------------------+ +| JSON_EXTRACT('{"foo": "bar", "aaa": 5}', '$.foo') | ++---------------------------------------------------+ +| "bar" | ++---------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## [->](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#operator_json-column-path) + +The `column->path` function returns the data in `column` that matches the `path` argument. It is an alias for [`JSON_EXTRACT()`](#json_extract). + +```sql +SELECT + j->'$.foo', + JSON_EXTRACT(j, '$.foo') +FROM ( + SELECT + '{"foo": "bar", "aaa": 5}' AS j + ) AS tbl; +``` + +``` ++------------+--------------------------+ +| j->'$.foo' | JSON_EXTRACT(j, '$.foo') | ++------------+--------------------------+ +| "bar" | "bar" | ++------------+--------------------------+ +1 row in set (0.00 sec) +``` + +## [->>](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#operator_json-inline-path) + +The `column->>path` function unquotes data in `column` that matches the `path` argument. It is an alias for `JSON_UNQUOTE(JSON_EXTRACT(doc, path_literal))`. + +```sql +SELECT + j->'$.foo', + JSON_EXTRACT(j, '$.foo') + j->>'$.foo', + JSON_UNQUOTE(JSON_EXTRACT(j, '$.foo')) +FROM ( + SELECT + '{"foo": "bar", "aaa": 5}' AS j + ) AS tbl; +``` + +``` ++------------+--------------------------+-------------+----------------------------------------+ +| j->'$.foo' | JSON_EXTRACT(j, '$.foo') | j->>'$.foo' | JSON_UNQUOTE(JSON_EXTRACT(j, '$.foo')) | ++------------+--------------------------+-------------+----------------------------------------+ +| "bar" | "bar" | bar | bar | ++------------+--------------------------+-------------+----------------------------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_KEYS()](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-keys) + +The `JSON_KEYS(json_doc [,path])` function returns the top-level keys of a JSON object as a JSON array. If a `path` argument is given, it returns the top-level keys from the selected path. + +Examples: + +The following example returns the two top-level keys in the JSON document. + +```sql +SELECT JSON_KEYS('{"name": {"first": "John", "last": "Doe"}, "type": "Person"}'); +``` + +``` ++---------------------------------------------------------------------------+ +| JSON_KEYS('{"name": {"first": "John", "last": "Doe"}, "type": "Person"}') | ++---------------------------------------------------------------------------+ +| ["name", "type"] | ++---------------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +The following example returns the top-level keys that are in the `$.name` path of the JSON document. + +```sql +SELECT JSON_KEYS('{"name": {"first": "John", "last": "Doe"}, "type": "Person"}', '$.name'); +``` + +``` ++-------------------------------------------------------------------------------------+ +| JSON_KEYS('{"name": {"first": "John", "last": "Doe"}, "type": "Person"}', '$.name') | ++-------------------------------------------------------------------------------------+ +| ["first", "last"] | ++-------------------------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_SEARCH()](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-search) + +The `JSON_SEARCH(json_doc, one_or_all, str)` function searches a JSON document for one or all matches of a string. + +Examples: + +In the following example, you can search for the first result for `cc`, which is at the position of index 2 in the `a` array. + +```sql +SELECT JSON_SEARCH('{"a": ["aa", "bb", "cc"], "b": ["cc", "dd"]}','one','cc'); +``` + +``` ++------------------------------------------------------------------------+ +| JSON_SEARCH('{"a": ["aa", "bb", "cc"], "b": ["cc", "dd"]}','one','cc') | ++------------------------------------------------------------------------+ +| "$.a[2]" | ++------------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +Now you do the same, but set `one_or_all` to `all` to get not just the first result, but all results. + +```sql +SELECT JSON_SEARCH('{"a": ["aa", "bb", "cc"], "b": ["cc", "dd"]}','all','cc'); +``` + +``` ++------------------------------------------------------------------------+ +| JSON_SEARCH('{"a": ["aa", "bb", "cc"], "b": ["cc", "dd"]}','all','cc') | ++------------------------------------------------------------------------+ +| ["$.a[2]", "$.b[0]"] | ++------------------------------------------------------------------------+ +1 row in set (0.01 sec) +``` + +## [MEMBER OF()](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#operator_member-of) + +The `str MEMBER OF (json_array)` function tests if the passed value `str` is an element of the `json_array`, it returns `1`. Otherwise, it returns `0`. It returns `NULL` if any of the arguments is `NULL`. + +``` +SELECT '🍍' MEMBER OF ('["🍍","🥥","🥭"]') AS 'Contains pineapple'; +``` + +``` ++--------------------+ +| Contains pineapple | ++--------------------+ +| 1 | ++--------------------+ +1 row in set (0.00 sec) + +``` + +## [JSON_OVERLAPS()](https://dev.mysql.com/doc/refman/8.0/en/json-search-functions.html#function_json-overlaps) + +The `JSON_OVERLAPS(json_doc, json_doc)` function indicates whether two JSON documents have overlapping part. If yes, it returns `1`. If not, it returns `0`. It returns `NULL` if any of the arguments is `NULL`. + +Examples: + +The following example shows that there is no overlap because the array value does not have the same number of elements. + +```sql +SELECT JSON_OVERLAPS( + '{"languages": ["Go","Rust","C#"]}', + '{"languages": ["Go","Rust"]}' +) AS 'Overlaps'; +``` + +``` ++----------+ +| Overlaps | ++----------+ +| 0 | ++----------+ +1 row in set (0.00 sec) +``` + +The following example shows that both JSON documents overlap as they are identical. + +```sql +SELECT JSON_OVERLAPS( + '{"languages": ["Go","Rust","C#"]}', + '{"languages": ["Go","Rust","C#"]}' +) AS 'Overlaps'; +``` + +``` ++----------+ +| Overlaps | ++----------+ +| 1 | ++----------+ +1 row in set (0.00 sec) +``` + +The following example shows that there is an overlap, while the second document has an extra attribute. + +```sql +SELECT JSON_OVERLAPS( + '{"languages": ["Go","Rust","C#"]}', + '{"languages": ["Go","Rust","C#"], "arch": ["arm64"]}' +) AS 'Overlaps'; +``` + +``` ++----------+ +| Overlaps | ++----------+ +| 1 | ++----------+ +1 row in set (0.00 sec) +``` + +## See also + +- [JSON Functions Overview](/functions-and-operators/json-functions.md) +- [JSON Data Type](/data-type-json.md) \ No newline at end of file diff --git a/functions-and-operators/json-functions/json-functions-utility.md b/functions-and-operators/json-functions/json-functions-utility.md new file mode 100644 index 0000000000000..4f0c0c27eb04e --- /dev/null +++ b/functions-and-operators/json-functions/json-functions-utility.md @@ -0,0 +1,73 @@ +--- +title: JSON Utility Functions +summary: Learn about JSON utility functions. +--- + +# JSON Utility Functions + +This document describes JSON utility functions. + +## [JSON_PRETTY()](https://dev.mysql.com/doc/refman/8.0/en/json-utility-functions.html#function_json-pretty) + +The `JSON_PRETTY(json_doc)` function does pretty formatting of a JSON document. + +```sql +SELECT JSON_PRETTY('{"person":{"name":{"first":"John","last":"Doe"},"age":23}}')\G +``` + +``` +*************************** 1. row *************************** +JSON_PRETTY('{"person":{"name":{"first":"John","last":"Doe"},"age":23}}'): { + "person": { + "age": 23, + "name": { + "first": "John", + "last": "Doe" + } + } +} +1 row in set (0.00 sec) +``` + +## [JSON_STORAGE_FREE()](https://dev.mysql.com/doc/refman/8.0/en/json-utility-functions.html#function_json-storage-free) + +The `JSON_STORAGE_FREE(json_doc)` function returns how much storage space is freed in the binary representation of the JSON value after it is updated in place. + +> **Note:** +> +> Because TiDB has a different storage architecture from MySQL, this function always returns `0` for a valid JSON value, and it is implemented for [compatibility with MySQL 8.0](/mysql-compatibility.md). Note that TiDB does not do in-place updates. For more information, see [RocksDB space usage](/storage-engine/rocksdb-overview.md#rocksdb-space-usage). + +```sql +SELECT JSON_STORAGE_FREE('{}'); +``` + +``` ++-------------------------+ +| JSON_STORAGE_FREE('{}') | ++-------------------------+ +| 0 | ++-------------------------+ +1 row in set (0.00 sec) +``` + +## [JSON_STORAGE_SIZE()](https://dev.mysql.com/doc/refman/8.0/en/json-utility-functions.html#function_json-storage-size) + +The `JSON_STORAGE_SIZE(json_doc)` function returns an approximate size of bytes required to store the JSON value. Because the size does not account for TiKV using compression, the output of this function is not strictly compatible with MySQL. + +```sql +SELECT JSON_STORAGE_SIZE('{}'); +``` + +``` ++-------------------------+ +| JSON_STORAGE_SIZE('{}') | ++-------------------------+ +| 9 | ++-------------------------+ +1 row in set (0.00 sec) +``` + +## See also + +- [JSON Functions Overview](/functions-and-operators/json-functions.md) +- [JSON Data Type](/data-type-json.md) \ No newline at end of file diff --git a/functions-and-operators/locking-functions.md b/functions-and-operators/locking-functions.md index 08a261cfa241c..14238fefa62aa 100644 --- a/functions-and-operators/locking-functions.md +++ b/functions-and-operators/locking-functions.md @@ -20,6 +20,6 @@ TiDB supports most of the user-level [locking functions](https://dev.mysql.com/d ## MySQL compatibility * The minimum timeout permitted by TiDB is 1 second, and the maximum timeout is 1 hour (3600 seconds). This differs from MySQL, where both 0 second and unlimited timeouts (`timeout=-1`) are permitted. TiDB will automatically convert out-of-range values to the nearest permitted value and convert `timeout=-1` to 3600 seconds. -* TiDB does not automatically detect deadlocks caused by user-level locks. Deadlocked sessions will timeout after a maximum of 1 hour, but can also be manually resolved by using [`KILL`](/sql-statements/sql-statement-kill.md) on one of the affected sessions. You can also prevent deadlocks by always acquiring user-level locks in the same order. +* TiDB does not automatically detect deadlocks caused by user-level locks. Deadlocked sessions will time out after a maximum of 1 hour, but can also be manually resolved by using [`KILL`](/sql-statements/sql-statement-kill.md) on one of the affected sessions. You can also prevent deadlocks by always acquiring user-level locks in the same order. * Locks take effect on all TiDB servers in the cluster. This differs from MySQL Cluster and Group Replication where locks are local to a single server. * `IS_USED_LOCK()` returns `1` if it is called from another session and is unable to return the ID of the process that is holding the lock. diff --git a/functions-and-operators/miscellaneous-functions.md b/functions-and-operators/miscellaneous-functions.md index c50284089cee2..ef414277d39b0 100644 --- a/functions-and-operators/miscellaneous-functions.md +++ b/functions-and-operators/miscellaneous-functions.md @@ -1,7 +1,6 @@ --- title: Miscellaneous Functions summary: Learn about miscellaneous functions in TiDB. -aliases: ['/docs/dev/functions-and-operators/miscellaneous-functions/','/docs/dev/reference/sql/functions-and-operators/miscellaneous-functions/'] --- # Miscellaneous Functions @@ -26,7 +25,7 @@ TiDB supports most of the [miscellaneous functions](https://dev.mysql.com/doc/re | [`IS_IPV6()`](#is_ipv6) | Whether argument is an IPv6 address | | [`IS_UUID()`](#is_uuid) | Whether argument is an UUID | | [`NAME_CONST()`](#name_const) | Can be used to rename a column name | -| [`SLEEP()`](#sleep) | Sleep for a number of seconds. Note that for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters, the `SLEEP()` function has a limitation wherein it can only support a maximum sleep time of 300 seconds. | +| [`SLEEP()`](#sleep) | Sleep for a number of seconds. Note that for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters, the `SLEEP()` function has a limitation wherein it can only support a maximum sleep time of 300 seconds. | | [`UUID()`](#uuid) | Return a Universal Unique Identifier (UUID) | | [`UUID_TO_BIN()`](#uuid_to_bin) | Convert UUID from text format to binary format | | [`VALUES()`](#values) | Defines the values to be used during an INSERT | diff --git a/functions-and-operators/numeric-functions-and-operators.md b/functions-and-operators/numeric-functions-and-operators.md index cf0b088ab1434..e0a6587d41953 100644 --- a/functions-and-operators/numeric-functions-and-operators.md +++ b/functions-and-operators/numeric-functions-and-operators.md @@ -1,7 +1,6 @@ --- title: Numeric Functions and Operators summary: Learn about the numeric functions and operators. -aliases: ['/docs/dev/functions-and-operators/numeric-functions-and-operators/','/docs/dev/reference/sql/functions-and-operators/numeric-functions-and-operators/'] --- # Numeric Functions and Operators diff --git a/functions-and-operators/operators.md b/functions-and-operators/operators.md index 665e30961135c..04c69dc98f25e 100644 --- a/functions-and-operators/operators.md +++ b/functions-and-operators/operators.md @@ -1,7 +1,6 @@ --- title: Operators summary: Learn about the operators precedence, comparison functions and operators, logical operators, and assignment operators. -aliases: ['/docs/dev/functions-and-operators/operators/','/docs/dev/reference/sql/functions-and-operators/operators/'] --- # Operators diff --git a/functions-and-operators/precision-math.md b/functions-and-operators/precision-math.md index e3916a1b75b92..cf8a67f326307 100644 --- a/functions-and-operators/precision-math.md +++ b/functions-and-operators/precision-math.md @@ -1,7 +1,6 @@ --- title: Precision Math summary: Learn about the precision math in TiDB. -aliases: ['/docs/dev/functions-and-operators/precision-math/','/docs/dev/reference/sql/functions-and-operators/precision-math/'] --- # Precision Math @@ -51,7 +50,7 @@ DECIMAL columns do not store a leading `+` character or `-` character or leading DECIMAL columns do not permit values larger than the range implied by the column definition. For example, a `DECIMAL(3,0)` column supports a range of `-999` to `999`. A `DECIMAL(M,D)` column permits at most `M - D` digits to the left of the decimal point. -For more information about the internal format of the DECIMAL values, see [`mydecimal.go`](https://github.com/pingcap/tidb/blob/master/pkg/types/mydecimal.go) in TiDB source code. +For more information about the internal format of the DECIMAL values, see [`mydecimal.go`](https://github.com/pingcap/tidb/blob/release-8.1/pkg/types/mydecimal.go) in TiDB source code. ## Expression handling diff --git a/functions-and-operators/sequence-functions.md b/functions-and-operators/sequence-functions.md index bbf14b6b2e624..f610a982f1719 100644 --- a/functions-and-operators/sequence-functions.md +++ b/functions-and-operators/sequence-functions.md @@ -7,11 +7,161 @@ summary: This document introduces sequence functions supported in TiDB. Sequence functions in TiDB are used to return or set values of sequence objects created using the [`CREATE SEQUENCE`](/sql-statements/sql-statement-create-sequence.md) statement. -| Function name | Feature description | +| Function name | Description | | :-------------- | :------------------------------------- | -| `NEXTVAL()` or `NEXT VALUE FOR` | Returns the next value of a sequence | -| `SETVAL()` | Sets the current value of a sequence | -| `LASTVAL()` | Returns the last used value of a sequence | +| [`NEXTVAL()`](#nextval) | Returns the next value of a sequence. | +| [`NEXT VALUE FOR`](#next-value-for) | Returns the next value of a sequence (alias for `NEXTVAL()`). | +| [`SETVAL()`](#setval) | Sets the current value of a sequence. | +| [`LASTVAL()`](#lastval) | Returns the last value generated by a sequence in the current session. | + +## `NEXTVAL()` + +The `NEXTVAL()` function returns the next value of a sequence. + +Example: + +Create a sequence named `s1`: + +```sql +CREATE SEQUENCE s1; +``` + +Get the next value from `s1`: + +```sql +SELECT NEXTVAL(s1); +``` + +The output is as follows: + +``` ++-------------+ +| NEXTVAL(s1) | ++-------------+ +| 1 | ++-------------+ +1 row in set (0.00 sec) +``` + +## `NEXT VALUE FOR` + +The `NEXT VALUE FOR` function is an alias for [`NEXTVAL()`](#nextval). + +Example: + +Get the next value from `s1` using `NEXTVAL()`: + +```sql +SELECT NEXTVAL(s1); +``` + +The output is as follows: + +``` ++-------------+ +| NEXTVAL(s1) | ++-------------+ +| 2 | ++-------------+ +1 row in set (0.00 sec) +``` + +Get the next value from `s1` using `NEXT VALUE FOR`: + +```sql +SELECT NEXT VALUE FOR s1; +``` + +The output is as follows: + +``` ++-------------------+ +| NEXT VALUE FOR s1 | ++-------------------+ +| 3 | ++-------------------+ +1 row in set (0.00 sec) +``` + +## `SETVAL()` + +The `SETVAL(n)` function sets the current value of a sequence. + +Example: + +Get the next value from `s1`: + +```sql +SELECT NEXTVAL(s1); +``` + +The output is as follows: + +``` ++-------------+ +| NEXTVAL(s1) | ++-------------+ +| 4 | ++-------------+ +1 row in set (0.00 sec) +``` + +Set the current value of `s1` to `10`: + +```sql +SELECT SETVAL(s1, 10); +``` + +The output is as follows: + +``` ++----------------+ +| SETVAL(s1, 10) | ++----------------+ +| 10 | ++----------------+ +1 row in set (0.00 sec) +``` + +Verify the next value after setting it to `10`: + +```sql +SELECT NEXTVAL(s1); +``` + +The output is as follows: + +``` ++-------------+ +| NEXTVAL(s1) | ++-------------+ +| 11 | ++-------------+ +1 row in set (0.00 sec) +``` + +## `LASTVAL()` + +The `LASTVAL()` function returns the last value generated by a sequence **in the current session**. + +Example: + +Get the last value generated by `s1` in the current session: + +```sql +SELECT LASTVAL(s1); +``` + +The output is as follows: + +``` ++-------------+ +| LASTVAL(s1) | ++-------------+ +| 11 | ++-------------+ +1 row in set (0.00 sec) +``` ## MySQL compatibility diff --git a/functions-and-operators/set-operators.md b/functions-and-operators/set-operators.md index e5177dcf7012a..2c1f869e8e515 100644 --- a/functions-and-operators/set-operators.md +++ b/functions-and-operators/set-operators.md @@ -116,7 +116,7 @@ TiDB supports using parentheses to specify the precedence of set operations. Exp ## Use `ORDER BY` and `LIMIT` -TiDB supports using [`ORDER BY`](/media/sqlgram/OrderByOptional.png) or [`LIMIT`](/media/sqlgram/LimitClause.png) clause in set operations. These two clauses must be at the end of the entire statement. +TiDB supports using `ORDER BY` or `LIMIT` clause on the entire result of set operations. These two clauses must be at the end of the entire statement. ```sql (SELECT * FROM t1 UNION ALL SELECT * FROM t1 INTERSECT SELECT * FROM t2) ORDER BY a LIMIT 2; diff --git a/functions-and-operators/string-functions.md b/functions-and-operators/string-functions.md index 746fb53c2b080..eb7a78ea2613e 100644 --- a/functions-and-operators/string-functions.md +++ b/functions-and-operators/string-functions.md @@ -1,7 +1,6 @@ --- title: String Functions summary: Learn about the string functions in TiDB. -aliases: ['/docs/dev/functions-and-operators/string-functions/','/docs/dev/reference/sql/functions-and-operators/string-functions/'] --- # String Functions @@ -134,7 +133,7 @@ SELECT CustomerName, BIT_LENGTH(CustomerName) AS BitLengthOfName FROM Customers; ### [`CHAR()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_char) -The `CHAR()` function is used to get the corresponding character of a specific ASCII value. It performs the opposite operation of `ASCII()`, which returns the ASCII value of a specific character. If multiple arguments are supplied, the function works on all arguments and are then concaternated together. +The `CHAR()` function is used to get the corresponding character of a specific ASCII value. It performs the opposite operation of `ASCII()`, which returns the ASCII value of a specific character. If multiple arguments are supplied, the function works on all arguments and are then concatenated together. Examples: @@ -541,7 +540,7 @@ Arguments: - `X`: the number to be formatted. It can be a direct numeric value, a numeric string, or a number in scientific notation. - `D`: the number of decimal places for the returned value. The function rounds the number `X` to `D` decimal places. If `D` is greater than the actual number of decimal places in `X`, the result is padded with zeros to the corresponding length. -- `[locale]`: specifies a locale setting to be used for grouping between decimal points, thousands separators, and separators for resultant numbers. A valid locale value is the same as the valid value of the [`lc_time_names`](https://dev.mysql.com/doc/refman/8.3/en/server-system-variables.html#sysvar_lc_time_names) system variable. If not specified or the region setting is `NULL`, the `'en_US'` region setting is used by default. This argument is optional. +- `[locale]`: specifies a locale setting to be used for grouping between decimal points, thousands separators, and separators for resultant numbers. A valid locale value is the same as the valid value of the [`lc_time_names`](https://dev.mysql.com/doc/refman/8.0/en/server-system-variables.html#sysvar_lc_time_names) system variable. If not specified or the region setting is `NULL`, the `'en_US'` region setting is used by default. This argument is optional. Behaviors: @@ -638,7 +637,11 @@ The `HEX()` function is used to convert the given argument into a string represe - If the argument is a number, `HEX(n)` returns a hexadecimal string representation of `n`. The function treats the argument `n` as a `BIGINT` number, equivalent to using `CONV(n, 10, 16)`. - If the argument is `NULL`, the function returns `NULL`. -Examples: +> **Note:** +> +> In the MySQL client, the [`--binary-as-hex`](https://dev.mysql.com/doc/refman/8.0/en/mysql-command-options.html#option_mysql_binary-as-hex) option is enabled by default in interactive mode, causing the client to display data with an unknown character set as a [hexadecimal literal](https://dev.mysql.com/doc/refman/8.0/en/hexadecimal-literals.html). You can use the `--skip-binary-as-hex` option to disable this behavior. + +Examples (with `mysql --skip-binary-as-hex`): ```sql SELECT X'616263', HEX('abc'), UNHEX(HEX('abc')), 0x616263; @@ -806,7 +809,7 @@ SELECT INSTR(0123, "12"); ### [`LCASE()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_lcase) -The `LCASE(str)` function is a synonym for [`LOWER(str)`](#lower), which returns the lowercase of the given argument. +The `LCASE(str)` function is a synonym for [`LOWER(str)`](#lower), which returns the lowercase of the given argument. ### [`LEFT()`](https://dev.mysql.com/doc/refman/8.0/en/string-functions.html#function_left) @@ -818,10 +821,10 @@ Syntax: LEFT(`str`, `len`) ``` -- `str`: the original string to extract characters. If `str` contains a multibyte character, the function counts it as a single code point. -- `len`: the length of characters to be returned. +- `str`: the original string to extract characters. If `str` contains a multibyte character, the function counts it as a single code point. +- `len`: the length of characters to be returned. - If `len` is equal to or less than 0, the function returns an empty string. - - If `len` is equal to or greater than the length of `str`, the function returns the original `str`. + - If `len` is equal to or greater than the length of `str`, the function returns the original `str`. - If any argument is `NULL`, the function returns `NULL`. Examples: @@ -1598,7 +1601,7 @@ SELECT CHAR(97), CHAR(65); 1 row in set (0.01 sec) ``` -The following example shows how `ORD()` handles multibyte characters. Here, both `101` and `0x65` are the UTF-8 encoded values for the `e` character, but in different formats. And both `50091` and `0xC3AB` represent the same values, but for the `ë` character. +The following example shows how `ORD()` handles multibyte characters. Here, both `101` and `0x65` are the UTF-8 encoded values for the `e` character, but in different formats. And both `50091` and `0xC3AB` represent the same values, but for the `ë` character. ```sql SELECT ORD('e'), ORD('ë'), HEX('e'), HEX('ë'); @@ -1652,7 +1655,7 @@ In this example a number of strings are matched against two regular expressions. ```sql WITH vals AS ( - SELECT 'TiDB' AS v + SELECT 'TiDB' AS v UNION ALL SELECT 'Titanium' UNION ALL @@ -1660,7 +1663,7 @@ WITH vals AS ( UNION ALL SELECT 'Rust' ) -SELECT +SELECT v, v REGEXP '^Ti' AS 'starts with "Ti"', v REGEXP '^.{4}$' AS 'Length is 4 characters' @@ -2018,8 +2021,8 @@ The following example generates a sequence of numbers from 1 to 20 using a [recu ```sql WITH RECURSIVE nr(n) AS ( - SELECT 1 AS n - UNION ALL + SELECT 1 AS n + UNION ALL SELECT n+1 FROM nr WHERE n<20 ) SELECT n, REPEAT('x',n) FROM nr; @@ -2240,7 +2243,8 @@ The `UNHEX()` function performs the reverse operation of the `HEX()` function. I > **Note:** > -> The argument must be a valid hexadecimal value that contains `0`–`9`, `A`–`F`, or `a`–`f`. If the argument is `NULL` or falls outside this range, the function returns `NULL`. +> - The argument must be a valid hexadecimal value that contains `0`–`9`, `A`–`F`, or `a`–`f`. If the argument is `NULL` or falls outside this range, the function returns `NULL`. +> - In the MySQL client, the [`--binary-as-hex`](https://dev.mysql.com/doc/refman/8.0/en/mysql-command-options.html#option_mysql_binary-as-hex) option is enabled by default in interactive mode, causing the client to display data with an unknown character set as a [hexadecimal literal](https://dev.mysql.com/doc/refman/8.0/en/hexadecimal-literals.html). You can use the `--skip-binary-as-hex` option to disable this behavior. Example: @@ -2373,4 +2377,4 @@ The difference between TiDB and MySQL support for the binary string type: ### Known issues -- [GitHub Issue #37981](https://github.com/pingcap/tidb/issues/37981) \ No newline at end of file +- [GitHub Issue #37981](https://github.com/pingcap/tidb/issues/37981) diff --git a/functions-and-operators/tidb-functions.md b/functions-and-operators/tidb-functions.md index 4f8683a050f85..48e4d5d3147e8 100644 --- a/functions-and-operators/tidb-functions.md +++ b/functions-and-operators/tidb-functions.md @@ -11,18 +11,21 @@ The following functions are TiDB extensions, and are not present in MySQL: | Function name | Function description | | :-------------- | :------------------------------------- | -| `TIDB_BOUNDED_STALENESS()` | The `TIDB_BOUNDED_STALENESS` function instructs TiDB to read the data as new as possible within the time range. See also: [Read Historical Data Using the `AS OF TIMESTAMP` Clause](/as-of-timestamp.md) | -| [`TIDB_DECODE_KEY(str)`](#tidb_decode_key) | The `TIDB_DECODE_KEY` function can be used to decode a TiDB-encoded key entry into a JSON structure containing `_tidb_rowid` and `table_id`. These encoded keys can be found in some system tables and in logging outputs. | -| [`TIDB_DECODE_PLAN(str)`](#tidb_decode_plan) | The `TIDB_DECODE_PLAN` function can be used to decode a TiDB execution plan. | -| `TIDB_IS_DDL_OWNER()` | The `TIDB_IS_DDL_OWNER` function can be used to check whether or not the TiDB instance you are connected to is the one that is the DDL Owner. The DDL Owner is the TiDB instance that is tasked with executing DDL statements on behalf of all other nodes in the cluster. | -| [`TIDB_PARSE_TSO(num)`](#tidb_parse_tso) | The `TIDB_PARSE_TSO` function can be used to extract the physical timestamp from a TiDB TSO timestamp. See also: [`tidb_current_ts`](/system-variables.md#tidb_current_ts). | -| `TIDB_PARSE_TSO_LOGICAL(num)` | The `TIDB_PARSE_TSO_LOGICAL` function can be used to extract the logical timestamp from a TiDB TSO timestamp. | -| [`TIDB_VERSION()`](#tidb_version) | The `TIDB_VERSION` function returns the TiDB version with additional build information. | -| [`TIDB_DECODE_SQL_DIGESTS(digests, stmtTruncateLength)`](#tidb_decode_sql_digests) | The `TIDB_DECODE_SQL_DIGESTS()` function is used to query the normalized SQL statements (a form without formats and arguments) corresponding to the set of SQL digests in the cluster. | -| `VITESS_HASH(str)` | The `VITESS_HASH` function returns the hash of a string that is compatible with Vitess' `HASH` function. This is intended to help the data migration from Vitess. | -| `TIDB_SHARD()` | The `TIDB_SHARD` function can be used to create a shard index to scatter the index hotspot. A shard index is an expression index with a `TIDB_SHARD` function as the prefix.| -| `TIDB_ROW_CHECKSUM()` | The `TIDB_ROW_CHECKSUM` function is used to query the checksum value of a row. This function can only be used in `SELECT` statements within the FastPlan process. That is, you can query through statements like `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id = ?` or `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id IN (?, ?, ...)`. See also: [Data integrity validation for single-row data](/ticdc/ticdc-integrity-check.md). | -| `CURRENT_RESOURCE_GROUP()` | The `CURRENT_RESOURCE_GROUP` function is used to return the resource group name that the current session is bound to. See also: [Use Resource Control to Achieve Resource Isolation](/tidb-resource-control.md). | +| [`CURRENT_RESOURCE_GROUP()`](#current_resource_group) | Returns the name of the resource group that the current session is bound to. See [using resource control to achieve resource isolation](/tidb-resource-control.md). | +| [`TIDB_BOUNDED_STALENESS()`](#tidb_bounded_staleness) | Instructs TiDB to read the most recent data within a specified time range. See [reading historical data using the `AS OF TIMESTAMP` clause](/as-of-timestamp.md). | +| [`TIDB_CURRENT_TSO()`](#tidb_current_tso) | Returns the current [TimeStamp Oracle (TSO) in TiDB](/tso.md). | +| [`TIDB_DECODE_BINARY_PLAN()`](#tidb_decode_binary_plan) | Decodes binary plans. | +| [`TIDB_DECODE_KEY()`](#tidb_decode_key) | Decodes a TiDB-encoded key entry into a JSON structure containing `_tidb_rowid` and `table_id`. These encoded keys can be found in some system tables and logging outputs. | +| [`TIDB_DECODE_PLAN()`](#tidb_decode_plan) | Decodes a TiDB execution plan. | +| [`TIDB_DECODE_SQL_DIGESTS()`](#tidb_decode_sql_digests) | Queries the normalized SQL statements (a form without formats and arguments) corresponding to a set of SQL digests in the cluster. | +| [`TIDB_ENCODE_SQL_DIGEST()`](#tidb_encode_sql_digest) | Gets a digest for a query string. | +| [`TIDB_IS_DDL_OWNER()`](#tidb_is_ddl_owner) | Checks whether or not the TiDB instance you are connected to is the DDL Owner. The DDL Owner is the TiDB instance that is tasked with executing DDL statements on behalf of all other nodes in the cluster. | +| [`TIDB_PARSE_TSO()`](#tidb_parse_tso) | Extracts the physical timestamp from a TiDB TSO timestamp. See also: [`tidb_current_ts`](/system-variables.md#tidb_current_ts). | +| [`TIDB_PARSE_TSO_LOGICAL()`](#tidb_parse_tso_logical) | Extracts the logical timestamp from a TiDB TSO timestamp. | +| [`TIDB_ROW_CHECKSUM()`](#tidb_row_checksum) | Queries the checksum value of a row. This function can only be used in `SELECT` statements within the FastPlan process. That is, you can query through statements like `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id = ?` or `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id IN (?, ?, ...)`. See also: [Data integrity validation for single-row data](/ticdc/ticdc-integrity-check.md). | +| [`TIDB_SHARD()`](#tidb_shard) | Creates a shard index to scatter the index hotspot. A shard index is an expression index with a `TIDB_SHARD` function as the prefix.| +| [`TIDB_VERSION()`](#tidb_version) | Returns the TiDB version with additional build information. | +| [`VITESS_HASH()`](#vitess_hash) | Returns the hash of a number. This function is compatible with the `HASH` function of Vitess, and is intended to help the data migration from Vitess. | @@ -30,28 +33,142 @@ The following functions are TiDB extensions, and are not present in MySQL: | Function name | Function description | | :-------------- | :------------------------------------- | -| `TIDB_BOUNDED_STALENESS()` | The `TIDB_BOUNDED_STALENESS` function instructs TiDB to read the data as new as possible within the time range. See also: [Read Historical Data Using the `AS OF TIMESTAMP` Clause](/as-of-timestamp.md) | -| [`TIDB_DECODE_KEY(str)`](#tidb_decode_key) | The `TIDB_DECODE_KEY` function can be used to decode a TiDB-encoded key entry into a JSON structure containing `_tidb_rowid` and `table_id`. These encoded keys can be found in some system tables and in logging outputs. | -| [`TIDB_DECODE_PLAN(str)`](#tidb_decode_plan) | The `TIDB_DECODE_PLAN` function can be used to decode a TiDB execution plan. | -| `TIDB_IS_DDL_OWNER()` | The `TIDB_IS_DDL_OWNER` function can be used to check whether or not the TiDB instance you are connected to is the one that is the DDL Owner. The DDL Owner is the TiDB instance that is tasked with executing DDL statements on behalf of all other nodes in the cluster. | -| [`TIDB_PARSE_TSO(num)`](#tidb_parse_tso) | The `TIDB_PARSE_TSO` function can be used to extract the physical timestamp from a TiDB TSO timestamp. See also: [`tidb_current_ts`](/system-variables.md#tidb_current_ts). | -| `TIDB_PARSE_TSO_LOGICAL(num)` | The `TIDB_PARSE_TSO_LOGICAL` function can be used to extract the logical timestamp from a TiDB TSO timestamp. | -| [`TIDB_VERSION()`](#tidb_version) | The `TIDB_VERSION` function returns the TiDB version with additional build information. | -| [`TIDB_DECODE_SQL_DIGESTS(digests, stmtTruncateLength)`](#tidb_decode_sql_digests) | The `TIDB_DECODE_SQL_DIGESTS()` function is used to query the normalized SQL statements (a form without formats and arguments) corresponding to the set of SQL digests in the cluster. | -| `VITESS_HASH(str)` | The `VITESS_HASH` function returns the hash of a string that is compatible with Vitess' `HASH` function. This is intended to help the data migration from Vitess. | -| `TIDB_SHARD()` | The `TIDB_SHARD` function can be used to create a shard index to scatter the index hotspot. A shard index is an expression index with a `TIDB_SHARD` function as the prefix.| -| `TIDB_ROW_CHECKSUM()` | The `TIDB_ROW_CHECKSUM` function is used to query the checksum value of a row. This function can only be used in `SELECT` statements within the FastPlan process. That is, you can query through statements like `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id = ?` or `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id IN (?, ?, ...)`. See also: [Data integrity validation for single-row data](https://docs.pingcap.com/tidb/stable/ticdc-integrity-check). | -| `CURRENT_RESOURCE_GROUP()` | The `CURRENT_RESOURCE_GROUP` function is used to return the resource group name that the current session is bound to. See also: [Use Resource Control to Achieve Resource Isolation](/tidb-resource-control.md). | +| [`CURRENT_RESOURCE_GROUP()`](#current_resource_group) | Returns the resource group name that the current session is bound to. See [using resource control to achieve resource isolation](/tidb-resource-control.md). | +| [`TIDB_BOUNDED_STALENESS()`](#tidb_bounded_staleness) | Instructs TiDB to read most recent data within a specified time range. See [reading historical data using the `AS OF TIMESTAMP` clause](/as-of-timestamp.md). | +| [`TIDB_CURRENT_TSO()`](#tidb_current_tso) | Returns the current [TimeStamp Oracle (TSO) in TiDB](/tso.md). | +| [`TIDB_DECODE_BINARY_PLAN()`](#tidb_decode_binary_plan) | Decodes binary plans. | +| [`TIDB_DECODE_KEY()`](#tidb_decode_key) | Decodes a TiDB-encoded key entry into a JSON structure containing `_tidb_rowid` and `table_id`. These encoded keys can be found in some system tables and logging outputs. | +| [`TIDB_DECODE_PLAN()`](#tidb_decode_plan) | Decodes a TiDB execution plan. | +| [`TIDB_DECODE_SQL_DIGESTS()`](#tidb_decode_sql_digests) | Queries the normalized SQL statements (a form without formats and arguments) corresponding to a set of SQL digests in the cluster. | +| [`TIDB_ENCODE_SQL_DIGEST()`](#tidb_encode_sql_digest) | Gets a digest for a query string. | +| [`TIDB_IS_DDL_OWNER()`](#tidb_is_ddl_owner) | Checks whether or not the TiDB instance you are connected to is the DDL Owner. The DDL Owner is the TiDB instance that is tasked with executing DDL statements on behalf of all other nodes in the cluster. | +| [`TIDB_PARSE_TSO()`](#tidb_parse_tso) | Extracts the physical timestamp from a TiDB TSO timestamp. See also: [`tidb_current_ts`](/system-variables.md#tidb_current_ts). | +| [`TIDB_PARSE_TSO_LOGICAL()`](#tidb_parse_tso_logical) | Extracts the logical timestamp from a TiDB TSO timestamp. | +| [`TIDB_ROW_CHECKSUM()`](#tidb_row_checksum) | Queries the checksum value of a row. This function can only be used in `SELECT` statements within the FastPlan process. That is, you can query through statements like `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id = ?` or `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id IN (?, ?, ...)`. See also: [Data integrity validation for single-row data](https://docs.pingcap.com/tidb/stable/ticdc-integrity-check). | +| [`TIDB_SHARD()`](#tidb_shard) | Creates a shard index to scatter the index hotspot. A shard index is an expression index with a `TIDB_SHARD` function as the prefix.| +| [`TIDB_VERSION()`](#tidb_version) | Returns the TiDB version with additional build information. | +| [`VITESS_HASH()`](#vitess_hash) | Returns the hash of a number. This function is compatible with the `HASH` function of Vitess, and is intended to help the data migration from Vitess. | -## Examples +## CURRENT_RESOURCE_GROUP -This section provides examples for some of the functions above. +The `CURRENT_RESOURCE_GROUP()` function is used to show the resource group name that the current session is bound to. When the [Resource control](/tidb-resource-control.md) feature is enabled, the available resources that can be used by SQL statements are restricted by the resource quota of the bound resource group. -### TIDB_DECODE_KEY +When a session is established, TiDB binds the session to the resource group that the login user is bound to by default. If the user is not bound to any resource groups, the session is bound to the `default` resource group. Once the session is established, the bound resource group will not change by default, even if the bound resource group of the user is changed via [modifying the resource group bound to the user](/sql-statements/sql-statement-alter-user.md#modify-basic-user-information). To change the bound resource group of the current session, you can use [`SET RESOURCE GROUP`](/sql-statements/sql-statement-set-resource-group.md). + +Examples: + +Create a user `user1`, create two resource groups `rg1` and `rg2`, and bind the user `user1` to the resource group `rg1`: + +```sql +CREATE USER 'user1'; +CREATE RESOURCE GROUP rg1 RU_PER_SEC = 1000; +CREATE RESOURCE GROUP rg2 RU_PER_SEC = 2000; +ALTER USER 'user1' RESOURCE GROUP `rg1`; +``` + +Use `user1` to log in and view the resource group bound to the current user: + +```sql +SELECT CURRENT_RESOURCE_GROUP(); +``` + +``` ++--------------------------+ +| CURRENT_RESOURCE_GROUP() | ++--------------------------+ +| rg1 | ++--------------------------+ +1 row in set (0.00 sec) +``` + +Execute `SET RESOURCE GROUP` to set the resource group for the current session to `rg2`, and then view the resource group bound to the current user: + +```sql +SET RESOURCE GROUP `rg2`; +SELECT CURRENT_RESOURCE_GROUP(); +``` + +``` ++--------------------------+ +| CURRENT_RESOURCE_GROUP() | ++--------------------------+ +| rg2 | ++--------------------------+ +1 row in set (0.00 sec) +``` + +## TIDB_BOUNDED_STALENESS + +The `TIDB_BOUNDED_STALENESS()` function is used as part of [`AS OF TIMESTAMP`](/as-of-timestamp.md) syntax. + +## TIDB_CURRENT_TSO + +The `TIDB_CURRENT_TSO()` function returns the [TSO](/tso.md) for the current transaction. This is similar to the [`tidb_current_ts`](/system-variables.md#tidb_current_ts) system variable. + +```sql +BEGIN; +``` + +``` +Query OK, 0 rows affected (0.00 sec) +``` + +```sql +SELECT TIDB_CURRENT_TSO(); +``` + +``` ++--------------------+ +| TIDB_CURRENT_TSO() | ++--------------------+ +| 450456244814610433 | ++--------------------+ +1 row in set (0.00 sec) +``` + +```sql +SELECT @@tidb_current_ts; +``` + +``` ++--------------------+ +| @@tidb_current_ts | ++--------------------+ +| 450456244814610433 | ++--------------------+ +1 row in set (0.00 sec) +``` + +## TIDB_DECODE_BINARY_PLAN + +The `TIDB_DECODE_BINARY_PLAN(binary_plan)` function decodes binary plans, like the ones in the `BINARY_PLAN` column of the [`STATEMENTS_SUMMARY`](/statement-summary-tables.md) table. + +The [`tidb_generate_binary_plan`](/system-variables.md#tidb_generate_binary_plan-new-in-v620) variable must be set to `ON` for the binary plans to be available. -In the following example, the table `t1` has a hidden `rowid` that is generated by TiDB. The `TIDB_DECODE_KEY` is used in the statement. From the result, you can see that the hidden `rowid` is decoded and output, which is a typical result for the non-clustered primary key. +Example: + +```sql +SELECT BINARY_PLAN,TIDB_DECODE_BINARY_PLAN(BINARY_PLAN) FROM information_schema.STATEMENTS_SUMMARY LIMIT 1\G +``` + +``` +*************************** 1. row *************************** + BINARY_PLAN: lQLwPgqQAgoMUHJvamVjdGlvbl8zEngKDk1lbVRhYmxlU2Nhbl80KQAAAAAAiMNAMAM4AUABSioKKAoSaW5mb3JtYQU00HNjaGVtYRISU1RBVEVNRU5UU19TVU1NQVJZWhV0aW1lOjI5LjPCtXMsIGxvb3BzOjJw////CQIEAXgJCBD///8BIQFnDOCb+EA6cQCQUjlDb2x1bW4jOTIsIHRpZGJfZGVjb2RlX2JpbmFyeV9wbGFuKBUjCCktPg0MEDEwM1oWBYAIMTA4NoEAeGINQ29uY3VycmVuY3k6NXDIZXj///////////8BGAE= +TIDB_DECODE_BINARY_PLAN(BINARY_PLAN): +| id | estRows | estCost | actRows | task | access object | execution info | operator info | memory | disk | +| Projection_3 | 10000.00 | 100798.00 | 3 | root | | time:108.3µs, loops:2, Concurrency:5 | Column#92, tidb_decode_binary_plan(Column#92)->Column#103 | 12.7 KB | N/A | +| └─MemTableScan_4 | 10000.00 | 0.00 | 3 | root | table:STATEMENTS_SUMMARY | time:29.3µs, loops:2 | | N/A | N/A | + +1 row in set (0.00 sec) +``` + +## TIDB_DECODE_KEY + +The `TIDB_DECODE_KEY()` function decodes a TiDB-encoded key entry into a JSON structure containing `_tidb_rowid` and `table_id`. These encoded keys exist in some system tables and logging outputs. + +In the following example, the table `t1` has a hidden `rowid` that is generated by TiDB. The `TIDB_DECODE_KEY()` function is used in the statement. From the result, you can see that the hidden `rowid` is decoded and output, which is a typical result for the non-clustered primary key. ```sql SELECT START_KEY, TIDB_DECODE_KEY(START_KEY) FROM information_schema.tikv_region_status WHERE table_name='t1' AND REGION_ID=2\G @@ -121,7 +238,7 @@ SELECT tidb_decode_key('7480000000000000FF3E5F720400000000FF0000000601633430FF33 1 row in set (0.001 sec) ``` -The first Region of a table starts with a key that only has the `table_id` of the table. The last Region of the table ends with `table_id + 1`. Any Regions in between have longer keys that includes a `_tidb_rowid` or `handle`. +In the following example, the first Region of a table starts with a key that only has the `table_id` of the table. The last Region of the table ends with `table_id + 1`. Any Regions in between have longer keys that includes a `_tidb_rowid` or `handle`. ```sql SELECT @@ -151,7 +268,7 @@ ORDER BY `TIDB_DECODE_KEY` returns valid JSON on success and returns the argument value if it fails to decode. -### TIDB_DECODE_PLAN +## TIDB_DECODE_PLAN You can find TiDB execution plans in encoded form in the slow query log. The `TIDB_DECODE_PLAN()` function is then used to decode the encoded plans into a human-readable form. @@ -169,9 +286,111 @@ SELECT tidb_decode_plan('8QIYMAkzMV83CQEH8E85LjA0CWRhdGE6U2VsZWN0aW9uXzYJOTYwCXR └─TableFullScan_5 cop[tikv] 960 table:t, keep order:false, stats:pseudo 960 tikv_task:{time:153µs, loops:960} N/A N/A ``` -### TIDB_PARSE_TSO +## TIDB_DECODE_SQL_DIGESTS + +The `TIDB_DECODE_SQL_DIGESTS()` function is used to query the normalized SQL statements (a form without formats and arguments) corresponding to the set of SQL digests in the cluster. This function accepts 1 or 2 arguments: + +* `digests`: A string. This parameter is in the format of a JSON string array, and each string in the array is a SQL digest. +* `stmtTruncateLength`: An integer (optional). It is used to limit the length of each SQL statement in the returned result. If a SQL statement exceeds the specified length, the statement is truncated. `0` means that the length is unlimited. + +This function returns a string, which is in the format of a JSON string array. The *i*-th item in the array is the normalized SQL statement corresponding to the *i*-th element in the `digests` parameter. If an element in the `digests` parameter is not a valid SQL digest or the system cannot find the corresponding SQL statement, the corresponding item in the returned result is `null`. If the truncation length is specified (`stmtTruncateLength > 0`), for each statement in the returned result that exceeds this length, the first `stmtTruncateLength` characters are retained and the suffix `"..."` is added at the end to indicate the truncation. If the `digests` parameter is `NULL`, the returned value of the function is `NULL`. + +> **Note:** +> +> * Only users with the [PROCESS](https://dev.mysql.com/doc/refman/8.0/en/privileges-provided.html#priv_process) privilege can use this function. +> * When `TIDB_DECODE_SQL_DIGESTS` is executed, TiDB queries the statement corresponding to each SQL digest from the statement summary tables, so there is no guarantee that the corresponding statement can always be found for any SQL digest. Only the statements that have been executed in the cluster can be found, and whether these SQL statements can be queried or not is also affected by the related configuration of the statement summary tables. For the detailed description of the statement summary table, see [Statement Summary Tables](/statement-summary-tables.md). +> * This function has a high overhead. In queries with a large number of rows (for example, querying the full table of `information_schema.cluster_tidb_trx` on a large and busy cluster), using this function might cause the queries to run for too long. Use it with caution. +> * This function has a high overhead because every time it is called, it internally queries the `STATEMENTS_SUMMARY`, `STATEMENTS_SUMMARY_HISTORY`, `CLUSTER_STATEMENTS_SUMMARY`, and `CLUSTER_STATEMENTS_SUMMARY_HISTORY` tables, and the query involves the `UNION` operation. This function currently does not support vectorization, that is, when calling this function for multiple rows of data, the above query is performed separately for each row. + +```sql +SET @digests = '["e6f07d43b5c21db0fbb9a31feac2dc599787763393dd5acbfad80e247eb02ad5","38b03afa5debbdf0326a014dbe5012a62c51957f1982b3093e748460f8b00821","e5796985ccafe2f71126ed6c0ac939ffa015a8c0744a24b7aee6d587103fd2f7"]'; + +SELECT TIDB_DECODE_SQL_DIGESTS(@digests); +``` + +```sql ++------------------------------------+ +| TIDB_DECODE_SQL_DIGESTS(@digests) | ++------------------------------------+ +| ["begin",null,"select * from `t`"] | ++------------------------------------+ +1 row in set (0.00 sec) +``` + +In the above example, the parameter is a JSON array containing 3 SQL digests, and the corresponding SQL statements are the three items in the query results. But the SQL statement corresponding to the second SQL digest cannot be found from the cluster, so the second item in the result is `null`. + +```sql +SELECT TIDB_DECODE_SQL_DIGESTS(@digests, 10); +``` + +```sql ++---------------------------------------+ +| TIDB_DECODE_SQL_DIGESTS(@digests, 10) | ++---------------------------------------+ +| ["begin",null,"select * f..."] | ++---------------------------------------+ +1 row in set (0.01 sec) +``` + +The above call specifies the second parameter (that is, the truncation length) as 10, and the length of the third statement in the query result is greater than 10. Therefore, only the first 10 characters are retained, and `"..."` is added at the end, which indicates the truncation. -The `TIDB_PARSE_TSO` function can be used to extract the physical timestamp from a TiDB TSO timestamp. TSO stands for Time Stamp Oracle and is a monotonically increasing timestamp given out by PD (Placement Driver) for every transaction. +See also: + +- [Statement summary tables](/statement-summary-tables.md) +- [`INFORMATION_SCHEMA.TIDB_TRX`](/information-schema/information-schema-tidb-trx.md) + +## TIDB_ENCODE_SQL_DIGEST + +The `TIDB_ENCODE_SQL_DIGEST(query_str)` returns the SQL digest for a query string. + +In the following example you can see that both queries get the same query digest, which is because the digest will be `select ?` for both of them. + +```sql +SELECT TIDB_ENCODE_SQL_DIGEST('SELECT 1'); +``` + +``` ++------------------------------------------------------------------+ +| TIDB_ENCODE_SQL_DIGEST('SELECT 1') | ++------------------------------------------------------------------+ +| e1c71d1661ae46e09b7aaec1c390957f0d6260410df4e4bc71b9c8d681021471 | ++------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +```sql +SELECT TIDB_ENCODE_SQL_DIGEST('SELECT 2'); +``` + +``` ++------------------------------------------------------------------+ +| TIDB_ENCODE_SQL_DIGEST('SELECT 2') | ++------------------------------------------------------------------+ +| e1c71d1661ae46e09b7aaec1c390957f0d6260410df4e4bc71b9c8d681021471 | ++------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +## TIDB_IS_DDL_OWNER + +The `TIDB_IS_DDL_OWNER()` function returns `1` if the instance you are connected to is the DDL owner. + +```sql +SELECT TIDB_IS_DDL_OWNER(); +``` + +``` ++---------------------+ +| TIDB_IS_DDL_OWNER() | ++---------------------+ +| 1 | ++---------------------+ +1 row in set (0.00 sec) +``` + +## TIDB_PARSE_TSO + +The `TIDB_PARSE_TSO()` function extracts the physical timestamp from a TiDB TSO timestamp. [TSO](/tso.md) stands for Time Stamp Oracle and is a monotonically increasing timestamp given out by PD (Placement Driver) for every transaction. A TSO is a number that consists of two parts: @@ -195,84 +414,76 @@ ROLLBACK; Here `TIDB_PARSE_TSO` is used to extract the physical timestamp from the timestamp number that is available in the `tidb_current_ts` session variable. Because timestamps are given out per transaction, this function is running in a transaction. -### TIDB_VERSION +## TIDB_PARSE_TSO_LOGICAL -The `TIDB_VERSION` function can be used to get the version and build details of the TiDB server that you are connected to. You can use this function when reporting issues on GitHub. +The `TIDB_PARSE_TSO_LOGICAL(tso)` function returns the logical part of a [TSO](/tso.md) timestamp. ```sql -SELECT TIDB_VERSION()\G +SELECT TIDB_PARSE_TSO_LOGICAL(450456244814610433); ``` -```sql -*************************** 1. row *************************** -TIDB_VERSION(): Release Version: v5.1.0-alpha-13-gd5e0ed0aa-dirty -Edition: Community -Git Commit Hash: d5e0ed0aaed72d2f2dfe24e9deec31cb6cb5fdf0 -Git Branch: master -UTC Build Time: 2021-05-24 14:39:20 -GoVersion: go1.13 -Race Enabled: false -TiKV Min Version: v3.0.0-60965b006877ca7234adaced7890d7b029ed1306 -Check Table Before Drop: false +``` ++--------------------------------------------+ +| TIDB_PARSE_TSO_LOGICAL(450456244814610433) | ++--------------------------------------------+ +| 1 | ++--------------------------------------------+ 1 row in set (0.00 sec) ``` -### TIDB_DECODE_SQL_DIGESTS +```sql +SELECT TIDB_PARSE_TSO_LOGICAL(450456244814610434); +``` -The `TIDB_DECODE_SQL_DIGESTS()` function is used to query the normalized SQL statements (a form without formats and arguments) corresponding to the set of SQL digests in the cluster. This function accepts 1 or 2 arguments: +``` ++--------------------------------------------+ +| TIDB_PARSE_TSO_LOGICAL(450456244814610434) | ++--------------------------------------------+ +| 2 | ++--------------------------------------------+ +1 row in set (0.00 sec) +``` -* `digests`: A string. This parameter is in the format of a JSON string array, and each string in the array is a SQL digest. -* `stmtTruncateLength`: An integer (optional). It is used to limit the length of each SQL statement in the returned result. If a SQL statement exceeds the specified length, the statement is truncated. `0` means that the length is unlimited. +## TIDB_ROW_CHECKSUM -This function returns a string, which is in the format of a JSON string array. The *i*-th item in the array is the normalized SQL statement corresponding to the *i*-th element in the `digests` parameter. If an element in the `digests` parameter is not a valid SQL digest or the system cannot find the corresponding SQL statement, the corresponding item in the returned result is `null`. If the truncation length is specified (`stmtTruncateLength > 0`), for each statement in the returned result that exceeds this length, the first `stmtTruncateLength` characters are retained and the suffix `"..."` is added at the end to indicate the truncation. If the `digests` parameter is `NULL`, the returned value of the function is `NULL`. +The `TIDB_ROW_CHECKSUM()` function is used to query the checksum value of a row. This function can only be used in `SELECT` statements within the FastPlan process. That is, you can query through statements like `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id = ?` or `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id IN (?, ?, ...)`. -> **Note:** -> -> * Only users with the [PROCESS](https://dev.mysql.com/doc/refman/8.0/en/privileges-provided.html#priv_process) privilege can use this function. -> * When `TIDB_DECODE_SQL_DIGESTS` is executed, TiDB queries the statement corresponding to each SQL digest from the statement summary tables, so there is no guarantee that the corresponding statement can always be found for any SQL digest. Only the statements that have been executed in the cluster can be found, and whether these SQL statements can be queried or not is also affected by the related configuration of the statement summary tables. For the detailed description of the statement summary table, see [Statement Summary Tables](/statement-summary-tables.md). -> * This function has a high overhead. In queries with a large number of rows (for example, querying the full table of `information_schema.cluster_tidb_trx` on a large and busy cluster), using this function might cause the queries to run for too long. Use it with caution. -> * This function has a high overhead because every time it is called, it internally queries the `STATEMENTS_SUMMARY`, `STATEMENTS_SUMMARY_HISTORY`, `CLUSTER_STATEMENTS_SUMMARY`, and `CLUSTER_STATEMENTS_SUMMARY_HISTORY` tables, and the query involves the `UNION` operation. This function currently does not support vectorization, that is, when calling this function for multiple rows of data, the above query is performed separately for each row. +To enable the checksum feature of single-row data in TiDB (controlled by the system variable [`tidb_enable_row_level_checksum`](/system-variables.md#tidb_enable_row_level_checksum-new-in-v710)), run the following statement: ```sql -set @digests = '["e6f07d43b5c21db0fbb9a31feac2dc599787763393dd5acbfad80e247eb02ad5","38b03afa5debbdf0326a014dbe5012a62c51957f1982b3093e748460f8b00821","e5796985ccafe2f71126ed6c0ac939ffa015a8c0744a24b7aee6d587103fd2f7"]'; - -select tidb_decode_sql_digests(@digests); +SET GLOBAL tidb_enable_row_level_checksum = ON; ``` -```sql -+------------------------------------+ -| tidb_decode_sql_digests(@digests) | -+------------------------------------+ -| ["begin",null,"select * from `t`"] | -+------------------------------------+ -1 row in set (0.00 sec) -``` +This configuration only takes effect for newly created sessions, so you need to reconnect to TiDB. -In the above example, the parameter is a JSON array containing 3 SQL digests, and the corresponding SQL statements are the three items in the query results. But the SQL statement corresponding to the second SQL digest cannot be found from the cluster, so the second item in the result is `null`. +Create table `t` and insert data: ```sql -select tidb_decode_sql_digests(@digests, 10); +USE test; +CREATE TABLE t (id INT PRIMARY KEY, k INT, c CHAR(1)); +INSERT INTO t VALUES (1, 10, 'a'); ``` +The following statement shows how to query the checksum value of the row where `id = 1` in table `t`: + ```sql -+---------------------------------------+ -| tidb_decode_sql_digests(@digests, 10) | -+---------------------------------------+ -| ["begin",null,"select * f..."] | -+---------------------------------------+ -1 row in set (0.01 sec) +SELECT *, TIDB_ROW_CHECKSUM() FROM t WHERE id = 1; ``` -The above call specifies the second parameter (that is, the truncation length) as 10, and the length of the third statement in the query result is greater than 10. Therefore, only the first 10 characters are retained, and `"..."` is added at the end, which indicates the truncation. - -See also: +The output is as follows: -- [`Statement Summary Tables`](/statement-summary-tables.md) -- [`INFORMATION_SCHEMA.TIDB_TRX`](/information-schema/information-schema-tidb-trx.md) +```sql ++----+------+------+---------------------+ +| id | k | c | TIDB_ROW_CHECKSUM() | ++----+------+------+---------------------+ +| 1 | 10 | a | 3813955661 | ++----+------+------+---------------------+ +1 row in set (0.000 sec) +``` -### TIDB_SHARD +## TIDB_SHARD -The `TIDB_SHARD` function can be used to create a shard index to scatter the index hotspot. A shard index is an expression index prefixed with a `TIDB_SHARD` function. +The `TIDB_SHARD()` function creates a shard index to scatter the index hotspot. A shard index is an expression index prefixed with a `TIDB_SHARD()` function. - Creation: @@ -296,11 +507,11 @@ The `TIDB_SHARD` function can be used to create a shard index to scatter the ind - Cannot go through FastPlan process, which affects optimizer performance. - Cannot be used to prepare the execution plan cache. -The following example shows how to use the `TIDB_SHARD` function. +The following example shows how to use the `TIDB_SHARD()` function. -- Use the `TIDB_SHARD` function to calculate the SHARD value. +- Use the `TIDB_SHARD()` function to calculate the SHARD value. - The following statement shows how to use the `TIDB_SHARD` function to calculate the SHARD value of `12373743746`: + The following statement shows how to use the `TIDB_SHARD()` function to calculate the SHARD value of `12373743746`: ```sql SELECT TIDB_SHARD(12373743746); @@ -317,91 +528,49 @@ The following example shows how to use the `TIDB_SHARD` function. 1 row in set (0.00 sec) ``` -- Create a shard index using the `TIDB_SHARD` function: +- Create a shard index using the `TIDB_SHARD()` function: ```sql CREATE TABLE test(id INT PRIMARY KEY CLUSTERED, a INT, b INT, UNIQUE KEY uk((tidb_shard(a)), a)); ``` -### TIDB_ROW_CHECKSUM - -The `TIDB_ROW_CHECKSUM` function is used to query the checksum value of a row. This function can only be used in `SELECT` statements within the FastPlan process. That is, you can query through statements like `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id = ?` or `SELECT TIDB_ROW_CHECKSUM() FROM t WHERE id IN (?, ?, ...)`. - -To enable the checksum feature of single-row data in TiDB (controlled by the system variable [`tidb_enable_row_level_checksum`](/system-variables.md#tidb_enable_row_level_checksum-new-in-v710)), run the following statement: - -```sql -SET GLOBAL tidb_enable_row_level_checksum = ON; -``` - -Create table `t` and insert data: - -```sql -USE test; -CREATE TABLE t (id INT PRIMARY KEY, k INT, c int); -INSERT INTO TABLE t values (1, 10, a); -``` +## TIDB_VERSION -The following statement shows how to query the checksum value of the row where `id = 1` in table `t`: +The `TIDB_VERSION()` function is used to get the version and build details of the TiDB server that you are connected to. You can use this function when reporting issues on GitHub. ```sql -SELECT *, TIDB_ROW_CHECKSUM() FROM t WHERE id = 1; +SELECT TIDB_VERSION()\G ``` -The output is as follows: - ```sql -+----+------+------+---------------------+ -| id | k | c | TIDB_ROW_CHECKSUM() | -+----+------+------+---------------------+ -| 1 | 10 | a | 3813955661 | -+----+------+------+---------------------+ -1 row in set (0.000 sec) -``` - -### CURRENT_RESOURCE_GROUP - -The `CURRENT_RESOURCE_GROUP` function is used to show the resource group name that the current session is bound to. When the [Resource control](/tidb-resource-control.md) feature is enabled, the available resources that can be used by SQL statements are restricted by the resource quota of the bound resource group. - -When a session is established, TiDB binds the session to the resource group that the login user is bound to by default. If the user is not bound to any resource groups, the session is bound to the `default` resource group. Once the session is established, the bound resource group will not change by default, even if the bound resource group of the user is changed via [modifying the resource group bound to the user](/sql-statements/sql-statement-alter-user.md#modify-basic-user-information). To change the bound resource group of the current session, you can use [`SET RESOURCE GROUP`](/sql-statements/sql-statement-set-resource-group.md). - -#### Example - -Create a user `user1`, create two resource groups `rg1` and `rg2`, and bind the user `user1` to the resource group `rg1`: - -```sql -CREATE USER 'user1'; -CREATE RESOURCE GROUP 'rg1' RU_PER_SEC = 1000; -CREATE RESOURCE GROUP 'rg2' RU_PER_SEC = 2000; -ALTER USER 'user1' RESOURCE GROUP `rg1`; +*************************** 1. row *************************** +TIDB_VERSION(): Release Version: v8.1.2 +Edition: Community +Git Commit Hash: 821e491a20fbab36604b36b647b5bae26a2c1418 +Git Branch: HEAD +UTC Build Time: 2024-12-26 19:16:25 +GoVersion: go1.21.10 +Race Enabled: false +Check Table Before Drop: false +Store: tikv +1 row in set (0.00 sec) ``` -Use `user1` to log in and view the resource group bound to the current user: +## VITESS_HASH -```sql -SELECT CURRENT_RESOURCE_GROUP(); -``` +The `VITESS_HASH(num)` function is used to hash a number in the same way Vitess does this. This is to aid migration from Vitess to TiDB. -``` -+--------------------------+ -| CURRENT_RESOURCE_GROUP() | -+--------------------------+ -| rg1 | -+--------------------------+ -1 row in set (0.00 sec) -``` - -Execute `SET RESOURCE GROUP` to set the resource group for the current session to `rg2`, and then view the resource group bound to the current user: +Example: ```sql -SET RESOURCE GROUP `rg2`; -SELECT CURRENT_RESOURCE_GROUP(); +SELECT VITESS_HASH(123); ``` ``` -+--------------------------+ -| CURRENT_RESOURCE_GROUP() | -+--------------------------+ -| rg2 | -+--------------------------+ ++---------------------+ +| VITESS_HASH(123) | ++---------------------+ +| 1155070131015363447 | ++---------------------+ 1 row in set (0.00 sec) -``` +``` \ No newline at end of file diff --git a/functions-and-operators/type-conversion-in-expression-evaluation.md b/functions-and-operators/type-conversion-in-expression-evaluation.md index 3ed050120d6ae..6c93c32f63601 100644 --- a/functions-and-operators/type-conversion-in-expression-evaluation.md +++ b/functions-and-operators/type-conversion-in-expression-evaluation.md @@ -1,7 +1,6 @@ --- title: Type Conversion in Expression Evaluation summary: Learn about the type conversion in expression evaluation. -aliases: ['/docs/dev/functions-and-operators/type-conversion-in-expression-evaluation/','/docs/dev/reference/sql/functions-and-operators/type-conversion/'] --- # Type Conversion in Expression Evaluation diff --git a/functions-and-operators/window-functions.md b/functions-and-operators/window-functions.md index 35282ebf4f020..2265e9ec11a1e 100644 --- a/functions-and-operators/window-functions.md +++ b/functions-and-operators/window-functions.md @@ -1,29 +1,476 @@ --- title: Window Functions summary: This document introduces window functions supported in TiDB. -aliases: ['/docs/dev/functions-and-operators/window-functions/','/docs/dev/reference/sql/functions-and-operators/window-functions/'] --- # Window Functions The usage of window functions in TiDB is similar to that in MySQL 8.0. For details, see [MySQL Window Functions](https://dev.mysql.com/doc/refman/8.0/en/window-functions.html). -Because window functions reserve additional words in the parser, TiDB provides an option to disable window functions. If you receive errors parsing SQL statements after upgrading, try setting `tidb_enable_window_function=0`. +In TiDB, you can control window functions using the following system variables: + +- [`tidb_enable_window_function`](/system-variables.md#tidb_enable_window_function): because window functions reserve additional [keywords](/keywords.md) in the parser, TiDB provides this variable to disable window functions. If you receive errors parsing SQL statements after upgrading TiDB, try setting this variable to `OFF`. +- [`tidb_enable_pipelined_window_function`](/system-variables.md#tidb_enable_pipelined_window_function): you can use this variable to disable pipeline execution algorithm for window functions. +- [`windowing_use_high_precision`](/system-variables.md#windowing_use_high_precision): you can use this variable to disable high precision mode for window functions. The window functions [listed here](/tiflash/tiflash-supported-pushdown-calculations.md) can be pushed down to TiFlash. -Except for `GROUP_CONCAT()` and `APPROX_PERCENTILE()`, TiDB supports all [`GROUP BY` aggregate functions](/functions-and-operators/aggregate-group-by-functions.md). In addition, TiDB supports the following window functions: - -| Function name | Feature description | -| :-------------- | :------------------------------------- | -| [`CUME_DIST()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_cume-dist) | Returns the cumulative distribution of a value within a group of values. | -| [`DENSE_RANK()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_dense-rank) | Returns the rank of the current row within the partition, and the rank is without gaps. | -| [`FIRST_VALUE()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_first-value) | Returns the expression value of the first row in the current window. | -| [`LAG()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_lag) | Returns the expression value from the row that precedes the current row by N rows within the partition. | -| [`LAST_VALUE()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_last-value) | Returns the expression value of the last row in the current window. | -| [`LEAD()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_lead) | Returns the expression value from the row that follows the current row by N rows within the partition. | -| [`NTH_VALUE()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_nth-value) | Returns the expression value from the N-th row of the current window. | -| [`NTILE()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_ntile)| Divides a partition into N buckets, assigns the bucket number to each row in the partition, and returns the bucket number of the current row within the partition. | -| [`PERCENT_RANK()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_percent-rank)| Returns the percentage of partition values that are less than the value in the current row. | -| [`RANK()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_rank)| Returns the rank of the current row within the partition. The rank may be with gaps. | -| [`ROW_NUMBER()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_row-number)| Returns the number of the current row in the partition. | +Except for `GROUP_CONCAT()` and `APPROX_PERCENTILE()`, TiDB supports using all [`GROUP BY` aggregate functions](/functions-and-operators/aggregate-group-by-functions.md) as window functions. In addition, TiDB supports the following window functions: + +| Function name | Feature description | +| :-------------------------------- | :------------------------------------- | +| [`CUME_DIST()`](#cume_dist) | Returns the cumulative distribution of a value within a group of values. | +| [`DENSE_RANK()`](#dense_rank) | Returns the rank of the current row within the partition, and the rank is without gaps. | +| [`FIRST_VALUE()`](#first_value) | Returns the expression value of the first row in the current window. | +| [`LAG()`](#lag) | Returns the expression value from the row that precedes the current row by N rows within the partition. | +| [`LAST_VALUE()`](#last_value) | Returns the expression value of the last row in the current window. | +| [`LEAD()`](#lead) | Returns the expression value from the row that follows the current row by N rows within the partition. | +| [`NTH_VALUE()`](#nth_value) | Returns the expression value from the N-th row of the current window. | +| [`NTILE()`](#ntile) | Divides a partition into N buckets, assigns the bucket number to each row in the partition, and returns the bucket number of the current row within the partition. | +| [`PERCENT_RANK()`](#percent_rank) | Returns the percentage of partition values that are less than the value in the current row. | +| [`RANK()`](#rank) | Returns the rank of the current row within the partition. The rank might have gaps. | +| [`ROW_NUMBER()`](#row_number) | Returns the number of the current row in the partition. | + +## [`CUME_DIST()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_cume-dist) + +`CUME_DIST()` calculates the cumulative distribution of a value within a group of values. Note that you need to use the `ORDER BY` clause with `CUME_DIST()` to sort the group of values. Otherwise, this function will not return the expected values. + +```sql +WITH RECURSIVE cte(n) AS ( + SELECT 1 + UNION + SELECT + n+2 + FROM + cte + WHERE + n<6 +) +SELECT + *, + CUME_DIST() OVER(ORDER BY n) +FROM + cte; +``` + +``` ++------+------------------------------+ +| n | CUME_DIST() OVER(ORDER BY n) | ++------+------------------------------+ +| 1 | 0.25 | +| 3 | 0.5 | +| 5 | 0.75 | +| 7 | 1 | ++------+------------------------------+ +4 rows in set (0.00 sec) +``` + +## [`DENSE_RANK()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_dense-rank) + +The `DENSE_RANK()` function returns the rank of the current row. It is similar to [`RANK()`](#rank) but does not leave any gaps in case of ties (rows that share the same values and order conditions). + +```sql +SELECT + *, + DENSE_RANK() OVER (ORDER BY n) +FROM ( + SELECT 5 AS 'n' + UNION ALL + SELECT 8 + UNION ALL + SELECT 5 + UNION ALL + SELECT 30 + UNION ALL + SELECT 31 + UNION ALL + SELECT 32) a; +``` + +``` ++----+--------------------------------+ +| n | DENSE_RANK() OVER (ORDER BY n) | ++----+--------------------------------+ +| 5 | 1 | +| 5 | 1 | +| 8 | 2 | +| 30 | 3 | +| 31 | 4 | +| 32 | 5 | ++----+--------------------------------+ +6 rows in set (0.00 sec) +``` + +## [`FIRST_VALUE()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_first-value) + +The `FIRST_VALUE(expr)` returns the first value in a window. + +The following example uses two different window definitions: + +- `PARTITION BY n MOD 2 ORDER BY n` partitions the data in table `a` into two groups: `1, 3` and `2, 4`. So it returns either `1` or `2` as those are the first values of those groups. +- `PARTITION BY n <= 2 ORDER BY n` partitions the data in table `a` into two groups: `1, 2` and `3, 4` So it returns either `1` or `3` depending on which group `n` belongs to. + +```sql +SELECT + n, + FIRST_VALUE(n) OVER (PARTITION BY n MOD 2 ORDER BY n), + FIRST_VALUE(n) OVER (PARTITION BY n <= 2 ORDER BY n) +FROM ( + SELECT 1 AS 'n' + UNION + SELECT 2 + UNION + SELECT 3 + UNION + SELECT 4 +) a +ORDER BY + n; +``` + +``` ++------+-------------------------------------------------------+------------------------------------------------------+ +| n | FIRST_VALUE(n) OVER (PARTITION BY n MOD 2 ORDER BY n) | FIRST_VALUE(n) OVER (PARTITION BY n <= 2 ORDER BY n) | ++------+-------------------------------------------------------+------------------------------------------------------+ +| 1 | 1 | 1 | +| 2 | 2 | 1 | +| 3 | 1 | 3 | +| 4 | 2 | 3 | ++------+-------------------------------------------------------+------------------------------------------------------+ +4 rows in set (0.00 sec) +``` + +## [`LAG()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_lag) + +The `LAG(expr [, num [, default]])` function returns the value of `expr` from the row that is `num` rows preceding the current row. If such row does not exist, `default` is returned. By default, `num` is `1` and `default` is `NULL` when they are not specified. + +In the following example, because `num` is not specified, `LAG(n)` returns the value of `n` in the previous row. When `n` is 1, because the previous row does not exist and `default` is not specified, `LAG(1)` returns `NULL`. + +```sql +WITH RECURSIVE cte(n) AS ( + SELECT 1 + UNION + SELECT + n+1 + FROM + cte + WHERE + n<10 +) +SELECT + n, + LAG(n) OVER () +FROM + cte; +``` + +``` ++------+----------------+ +| n | LAG(n) OVER () | ++------+----------------+ +| 1 | NULL | +| 2 | 1 | +| 3 | 2 | +| 4 | 3 | +| 5 | 4 | +| 6 | 5 | +| 7 | 6 | +| 8 | 7 | +| 9 | 8 | +| 10 | 9 | ++------+----------------+ +10 rows in set (0.01 sec) +``` + +## [`LAST_VALUE()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_last-value) + +The `LAST_VALUE()` function returns the last value in the window. + +```sql +WITH RECURSIVE cte(n) AS ( + SELECT + 1 + UNION + SELECT + n+1 + FROM + cte + WHERE + n<10 +) +SELECT + n, + LAST_VALUE(n) OVER (PARTITION BY n<=5) +FROM + cte +ORDER BY + n; +``` + +``` ++------+----------------------------------------+ +| n | LAST_VALUE(n) OVER (PARTITION BY n<=5) | ++------+----------------------------------------+ +| 1 | 5 | +| 2 | 5 | +| 3 | 5 | +| 4 | 5 | +| 5 | 5 | +| 6 | 10 | +| 7 | 10 | +| 8 | 10 | +| 9 | 10 | +| 10 | 10 | ++------+----------------------------------------+ +10 rows in set (0.00 sec) +``` + +## [`LEAD()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_lead) + +The `LEAD(expr [, num [,default]])` function returns the value of `expr` from the row that is `num` rows following the current row. If such row does not exist, `default` is returned. By default, `num` is `1` and `default` is `NULL` when they are not specified. + +In the following example, because `num` is not specified, `LEAD(n)` returns the value of `n` in the next row following the current row. When `n` is 10, because the next row does not exist and `default` is not specified, `LEAD(10)` returns `NULL`. + +```sql +WITH RECURSIVE cte(n) AS ( + SELECT + 1 + UNION + SELECT + n+1 + FROM + cte + WHERE + n<10 +) +SELECT + n, + LEAD(n) OVER () +FROM + cte; +``` + +``` ++------+-----------------+ +| n | LEAD(n) OVER () | ++------+-----------------+ +| 1 | 2 | +| 2 | 3 | +| 3 | 4 | +| 4 | 5 | +| 5 | 6 | +| 6 | 7 | +| 7 | 8 | +| 8 | 9 | +| 9 | 10 | +| 10 | NULL | ++------+-----------------+ +10 rows in set (0.00 sec) +``` + +## [`NTH_VALUE()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_nth-value) + +The `NTH_VALUE(expr, n)` function returns the `n`-th value of the window. + +```sql +WITH RECURSIVE cte(n) AS ( + SELECT + 1 + UNION + SELECT + n+1 + FROM + cte + WHERE + n<10 +) +SELECT + n, + FIRST_VALUE(n) OVER w AS 'First', + NTH_VALUE(n, 2) OVER w AS 'Second', + NTH_VALUE(n, 3) OVER w AS 'Third', + LAST_VALUE(n) OVER w AS 'Last' +FROM + cte +WINDOW + w AS (PARTITION BY n<=5) +ORDER BY + n; +``` + +``` ++------+-------+--------+-------+------+ +| n | First | Second | Third | Last | ++------+-------+--------+-------+------+ +| 1 | 1 | 2 | 3 | 5 | +| 2 | 1 | 2 | 3 | 5 | +| 3 | 1 | 2 | 3 | 5 | +| 4 | 1 | 2 | 3 | 5 | +| 5 | 1 | 2 | 3 | 5 | +| 6 | 6 | 7 | 8 | 10 | +| 7 | 6 | 7 | 8 | 10 | +| 8 | 6 | 7 | 8 | 10 | +| 9 | 6 | 7 | 8 | 10 | +| 10 | 6 | 7 | 8 | 10 | ++------+-------+--------+-------+------+ +10 rows in set (0.00 sec) +``` + +## [`NTILE()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_ntile) + +The `NTILE(n)` function divides the window into `n` groups and returns the group number of each row. + +```sql +WITH RECURSIVE cte(n) AS ( + SELECT + 1 + UNION + SELECT + n+1 + FROM + cte + WHERE + n<10 +) +SELECT + n, + NTILE(5) OVER (), + NTILE(2) OVER () +FROM + cte; +``` + +``` ++------+------------------+------------------+ +| n | NTILE(5) OVER () | NTILE(2) OVER () | ++------+------------------+------------------+ +| 1 | 1 | 1 | +| 2 | 1 | 1 | +| 3 | 2 | 1 | +| 4 | 2 | 1 | +| 5 | 3 | 1 | +| 6 | 3 | 2 | +| 7 | 4 | 2 | +| 8 | 4 | 2 | +| 9 | 5 | 2 | +| 10 | 5 | 2 | ++------+------------------+------------------+ +10 rows in set (0.00 sec) + +``` + +## [`PERCENT_RANK()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_percent-rank) + +The `PERCENT_RANK()` function returns a number between 0 and 1 indicating the percentage of rows with a value less than the value of the current row. + +```sql +SELECT + *, + PERCENT_RANK() OVER (ORDER BY n), + PERCENT_RANK() OVER (ORDER BY n DESC) +FROM ( + SELECT 5 AS 'n' + UNION ALL + SELECT 8 + UNION ALL + SELECT 5 + UNION ALL + SELECT 30 + UNION ALL + SELECT 31 + UNION ALL + SELECT 32) a; +``` + +``` ++----+----------------------------------+---------------------------------------+ +| n | PERCENT_RANK() OVER (ORDER BY n) | PERCENT_RANK() OVER (ORDER BY n DESC) | ++----+----------------------------------+---------------------------------------+ +| 5 | 0 | 0.8 | +| 5 | 0 | 0.8 | +| 8 | 0.4 | 0.6 | +| 30 | 0.6 | 0.4 | +| 31 | 0.8 | 0.2 | +| 32 | 1 | 0 | ++----+----------------------------------+---------------------------------------+ +6 rows in set (0.00 sec) +``` + +## [`RANK()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_rank) + +The `RANK()` function is similar to [`DENSE_RANK()`](#dense_rank) but will leave gaps in case of ties (rows that share the same values and order conditions). This means it provides an absolute ranking. For example, a rank of 7 means that there are 6 rows with lower ranks. + +```sql +SELECT + *, + RANK() OVER (ORDER BY n), + DENSE_RANK() OVER (ORDER BY n) +FROM ( + SELECT 5 AS 'n' + UNION ALL + SELECT 8 + UNION ALL + SELECT 5 + UNION ALL + SELECT 30 + UNION ALL + SELECT 31 + UNION ALL + SELECT 32) a; +``` + +``` ++----+--------------------------+--------------------------------+ +| n | RANK() OVER (ORDER BY n) | DENSE_RANK() OVER (ORDER BY n) | ++----+--------------------------+--------------------------------+ +| 5 | 1 | 1 | +| 5 | 1 | 1 | +| 8 | 3 | 2 | +| 30 | 4 | 3 | +| 31 | 5 | 4 | +| 32 | 6 | 5 | ++----+--------------------------+--------------------------------+ +6 rows in set (0.00 sec) +``` + +## [`ROW_NUMBER()`](https://dev.mysql.com/doc/refman/8.0/en/window-function-descriptions.html#function_row-number) + +The `ROW_NUMBER()` returns the row number of the current row in the result set. + +```sql +WITH RECURSIVE cte(n) AS ( + SELECT + 1 + UNION + SELECT + n+3 + FROM + cte + WHERE + n<30 +) +SELECT + n, + ROW_NUMBER() OVER () +FROM + cte; +``` + +``` ++------+----------------------+ +| n | ROW_NUMBER() OVER () | ++------+----------------------+ +| 1 | 1 | +| 4 | 2 | +| 7 | 3 | +| 10 | 4 | +| 13 | 5 | +| 16 | 6 | +| 19 | 7 | +| 22 | 8 | +| 25 | 9 | +| 28 | 10 | +| 31 | 11 | ++------+----------------------+ +11 rows in set (0.00 sec) +``` \ No newline at end of file diff --git a/garbage-collection-configuration.md b/garbage-collection-configuration.md index 19479c058ce75..3d5cc532df7a6 100644 --- a/garbage-collection-configuration.md +++ b/garbage-collection-configuration.md @@ -1,7 +1,6 @@ --- title: Garbage Collection Configuration summary: Learn about GC configuration parameters. -aliases: ['/docs/dev/garbage-collection-configuration/','/docs/dev/reference/garbage-collection/configuration/'] --- # Garbage Collection Configuration @@ -23,7 +22,7 @@ For more information about how to modify the value of a system variable, see [Sy > **Note:** > -> This section is only applicable to TiDB Self-Hosted. TiDB Cloud does not have a GC I/O limit by default. +> This section is only applicable to TiDB Self-Managed. TiDB Cloud does not have a GC I/O limit by default. @@ -61,7 +60,7 @@ Based on the `DISTRIBUTED` GC mode, the mechanism of GC in Compaction Filter use > **Note:** > -> The following examples of modifying TiKV configurations are only applicable to TiDB Self-Hosted. For TiDB Cloud, the mechanism of GC in Compaction Filter is enabled by default. +> The following examples of modifying TiKV configurations are only applicable to TiDB Self-Managed. For TiDB Cloud, the mechanism of GC in Compaction Filter is enabled by default. diff --git a/garbage-collection-overview.md b/garbage-collection-overview.md index b4ac57ac6363b..2c2c0d82f7500 100644 --- a/garbage-collection-overview.md +++ b/garbage-collection-overview.md @@ -1,7 +1,6 @@ --- title: GC Overview summary: Learn about Garbage Collection in TiDB. -aliases: ['/docs/dev/garbage-collection-overview/','/docs/dev/reference/garbage-collection/overview/'] --- # GC Overview diff --git a/generate-self-signed-certificates.md b/generate-self-signed-certificates.md index b286c5ee01d0e..f66809dd01a80 100644 --- a/generate-self-signed-certificates.md +++ b/generate-self-signed-certificates.md @@ -1,7 +1,6 @@ --- title: Generate Self-signed Certificates summary: Use `openssl` to generate self-signed certificates. -aliases: ['/docs/dev/generate-self-signed-certificates/','/docs/dev/how-to/secure/generate-self-signed-certificates/'] --- # Generate Self-Signed Certificates diff --git a/generated-columns.md b/generated-columns.md index 56238ddd2c034..35e7984262646 100644 --- a/generated-columns.md +++ b/generated-columns.md @@ -1,7 +1,6 @@ --- title: Generated Columns summary: Learn how to use generated columns. -aliases: ['/docs/dev/generated-columns/','/docs/dev/reference/sql/generated-columns/'] --- # Generated Columns @@ -153,5 +152,6 @@ The current limitations of JSON and generated columns are as follows: - You cannot add a stored generated column through `ALTER TABLE`. - You can neither convert a stored generated column to a normal column through the `ALTER TABLE` statement nor convert a normal column to a stored generated column. - You cannot modify the expression of a stored generated column through the `ALTER TABLE` statement. -- Not all [JSON functions](/functions-and-operators/json-functions.md) are supported; +- Not all [JSON functions](/functions-and-operators/json-functions.md) are supported. +- The [`NULLIF()` function](/functions-and-operators/control-flow-functions.md#nullif) is not supported. You can use the [`CASE` function](/functions-and-operators/control-flow-functions.md#case) instead. - Currently, the generated column index replacement rule is valid only when the generated column is a virtual generated column. It is not valid on the stored generated column, but the index can still be used by directly using the generated column itself. diff --git a/geo-distributed-deployment-topology.md b/geo-distributed-deployment-topology.md index 6846a757f8c44..ac217e7797a66 100644 --- a/geo-distributed-deployment-topology.md +++ b/geo-distributed-deployment-topology.md @@ -1,7 +1,6 @@ --- title: Geo-distributed Deployment topology summary: Learn the geo-distributed deployment topology of TiDB. -aliases: ['/docs/dev/geo-distributed-deployment-topology/'] --- # Geo-Distributed Deployment Topology @@ -17,6 +16,10 @@ This document takes the typical architecture of three data centers (DC) in two c | TiKV | 5 | 16 VCore 32GB 4TB (nvme ssd) * 1 | 10.0.1.11
10.0.1.12
10.0.1.13
10.0.1.14 | 10.0.1.15 | Default port
Global directory configuration | | Monitoring & Grafana | 1 | 4 VCore 8GB * 1 500GB (ssd) | 10.0.1.16 | | Default port
Global directory configuration | +> **Note:** +> +> The IP addresses of the instances are given as examples only. In your actual deployment, replace the IP addresses with your actual IP addresses. + ### Topology templates - [The geo-distributed topology template](https://github.com/pingcap/docs/blob/master/config-templates/geo-redundancy-deployment.yaml) @@ -63,7 +66,7 @@ This section describes the key parameter configuration of the TiDB geo-distribut > **Note:** > -> Using `raftstore.raft-min-election-timeout-ticks` and `raftstore.raft-max-election-timeout-ticks` to configure larger election timeout ticks for a TiKV node can significantly decrease the likelihood of Regions on that node becoming Leaders. However, in a disaster scenario where some TiKV nodes are offline and the remaining active TiKV nodes lag behind in Raft logs, only Regions on this TiKV node with large election timeout ticks can become Leaders. Because Regions on this TiKV node must wait for at least the duration set by `raftstore.raft-min-election-timeout-ticks' before initiating an election, it is recommended to avoid setting these values excessively large to prevent potential impact on the cluster availability in such scenarios. +> Using `raftstore.raft-min-election-timeout-ticks` and `raftstore.raft-max-election-timeout-ticks` to configure larger election timeout ticks for a TiKV node can significantly decrease the likelihood of Regions on that node becoming Leaders. However, in a disaster scenario where some TiKV nodes are offline and the remaining active TiKV nodes lag behind in Raft logs, only Regions on this TiKV node with large election timeout ticks can become Leaders. Because Regions on this TiKV node must wait for at least the duration set by `raftstore.raft-min-election-timeout-ticks` before initiating an election, it is recommended to avoid setting these values excessively large to prevent potential impact on the cluster availability in such scenarios. #### PD parameters diff --git a/get-started-with-tidb-lightning.md b/get-started-with-tidb-lightning.md index 55605a5577bd6..076f4e54b23ec 100644 --- a/get-started-with-tidb-lightning.md +++ b/get-started-with-tidb-lightning.md @@ -1,6 +1,5 @@ --- title: Quick Start for TiDB Lightning -aliases: ['/docs/dev/get-started-with-tidb-lightning/','/docs/dev/how-to/get-started/tidb-lightning/'] summary: TiDB Lightning is a tool for importing MySQL data into a TiDB cluster. It is recommended for test and trial purposes only, not for production or development environments. The process involves preparing full backup data, deploying the TiDB cluster, installing TiDB Lightning, starting TiDB Lightning, and checking data integrity. For detailed features and usage, refer to the TiDB Lightning Overview. --- @@ -28,7 +27,7 @@ First, you can use [dumpling](/dumpling-overview.md) to export data from MySQL. tiup install dumpling ``` -3. To export data from MySQL, you can refer to the detailed steps provided in [Use Dumpling to Export Data](/dumpling-overview.md#export-to-sql-files)： +3. To export data from MySQL, you can refer to the detailed steps provided in [Use Dumpling to Export Data](/dumpling-overview.md#export-to-sql-files): ```sh tiup dumpling -h 127.0.0.1 -P 3306 -u root -t 16 -F 256MB -B test -f 'test.t[12]' -o /data/my_database/ diff --git a/glossary.md b/glossary.md index 2f5b5ee5f6b98..d85d3afc154b1 100644 --- a/glossary.md +++ b/glossary.md @@ -1,7 +1,6 @@ --- title: Glossary summary: Glossaries about TiDB. -aliases: ['/docs/dev/glossary/'] --- # Glossary @@ -16,7 +15,7 @@ ACID refers to the four key properties of a transaction: atomicity, consistency, - **Consistency** means that transactions always bring the database from one consistent state to another. In TiDB, data consistency is ensured before writing data to the memory. -- **Isolation** means that a transaction in process is invisible to other transactions until it completes. This allows concurrent transactions to read and write data without sacrificing consistency. TiDB currently supports the isolation level of `REPEATABLE READ`. +- **Isolation** means that a transaction in process is invisible to other transactions until it completes. This allows concurrent transactions to read and write data without sacrificing consistency. For more information, see [TiDB transaction isolation levels](/transaction-isolation-levels.md#tidb-transaction-isolation-levels). - **Durability** means that once a transaction is committed, it remains committed even in the event of a system failure. TiKV uses persistent storage to ensure durability. @@ -76,6 +75,10 @@ Leader/Follower/Learner each corresponds to a role in a Raft group of [peers](#r Starting from v5.0, TiDB introduces Massively Parallel Processing (MPP) architecture through TiFlash nodes, which shares the execution workloads of large join queries among TiFlash nodes. When the MPP mode is enabled, TiDB, based on cost, determines whether to use the MPP framework to perform the calculation. In the MPP mode, the join keys are redistributed through the Exchange operation while being calculated, which distributes the calculation pressure to each TiFlash node and speeds up the calculation. For more information, see [Use TiFlash MPP Mode](/tiflash/use-tiflash-mpp-mode.md). +### Multi-version concurrency control (MVCC) + +[MVCC](https://en.wikipedia.org/wiki/Multiversion_concurrency_control) is a concurrency control mechanism in TiDB and other databases. It processes the memory read by transactions to achieve concurrent access to TiDB, thereby avoiding blocking caused by conflicts between concurrent reads and writes. + ## O ### Old value @@ -169,7 +172,3 @@ Top SQL helps locate SQL queries that contribute to a high load of a TiDB or TiK ### TSO Because TiKV is a distributed storage system, it requires a global timing service, Timestamp Oracle (TSO), to assign a monotonically increasing timestamp. In TiKV, such a feature is provided by PD, and in Google [Spanner](http://static.googleusercontent.com/media/research.google.com/en//archive/spanner-osdi2012.pdf), this feature is provided by multiple atomic clocks and GPS. - -### TTL - -[Time to live (TTL)](/time-to-live.md) is a feature that allows you to manage TiDB data lifetime at the row level. For a table with the TTL attribute, TiDB automatically checks data lifetime and deletes expired data at the row level. diff --git a/grafana-overview-dashboard.md b/grafana-overview-dashboard.md index daf0ba5c74976..eef5ff710d8c3 100644 --- a/grafana-overview-dashboard.md +++ b/grafana-overview-dashboard.md @@ -1,7 +1,6 @@ --- title: Key Metrics summary: Learn some key metrics displayed on the Grafana Overview dashboard. -aliases: ['/docs/dev/grafana-overview-dashboard/','/docs/dev/reference/key-monitoring-metrics/overview-dashboard/'] --- # Key Metrics @@ -42,7 +41,7 @@ To understand the key metrics displayed on the Overview dashboard, check the fol | TiDB | Transaction Duration | The execution time of a transaction | | TiDB | KV Cmd OPS | The number of executed KV commands. | | TiDB | KV Cmd Duration 99 | The execution time of the KV command. | -| TiDB | PD TSO OPS | The number of TSO that TiDB obtains from PD per second. | +| TiDB | PD TSO OPS | The number of gRPC requests per second that TiDB sends to PD (cmd) and the number of TSO requests (request); each gRPC request contains a batch of TSO requests. | | TiDB | PD TSO Wait Duration | The duration that TiDB waits for PD to return TSO. | | TiDB | TiClient Region Error OPS | The number of Region related errors returned by TiKV. | | TiDB | Lock Resolve OPS | The number of TiDB operations that resolve locks. When TiDB's read or write request encounters a lock, it tries to resolve the lock. | diff --git a/grafana-pd-dashboard.md b/grafana-pd-dashboard.md index 048dc0c7b3d69..c1e6029a3f466 100644 --- a/grafana-pd-dashboard.md +++ b/grafana-pd-dashboard.md @@ -1,7 +1,6 @@ --- title: Key Monitoring Metrics of PD summary: Learn some key metrics displayed on the Grafana PD dashboard. -aliases: ['/docs/dev/grafana-pd-dashboard/','/docs/dev/reference/key-monitoring-metrics/pd-dashboard/'] --- # Key Monitoring Metrics of PD diff --git a/grafana-performance-overview-dashboard.md b/grafana-performance-overview-dashboard.md index f2e6c1005e5e6..3aa9ae07d9348 100644 --- a/grafana-performance-overview-dashboard.md +++ b/grafana-performance-overview-dashboard.md @@ -67,11 +67,11 @@ Number of commands processed by all TiDB instances per second based on type ### KV/TSO Request OPS - kv request total: Total number of KV requests per second in all TiDB instances -- kv request by type: Number of KV requests per second in all TiDB instances based on such types as `Get`, `Prewrite`, and `Commit`. -- tso - cmd: Number of `tso cmd` requests per second in all TiDB instances -- tso - request: Number of `tso request` requests per second in all TiDB instances +- kv request by type: Number of KV requests per second in all TiDB instances based on such types as `Get`, `Prewrite`, and `Commit` +- tso - cmd: Number of gRPC requests per second that TiDB sends to PD in all TiDB instances; each gRPC request contains a batch of TSO requests +- tso - request: Number of TSO requests per second in all TiDB instances -Generally, dividing `tso - cmd` by `tso - request` yields the average batch size of requests per second. +Generally, `tso - request` divided by `tso - cmd` is the average size of TSO request batches per second. ### KV Request Time By Source @@ -138,10 +138,10 @@ Average time consumed in executing gRPC requests in all TiKV instances based on ### PD TSO Wait/RPC Duration -- wait - avg: Average time in waiting for PD to return TSO in all TiDB instances -- rpc - avg: Average time from sending TSO requests to PD to receiving TSO in all TiDB instances -- wait - 99: P99 time in waiting for PD to return TSO in all TiDB instances -- rpc - 99: P99 time from sending TSO requests to PD to receiving TSO in all TiDB instances +- wait - avg: Average duration of waiting for PD to return TSO in all TiDB instances +- rpc - avg: Average duration from the time that TiDB sends gRPC requests to PD to get TSO to the time that TiDB receives TSO in all TiDB instances +- wait - 99: P99 duration of waiting for PD to return TSO in all TiDB instances +- rpc - 99: P99 duration from the time that TiDB sends gRPC requests to PD to get TSO to the time that TiDB receives TSO in all TiDB instances ### Storage Async Write Duration, Store Duration, and Apply Duration diff --git a/grafana-resource-control-dashboard.md b/grafana-resource-control-dashboard.md index 3cca223262ba3..4793c6e96e645 100644 --- a/grafana-resource-control-dashboard.md +++ b/grafana-resource-control-dashboard.md @@ -11,7 +11,7 @@ The Grafana dashboard is divided into a series of sub dashboards which include O If your cluster has used the [Resource Control](/tidb-resource-control.md) feature, you can get an overview of the resource consumption status from the Resource Control dashboard. -TiDB uses the [token bucket algorithm](https://en.wikipedia.org/wiki/Token_bucket) for flow control. As described in the [RFC: Global Resource Control in TiDB](https://github.com/pingcap/tidb/blob/master/docs/design/2022-11-25-global-resource-control.md#distributed-token-buckets), a TiDB node might have multiple Resource Groups, which are flow controlled by GAC (Global Admission Control) on the PD side. The Local Token Buckets in each TiDB node periodically (5 seconds by default) communicate with the GAC on the PD side to reconfigure the local tokens. In TiDB, the Local Token Buckets are implemented as Resource Controller Clients. +TiDB uses the [token bucket algorithm](https://en.wikipedia.org/wiki/Token_bucket) for flow control. As described in the [RFC: Global Resource Control in TiDB](https://github.com/pingcap/tidb/blob/release-8.1/docs/design/2022-11-25-global-resource-control.md#distributed-token-buckets), a TiDB node might have multiple Resource Groups, which are flow controlled by GAC (Global Admission Control) on the PD side. The Local Token Buckets in each TiDB node periodically (5 seconds by default) communicate with the GAC on the PD side to reconfigure the local tokens. In TiDB, the Local Token Buckets are implemented as Resource Controller Clients. This document describes some key monitoring metrics displayed on the Resource Control dashboard. diff --git a/grafana-tidb-dashboard.md b/grafana-tidb-dashboard.md index 3e6427576186f..08fca1da11335 100644 --- a/grafana-tidb-dashboard.md +++ b/grafana-tidb-dashboard.md @@ -1,7 +1,6 @@ --- title: TiDB Monitoring Metrics summary: Learn some key metrics displayed on the Grafana TiDB dashboard. -aliases: ['/docs/dev/grafana-tidb-dashboard/','/docs/dev/reference/key-monitoring-metrics/tidb-dashboard/'] --- # TiDB Monitoring Metrics @@ -133,10 +132,10 @@ The following metrics relate to requests sent to TiKV. Retry requests are counte - PD Client CMD OPS: the statistics of commands executed by PD Client per second - PD Client CMD Duration: the time it takes for PD Client to execute commands - PD Client CMD Fail OPS: the statistics of failed commands executed by PD Client per second -- PD TSO OPS: the number of TSO that TiDB obtains from PD per second +- PD TSO OPS: the number of gRPC requests per second that TiDB sends to PD (cmd) and the number of TSO requests (request); each gRPC request contains a batch of TSO requests - PD TSO Wait Duration: the time that TiDB waits for PD to return TSO -- PD TSO RPC duration: the duration from the time that TiDB sends request to PD (to get TSO) to the time that TiDB receives TSO -- Start TSO Wait Duration: the duration from the time that TiDB sends request to PD (to get `start TSO`) to the time that TiDB receives `start TSO` +- PD TSO RPC duration: the duration from the time that TiDB sends gRPC requests to PD to get TSO to the time that TiDB receives the gRPC response from PD +- Async TSO Duration: the duration from the time that TiDB prepares to get TSO to the time that TiDB actually starts to wait for PD to return TSO ### Schema Load diff --git a/grafana-tikv-dashboard.md b/grafana-tikv-dashboard.md index 18579e8bd2fcf..a9bfac112787c 100644 --- a/grafana-tikv-dashboard.md +++ b/grafana-tikv-dashboard.md @@ -1,7 +1,6 @@ --- title: Key Monitoring Metrics of TiKV summary: Learn some key metrics displayed on the Grafana TiKV dashboard. -aliases: ['/docs/dev/grafana-tikv-dashboard/','/docs/dev/reference/key-monitoring-metrics/tikv-dashboard/'] --- # Key Monitoring Metrics of TiKV @@ -494,6 +493,44 @@ This section provides a detailed description of these key metrics on the **TiKV- - Get Region Operation Count: The number of times the coordinator requests Region information from the PD - Try Advance Trigger Time: The time taken for the coordinator to attempt to advance the checkpoint +### Backup & Import + +- Import CPU Utilization: The CPU utilization aggregated by SST importer. +- Import Thread Count: The number of threads used by SST importer. +- Import Errors: The number of errors encountered during SST import. +- Import RPC Duration: The time spent on various RPC calls in SST importer. +- Import RPC Ops: The total number of RPC calls in SST importer. +- Import RPC Count: The number of RPC calls being processed by SST importer. +- Import Write/Download RPC Duration: The RPC time for write or download operations in SST importer. +- Import Wait Duration: The time spent waiting in queue for download task execution. +- Import Read SST Duration: The time spent reading an SST file from external storage and downloading it to TiKV. +- Import Rewrite SST Duration: The time spent rewriting the SST file based on rewrite rules. +- Import Ingest RPC Duration: The time spent handling ingest RPC requests on TiKV. +- Import Ingest SST Duration: The time spent ingesting the SST file into RocksDB. +- Import Ingest SST Bytes: The number of bytes ingested. +- Import Download SST Throughput: The SST download throughput in bytes per second. +- cloud request: The number of requests to cloud providers. + +### Point In Time Restore + +- CPU Usage: The CPU utilization by point-in-time recovery (PITR). +- P99 RPC Duration: The 99th percentile of RPC request duration. +- Import RPC Ops: The total number of RPC calls in SST importer. +- Import RPC Count: The number of RPC calls being processed by SST importer. +- Cache Events: The number of events in the file cache during SST import. +- Overall RPC Duration: The time spent on RPC calls. +- Read File into Memory Duration: The time spent downloading files from external storage and loading them into memory. +- Queuing Time: The time spent waiting to be scheduled on a thread. +- Apply Request Throughput: The rate of applying requests in bytes. +- Downloaded File Size: The size of downloaded file in bytes. +- Apply Batch Size: The number of bytes for applying to Raft store in one batch. +- Blocked by Concurrency Time: The time spent waiting for execution due to concurrency constraints. +- Apply Request Speed: The speed of applying request to Raft store. +- Cached File in Memory: The files cached by the applying requests of SST importer. +- Engine Requests Unfinished: The number of pending requests to Raft store. +- Apply Time: The time spent writing data to Raft store. +- Raft Store Memory Usage: The memory usage for Raft store. + ### Explanation of Common Parameters #### gRPC Message Type diff --git a/hardware-and-software-requirements.md b/hardware-and-software-requirements.md index cec210fca4b02..275e15b5d43dd 100644 --- a/hardware-and-software-requirements.md +++ b/hardware-and-software-requirements.md @@ -1,7 +1,6 @@ --- title: Software and Hardware Recommendations summary: Learn the software and hardware recommendations for deploying and running TiDB. -aliases: ['/docs/dev/hardware-and-software-requirements/','/docs/dev/how-to/deploy/hardware-recommendations/'] --- # Software and Hardware Recommendations @@ -18,84 +17,112 @@ As an open-source distributed SQL database with high performance, TiDB can be de ## OS and platform requirements - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
Operating systemsSupported CPU architectures
Red Hat Enterprise Linux 8.4 or a later 8.x version
  • x86_64
  • ARM 64
  • Red Hat Enterprise Linux 7.3 or a later 7.x version
  • CentOS 7.3 or a later 7.x version
  • x86_64
  • ARM 64
Amazon Linux 2
  • x86_64
  • ARM 64
Rocky Linux 9.1 or later
  • x86_64
  • ARM 64
Kylin Euler V10 SP1/SP2
  • x86_64
  • ARM 64
UnionTech OS (UOS) V20
  • x86_64
  • ARM 64
openEuler 22.03 LTS SP1/SP3
  • x86_64
  • ARM 64
macOS 12 (Monterey) or later
  • x86_64
  • ARM 64
Oracle Enterprise Linux 8 or a laterx86_64
Ubuntu LTS 20.04 or laterx86_64
CentOS 8 Stream
  • x86_64
  • ARM 64
Debian 10 (Buster) or laterx86_64
Fedora 38 or laterx86_64
openSUSE Leap later than v15.5 (not including Tumbleweed)x86_64
SUSE Linux Enterprise Server 15x86_64
- -> **Note:** -> -> - For Oracle Enterprise Linux, TiDB supports the Red Hat Compatible Kernel (RHCK) and does not support the Unbreakable Enterprise Kernel provided by Oracle Enterprise Linux. -> - According to [CentOS Linux EOL](https://www.centos.org/centos-linux-eol/), the upstream support for CentOS Linux 8 ended on December 31, 2021. CentOS Stream 8 continues to be supported by the CentOS organization. -> - Support for Ubuntu 16.04 will be removed in future versions of TiDB. Upgrading to Ubuntu 18.04 or later is strongly recommended. -> - If you are using the 32-bit version of an operating system listed in the preceding table, TiDB **is not guaranteed** to be compilable, buildable or deployable on the 32-bit operating system and the corresponding CPU architecture, or TiDB does not actively adapt to the 32-bit operating system. -> - Other operating system versions not mentioned above might work but are not officially supported. +In v8.1 LTS, TiDB ensures multi-level quality standards for various combinations of operating systems and CPU architectures. + ++ For the following combinations of operating systems and CPU architectures, TiDB **provides enterprise-level production quality**, and the product features have been comprehensively and systematically verified: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Operating systemsSupported CPU architectures
Red Hat Enterprise Linux 8.4 or a later 8.x version
  • x86_64
  • ARM 64
  • Red Hat Enterprise Linux 7.3 or a later 7.x version (TiDB ends support for it starting from 8.4 DMR)
  • CentOS 7.3 or a later 7.x version (TiDB ends support for it starting from 8.4 DMR)
  • x86_64
  • ARM 64
Amazon Linux 2
  • x86_64
  • ARM 64
Amazon Linux 2023
  • x86_64
  • ARM 64
Rocky Linux 9.1 or later
  • x86_64
  • ARM 64
Kylin V10 SP1/SP2/SP3 (SP3 is supported starting from v7.5.5)
  • x86_64
  • ARM 64
UnionTech OS (UOS) V20
  • x86_64
  • ARM 64
openEuler 22.03 LTS SP1/SP3
  • x86_64
  • ARM 64
+ + > **Note:** + > + > - According to [CentOS Linux EOL](https://blog.centos.org/2023/04/end-dates-are-coming-for-centos-stream-8-and-centos-linux-7/), the upstream support for CentOS Linux 7 ends on June 30, 2024. TiDB ends the support for CentOS 7 starting from the 8.4 DMR version. It is recommended to use Rocky Linux 9.1 or a later version. + > - According to [Red Hat Enterprise Linux Life Cycle](https://access.redhat.com/support/policy/updates/errata/#Life_Cycle_Dates), the maintenance support for Red Hat Enterprise Linux 7 ends on June 30, 2024. TiDB ends the support for Red Hat Enterprise Linux 7 starting from the 8.4 DMR version. It is recommended to use Rocky Linux 9.1 or a later version. + ++ For the following combinations of operating systems and CPU architectures, you can compile, build, and deploy TiDB. In addition, you can also use the basic features of OLTP, OLAP, and the data tools. However, TiDB **does not guarantee enterprise-level production quality**: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Operating systemsSupported CPU architectures
macOS 12 (Monterey) or later
  • x86_64
  • ARM 64
Oracle Enterprise Linux 8 or a laterx86_64
Ubuntu LTS 20.04 or laterx86_64
CentOS Stream 8
  • x86_64
  • ARM 64
Debian 10 (Buster) or laterx86_64
Fedora 38 or laterx86_64
openSUSE Leap later than v15.5 (not including Tumbleweed)x86_64
SUSE Linux Enterprise Server 15x86_64
+ + > **Note:** + > + > - For Oracle Enterprise Linux, TiDB supports the Red Hat Compatible Kernel (RHCK) and does not support the Unbreakable Enterprise Kernel provided by Oracle Enterprise Linux. + > - Support for Ubuntu 16.04 will be removed in future versions of TiDB. Upgrading to Ubuntu 18.04 or later is strongly recommended. + > - CentOS Stream 8 reaches [End of Builds](https://blog.centos.org/2023/04/end-dates-are-coming-for-centos-stream-8-and-centos-linux-7/) on May 31, 2024. + ++ If you are using the 32-bit version of an operating system listed in the preceding two tables, TiDB **is not guaranteed** to be compilable, buildable or deployable on the 32-bit operating system and the corresponding CPU architecture, or TiDB does not actively adapt to the 32-bit operating system. + ++ Other operating system versions not mentioned above might work but are not officially supported. ### Libraries required for compiling and running TiDB @@ -174,6 +201,7 @@ You can deploy and run TiDB on the 64-bit generic hardware server platform in th > - In the production environment, the TiDB and PD instances can be deployed on the same server. If you have a higher requirement for performance and reliability, try to deploy them separately. > - It is strongly recommended to configure TiDB, TiKV, and TiFlash with at least 8 CPU cores each in the production environment. To get better performance, a higher configuration is recommended. > - It is recommended to keep the size of TiKV hard disk within 4 TB if you are using PCIe SSDs or within 1.5 TB if you are using regular SSDs. +> - If you deploy TiKV on a cloud provider, such as AWS, Google Cloud, or Azure, it is recommended to use cloud disks for TiKV nodes. Data on local disks might be lost if the TiKV instance crashes in the cloud environment. Before you deploy TiFlash, note the following items: @@ -233,7 +261,7 @@ As an open-source distributed SQL database, TiDB requires the following network TiDB -
  • At least 30 GB for the log disk
  • Starting from v6.5.0, Fast Online DDL (controlled by the tidb_ddl_enable_fast_reorg variable) is enabled by default to accelerate DDL operations, such as adding indexes. If DDL operations involving large objects exist in your application, or you want to use IMPORT INTO to import data, it is highly recommended to prepare additional SSD disk space for TiDB (100 GB or more). For detailed configuration instructions, see Set a temporary space for a TiDB instance
+
  • At least 30 GB for the log disk
  • Starting from v6.5.0, Fast Online DDL (controlled by the tidb_ddl_enable_fast_reorg variable) is enabled by default to accelerate DDL operations, such as adding indexes. If DDL operations involving large objects exist in your application, or you want to use IMPORT INTO to import data, it is highly recommended to prepare additional SSD disk space for TiDB (100 GB or more). For detailed configuration instructions, see Set a temporary space for a TiDB instance
Lower than 90% diff --git a/hybrid-deployment-topology.md b/hybrid-deployment-topology.md index db2fe2028b687..0b55cc0678752 100644 --- a/hybrid-deployment-topology.md +++ b/hybrid-deployment-topology.md @@ -1,7 +1,6 @@ --- title: Hybrid Deployment Topology summary: Learn the hybrid deployment topology of TiDB clusters. -aliases: ['/docs/dev/hybrid-deployment-topology/'] --- # Hybrid Deployment Topology @@ -21,6 +20,10 @@ The deployment machine has multiple CPU processors with sufficient memory. To im | TiKV | 6 | 32 VCore 64GB | 10.0.1.7
10.0.1.8
10.0.1.9 | 1. Separate the instance-level port and status_port;
2. Configure the global parameters `readpool`, `storage` and `raftstore`;
3. Configure labels of the instance-level host;
4. Configure NUMA to bind CPU cores | | Monitoring & Grafana | 1 | 4 VCore 8GB * 1 500GB (ssd) | 10.0.1.10 | Default configuration | +> **Note:** +> +> The IP addresses of the instances are given as examples only. In your actual deployment, replace the IP addresses with your actual IP addresses. + ### Topology templates - [The simple template for the hybrid deployment](https://github.com/pingcap/docs/blob/master/config-templates/simple-multi-instance.yaml) diff --git a/identify-expensive-queries.md b/identify-expensive-queries.md index 6faf1953beea0..c1b7919be26ce 100644 --- a/identify-expensive-queries.md +++ b/identify-expensive-queries.md @@ -1,6 +1,5 @@ --- title: Identify Expensive Queries -aliases: ['/docs/dev/identify-expensive-queries/','/docs/dev/how-to/maintain/identify-abnormal-queries/identify-expensive-queries/'] summary: TiDB helps identify expensive queries by printing information about statements that exceed the execution time or memory usage threshold. This allows for diagnosing and improving SQL performance. The expensive query log includes details such as execution time, memory usage, user, database, and TiKV Coprocessor task information. This log differs from the slow query log as it prints information as soon as the statement exceeds the resource threshold. --- diff --git a/identify-slow-queries.md b/identify-slow-queries.md index 8e917d127e890..52bd739f867cd 100644 --- a/identify-slow-queries.md +++ b/identify-slow-queries.md @@ -1,7 +1,6 @@ --- title: Identify Slow Queries summary: Use the slow query log to identify problematic SQL statements. -aliases: ['/docs/dev/identify-slow-queries/','/docs/dev/how-to/maintain/identify-abnormal-queries/identify-slow-queries/','/docs/dev/how-to/maintain/identify-slow-queries'] --- # Identify Slow Queries @@ -74,7 +73,7 @@ Slow query basics: * `Succ`: Whether a statement is executed successfully. * `Backoff_time`: The waiting time before retry when a statement encounters errors that require a retry. The common errors as such include: `lock occurs`, `Region split`, and `tikv server is busy`. * `Plan`: The execution plan of a statement. Execute the `SELECT tidb_decode_plan('xxx...')` statement to parse the specific execution plan. -* `Binary_plan`: The execution plan of a binary-encoded statement. Execute the `SELECT tidb_decode_binary_plan('xxx...')` statement to parse the specific execution plan. The `Plan` and `Binary_plan` fields carry the same information. However, the format of execution plans parsed from the two fields are different. +* `Binary_plan`: The execution plan of a binary-encoded statement. Execute the [`SELECT tidb_decode_binary_plan('xxx...')`](/functions-and-operators/tidb-functions.md#tidb_decode_binary_plan) statement to parse the specific execution plan. The `Plan` and `Binary_plan` fields carry the same information. However, the format of execution plans parsed from the two fields are different. * `Prepared`: Whether this statement is a `Prepare` or `Execute` request or not. * `Plan_from_cache`: Whether this statement hits the execution plan cache. * `Plan_from_binding`: Whether this statement uses the bound execution plans. @@ -593,7 +592,7 @@ ADMIN SHOW SLOW TOP [internal | all] N ADMIN SHOW SLOW recent 10 ``` -`top N` shows the slowest N query records recently (within a few days). If the `internal` option is provided, the returned results would be the inner SQL executed by the system; If the `all` option is provided, the returned results would be the user's SQL combinated with inner SQL; Otherwise, this command would only return the slow query records from the user's SQL. +`top N` shows the slowest N query records recently (within a few days). If the `internal` option is provided, the returned results would be the inner SQL executed by the system; If the `all` option is provided, the returned results would be the user's SQL combined with inner SQL; Otherwise, this command would only return the slow query records from the user's SQL. {{< copyable "sql" >}} diff --git a/import-example-data.md b/import-example-data.md index 8556ccda969fd..aba2472367787 100644 --- a/import-example-data.md +++ b/import-example-data.md @@ -1,7 +1,6 @@ --- title: Import Example Database summary: Install the Bikeshare example database. -aliases: ['/docs/dev/import-example-data/','/docs/dev/how-to/get-started/import-example-database/'] --- # Import Example Database diff --git a/information-schema/client-errors-summary-by-user.md b/information-schema/client-errors-summary-by-user.md index 797897bae8502..b0cf4bb59e7aa 100644 --- a/information-schema/client-errors-summary-by-user.md +++ b/information-schema/client-errors-summary-by-user.md @@ -13,7 +13,7 @@ The table `CLIENT_ERRORS_SUMMARY_BY_USER` provides a summary of SQL errors and w * Permission errors. * A table that does not exist. -Client errors are returned to the client via the MySQL server protocol, where applications are expected to take appropriate action. The `INFORMATION_SCHEMA.CLIENT_ERRORS_SUMMARY_BY_USER` table provides an useful method to inspect errors in the scenario where applications are not correctly handling (or logging) errors returned by the TiDB server. +Client errors are returned to the client via the MySQL server protocol, where applications are expected to take appropriate action. The `INFORMATION_SCHEMA.CLIENT_ERRORS_SUMMARY_BY_USER` table provides a useful method to inspect errors in the scenario where applications are not correctly handling (or logging) errors returned by the TiDB server. Because `CLIENT_ERRORS_SUMMARY_BY_USER` summarizes the errors on a per-user basis, it can be useful to diagnose scenarios where one user server is generating more errors than other servers. Possible scenarios include: diff --git a/information-schema/information-schema-check-constraints.md b/information-schema/information-schema-check-constraints.md index 9b1a2088ac47a..72014bc10858b 100644 --- a/information-schema/information-schema-check-constraints.md +++ b/information-schema/information-schema-check-constraints.md @@ -29,6 +29,7 @@ The output is as follows: The following example adds a `CHECK` constraint using the `CREATE TABLE` statement: ```sql +SET GLOBAL tidb_enable_check_constraint = ON; CREATE TABLE test.t1 (id INT PRIMARY KEY, CHECK (id%2 = 0)); SELECT * FROM CHECK_CONSTRAINTS\G ``` diff --git a/information-schema/information-schema-cluster-config.md b/information-schema/information-schema-cluster-config.md index c5e1bfa1db225..dc414ab6bd901 100644 --- a/information-schema/information-schema-cluster-config.md +++ b/information-schema/information-schema-cluster-config.md @@ -1,7 +1,6 @@ --- title: CLUSTER_CONFIG summary: Learn the `CLUSTER_CONFIG` information_schema table. -aliases: ['/docs/dev/system-tables/system-table-cluster-config/','/docs/dev/reference/system-databases/cluster-config/','/tidb/dev/system-table-cluster-config/'] --- # CLUSTER_CONFIG @@ -10,7 +9,7 @@ You can use the `CLUSTER_CONFIG` cluster configuration table to get the current > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). {{< copyable "sql" >}} diff --git a/information-schema/information-schema-cluster-hardware.md b/information-schema/information-schema-cluster-hardware.md index d3b1caa939e95..8232602b1b3c4 100644 --- a/information-schema/information-schema-cluster-hardware.md +++ b/information-schema/information-schema-cluster-hardware.md @@ -1,7 +1,6 @@ --- title: CLUSTER_HARDWARE summary: Learn the `CLUSTER_HARDWARE` information_schema table. -aliases: ['/docs/dev/system-tables/system-table-cluster-hardware/','/docs/dev/reference/system-databases/cluster-hardware/','/tidb/dev/system-table-cluster-hardware/'] --- # CLUSTER_HARDWARE @@ -10,7 +9,7 @@ The `CLUSTER_HARDWARE` hardware system table provides the hardware information o > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). {{< copyable "sql" >}} diff --git a/information-schema/information-schema-cluster-info.md b/information-schema/information-schema-cluster-info.md index a1781a2f16016..3bf800d1c9171 100644 --- a/information-schema/information-schema-cluster-info.md +++ b/information-schema/information-schema-cluster-info.md @@ -1,7 +1,6 @@ --- title: CLUSTER_INFO summary: Learn the `CLUSTER_INFO` cluster topology information table. -aliases: ['/docs/dev/system-tables/system-table-cluster-info/','/docs/dev/reference/system-databases/cluster-info/','/tidb/dev/system-table-cluster-info/'] --- # CLUSTER_INFO @@ -10,7 +9,7 @@ The `CLUSTER_INFO` cluster topology table provides the current topology informat > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. {{< copyable "sql" >}} @@ -39,7 +38,7 @@ Field description: * `TYPE`: The instance type. The optional values are `tidb`, `pd`, and `tikv`. * `INSTANCE`: The instance address, which is a string in the format of `IP:PORT`. -* `STATUS_ADDRESS`: The service address of HTTP API. Some commands in tikv-ctl, pd-ctl, or tidb-ctl might use this API and this address. You can also get more cluster information via this address. Refer to [TiDB HTTP API document](https://github.com/pingcap/tidb/blob/master/docs/tidb_http_api.md) for details. +* `STATUS_ADDRESS`: The service address of HTTP API. Some commands in tikv-ctl, pd-ctl, or tidb-ctl might use this API and this address. You can also get more cluster information via this address. Refer to [TiDB HTTP API document](https://github.com/pingcap/tidb/blob/release-8.1/docs/tidb_http_api.md) for details. * `VERSION`: The semantic version number of the corresponding instance. To be compatible with the MySQL version number, the TiDB version is displayed in the format of `${mysql-version}-${tidb-version}`. * `GIT_HASH`: The Git Commit Hash when compiling the instance version, which is used to identify whether two instances are of the absolutely consistent version. * `START_TIME`: The starting time of the corresponding instance. diff --git a/information-schema/information-schema-cluster-load.md b/information-schema/information-schema-cluster-load.md index 21faa9a49ff41..69d2b1d173dc7 100644 --- a/information-schema/information-schema-cluster-load.md +++ b/information-schema/information-schema-cluster-load.md @@ -1,7 +1,6 @@ --- title: CLUSTER_LOAD summary: Learn the `CLUSTER_LOAD` information_schema table. -aliases: ['/docs/dev/system-tables/system-table-cluster-load/','/docs/dev/reference/system-databases/cluster-load/','/tidb/dev/system-table-cluster-load/'] --- # CLUSTER_LOAD @@ -10,7 +9,7 @@ The `CLUSTER_LOAD` cluster load table provides the current load information of t > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). {{< copyable "sql" >}} diff --git a/information-schema/information-schema-cluster-log.md b/information-schema/information-schema-cluster-log.md index b9e7899dabd23..f9d94dd0d065e 100644 --- a/information-schema/information-schema-cluster-log.md +++ b/information-schema/information-schema-cluster-log.md @@ -1,7 +1,6 @@ --- title: CLUSTER_LOG summary: Learn the `CLUSTER_LOG` information_schema table. -aliases: ['/docs/dev/system-tables/system-table-cluster-log/','/docs/dev/reference/system-databases/cluster-log/','/tidb/dev/system-table-cluster-log/'] --- # CLUSTER_LOG @@ -10,7 +9,7 @@ You can query cluster logs on the `CLUSTER_LOG` cluster log table. By pushing do > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). To get the logs of the TiDB cluster before v4.0, you need to log in to each instance to summarize logs. This cluster log table in 4.0 provides the global and time-ordered log search result, which makes it easier to track full-link events. For example, by searching logs according to the `region id`, you can query all logs in the life cycle of this Region. Similarly, by searching the full link log through the slow log's `txn id`, you can query the flow and the number of keys scanned by this transaction at each instance. diff --git a/information-schema/information-schema-cluster-systeminfo.md b/information-schema/information-schema-cluster-systeminfo.md index 2f84af2406f62..c915561fb98d0 100644 --- a/information-schema/information-schema-cluster-systeminfo.md +++ b/information-schema/information-schema-cluster-systeminfo.md @@ -1,7 +1,6 @@ --- title: CLUSTER_SYSTEMINFO summary: Learn the `CLUSTER_SYSTEMINFO` kernel parameter table. -aliases: ['/docs/dev/system-tables/system-table-cluster-systeminfo/','/docs/dev/reference/system-databases/cluster-systeminfo/','/tidb/dev/system-table-cluster-systeminfo/'] --- # CLUSTER_SYSTEMINFO @@ -10,7 +9,7 @@ You can use the `CLUSTER_SYSTEMINFO` kernel parameter table to query the kernel > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). {{< copyable "sql" >}} diff --git a/information-schema/information-schema-data-lock-waits.md b/information-schema/information-schema-data-lock-waits.md index c6a31fb4a315f..9b98df5b99fd3 100644 --- a/information-schema/information-schema-data-lock-waits.md +++ b/information-schema/information-schema-data-lock-waits.md @@ -88,4 +88,4 @@ CURRENT_HOLDING_TRX_ID: 426790590082449409 1 row in set (0.01 sec) ``` -The above query result shows that the transaction of the ID `426790594290122753` is trying to obtain the pessimistic lock on the key `"7480000000000000355F728000000000000001"` when executing a statement that has digest `"38b03afa5debbdf0326a014dbe5012a62c51957f1982b3093e748460f8b00821"` and is in the form of ``update `t` set `v` = `v` + ? where `id` = ?``, but the lock on this key was held by the transaction of the ID `426790590082449409`. +The above query result shows that the transaction of the ID `426790594290122753` is trying to obtain the pessimistic lock on the key `"7480000000000000355F728000000000000001"` when executing a statement that has digest `"38b03afa5debbdf0326a014dbe5012a62c51957f1982b3093e748460f8b00821"` and is in the form of ``update `t` set `v` = `v` + ? where `id` = ?``, but the lock on this key was held by the transaction of the ID `426790590082449409`. diff --git a/information-schema/information-schema-inspection-result.md b/information-schema/information-schema-inspection-result.md index ab0ed11c581d6..2ccffb8b5ec61 100644 --- a/information-schema/information-schema-inspection-result.md +++ b/information-schema/information-schema-inspection-result.md @@ -1,7 +1,6 @@ --- title: INSPECTION_RESULT summary: Learn the `INSPECTION_RESULT` diagnostic result table. -aliases: ['/docs/dev/system-tables/system-table-inspection-result/','/docs/dev/reference/system-databases/inspection-result/','/tidb/dev/system-table-inspection-result/'] --- # INSPECTION_RESULT @@ -12,7 +11,7 @@ The `INSPECTION_RESULT` diagnostic table can help you quickly find problems and > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). The structure of the `information_schema.inspection_result` diagnostic result table `information_schema.inspection_result` is as follows: @@ -264,7 +263,7 @@ In `critical-error` diagnostic rule, the following two diagnostic rules are exec | ---- | ---- | ---- | ---- | | TiDB | panic-count | tidb_panic_count_total_count | Panic occurs in TiDB. | | TiDB | binlog-error | tidb_binlog_error_total_count | An error occurs when TiDB writes binlog. | - | TiKV | critical-error | tikv_critical_error_total_coun | The critical error of TiKV. | + | TiKV | critical-error | tikv_critical_error_total_count | The critical error of TiKV. | | TiKV | scheduler-is-busy | tikv_scheduler_is_busy_total_count | The TiKV scheduler is too busy, which makes TiKV temporarily unavailable. | | TiKV | coprocessor-is-busy | tikv_coprocessor_is_busy_total_count | The TiKV Coprocessor is too busy. | | TiKV | channel-is-full | tikv_channel_full_total_count | The "channel full" error occurs in TiKV. | diff --git a/information-schema/information-schema-inspection-rules.md b/information-schema/information-schema-inspection-rules.md index fec49f99ad45a..15f3e9b49b582 100644 --- a/information-schema/information-schema-inspection-rules.md +++ b/information-schema/information-schema-inspection-rules.md @@ -9,7 +9,7 @@ The `INSPECTION_RULES` table provides information about which diagnostic tests a > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). {{< copyable "sql" >}} diff --git a/information-schema/information-schema-inspection-summary.md b/information-schema/information-schema-inspection-summary.md index 0ba9686b0d397..2ccfcaa001ae0 100644 --- a/information-schema/information-schema-inspection-summary.md +++ b/information-schema/information-schema-inspection-summary.md @@ -1,7 +1,6 @@ --- title: INSPECTION_SUMMARY summary: Learn the `INSPECTION_SUMMARY` inspection summary table. -aliases: ['/docs/dev/system-tables/system-table-inspection-summary/','/docs/dev/reference/system-databases/inspection-summary/','/tidb/dev/system-table-inspection-summary/'] --- # INSPECTION_SUMMARY @@ -10,7 +9,7 @@ In some scenarios, you might need to pay attention only to the monitoring summar > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). The structure of the `information_schema.inspection_summary` inspection summary table is as follows: diff --git a/information-schema/information-schema-keywords.md b/information-schema/information-schema-keywords.md index 1c2668b2275bf..6d35e7284cfd7 100644 --- a/information-schema/information-schema-keywords.md +++ b/information-schema/information-schema-keywords.md @@ -5,7 +5,7 @@ summary: Learn the `KEYWORDS` INFORMATION_SCHEMA table. # KEYWORDS -Starting from v7.6.0, TiDB provides the `KEYWORDS` table. You can use this table to get information about [keywords](/keywords.md) in TiDB. +Starting from v7.5.3 and v7.6.0, TiDB provides the `KEYWORDS` table. You can use this table to get information about [keywords](/keywords.md) in TiDB. ```sql USE INFORMATION_SCHEMA; diff --git a/information-schema/information-schema-metrics-summary.md b/information-schema/information-schema-metrics-summary.md index e8bc9d5aeb9e5..9f9f2639a6be9 100644 --- a/information-schema/information-schema-metrics-summary.md +++ b/information-schema/information-schema-metrics-summary.md @@ -1,7 +1,6 @@ --- title: METRICS_SUMMARY summary: Learn the METRICS_SUMMARY system table. -aliases: ['/docs/dev/system-tables/system-table-metrics-summary/','/docs/dev/reference/system-databases/metrics-summary/','/tidb/dev/system-table-metrics-summary'] --- # METRICS_SUMMARY @@ -13,7 +12,7 @@ The TiDB cluster has many monitoring metrics. To make it easy to detect abnormal > **Note:** > -> The preceding two monitoring summary tables are only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> The preceding two monitoring summary tables are only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). The two tables summarize all monitoring data for you to check each monitoring metric efficiently. Compared with `information_schema.metrics_summary`, the `information_schema.metrics_summary_by_label` table has an additional `label` column and performs differentiated statistics according to different labels. diff --git a/information-schema/information-schema-metrics-tables.md b/information-schema/information-schema-metrics-tables.md index 9d3226bc40d9d..9a31e8407814a 100644 --- a/information-schema/information-schema-metrics-tables.md +++ b/information-schema/information-schema-metrics-tables.md @@ -1,7 +1,6 @@ --- title: METRICS_TABLES summary: Learn the `METRICS_TABLES` system table. -aliases: ['/docs/dev/system-tables/system-table-metrics-tables/','/docs/dev/reference/system-databases/metrics-tables/','/tidb/dev/system-table-metrics-tables/'] --- # METRICS_TABLES @@ -10,7 +9,7 @@ The `METRICS_TABLES` table provides the PromQL (Prometheus Query Language) defin > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). ```sql USE INFORMATION_SCHEMA; diff --git a/information-schema/information-schema-placement-policies.md b/information-schema/information-schema-placement-policies.md index 59130a1896194..adf331c4dfdfd 100644 --- a/information-schema/information-schema-placement-policies.md +++ b/information-schema/information-schema-placement-policies.md @@ -1,7 +1,6 @@ --- title: PLACEMENT_POLICIES summary: Learn the `PLACEMENT_POLICIES` information_schema table. -aliases: ['/tidb/dev/information-schema-placement-rules'] --- # PLACEMENT_POLICIES @@ -10,7 +9,7 @@ The `PLACEMENT_POLICIES` table provides information on all placement policies. F > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. {{< copyable "sql" >}} diff --git a/information-schema/information-schema-resource-groups.md b/information-schema/information-schema-resource-groups.md index e2b86a07d2f91..3d54953fd92d8 100644 --- a/information-schema/information-schema-resource-groups.md +++ b/information-schema/information-schema-resource-groups.md @@ -9,7 +9,7 @@ The `RESOURCE_GROUPS` table shows the information about all resource groups. For > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ```sql USE information_schema; diff --git a/information-schema/information-schema-runaway-watches.md b/information-schema/information-schema-runaway-watches.md index f65d557e574b5..1400d05c0bc15 100644 --- a/information-schema/information-schema-runaway-watches.md +++ b/information-schema/information-schema-runaway-watches.md @@ -9,7 +9,7 @@ The `RUNAWAY_WATCHES` table shows the watch list of runaway queries that consume > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ```sql USE INFORMATION_SCHEMA; diff --git a/information-schema/information-schema-slow-query.md b/information-schema/information-schema-slow-query.md index b8fe57c66fe87..b4eca2456e4e9 100644 --- a/information-schema/information-schema-slow-query.md +++ b/information-schema/information-schema-slow-query.md @@ -19,7 +19,7 @@ The `SLOW_QUERY` table provides the slow query information of the current node, > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. @@ -129,7 +129,7 @@ The `CLUSTER_SLOW_QUERY` table provides the slow query information of all nodes > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. diff --git a/information-schema/information-schema-sql-diagnostics.md b/information-schema/information-schema-sql-diagnostics.md index ab65d80a104df..77185c5419add 100644 --- a/information-schema/information-schema-sql-diagnostics.md +++ b/information-schema/information-schema-sql-diagnostics.md @@ -1,7 +1,6 @@ --- title: SQL Diagnostics summary: Understand SQL diagnostics in TiDB. -aliases: ['/docs/dev/system-tables/system-table-sql-diagnostics/','/docs/dev/reference/system-databases/sql-diagnosis/','/docs/dev/system-tables/system-table-sql-diagnosis/','/tidb/dev/system-table-sql-diagnostics/','/tidb/dev/check-cluster-status-using-sql-statements','/docs/dev/check-cluster-status-using-sql-statements/','/docs/dev/reference/performance/check-cluster-status-using-sql-statements/'] --- # SQL Diagnostics @@ -19,7 +18,7 @@ The SQL diagnostic system has the following advantages: The SQL diagnostic system consists of three major parts: -+ **Cluster information table**: The SQL diagnostics system introduces cluster information tables that provide a unified way to get the discrete information of each instance. This system fully integrates the cluster topology, hardware information, software information, kernel parameters, monitoring, system information, slow queries, statements, and logs of the entire cluster into the table. So you can query these information using SQL statements. ++ **Cluster information table**: The SQL diagnostics system introduces cluster information tables that provide a unified way to get the discrete information of each instance. This system fully integrates the cluster topology, hardware information, software information, kernel parameters, monitoring, system information, slow queries, statements, and logs of the entire cluster into the table. So you can query this information using SQL statements. + **Cluster monitoring table**: The SQL diagnostic system introduces cluster monitoring tables. All of these tables are in `metrics_schema`, and you can query monitoring information using SQL statements. Compared to the visualized monitoring before v4.0, you can use this SQL-based method to perform correlated queries on all the monitoring information of the entire cluster, and compare the results of different time periods to quickly identify performance bottlenecks. Because the TiDB cluster has many monitoring metrics, the SQL diagnostic system also provides monitoring summary tables, so you can find abnormal monitoring items more easily. diff --git a/information-schema/information-schema-tables.md b/information-schema/information-schema-tables.md index d3d3613f2f25f..8ae5df9a46bf1 100644 --- a/information-schema/information-schema-tables.md +++ b/information-schema/information-schema-tables.md @@ -106,7 +106,7 @@ The description of columns in the `TABLES` table is as follows: * `MAX_DATA_LENGTH`: The maximum data length. The value is currently `0`, which means the data length has no upper limit. * `INDEX_LENGTH`: The index length. `INDEX_LENGTH` = `TABLE_ROWS` \* the sum of lengths of the columns in the index tuple. The replicas of TiKV are not taken into account. * `DATA_FREE`: Data fragment. The value is currently `0`. -* `AUTO_INCREMENT`: The current step of the auto- increment primary key. +* `AUTO_INCREMENT`: The current step of the auto-increment primary key. * `CREATE_TIME`: The time at which the table is created. * `UPDATE_TIME`: The time at which the table is updated. * `CHECK_TIME`: The time at which the table is checked. diff --git a/information-schema/information-schema-tidb-check-constraints.md b/information-schema/information-schema-tidb-check-constraints.md new file mode 100644 index 0000000000000..c979c41ee2ce4 --- /dev/null +++ b/information-schema/information-schema-tidb-check-constraints.md @@ -0,0 +1,59 @@ +--- +title: TIDB_CHECK_CONSTRAINTS +summary: Learn the `TIDB_CHECK_CONSTRAINTS` INFORMATION_SCHEMA table. +--- + +# TIDB\_CHECK\_CONSTRAINTS + +The `TIDB_CHECK_CONSTRAINTS` table provides information about [`CHECK` constraints](/constraints.md#check) on tables. In addition to the columns in [`CHECK_CONSTRAINTS`](/information-schema/information-schema-check-constraints.md), `TIDB_CHECK_CONSTRAINTS` provides the name and ID of the table that defines the `CHECK` constraint. + +```sql +USE INFORMATION_SCHEMA; +DESC TIDB_CHECK_CONSTRAINTS; +``` + +The output is as follows: + +```sql ++--------------------+-------------+------+------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++--------------------+-------------+------+------+---------+-------+ +| CONSTRAINT_CATALOG | varchar(64) | NO | | NULL | | +| CONSTRAINT_SCHEMA | varchar(64) | NO | | NULL | | +| CONSTRAINT_NAME | varchar(64) | NO | | NULL | | +| CHECK_CLAUSE | longtext | NO | | NULL | | +| TABLE_NAME | varchar(64) | YES | | NULL | | +| TABLE_ID | bigint(21) | YES | | NULL | | ++--------------------+-------------+------+------+---------+-------+ +6 rows in set (0.00 sec) +``` + +The following example adds a `CHECK` constraint using the `CREATE TABLE` statement: + +```sql +SET GLOBAL tidb_enable_check_constraint = ON; +CREATE TABLE test.t1 (id INT PRIMARY KEY, CHECK (id%2 = 0)); +SELECT * FROM TIDB_CHECK_CONSTRAINTS\G +``` + +The output is as follows: + +```sql +*************************** 1. row *************************** +CONSTRAINT_CATALOG: def + CONSTRAINT_SCHEMA: test + CONSTRAINT_NAME: t1_chk_1 + CHECK_CLAUSE: (`id` % 2 = 0) + TABLE_NAME: t1 + TABLE_ID: 107 +1 row in set (0.02 sec) +``` + +Fields in the `TIDB_CHECK_CONSTRAINTS` table are described as follows: + +* `CONSTRAINT_CATALOG`: The catalog of the constraint, which is always `def`. +* `CONSTRAINT_SCHEMA`: The schema of the constraint. +* `CONSTRAINT_NAME`: The name of the constraint. +* `CHECK_CLAUSE`: The clause of the check constraint. +* `TABLE_NAME`: The name of the table where the constraint is located. +* `TABLE_ID`: The ID of the table where the constraint is located. diff --git a/information-schema/information-schema-tidb-hot-regions-history.md b/information-schema/information-schema-tidb-hot-regions-history.md index 314a8091be243..38947cb035d65 100644 --- a/information-schema/information-schema-tidb-hot-regions-history.md +++ b/information-schema/information-schema-tidb-hot-regions-history.md @@ -9,7 +9,7 @@ The `TIDB_HOT_REGIONS_HISTORY` table provides information about history hot Regi > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. diff --git a/information-schema/information-schema-tidb-hot-regions.md b/information-schema/information-schema-tidb-hot-regions.md index 60084f3ca4218..127cc9188d8e1 100644 --- a/information-schema/information-schema-tidb-hot-regions.md +++ b/information-schema/information-schema-tidb-hot-regions.md @@ -9,7 +9,7 @@ The `TIDB_HOT_REGIONS` table provides information about the current hot Regions. > **Note:** > -> This table is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This table is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). {{< copyable "sql" >}} diff --git a/information-schema/information-schema-tidb-index-usage.md b/information-schema/information-schema-tidb-index-usage.md index a98f23667c2e2..8cc84fe5ba815 100644 --- a/information-schema/information-schema-tidb-index-usage.md +++ b/information-schema/information-schema-tidb-index-usage.md @@ -75,25 +75,25 @@ DESC CLUSTER_TIDB_INDEX_USAGE; The output is as follows: ```sql -+-------------------------+-----------------------------------------------------------------+------+------+---------+-------+ -| Field | Type | Null | Key | Default | Extra | -+-------------------------+-----------------------------------------------------------------+------+------+---------+-------+ -| INSTANCE | varchar(64) | YES | | NULL | | -| ID | bigint(21) unsigned | NO | PRI | NULL | | -| START_TIME | timestamp(6) | YES | | NULL | | -| CURRENT_SQL_DIGEST | varchar(64) | YES | | NULL | | -| CURRENT_SQL_DIGEST_TEXT | text | YES | | NULL | | -| STATE | enum('Idle','Running','LockWaiting','Committing','RollingBack') | YES | | NULL | | -| WAITING_START_TIME | timestamp(6) | YES | | NULL | | -| MEM_BUFFER_KEYS | bigint(64) | YES | | NULL | | -| MEM_BUFFER_BYTES | bigint(64) | YES | | NULL | | -| SESSION_ID | bigint(21) unsigned | YES | | NULL | | -| USER | varchar(16) | YES | | NULL | | -| DB | varchar(64) | YES | | NULL | | -| ALL_SQL_DIGESTS | text | YES | | NULL | | -| RELATED_TABLE_IDS | text | YES | | NULL | | -| WAITING_TIME | double | YES | | NULL | | -+-------------------------+-----------------------------------------------------------------+------+------+---------+-------+ ++--------------------------+-------------+------+------+---------+-------+ +| Field | Type | Null | Key | Default | Extra | ++--------------------------+-------------+------+------+---------+-------+ +| INSTANCE | varchar(64) | YES | | NULL | | +| TABLE_SCHEMA | varchar(64) | YES | | NULL | | +| TABLE_NAME | varchar(64) | YES | | NULL | | +| INDEX_NAME | varchar(64) | YES | | NULL | | +| QUERY_TOTAL | bigint(21) | YES | | NULL | | +| KV_REQ_TOTAL | bigint(21) | YES | | NULL | | +| ROWS_ACCESS_TOTAL | bigint(21) | YES | | NULL | | +| PERCENTAGE_ACCESS_0 | bigint(21) | YES | | NULL | | +| PERCENTAGE_ACCESS_0_1 | bigint(21) | YES | | NULL | | +| PERCENTAGE_ACCESS_1_10 | bigint(21) | YES | | NULL | | +| PERCENTAGE_ACCESS_10_20 | bigint(21) | YES | | NULL | | +| PERCENTAGE_ACCESS_20_50 | bigint(21) | YES | | NULL | | +| PERCENTAGE_ACCESS_50_100 | bigint(21) | YES | | NULL | | +| PERCENTAGE_ACCESS_100 | bigint(21) | YES | | NULL | | +| LAST_ACCESS_TIME | datetime | YES | | NULL | | ++--------------------------+-------------+------+------+---------+-------+ 15 rows in set (0.00 sec) ``` @@ -101,6 +101,7 @@ The output is as follows: - The data in the `TIDB_INDEX_USAGE` table might be delayed by up to 5 minutes. - After TiDB restarts, the data in the `TIDB_INDEX_USAGE` table is cleared. +- TiDB records index usage for a table only when the table has valid statistics. ## Read more diff --git a/information-schema/information-schema-tidb-servers-info.md b/information-schema/information-schema-tidb-servers-info.md index c51b6f8ff7806..2d17d496633a6 100644 --- a/information-schema/information-schema-tidb-servers-info.md +++ b/information-schema/information-schema-tidb-servers-info.md @@ -9,7 +9,7 @@ The `TIDB_SERVERS_INFO` table provides information about TiDB servers in the TiD > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ```sql USE INFORMATION_SCHEMA; @@ -50,7 +50,7 @@ The output is as follows: PORT: 4000 STATUS_PORT: 10080 LEASE: 45s - VERSION: 8.0.11-TiDB-v8.0.0 + VERSION: 8.0.11-TiDB-v8.1.2 GIT_HASH: 827d8ff2d22ac4c93ae1b841b79d468211e1d393 BINLOG_STATUS: Off LABELS: diff --git a/information-schema/information-schema-tiflash-segments.md b/information-schema/information-schema-tiflash-segments.md index d3d7f585d12c1..54ca2a18f82e9 100644 --- a/information-schema/information-schema-tiflash-segments.md +++ b/information-schema/information-schema-tiflash-segments.md @@ -9,10 +9,6 @@ summary: Learn the `TIFLASH_SEGMENTS` information_schema table. > > Do not use this table in production environments, as the fields of the table are unstable, and subject to change in new releases of TiDB, without prior notice. -> **Note:** -> -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. - The `TIFLASH_SEGMENTS` table provides statistical information about data tables in TiFlash. ```sql diff --git a/information-schema/information-schema-tiflash-tables.md b/information-schema/information-schema-tiflash-tables.md index f7971996cb86c..42992d07809c5 100644 --- a/information-schema/information-schema-tiflash-tables.md +++ b/information-schema/information-schema-tiflash-tables.md @@ -9,10 +9,6 @@ summary: Learn the `TIFLASH_TABLES` information_schema table. > > Do not use this table in production environments, as the fields of the table are unstable, and subject to change in new releases of TiDB, without prior notice. -> **Note:** -> -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. - The `TIFLASH_TABLES` table provides statistical information about data tables in TiFlash. ```sql diff --git a/information-schema/information-schema-tikv-region-peers.md b/information-schema/information-schema-tikv-region-peers.md index c9018c20caeb2..e0cf38b69b736 100644 --- a/information-schema/information-schema-tikv-region-peers.md +++ b/information-schema/information-schema-tikv-region-peers.md @@ -9,7 +9,7 @@ The `TIKV_REGION_PEERS` table shows detailed information of a single Region node > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ```sql USE INFORMATION_SCHEMA; diff --git a/information-schema/information-schema-tikv-region-status.md b/information-schema/information-schema-tikv-region-status.md index f9d225ee66b5d..0d87a988aa225 100644 --- a/information-schema/information-schema-tikv-region-status.md +++ b/information-schema/information-schema-tikv-region-status.md @@ -9,15 +9,15 @@ The `TIKV_REGION_STATUS` table shows some basic information of TiKV Regions via > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. - -{{< copyable "sql" >}} +> This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ```sql -USE information_schema; -DESC tikv_region_status; +USE INFORMATION_SCHEMA; +DESC TIKV_REGION_STATUS; ``` +The output is as follows: + ```sql +---------------------------+-------------+------+------+---------+-------+ | Field | Type | Null | Key | Default | Extra | @@ -31,6 +31,9 @@ DESC tikv_region_status; | IS_INDEX | tinyint(1) | NO | | 0 | | | INDEX_ID | bigint(21) | YES | | NULL | | | INDEX_NAME | varchar(64) | YES | | NULL | | +| IS_PARTITION | tinyint(1) | NO | | 0 | | +| PARTITION_ID | bigint(21) | YES | | NULL | | +| PARTITION_NAME | varchar(64) | YES | | NULL | | | EPOCH_CONF_VER | bigint(21) | YES | | NULL | | | EPOCH_VERSION | bigint(21) | YES | | NULL | | | WRITTEN_BYTES | bigint(21) | YES | | NULL | | @@ -40,7 +43,7 @@ DESC tikv_region_status; | REPLICATIONSTATUS_STATE | varchar(64) | YES | | NULL | | | REPLICATIONSTATUS_STATEID | bigint(21) | YES | | NULL | | +---------------------------+-------------+------+------+---------+-------+ -17 rows in set (0.00 sec) +20 rows in set (0.00 sec) ``` The descriptions of the columns in the `TIKV_REGION_STATUS` table are as follows: @@ -54,6 +57,9 @@ The descriptions of the columns in the `TIKV_REGION_STATUS` table are as follows * `IS_INDEX`: Whether the Region data is an index. 0 means that it is not an index, while 1 means that it is an index. If the current Region contains both table data and index data, there will be multiple rows of records, and `IS_INDEX` is 0 and 1 respectively. * `INDEX_ID`: The ID of the index to which the Region belongs. If `IS_INDEX` is 0, the value of this column is NULL. * `INDEX_NAME`: The name of the index to which the Region belongs. If `IS_INDEX` is 0, the value of this column is NULL. +* `IS_PARTITION`: Whether the table to which the Region belongs is partitioned. +* `PARTITION_ID`: If the table to which the Region belongs is partitioned, this column displays the ID of the partition to which the Region belongs. +* `PARTITION_NAME`: If the table to which the Region belongs is partitioned, this column displays the name of the partition to which the Region belongs. * `EPOCH_CONF_VER`: The version number of the Region configuration. The version number increases when a peer is added or removed. * `EPOCH_VERSION`: The current version number of the Region. The version number increases when the Region is split or merged. * `WRITTEN_BYTES`: The amount of data (bytes) written to the Region. diff --git a/information-schema/information-schema-tikv-store-status.md b/information-schema/information-schema-tikv-store-status.md index daee44e63533d..7e1600c83d4ef 100644 --- a/information-schema/information-schema-tikv-store-status.md +++ b/information-schema/information-schema-tikv-store-status.md @@ -9,7 +9,7 @@ The `TIKV_STORE_STATUS` table shows some basic information of TiKV nodes via PD' > **Note:** > -> This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ```sql USE INFORMATION_SCHEMA; diff --git a/information-schema/information-schema-user-privileges.md b/information-schema/information-schema-user-privileges.md index d65ca027078cf..1affd5d785ce1 100644 --- a/information-schema/information-schema-user-privileges.md +++ b/information-schema/information-schema-user-privileges.md @@ -79,7 +79,7 @@ The output is as follows: - + ```sql +------------+---------------+-------------------------+--------------+ diff --git a/information-schema/information-schema.md b/information-schema/information-schema.md index 16ce9f0e13812..b74275c06f2b9 100644 --- a/information-schema/information-schema.md +++ b/information-schema/information-schema.md @@ -1,7 +1,6 @@ --- title: Information Schema summary: TiDB implements the ANSI-standard information_schema for viewing system metadata. -aliases: ['/docs/dev/system-tables/system-table-information-schema/','/docs/dev/reference/system-databases/information-schema/','/tidb/dev/system-table-information-schema/'] --- # Information Schema @@ -103,7 +102,7 @@ Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` statement. The bene > **Note:** > -> Some of the following tables are only supported on TiDB Self-Hosted and not supported on TiDB Cloud. To get a full list of unsupported tables on TiDB Cloud, see [System tables](https://docs.pingcap.com/tidbcloud/limited-sql-features#system-tables). +> Some of the following tables are only supported on TiDB Self-Managed and not supported on TiDB Cloud. To get a full list of unsupported tables on TiDB Cloud, see [System tables](https://docs.pingcap.com/tidbcloud/limited-sql-features#system-tables). | Table name | Description | |-----------------------------------------------------------------------------------------|-------------| @@ -114,15 +113,15 @@ Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` statement. The bene | [`CLUSTER_CONFIG`](/information-schema/information-schema-cluster-config.md) | Provides details about configuration settings for the entire TiDB cluster. This table is not applicable to TiDB Cloud. | | `CLUSTER_DEADLOCKS` | Provides a cluster-level view of the `DEADLOCKS` table. | | [`CLUSTER_HARDWARE`](/information-schema/information-schema-cluster-hardware.md) | Provides details on the underlying physical hardware discovered on each TiDB component. This table is not applicable to TiDB Cloud. | -| [`CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) | Provides details on the current cluster topology. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | +| [`CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) | Provides details on the current cluster topology. This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. | | [`CLUSTER_LOAD`](/information-schema/information-schema-cluster-load.md) | Provides current load information for TiDB servers in the cluster. This table is not applicable to TiDB Cloud. | | [`CLUSTER_LOG`](/information-schema/information-schema-cluster-log.md) | Provides a log for the entire TiDB cluster. This table is not applicable to TiDB Cloud. | | `CLUSTER_MEMORY_USAGE` | Provides a cluster-level view of the `MEMORY_USAGE` table. | | `CLUSTER_MEMORY_USAGE_OPS_HISTORY` | Provides a cluster-level view of the `MEMORY_USAGE_OPS_HISTORY` table. | | `CLUSTER_PROCESSLIST` | Provides a cluster-level view of the `PROCESSLIST` table. | -| `CLUSTER_SLOW_QUERY` | Provides a cluster-level view of the `SLOW_QUERY` table. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | -| `CLUSTER_STATEMENTS_SUMMARY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY` table. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | -| `CLUSTER_STATEMENTS_SUMMARY_HISTORY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY_HISTORY` table. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | +| `CLUSTER_SLOW_QUERY` | Provides a cluster-level view of the `SLOW_QUERY` table. This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. | +| `CLUSTER_STATEMENTS_SUMMARY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY` table. This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. | +| `CLUSTER_STATEMENTS_SUMMARY_HISTORY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY_HISTORY` table. This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. | | `CLUSTER_TIDB_INDEX_USAGE` | Provides a cluster-level view of the `TIDB_INDEX_USAGE` table. | | `CLUSTER_TIDB_TRX` | Provides a cluster-level view of the `TIDB_TRX` table. | | [`CLUSTER_SYSTEMINFO`](/information-schema/information-schema-cluster-systeminfo.md) | Provides details about kernel parameter configuration for servers in the cluster. This table is not applicable to TiDB Cloud. | @@ -137,11 +136,11 @@ Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` statement. The bene | [`METRICS_SUMMARY`](/information-schema/information-schema-metrics-summary.md) | A summary of metrics extracted from Prometheus. This table is not applicable to TiDB Cloud. | | `METRICS_SUMMARY_BY_LABEL` | See `METRICS_SUMMARY` table. This table is not applicable to TiDB Cloud. | | [`METRICS_TABLES`](/information-schema/information-schema-metrics-tables.md) | Provides the PromQL definitions for tables in `METRICS_SCHEMA`. This table is not applicable to TiDB Cloud. | -| [`PLACEMENT_POLICIES`](/information-schema/information-schema-placement-policies.md) | Provides information on all placement policies. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | +| [`PLACEMENT_POLICIES`](/information-schema/information-schema-placement-policies.md) | Provides information on all placement policies. This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. | | [`SEQUENCES`](/information-schema/information-schema-sequences.md) | The TiDB implementation of sequences is based on MariaDB. | -| [`SLOW_QUERY`](/information-schema/information-schema-slow-query.md) | Provides information on slow queries on the current TiDB server. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | -| [`STATEMENTS_SUMMARY`](/statement-summary-tables.md) | Similar to performance_schema statement summary in MySQL. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | -| [`STATEMENTS_SUMMARY_HISTORY`](/statement-summary-tables.md) | Similar to performance_schema statement summary history in MySQL. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | +| [`SLOW_QUERY`](/information-schema/information-schema-slow-query.md) | Provides information on slow queries on the current TiDB server. This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. | +| [`STATEMENTS_SUMMARY`](/statement-summary-tables.md) | Similar to performance_schema statement summary in MySQL. This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. | +| [`STATEMENTS_SUMMARY_HISTORY`](/statement-summary-tables.md) | Similar to performance_schema statement summary history in MySQL. This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. | | [`TABLE_STORAGE_STATS`](/information-schema/information-schema-table-storage-stats.md) | Provides details about table sizes in storage. | | [`TIDB_HOT_REGIONS`](/information-schema/information-schema-tidb-hot-regions.md) | Provides statistics about which regions are hot. | | [`TIDB_HOT_REGIONS_HISTORY`](/information-schema/information-schema-tidb-hot-regions-history.md) | Provides history statistics about which Regions are hot. | @@ -167,15 +166,15 @@ Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` statement. The bene | [`CLUSTER_CONFIG`](https://docs.pingcap.com/tidb/stable/information-schema-cluster-config) | Provides details about configuration settings for the entire TiDB cluster. This table is not applicable to TiDB Cloud. | | `CLUSTER_DEADLOCKS` | Provides a cluster-level view of the `DEADLOCKS` table. | | [`CLUSTER_HARDWARE`](https://docs.pingcap.com/tidb/stable/information-schema-cluster-hardware) | Provides details on the underlying physical hardware discovered on each TiDB component. This table is not applicable to TiDB Cloud. | -| [`CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) | Provides details on the current cluster topology. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | +| [`CLUSTER_INFO`](/information-schema/information-schema-cluster-info.md) | Provides details on the current cluster topology. This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. | | [`CLUSTER_LOAD`](https://docs.pingcap.com/tidb/stable/information-schema-cluster-load) | Provides current load information for TiDB servers in the cluster. This table is not applicable to TiDB Cloud. | | [`CLUSTER_LOG`](https://docs.pingcap.com/tidb/stable/information-schema-cluster-log) | Provides a log for the entire TiDB cluster. This table is not applicable to TiDB Cloud. | | `CLUSTER_MEMORY_USAGE` | Provides a cluster-level view of the `MEMORY_USAGE` table. This table is not applicable to TiDB Cloud. | | `CLUSTER_MEMORY_USAGE_OPS_HISTORY` | Provides a cluster-level view of the `MEMORY_USAGE_OPS_HISTORY` table. This table is not applicable to TiDB Cloud. | | `CLUSTER_PROCESSLIST` | Provides a cluster-level view of the `PROCESSLIST` table. | -| `CLUSTER_SLOW_QUERY` | Provides a cluster-level view of the `SLOW_QUERY` table. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | -| `CLUSTER_STATEMENTS_SUMMARY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY` table. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | -| `CLUSTER_STATEMENTS_SUMMARY_HISTORY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY_HISTORY` table. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | +| `CLUSTER_SLOW_QUERY` | Provides a cluster-level view of the `SLOW_QUERY` table. This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. | +| `CLUSTER_STATEMENTS_SUMMARY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY` table. This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. | +| `CLUSTER_STATEMENTS_SUMMARY_HISTORY` | Provides a cluster-level view of the `STATEMENTS_SUMMARY_HISTORY` table. This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. | | `CLUSTER_TIDB_TRX` | Provides a cluster-level view of the `TIDB_TRX` table. | | [`CLUSTER_SYSTEMINFO`](https://docs.pingcap.com/tidb/stable/information-schema-cluster-systeminfo) | Provides details about kernel parameter configuration for servers in the cluster. This table is not applicable to TiDB Cloud. | | [`DATA_LOCK_WAITS`](/information-schema/information-schema-data-lock-waits.md) | Provides the lock-waiting information on the TiKV server. | @@ -189,11 +188,11 @@ Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` statement. The bene | [`METRICS_SUMMARY`](https://docs.pingcap.com/tidb/stable/information-schema-metrics-summary) | A summary of metrics extracted from Prometheus. This table is not applicable to TiDB Cloud. | | `METRICS_SUMMARY_BY_LABEL` | See `METRICS_SUMMARY` table. This table is not applicable to TiDB Cloud. | | [`METRICS_TABLES`](https://docs.pingcap.com/tidb/stable/information-schema-metrics-tables) | Provides the PromQL definitions for tables in `METRICS_SCHEMA`. This table is not applicable to TiDB Cloud. | -| [`PLACEMENT_POLICIES`](https://docs.pingcap.com/tidb/stable/information-schema-placement-policies) | Provides information on all placement policies. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | +| [`PLACEMENT_POLICIES`](https://docs.pingcap.com/tidb/stable/information-schema-placement-policies) | Provides information on all placement policies. This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. | | [`SEQUENCES`](/information-schema/information-schema-sequences.md) | The TiDB implementation of sequences is based on MariaDB. | -| [`SLOW_QUERY`](/information-schema/information-schema-slow-query.md) | Provides information on slow queries on the current TiDB server. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | -| [`STATEMENTS_SUMMARY`](/statement-summary-tables.md) | Similar to performance_schema statement summary in MySQL. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. | -| [`STATEMENTS_SUMMARY_HISTORY`](/statement-summary-tables.md) | Similar to performance_schema statement summary history in MySQL. This table is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters.| +| [`SLOW_QUERY`](/information-schema/information-schema-slow-query.md) | Provides information on slow queries on the current TiDB server. This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. | +| [`STATEMENTS_SUMMARY`](/statement-summary-tables.md) | Similar to performance_schema statement summary in MySQL. This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. | +| [`STATEMENTS_SUMMARY_HISTORY`](/statement-summary-tables.md) | Similar to performance_schema statement summary history in MySQL. This table is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters.| | [`TABLE_STORAGE_STATS`](/information-schema/information-schema-table-storage-stats.md) | Provides details about table sizes in storage. | | [`TIDB_HOT_REGIONS`](https://docs.pingcap.com/tidb/stable/information-schema-tidb-hot-regions) | Provides statistics about which regions are hot. This table is not applicable to TiDB Cloud. | | [`TIDB_HOT_REGIONS_HISTORY`](/information-schema/information-schema-tidb-hot-regions-history.md) | Provides history statistics about which Regions are hot. | diff --git a/join-reorder.md b/join-reorder.md index 747c18365e058..295da0046927c 100644 --- a/join-reorder.md +++ b/join-reorder.md @@ -1,7 +1,6 @@ --- title: Introduction to Join Reorder summary: Use the Join Reorder algorithm to join multiple tables in TiDB. -aliases: ['/docs/dev/join-reorder/','/docs/dev/reference/performance/join-reorder/'] --- # Introduction to Join Reorder diff --git a/keywords.md b/keywords.md index b468aa0e972c4..3f925d4abeb9f 100644 --- a/keywords.md +++ b/keywords.md @@ -1,19 +1,16 @@ --- title: Keywords summary: Keywords and Reserved Words -aliases: ['/docs/dev/keywords-and-reserved-words/','/docs/dev/reference/sql/language-structure/keywords-and-reserved-words/','/tidb/dev/keywords-and-reserved-words/'] --- # Keywords This article introduces the keywords in TiDB, the differences between reserved words and non-reserved words and summarizes all keywords for the query. -Keywords are words that have special meanings in SQL statements, such as `SELECT`, `UPDATE`, and `DELETE`. Some of them can be used as identifiers directly, which are called **non-reserved keywords**. Some of them require special treatment before being used as identifiers, which are called **reserved keywords**. However, there are special non-reserved keywords that might still require special treatment. It is recommended that you treat them as reserved keywords. +Keywords are words that have special meanings in SQL statements, such as [`SELECT`](/sql-statements/sql-statement-select.md), [`UPDATE`](/sql-statements/sql-statement-update.md), and [`DELETE`](/sql-statements/sql-statement-delete.md). Some of them can be used as identifiers directly, which are called **non-reserved keywords**. Some of them require special treatment before being used as identifiers, which are called **reserved keywords**. However, there are special non-reserved keywords that might still require special treatment. It is recommended that you treat them as reserved keywords. To use the reserved keywords as identifiers, you must enclose them in backticks `` ` ``: -{{< copyable "sql" >}} - ```sql CREATE TABLE select (a INT); ``` @@ -22,8 +19,6 @@ CREATE TABLE select (a INT); ERROR 1105 (HY000): line 0 column 19 near " (a INT)" (total length 27) ``` -{{< copyable "sql" >}} - ```sql CREATE TABLE `select` (a INT); ``` @@ -34,8 +29,6 @@ Query OK, 0 rows affected (0.09 sec) The non-reserved keywords do not require backticks, such as `BEGIN` and `END`, which can be successfully used as identifiers in the following statement: -{{< copyable "sql" >}} - ```sql CREATE TABLE `select` (BEGIN int, END int); ``` @@ -46,8 +39,6 @@ Query OK, 0 rows affected (0.09 sec) In the special case, the reserved keywords do not need backticks if they are used with the `.` delimiter: -{{< copyable "sql" >}} - ```sql CREATE TABLE test.select (BEGIN int, END int); ``` @@ -56,7 +47,7 @@ CREATE TABLE test.select (BEGIN int, END int); Query OK, 0 rows affected (0.08 sec) ``` -Starting from v7.6.0, TiDB provides a full list of keywords in the [`INFORMATION_SCHEMA.KEYWORDS`](/information-schema/information-schema-keywords.md) table. +Starting from v7.5.3 and v7.6.0, TiDB provides a full list of keywords in the [`INFORMATION_SCHEMA.KEYWORDS`](/information-schema/information-schema-keywords.md) table. ## Keyword list @@ -508,7 +499,6 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - PER_DB - PER_TABLE - PESSIMISTIC -- PLACEMENT (S) - PLUGINS - POINT - POLICY @@ -687,8 +677,6 @@ The following list shows the keywords in TiDB. Reserved keywords are marked with - TABLESAMPLE (R) - TABLESPACE - TABLE_CHECKSUM -- TELEMETRY -- TELEMETRY_ID - TEMPORARY - TEMPTABLE - TERMINATED (R) diff --git a/literal-values.md b/literal-values.md index f0048492460a6..94e6881e0e3c8 100644 --- a/literal-values.md +++ b/literal-values.md @@ -1,7 +1,6 @@ --- title: Literal Values summary: This article introduces the literal values ​​of TiDB SQL statements. -aliases: ['/docs/dev/literal-values/','/docs/dev/reference/sql/language-structure/literal-values/'] --- # Literal Values @@ -94,7 +93,7 @@ Date and time literal values can be represented in several formats, such as quot TiDB supports the following date formats: -* `'YYYY-MM-DD'` or `'YY-MM-DD'`: The `-` delimiter here is not strict. It can be any punctuation. For example, `'2017-08-24'`, `'2017&08&24'`, `'2012@12^31'` are all valid date formats. The only special punctuation is '.', which is is treated as a decimal point to separate the integer and fractional parts. Date and time can be separated by `T` or a white space. For example, `2017-8-24 10:42:00` and `2017-8-24T10:42:00` represents the same date and time. +* `'YYYY-MM-DD'` or `'YY-MM-DD'`: The `-` delimiter here is not strict. It can be any punctuation. For example, `'2017-08-24'`, `'2017&08&24'`, `'2012@12^31'` are all valid date formats. The only special punctuation is '.', which is treated as a decimal point to separate the integer and fractional parts. Date and time can be separated by `T` or a white space. For example, `2017-8-24 10:42:00` and `2017-8-24T10:42:00` represents the same date and time. * `'YYYYMMDDHHMMSS'` or `'YYMMDDHHMMSS'`: For example, `'20170824104520'` and `'170824104520'` are regarded as `'2017-08-24 10:45:20'`. However, if you provide a value out of range, such as `'170824304520'`, it is not treated as a valid date. Note that incorrect formats such as `YYYYMMDD HHMMSS`, `YYYYMMDD HH:MM:DD`, or `YYYY-MM-DD HHMMSS` will fail to insert. * `YYYYMMDDHHMMSS` or `YYMMDDHHMMSS`: Note that these formats have no single or double quotes, but a number. For example, `20170824104520` is interpreted as `'2017-08-24 10:45:20'`. diff --git a/maintain-tidb-using-tiup.md b/maintain-tidb-using-tiup.md index 18e30b89617fe..5458dd0bfde5a 100644 --- a/maintain-tidb-using-tiup.md +++ b/maintain-tidb-using-tiup.md @@ -1,7 +1,6 @@ --- title: TiUP Common Operations summary: Learn the common operations to operate and maintain a TiDB cluster using TiUP. -aliases: ['/docs/dev/maintain-tidb-using-tiup/','/docs/dev/how-to/maintain/tiup-operations/'] --- # TiUP Common Operations @@ -113,7 +112,7 @@ When the cluster is in operation, if you need to modify the parameters of a comp **Use `.` to represent the hierarchy of the configuration items**. - For more information on the configuration parameters of components, refer to [TiDB `config.toml.example`](https://github.com/pingcap/tidb/blob/master/pkg/config/config.toml.example), [TiKV `config.toml.example`](https://github.com/tikv/tikv/blob/master/etc/config-template.toml), and [PD `config.toml.example`](https://github.com/tikv/pd/blob/master/conf/config.toml). + For more information on the configuration parameters of components, refer to [TiDB `config.toml.example`](https://github.com/pingcap/tidb/blob/release-8.1/pkg/config/config.toml.example), [TiKV `config.toml.example`](https://github.com/tikv/tikv/blob/release-8.1/etc/config-template.toml), and [PD `config.toml.example`](https://github.com/tikv/pd/blob/release-8.1/conf/config.toml). 3. Rolling update the configuration and restart the corresponding components by running the `reload` command: @@ -125,7 +124,7 @@ When the cluster is in operation, if you need to modify the parameters of a comp ### Example -If you want to set the transaction size limit parameter (`txn-total-size-limit` in the [performance](https://github.com/pingcap/tidb/blob/master/pkg/config/config.toml.example) module) to `1G` in tidb-server, edit the configuration as follows: +If you want to set the transaction size limit parameter (`txn-total-size-limit` in the [performance](https://github.com/pingcap/tidb/blob/release-8.1/pkg/config/config.toml.example) module) to `1G` in tidb-server, edit the configuration as follows: ``` server_configs: diff --git a/media/best-practices/commit-log-duration.png b/media/best-practices/commit-log-duration.png new file mode 100644 index 0000000000000..de889199aeb62 Binary files /dev/null and b/media/best-practices/commit-log-duration.png differ diff --git a/media/dashboard/dashboard-statement-detail0.png b/media/dashboard/dashboard-statement-detail0.png new file mode 100644 index 0000000000000..b066c5e3c4a88 Binary files /dev/null and b/media/dashboard/dashboard-statement-detail0.png differ diff --git a/media/dashboard/dashboard-statement-detail1.png b/media/dashboard/dashboard-statement-detail1.png new file mode 100644 index 0000000000000..b17cb3797711f Binary files /dev/null and b/media/dashboard/dashboard-statement-detail1.png differ diff --git a/media/dashboard/dashboard-statement-detail2.png b/media/dashboard/dashboard-statement-detail2.png new file mode 100644 index 0000000000000..8cfedf64b1c0c Binary files /dev/null and b/media/dashboard/dashboard-statement-detail2.png differ diff --git a/media/develop/aws-lambda-cf-create-stack.png b/media/develop/aws-lambda-cf-create-stack.png new file mode 100644 index 0000000000000..7b57c754467fb Binary files /dev/null and b/media/develop/aws-lambda-cf-create-stack.png differ diff --git a/media/develop/aws-lambda-cf-stack-config-option.png b/media/develop/aws-lambda-cf-stack-config-option.png new file mode 100644 index 0000000000000..116f8e1bfc8a6 Binary files /dev/null and b/media/develop/aws-lambda-cf-stack-config-option.png differ diff --git a/media/develop/aws-lambda-cf-stack-config-review.png b/media/develop/aws-lambda-cf-stack-config-review.png new file mode 100644 index 0000000000000..120fc5b024b87 Binary files /dev/null and b/media/develop/aws-lambda-cf-stack-config-review.png differ diff --git a/media/develop/aws-lambda-cf-stack-config.png b/media/develop/aws-lambda-cf-stack-config.png new file mode 100644 index 0000000000000..dc542282b62b8 Binary files /dev/null and b/media/develop/aws-lambda-cf-stack-config.png differ diff --git a/media/develop/aws-lambda-get-apigateway-invoke-url.png b/media/develop/aws-lambda-get-apigateway-invoke-url.png new file mode 100644 index 0000000000000..586a89a40cd5a Binary files /dev/null and b/media/develop/aws-lambda-get-apigateway-invoke-url.png differ diff --git a/media/develop/aws-lambda-structure-overview.png b/media/develop/aws-lambda-structure-overview.png new file mode 100644 index 0000000000000..0ff0933d1913e Binary files /dev/null and b/media/develop/aws-lambda-structure-overview.png differ diff --git a/media/develop/aws-lambda-tidbcloud-connection-info.png b/media/develop/aws-lambda-tidbcloud-connection-info.png new file mode 100644 index 0000000000000..f7ac6a1078837 Binary files /dev/null and b/media/develop/aws-lambda-tidbcloud-connection-info.png differ diff --git a/media/develop/navicat-premium-add-new-connection.png b/media/develop/navicat-premium-add-new-connection.png new file mode 100644 index 0000000000000..f91f1f495719b Binary files /dev/null and b/media/develop/navicat-premium-add-new-connection.png differ diff --git a/media/develop/navicat-premium-connection-config-dedicated-general.png b/media/develop/navicat-premium-connection-config-dedicated-general.png new file mode 100644 index 0000000000000..2dfb34692d661 Binary files /dev/null and b/media/develop/navicat-premium-connection-config-dedicated-general.png differ diff --git a/media/develop/navicat-premium-connection-config-dedicated-ssl.png b/media/develop/navicat-premium-connection-config-dedicated-ssl.png new file mode 100644 index 0000000000000..5ea365072fe51 Binary files /dev/null and b/media/develop/navicat-premium-connection-config-dedicated-ssl.png differ diff --git a/media/develop/navicat-premium-connection-config-self-hosted-general.png b/media/develop/navicat-premium-connection-config-self-hosted-general.png new file mode 100644 index 0000000000000..5e3cd7df34074 Binary files /dev/null and b/media/develop/navicat-premium-connection-config-self-hosted-general.png differ diff --git a/media/develop/navicat-premium-connection-config-serverless-general.png b/media/develop/navicat-premium-connection-config-serverless-general.png new file mode 100644 index 0000000000000..aa14fa3c60c7d Binary files /dev/null and b/media/develop/navicat-premium-connection-config-serverless-general.png differ diff --git a/media/develop/navicat-premium-connection-config-serverless-ssl.png b/media/develop/navicat-premium-connection-config-serverless-ssl.png new file mode 100644 index 0000000000000..7bdb79e16fc93 Binary files /dev/null and b/media/develop/navicat-premium-connection-config-serverless-ssl.png differ diff --git a/media/grafana-password-reset1.png b/media/grafana-password-reset1.png new file mode 100644 index 0000000000000..b8559c659cf7c Binary files /dev/null and b/media/grafana-password-reset1.png differ diff --git a/media/grafana-password-reset2.png b/media/grafana-password-reset2.png new file mode 100644 index 0000000000000..0c7c604d0fda6 Binary files /dev/null and b/media/grafana-password-reset2.png differ diff --git a/media/performance/titan-rocksdb-compactions.png b/media/performance/titan-rocksdb-compactions.png new file mode 100644 index 0000000000000..1d57fc279aa49 Binary files /dev/null and b/media/performance/titan-rocksdb-compactions.png differ diff --git a/media/sql-tuning/cost-based-optimization.png b/media/sql-tuning/cost-based-optimization.png new file mode 100644 index 0000000000000..ef333897f6b06 Binary files /dev/null and b/media/sql-tuning/cost-based-optimization.png differ diff --git a/media/sql-tuning/execution-plan-traverse.png b/media/sql-tuning/execution-plan-traverse.png new file mode 100644 index 0000000000000..373923256964f Binary files /dev/null and b/media/sql-tuning/execution-plan-traverse.png differ diff --git a/media/sql-tuning/slow-query-default.png b/media/sql-tuning/slow-query-default.png new file mode 100644 index 0000000000000..23aaf26cec2cc Binary files /dev/null and b/media/sql-tuning/slow-query-default.png differ diff --git a/media/sql-tuning/slow-query-dimension.png b/media/sql-tuning/slow-query-dimension.png new file mode 100644 index 0000000000000..e7d34c6c97de9 Binary files /dev/null and b/media/sql-tuning/slow-query-dimension.png differ diff --git a/media/sql-tuning/sql-statement-dimension.png b/media/sql-tuning/sql-statement-dimension.png new file mode 100644 index 0000000000000..ad8b30b9008b4 Binary files /dev/null and b/media/sql-tuning/sql-statement-dimension.png differ diff --git a/media/sql-tuning/sql-statements-default.png b/media/sql-tuning/sql-statements-default.png new file mode 100644 index 0000000000000..df31ffac1b845 Binary files /dev/null and b/media/sql-tuning/sql-statements-default.png differ diff --git a/media/sql-tuning/workflow-tiflash.png b/media/sql-tuning/workflow-tiflash.png new file mode 100644 index 0000000000000..60bdcd383d561 Binary files /dev/null and b/media/sql-tuning/workflow-tiflash.png differ diff --git a/media/sql-tuning/workflow.png b/media/sql-tuning/workflow.png new file mode 100644 index 0000000000000..fcd6223c5cfe7 Binary files /dev/null and b/media/sql-tuning/workflow.png differ diff --git a/media/tidb-cloud/Project-CIDR2.png b/media/tidb-cloud/Project-CIDR2.png index b412a4889f229..b0739027bda67 100644 Binary files a/media/tidb-cloud/Project-CIDR2.png and b/media/tidb-cloud/Project-CIDR2.png differ diff --git a/media/tidb-cloud/Project-CIDR4.png b/media/tidb-cloud/Project-CIDR4.png index 0879853b29d08..e73e2b3bb87a5 100644 Binary files a/media/tidb-cloud/Project-CIDR4.png and b/media/tidb-cloud/Project-CIDR4.png differ diff --git a/media/tidb-cloud/VPC-Peering3.png b/media/tidb-cloud/VPC-Peering3.png deleted file mode 100644 index b3a8aacbb72c3..0000000000000 Binary files a/media/tidb-cloud/VPC-Peering3.png and /dev/null differ diff --git a/media/tidb-cloud/blank_transparent_placeholder.png b/media/tidb-cloud/blank_transparent_placeholder.png new file mode 100644 index 0000000000000..4d8ec8f7b4a1c Binary files /dev/null and b/media/tidb-cloud/blank_transparent_placeholder.png differ diff --git a/media/tidb-cloud/changefeed/connect-to-aws-self-hosted-kafka-privatelink-service.jpeg b/media/tidb-cloud/changefeed/connect-to-aws-self-hosted-kafka-privatelink-service.jpeg new file mode 100644 index 0000000000000..d0d5fa35a0555 Binary files /dev/null and b/media/tidb-cloud/changefeed/connect-to-aws-self-hosted-kafka-privatelink-service.jpeg differ diff --git a/media/tidb-cloud/changefeed/connect-to-google-cloud-self-hosted-kafka-private-service-connect-by-kafka-proxy.jpeg b/media/tidb-cloud/changefeed/connect-to-google-cloud-self-hosted-kafka-private-service-connect-by-kafka-proxy.jpeg new file mode 100644 index 0000000000000..290ee89bde5e3 Binary files /dev/null and b/media/tidb-cloud/changefeed/connect-to-google-cloud-self-hosted-kafka-private-service-connect-by-kafka-proxy.jpeg differ diff --git a/media/tidb-cloud/changefeed/connect-to-google-cloud-self-hosted-kafka-private-service-connect-by-portmapping.jpeg b/media/tidb-cloud/changefeed/connect-to-google-cloud-self-hosted-kafka-private-service-connect-by-portmapping.jpeg new file mode 100644 index 0000000000000..d0b3595663820 Binary files /dev/null and b/media/tidb-cloud/changefeed/connect-to-google-cloud-self-hosted-kafka-private-service-connect-by-portmapping.jpeg differ diff --git a/media/tidb-cloud/connected-ai-chat-example.png b/media/tidb-cloud/connected-ai-chat-example.png new file mode 100644 index 0000000000000..1f8aafbcc91ab Binary files /dev/null and b/media/tidb-cloud/connected-ai-chat-example.png differ diff --git a/media/tidb-cloud/connected-lark-ticket-creation-1.png b/media/tidb-cloud/connected-lark-ticket-creation-1.png new file mode 100644 index 0000000000000..ef7cc8689ab9a Binary files /dev/null and b/media/tidb-cloud/connected-lark-ticket-creation-1.png differ diff --git a/media/tidb-cloud/connected-lark-ticket-creation-2.png b/media/tidb-cloud/connected-lark-ticket-creation-2.png new file mode 100644 index 0000000000000..f31ec78a5f5cc Binary files /dev/null and b/media/tidb-cloud/connected-lark-ticket-creation-2.png differ diff --git a/media/tidb-cloud/connected-lark-ticket-creation-3.png b/media/tidb-cloud/connected-lark-ticket-creation-3.png new file mode 100644 index 0000000000000..f596edef66fd6 Binary files /dev/null and b/media/tidb-cloud/connected-lark-ticket-creation-3.png differ diff --git a/media/tidb-cloud/connected-lark-ticket-creation-4.png b/media/tidb-cloud/connected-lark-ticket-creation-4.png new file mode 100644 index 0000000000000..e73b07766d0a5 Binary files /dev/null and b/media/tidb-cloud/connected-lark-ticket-creation-4.png differ diff --git a/media/tidb-cloud/connected-lark-ticket-interaction-1.png b/media/tidb-cloud/connected-lark-ticket-interaction-1.png new file mode 100644 index 0000000000000..ad8557d371bc5 Binary files /dev/null and b/media/tidb-cloud/connected-lark-ticket-interaction-1.png differ diff --git a/media/tidb-cloud/connected-slack-ticket-creation-1.png b/media/tidb-cloud/connected-slack-ticket-creation-1.png new file mode 100644 index 0000000000000..d1807127694b3 Binary files /dev/null and b/media/tidb-cloud/connected-slack-ticket-creation-1.png differ diff --git a/media/tidb-cloud/connected-slack-ticket-creation-2.png b/media/tidb-cloud/connected-slack-ticket-creation-2.png new file mode 100644 index 0000000000000..a95885bc46858 Binary files /dev/null and b/media/tidb-cloud/connected-slack-ticket-creation-2.png differ diff --git a/media/tidb-cloud/connected-slack-ticket-creation-3.png b/media/tidb-cloud/connected-slack-ticket-creation-3.png new file mode 100644 index 0000000000000..3b66915f287f0 Binary files /dev/null and b/media/tidb-cloud/connected-slack-ticket-creation-3.png differ diff --git a/media/tidb-cloud/connected-slack-ticket-creation-4.png b/media/tidb-cloud/connected-slack-ticket-creation-4.png new file mode 100644 index 0000000000000..c32788f962b26 Binary files /dev/null and b/media/tidb-cloud/connected-slack-ticket-creation-4.png differ diff --git a/media/tidb-cloud/connected-slack-ticket-creation-5.png b/media/tidb-cloud/connected-slack-ticket-creation-5.png new file mode 100644 index 0000000000000..289e57a9d6ed6 Binary files /dev/null and b/media/tidb-cloud/connected-slack-ticket-creation-5.png differ diff --git a/media/tidb-cloud/connected-slack-ticket-interaction-1.png b/media/tidb-cloud/connected-slack-ticket-interaction-1.png new file mode 100644 index 0000000000000..eac521f40fae9 Binary files /dev/null and b/media/tidb-cloud/connected-slack-ticket-interaction-1.png differ diff --git a/media/tidb-cloud/connected-slack-ticket-interaction-2.png b/media/tidb-cloud/connected-slack-ticket-interaction-2.png new file mode 100644 index 0000000000000..b5421128d59fb Binary files /dev/null and b/media/tidb-cloud/connected-slack-ticket-interaction-2.png differ diff --git a/media/tidb-cloud/connected-slack-ticket-interaction-3.png b/media/tidb-cloud/connected-slack-ticket-interaction-3.png new file mode 100644 index 0000000000000..58e508bc93612 Binary files /dev/null and b/media/tidb-cloud/connected-slack-ticket-interaction-3.png differ diff --git a/media/tidb-cloud/connected-slack-ticket-interaction-4.png b/media/tidb-cloud/connected-slack-ticket-interaction-4.png new file mode 100644 index 0000000000000..16f6b648388e0 Binary files /dev/null and b/media/tidb-cloud/connected-slack-ticket-interaction-4.png differ diff --git a/media/tidb-cloud/connected-slack-ticket-interaction-5.png b/media/tidb-cloud/connected-slack-ticket-interaction-5.png new file mode 100644 index 0000000000000..f4bf250ece6fd Binary files /dev/null and b/media/tidb-cloud/connected-slack-ticket-interaction-5.png differ diff --git a/media/tidb-cloud/connected-slack-ticket-interaction-6.png b/media/tidb-cloud/connected-slack-ticket-interaction-6.png new file mode 100644 index 0000000000000..74daf12c3b07d Binary files /dev/null and b/media/tidb-cloud/connected-slack-ticket-interaction-6.png differ diff --git a/media/tidb-cloud/connected-slack-ticket-interaction-7.png b/media/tidb-cloud/connected-slack-ticket-interaction-7.png new file mode 100644 index 0000000000000..9197e6b8527f2 Binary files /dev/null and b/media/tidb-cloud/connected-slack-ticket-interaction-7.png differ diff --git a/media/tidb-cloud/connected-slack-ticket-interaction-8.png b/media/tidb-cloud/connected-slack-ticket-interaction-8.png new file mode 100644 index 0000000000000..3826a5d4d3369 Binary files /dev/null and b/media/tidb-cloud/connected-slack-ticket-interaction-8.png differ diff --git a/media/tidb-cloud/data-service/GPTs1.png b/media/tidb-cloud/data-service/GPTs1.png index ba56ed8126c1d..b9b5d51ebf44d 100644 Binary files a/media/tidb-cloud/data-service/GPTs1.png and b/media/tidb-cloud/data-service/GPTs1.png differ diff --git a/media/tidb-cloud/implementation-of-tidb-node-group.png b/media/tidb-cloud/implementation-of-tidb-node-group.png new file mode 100644 index 0000000000000..c29f3862550d3 Binary files /dev/null and b/media/tidb-cloud/implementation-of-tidb-node-group.png differ diff --git a/media/tidb-cloud/import-data-csv-config.png b/media/tidb-cloud/import-data-csv-config.png index bf469faa0112f..0f769a5d3aa17 100644 Binary files a/media/tidb-cloud/import-data-csv-config.png and b/media/tidb-cloud/import-data-csv-config.png differ diff --git a/media/tidb-cloud/private-endpoint/create-endpoint-2.png b/media/tidb-cloud/private-endpoint/create-endpoint-2.png index 3bf524416f965..5f6ae32c30cb7 100644 Binary files a/media/tidb-cloud/private-endpoint/create-endpoint-2.png and b/media/tidb-cloud/private-endpoint/create-endpoint-2.png differ diff --git a/media/tidb-cloud/recovery-group/recovery-group-failover.png b/media/tidb-cloud/recovery-group/recovery-group-failover.png new file mode 100644 index 0000000000000..ff2c15c37a520 Binary files /dev/null and b/media/tidb-cloud/recovery-group/recovery-group-failover.png differ diff --git a/media/tidb-cloud/recovery-group/recovery-group-overview.png b/media/tidb-cloud/recovery-group/recovery-group-overview.png new file mode 100644 index 0000000000000..fe97b0067ce39 Binary files /dev/null and b/media/tidb-cloud/recovery-group/recovery-group-overview.png differ diff --git a/media/tidb-cloud/recovery-group/recovery-group-protected.png b/media/tidb-cloud/recovery-group/recovery-group-protected.png new file mode 100644 index 0000000000000..5e44702cb732e Binary files /dev/null and b/media/tidb-cloud/recovery-group/recovery-group-protected.png differ diff --git a/media/tidb-cloud/recovery-group/recovery-group-reprotected.png b/media/tidb-cloud/recovery-group/recovery-group-reprotected.png new file mode 100644 index 0000000000000..7d1afe53ed384 Binary files /dev/null and b/media/tidb-cloud/recovery-group/recovery-group-reprotected.png differ diff --git a/media/tidb-cloud/recovery-group/recovery-group-unprotected.png b/media/tidb-cloud/recovery-group/recovery-group-unprotected.png new file mode 100644 index 0000000000000..9597a81be24a5 Binary files /dev/null and b/media/tidb-cloud/recovery-group/recovery-group-unprotected.png differ diff --git a/media/tidb-cloud/serverless-external-storage/azure-sas-create.png b/media/tidb-cloud/serverless-external-storage/azure-sas-create.png new file mode 100644 index 0000000000000..f8c50e42359d9 Binary files /dev/null and b/media/tidb-cloud/serverless-external-storage/azure-sas-create.png differ diff --git a/media/tidb-cloud/serverless-external-storage/azure-sas-position.png b/media/tidb-cloud/serverless-external-storage/azure-sas-position.png new file mode 100644 index 0000000000000..01fbb2edaf7b0 Binary files /dev/null and b/media/tidb-cloud/serverless-external-storage/azure-sas-position.png differ diff --git a/media/tidb-cloud/serverless-external-storage/azure-storage-account-overview.png b/media/tidb-cloud/serverless-external-storage/azure-storage-account-overview.png new file mode 100644 index 0000000000000..a12f85adee548 Binary files /dev/null and b/media/tidb-cloud/serverless-external-storage/azure-storage-account-overview.png differ diff --git a/media/tidb-cloud/serverless-external-storage/gcs-service-account-key.png b/media/tidb-cloud/serverless-external-storage/gcs-service-account-key.png new file mode 100644 index 0000000000000..655c502fbab4b Binary files /dev/null and b/media/tidb-cloud/serverless-external-storage/gcs-service-account-key.png differ diff --git a/media/tidb-cloud/serverless-external-storage/gcs-service-account.png b/media/tidb-cloud/serverless-external-storage/gcs-service-account.png new file mode 100644 index 0000000000000..7ab07dc7c8885 Binary files /dev/null and b/media/tidb-cloud/serverless-external-storage/gcs-service-account.png differ diff --git a/media/tidb-cloud/serverless-external-storage/serverless-role-arn.png b/media/tidb-cloud/serverless-external-storage/serverless-role-arn.png new file mode 100644 index 0000000000000..509f1bf06940d Binary files /dev/null and b/media/tidb-cloud/serverless-external-storage/serverless-role-arn.png differ diff --git a/media/tidb-cloud/serverless-regional-high-avaliability-aws.png b/media/tidb-cloud/serverless-regional-high-avaliability-aws.png new file mode 100644 index 0000000000000..368373b69bef2 Binary files /dev/null and b/media/tidb-cloud/serverless-regional-high-avaliability-aws.png differ diff --git a/media/tidb-cloud/serverless-zonal-high-avaliability-aws.png b/media/tidb-cloud/serverless-zonal-high-avaliability-aws.png new file mode 100644 index 0000000000000..2a2d5739640c6 Binary files /dev/null and b/media/tidb-cloud/serverless-zonal-high-avaliability-aws.png differ diff --git a/media/tidb-cloud/tidb-cloud-alert-subscription.png b/media/tidb-cloud/tidb-cloud-alert-subscription.png new file mode 100644 index 0000000000000..6dfaa96d269c5 Binary files /dev/null and b/media/tidb-cloud/tidb-cloud-alert-subscription.png differ diff --git a/media/tidb-cloud/tidb-cloud-clinic.png b/media/tidb-cloud/tidb-cloud-clinic.png new file mode 100644 index 0000000000000..6b7944062c0db Binary files /dev/null and b/media/tidb-cloud/tidb-cloud-clinic.png differ diff --git a/media/tidb-cloud/tidb-cloud-dedicated-architecture.png b/media/tidb-cloud/tidb-cloud-dedicated-architecture.png new file mode 100644 index 0000000000000..af2fb05877d57 Binary files /dev/null and b/media/tidb-cloud/tidb-cloud-dedicated-architecture.png differ diff --git a/media/tidb-cloud/tidb-node-group-change-name.png b/media/tidb-cloud/tidb-node-group-change-name.png new file mode 100644 index 0000000000000..9c21096892ab2 Binary files /dev/null and b/media/tidb-cloud/tidb-node-group-change-name.png differ diff --git a/media/tidb-cloud/tidb-node-group-change-node-count.png b/media/tidb-cloud/tidb-node-group-change-node-count.png new file mode 100644 index 0000000000000..bac403e79b36c Binary files /dev/null and b/media/tidb-cloud/tidb-node-group-change-node-count.png differ diff --git a/media/tidb-cloud/tidb-node-group-connect-public-endpoint.png b/media/tidb-cloud/tidb-node-group-connect-public-endpoint.png new file mode 100644 index 0000000000000..ce32c851eff07 Binary files /dev/null and b/media/tidb-cloud/tidb-node-group-connect-public-endpoint.png differ diff --git a/media/tidb-cloud/tidb-node-group-create.png b/media/tidb-cloud/tidb-node-group-create.png new file mode 100644 index 0000000000000..071f1bc1c4a8a Binary files /dev/null and b/media/tidb-cloud/tidb-node-group-create.png differ diff --git a/media/tidb-cloud/tidb-node-group-delete.png b/media/tidb-cloud/tidb-node-group-delete.png new file mode 100644 index 0000000000000..79ca9a39e62c8 Binary files /dev/null and b/media/tidb-cloud/tidb-node-group-delete.png differ diff --git a/media/tidb-cloud/v.7.1.0-oltp_read_write.png b/media/tidb-cloud/v.7.1.0-oltp_read_write.png deleted file mode 100644 index 0b56a59afcac0..0000000000000 Binary files a/media/tidb-cloud/v.7.1.0-oltp_read_write.png and /dev/null differ diff --git a/media/tidb-cloud/v6.5.6-oltp_insert.png b/media/tidb-cloud/v6.5.6-oltp_insert.png new file mode 100644 index 0000000000000..0ee03639ce2bf Binary files /dev/null and b/media/tidb-cloud/v6.5.6-oltp_insert.png differ diff --git a/media/tidb-cloud/v6.5.6-oltp_read_write.png b/media/tidb-cloud/v6.5.6-oltp_read_write.png new file mode 100644 index 0000000000000..82e5ab9075603 Binary files /dev/null and b/media/tidb-cloud/v6.5.6-oltp_read_write.png differ diff --git a/media/tidb-cloud/v6.5.6-oltp_select_point.png b/media/tidb-cloud/v6.5.6-oltp_select_point.png new file mode 100644 index 0000000000000..19496cb0e42b0 Binary files /dev/null and b/media/tidb-cloud/v6.5.6-oltp_select_point.png differ diff --git a/media/tidb-cloud/v6.5.6-oltp_update_index.png b/media/tidb-cloud/v6.5.6-oltp_update_index.png new file mode 100644 index 0000000000000..9cd6ab54f7cc9 Binary files /dev/null and b/media/tidb-cloud/v6.5.6-oltp_update_index.png differ diff --git a/media/tidb-cloud/v6.5.6-oltp_update_non_index.png b/media/tidb-cloud/v6.5.6-oltp_update_non_index.png new file mode 100644 index 0000000000000..235529282bc2a Binary files /dev/null and b/media/tidb-cloud/v6.5.6-oltp_update_non_index.png differ diff --git a/media/tidb-cloud/v6.5.6-tpmC.png b/media/tidb-cloud/v6.5.6-tpmC.png new file mode 100644 index 0000000000000..d454c4492ed21 Binary files /dev/null and b/media/tidb-cloud/v6.5.6-tpmC.png differ diff --git a/media/tidb-cloud/v7.1.3-oltp_insert.png b/media/tidb-cloud/v7.1.3-oltp_insert.png new file mode 100644 index 0000000000000..b98251fbae658 Binary files /dev/null and b/media/tidb-cloud/v7.1.3-oltp_insert.png differ diff --git a/media/tidb-cloud/v7.1.3-oltp_read_write.png b/media/tidb-cloud/v7.1.3-oltp_read_write.png new file mode 100644 index 0000000000000..1d21b6573d51b Binary files /dev/null and b/media/tidb-cloud/v7.1.3-oltp_read_write.png differ diff --git a/media/tidb-cloud/v7.1.3-oltp_select_point.png b/media/tidb-cloud/v7.1.3-oltp_select_point.png new file mode 100644 index 0000000000000..5d619973bdcd0 Binary files /dev/null and b/media/tidb-cloud/v7.1.3-oltp_select_point.png differ diff --git a/media/tidb-cloud/v7.1.3-oltp_update_index.png b/media/tidb-cloud/v7.1.3-oltp_update_index.png new file mode 100644 index 0000000000000..2ed956c736dde Binary files /dev/null and b/media/tidb-cloud/v7.1.3-oltp_update_index.png differ diff --git a/media/tidb-cloud/v7.1.3-oltp_update_non_index.png b/media/tidb-cloud/v7.1.3-oltp_update_non_index.png new file mode 100644 index 0000000000000..74aa87acdecdd Binary files /dev/null and b/media/tidb-cloud/v7.1.3-oltp_update_non_index.png differ diff --git a/media/tidb-cloud/v7.1.3-tpmC.png b/media/tidb-cloud/v7.1.3-tpmC.png new file mode 100644 index 0000000000000..d8bf49e923c69 Binary files /dev/null and b/media/tidb-cloud/v7.1.3-tpmC.png differ diff --git a/media/tidb-cloud/v7.5.0_oltp_insert.png b/media/tidb-cloud/v7.5.0-oltp_insert.png similarity index 100% rename from media/tidb-cloud/v7.5.0_oltp_insert.png rename to media/tidb-cloud/v7.5.0-oltp_insert.png diff --git a/media/tidb-cloud/v7.5.0-oltp_point_select.png b/media/tidb-cloud/v7.5.0-oltp_point_select.png new file mode 100644 index 0000000000000..12c34021f206b Binary files /dev/null and b/media/tidb-cloud/v7.5.0-oltp_point_select.png differ diff --git a/media/tidb-cloud/v7.5.0-oltp_read_write.png b/media/tidb-cloud/v7.5.0-oltp_read_write.png new file mode 100644 index 0000000000000..b423451bd21ff Binary files /dev/null and b/media/tidb-cloud/v7.5.0-oltp_read_write.png differ diff --git a/media/tidb-cloud/v7.5.0_oltp_update_index.png b/media/tidb-cloud/v7.5.0-oltp_update_index.png similarity index 100% rename from media/tidb-cloud/v7.5.0_oltp_update_index.png rename to media/tidb-cloud/v7.5.0-oltp_update_index.png diff --git a/media/tidb-cloud/v7.5.0_oltp_update_non_index.png b/media/tidb-cloud/v7.5.0-oltp_update_non_index.png similarity index 100% rename from media/tidb-cloud/v7.5.0_oltp_update_non_index.png rename to media/tidb-cloud/v7.5.0-oltp_update_non_index.png diff --git a/media/tidb-cloud/v7.5.0_oltp_point_select.png b/media/tidb-cloud/v7.5.0_oltp_point_select.png deleted file mode 100644 index f3f59e7c85321..0000000000000 Binary files a/media/tidb-cloud/v7.5.0_oltp_point_select.png and /dev/null differ diff --git a/media/tidb-cloud/v7.5.0_oltp_read_write.png b/media/tidb-cloud/v7.5.0_oltp_read_write.png deleted file mode 100644 index d796eadc62276..0000000000000 Binary files a/media/tidb-cloud/v7.5.0_oltp_read_write.png and /dev/null differ diff --git a/media/tidb-cloud/v7.5.0_tpcc.png b/media/tidb-cloud/v7.5.0_tpcc.png index 9078c9178e003..26089ecb4408c 100644 Binary files a/media/tidb-cloud/v7.5.0_tpcc.png and b/media/tidb-cloud/v7.5.0_tpcc.png differ diff --git a/media/tidb-cloud/v8.1.0_oltp_insert.png b/media/tidb-cloud/v8.1.0_oltp_insert.png new file mode 100644 index 0000000000000..fe0b422966171 Binary files /dev/null and b/media/tidb-cloud/v8.1.0_oltp_insert.png differ diff --git a/media/tidb-cloud/v8.1.0_oltp_point_select.png b/media/tidb-cloud/v8.1.0_oltp_point_select.png new file mode 100644 index 0000000000000..a1eb7e3029d76 Binary files /dev/null and b/media/tidb-cloud/v8.1.0_oltp_point_select.png differ diff --git a/media/tidb-cloud/v8.1.0_oltp_read_write.png b/media/tidb-cloud/v8.1.0_oltp_read_write.png new file mode 100644 index 0000000000000..75e9278d1f290 Binary files /dev/null and b/media/tidb-cloud/v8.1.0_oltp_read_write.png differ diff --git a/media/tidb-cloud/v8.1.0_oltp_update_index.png b/media/tidb-cloud/v8.1.0_oltp_update_index.png new file mode 100644 index 0000000000000..b5d58fe531600 Binary files /dev/null and b/media/tidb-cloud/v8.1.0_oltp_update_index.png differ diff --git a/media/tidb-cloud/v8.1.0_oltp_update_non_index.png b/media/tidb-cloud/v8.1.0_oltp_update_non_index.png new file mode 100644 index 0000000000000..b8e760933506d Binary files /dev/null and b/media/tidb-cloud/v8.1.0_oltp_update_non_index.png differ diff --git a/media/tidb-cloud/v8.1.0_tpcc.png b/media/tidb-cloud/v8.1.0_tpcc.png new file mode 100644 index 0000000000000..10eb8ef603758 Binary files /dev/null and b/media/tidb-cloud/v8.1.0_tpcc.png differ diff --git a/media/tidb-cloud/v8.5.0_oltp_insert.png b/media/tidb-cloud/v8.5.0_oltp_insert.png new file mode 100644 index 0000000000000..8f3b77ab5d99d Binary files /dev/null and b/media/tidb-cloud/v8.5.0_oltp_insert.png differ diff --git a/media/tidb-cloud/v8.5.0_oltp_point_select.png b/media/tidb-cloud/v8.5.0_oltp_point_select.png new file mode 100644 index 0000000000000..719d9f80c2ab3 Binary files /dev/null and b/media/tidb-cloud/v8.5.0_oltp_point_select.png differ diff --git a/media/tidb-cloud/v8.5.0_oltp_read_write.png b/media/tidb-cloud/v8.5.0_oltp_read_write.png new file mode 100644 index 0000000000000..bc3ce8441006d Binary files /dev/null and b/media/tidb-cloud/v8.5.0_oltp_read_write.png differ diff --git a/media/tidb-cloud/v8.5.0_oltp_update_index.png b/media/tidb-cloud/v8.5.0_oltp_update_index.png new file mode 100644 index 0000000000000..4b22348748cdc Binary files /dev/null and b/media/tidb-cloud/v8.5.0_oltp_update_index.png differ diff --git a/media/tidb-cloud/v8.5.0_oltp_update_non_index.png b/media/tidb-cloud/v8.5.0_oltp_update_non_index.png new file mode 100644 index 0000000000000..9dcfb8ff2da16 Binary files /dev/null and b/media/tidb-cloud/v8.5.0_oltp_update_non_index.png differ diff --git a/media/tidb-cloud/v8.5.0_tpcc.png b/media/tidb-cloud/v8.5.0_tpcc.png new file mode 100644 index 0000000000000..a664cda6c3568 Binary files /dev/null and b/media/tidb-cloud/v8.5.0_tpcc.png differ diff --git a/media/tiflash/tiflash_mintso_v1.png b/media/tiflash/tiflash_mintso_v1.png new file mode 100644 index 0000000000000..964dcc6daacfe Binary files /dev/null and b/media/tiflash/tiflash_mintso_v1.png differ diff --git a/media/tiflash/tiflash_mintso_v2.png b/media/tiflash/tiflash_mintso_v2.png new file mode 100644 index 0000000000000..28d49cf7ab866 Binary files /dev/null and b/media/tiflash/tiflash_mintso_v2.png differ diff --git a/media/vector-search/embedding-search.png b/media/vector-search/embedding-search.png new file mode 100644 index 0000000000000..0035ada1b7479 Binary files /dev/null and b/media/vector-search/embedding-search.png differ diff --git a/media/vector-search/hybrid-search-overview.svg b/media/vector-search/hybrid-search-overview.svg new file mode 100644 index 0000000000000..478aa3b2bed26 --- /dev/null +++ b/media/vector-search/hybrid-search-overview.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/metadata-lock.md b/metadata-lock.md index fbf663a184813..c16175a227a82 100644 --- a/metadata-lock.md +++ b/metadata-lock.md @@ -64,7 +64,7 @@ Starting from v6.5.0, TiDB enables metadata lock by default. When you upgrade yo | `COMMIT;` | | | `BEGIN;` | | | | `ALTER TABLE t MODIFY COLUMN a CHAR(10);` | - | `SELECT * FROM t;` (returns `Error 8028: Information schema is changed`) | | + | `SELECT * FROM t;` (returns `ERROR 8028 (HY000): public column a has changed`) | | ## Observability diff --git a/metrics-schema.md b/metrics-schema.md index 8462745adb995..3a8b19fdf892c 100644 --- a/metrics-schema.md +++ b/metrics-schema.md @@ -1,7 +1,6 @@ --- title: Metrics Schema summary: Learn the `METRICS_SCHEMA` schema. -aliases: ['/docs/dev/system-tables/system-table-metrics-schema/','/docs/dev/reference/system-databases/metrics-schema/','/tidb/dev/system-table-metrics-schema/'] --- # Metrics Schema diff --git a/migrate-aurora-to-tidb.md b/migrate-aurora-to-tidb.md index cb6de953e2d7b..12cbc69b97a58 100644 --- a/migrate-aurora-to-tidb.md +++ b/migrate-aurora-to-tidb.md @@ -1,7 +1,6 @@ --- title: Migrate Data from Amazon Aurora to TiDB summary: Learn how to migrate data from Amazon Aurora to TiDB using DB snapshot. -aliases: ['/tidb/dev/migrate-from-aurora-using-lightning','/docs/dev/migrate-from-aurora-mysql-database/','/docs/dev/how-to/migrate/from-mysql-aurora/','/docs/dev/how-to/migrate/from-aurora/', '/tidb/dev/migrate-from-aurora-mysql-database/', '/tidb/dev/migrate-from-mysql-aurora/','/tidb/stable/migrate-from-aurora-using-lightning/'] --- # Migrate Data from Amazon Aurora to TiDB @@ -51,7 +50,7 @@ Create a new `tidb-lightning-schema.toml` file, copy the following content into # The target TiDB cluster information. host = ${host} port = ${port} -user = "${user_name} +user = "${user_name}" password = "${password}" status-port = ${status-port} # The TiDB status port. Usually the port is 10080. pd-addr = "${ip}:${port}" # The cluster PD address. Usually the port is 2379. @@ -120,7 +119,7 @@ Create a new `tidb-lightning-data.toml` configuration file, copy the following c # The target TiDB cluster information. host = ${host} port = ${port} -user = "${user_name} +user = "${user_name}" password = "${password}" status-port = ${status-port} # The TiDB status port. Usually the port is 10080. pd-addr = "${ip}:${port}" # The cluster PD address. Usually the port is 2379. diff --git a/migrate-from-csv-files-to-tidb.md b/migrate-from-csv-files-to-tidb.md index 38b8afa009f05..5a1f01526f83e 100644 --- a/migrate-from-csv-files-to-tidb.md +++ b/migrate-from-csv-files-to-tidb.md @@ -1,7 +1,6 @@ --- title: Migrate Data from CSV Files to TiDB summary: Learn how to migrate data from CSV files to TiDB. -aliases: ['/docs/dev/tidb-lightning/migrate-from-csv-using-tidb-lightning/','/docs/dev/reference/tools/tidb-lightning/csv/','/tidb/dev/migrate-from-csv-using-tidb-lightning'] --- # Migrate Data from CSV Files to TiDB @@ -22,6 +21,8 @@ Put all the CSV files in the same directory. If you need TiDB Lightning to recog - If a CSV file contains the data for an entire table, name the file `${db_name}.${table_name}.csv`. - If the data of one table is separated into multiple CSV files, append a numeric suffix to these CSV files. For example, `${db_name}.${table_name}.003.csv`. The numeric suffixes can be inconsecutive but must be in ascending order. You also need to add extra zeros before the number to ensure all the suffixes are in the same length. +TiDB Lightning recursively searches for all `.csv` files in this directory and its subdirectories. + ## Step 2. Create the target table schema Because CSV files do not contain schema information, before importing data from CSV files into TiDB, you need to create the target table schema. You can create the target table schema by either of the following two methods: @@ -63,6 +64,8 @@ data-source-dir = "${data-path}" # A local path or S3 path. For example, 's3://m separator = ',' # Delimiter. Can be zero or multiple characters. delimiter = '"' +# Line terminator. By default, \r, \n, and \r\n are all treated as line terminators. +# terminator = "\r\n" # Configures whether the CSV file has a table header. # If this item is set to true, TiDB Lightning uses the first line of the CSV file to parse the corresponding relationship of fields. header = true @@ -104,6 +107,8 @@ In a strict-format CSV file, each field only takes up one line. It must meet the - The delimiter is empty. - Each field does not contain CR (`\r`) or LF (`\n`). +You need to explicitly specify the line terminator `terminator` for a `strict-format` CSV file. + If your CSV file meets the above requirements, you can speed up the import by enabling the `strict-format` mode as follows: ```toml diff --git a/migrate-from-sql-files-to-tidb.md b/migrate-from-sql-files-to-tidb.md index 877656adbeb24..1f43d9331d68d 100644 --- a/migrate-from-sql-files-to-tidb.md +++ b/migrate-from-sql-files-to-tidb.md @@ -1,7 +1,6 @@ --- title: Migrate Data from SQL Files to TiDB summary: Learn how to migrate data from SQL files to TiDB. -aliases: ['/docs/dev/migrate-from-mysql-mydumper-files/','/tidb/dev/migrate-from-mysql-mydumper-files/','/tidb/dev/migrate-from-mysql-dumpling-files'] --- # Migrate Data from SQL Files to TiDB diff --git a/migrate-from-tidb-to-mysql.md b/migrate-from-tidb-to-mysql.md index 716b167ae18d1..a2c35105e7484 100644 --- a/migrate-from-tidb-to-mysql.md +++ b/migrate-from-tidb-to-mysql.md @@ -56,7 +56,7 @@ After setting up the environment, you can use [Dumpling](/dumpling-overview.md) 1. Disable Garbage Collection (GC). - To ensure that newly written data is not deleted during incremental migration, you should disable GC for the upstream cluster before exporting full data. In this way, history data is not deleted. For TiDB v4.0.0 and later versions, Dumpling might [automatically extend the GC time](/dumpling-overview.md#manually-set-the-tidb-gc-time). Nevertheless, manually disabling GC is still necessary because the GC process might begin after Dumpling exits, leading to the failure of incremental changes migration. + To ensure that newly written data is not deleted during incremental migration, you should disable GC for the upstream cluster before exporting full data. In this way, history data is not deleted. For TiDB v4.0.0 and later versions, Dumpling might [automatically adjust the GC safe point to block GC](/dumpling-overview.md#manually-set-the-tidb-gc-time). Nevertheless, manually disabling GC is still necessary because the GC process might begin after Dumpling exits, leading to the failure of incremental changes migration. Run the following command to disable GC: @@ -75,7 +75,7 @@ After setting up the environment, you can use [Dumpling](/dumpling-overview.md) ``` ``` - +-------------------------+： + +-------------------------+ | @@global.tidb_gc_enable | +-------------------------+ | 0 | diff --git a/migrate-from-tidb-to-tidb.md b/migrate-from-tidb-to-tidb.md index 3e3c0de72d67d..e6eb255c31655 100644 --- a/migrate-from-tidb-to-tidb.md +++ b/migrate-from-tidb-to-tidb.md @@ -99,7 +99,7 @@ This document exemplifies the whole migration process and contains the following ## Step 2. Migrate full data -After setting up the environment, you can use the backup and restore functions of [BR](https://github.com/pingcap/tidb/tree/master/br) to migrate full data. BR can be started in [three ways](/br/br-use-overview.md#deploy-and-use-br). In this document, we use the SQL statements, `BACKUP` and `RESTORE`. +After setting up the environment, you can use the backup and restore functions of [BR](https://github.com/pingcap/tidb/tree/release-8.1/br) to migrate full data. BR can be started in [three ways](/br/br-use-overview.md#deploy-and-use-br). In this document, we use the SQL statements, `BACKUP` and `RESTORE`. > **Note:** > @@ -136,6 +136,10 @@ After setting up the environment, you can use the backup and restore functions o 1 row in set (0.00 sec) ``` + > **Note:** + > + > TiCDC `gc-ttl` is 24 hours by default. If the backup and restore takes a long time, the default `gc-ttl` might not be sufficient, which could cause the subsequent [incremental replication task](#step-3-migrate-incremental-data) to fail. To avoid this situation, adjust the `gc-ttl` value according to your specific needs when starting the TiCDC server. For more information, see [What is `gc-ttl` in TiCDC](/ticdc/ticdc-faq.md#what-is-gc-ttl-in-ticdc). + 2. Back up data. Run the `BACKUP` statement in the upstream cluster to back up data: diff --git a/migrate-large-mysql-shards-to-tidb.md b/migrate-large-mysql-shards-to-tidb.md index b5f3513e322f6..887c06776f550 100644 --- a/migrate-large-mysql-shards-to-tidb.md +++ b/migrate-large-mysql-shards-to-tidb.md @@ -281,7 +281,7 @@ name: task-test # The name of the task. Should be globally unique. task-mode: incremental # The mode of the task. "incremental" means full data migration is skipped and only incremental replication is performed. # Required for incremental replication from sharded tables. By default, the "pessimistic" mode is used. # If you have a deep understanding of the principles and usage limitations of the optimistic mode, you can also use the "optimistic" mode. -# For more information, see [Merge and Migrate Data from Sharded Tables](https://docs.pingcap.com/zh/tidb/dev/feature-shard-merge/). +# For more information, see [Merge and Migrate Data from Sharded Tables](https://docs.pingcap.com/tidb/dev/feature-shard-merge/). shard-mode: "pessimistic" diff --git a/migrate-small-mysql-shards-to-tidb.md b/migrate-small-mysql-shards-to-tidb.md index 36a08188b0057..44a3c9719612a 100644 --- a/migrate-small-mysql-shards-to-tidb.md +++ b/migrate-small-mysql-shards-to-tidb.md @@ -1,7 +1,6 @@ --- title: Migrate and Merge MySQL Shards of Small Datasets to TiDB summary: Learn how to migrate and merge small datasets of shards from MySQL to TiDB. -aliases: ['/tidb/dev/usage-scenario-shard-merge/', '/tidb/dev/usage-scenario-simple-migration/'] --- # Migrate and Merge MySQL Shards of Small Datasets to TiDB diff --git a/migrate-small-mysql-to-tidb.md b/migrate-small-mysql-to-tidb.md index 01c8731560ee2..23078f98c4245 100644 --- a/migrate-small-mysql-to-tidb.md +++ b/migrate-small-mysql-to-tidb.md @@ -1,7 +1,6 @@ --- title: Migrate Small Datasets from MySQL to TiDB summary: Learn how to migrate small datasets from MySQL to TiDB. -aliases: ['/tidb/dev/usage-scenario-incremental-migration/'] --- # Migrate Small Datasets from MySQL to TiDB diff --git a/migrate-with-more-columns-downstream.md b/migrate-with-more-columns-downstream.md index 72130adf3cc56..10fd50c2f84de 100644 --- a/migrate-with-more-columns-downstream.md +++ b/migrate-with-more-columns-downstream.md @@ -1,7 +1,6 @@ --- title: Migrate Data to a Downstream TiDB Table with More Columns summary: Learn how to migrate data to a downstream TiDB table with more columns than the corresponding upstream table. -aliases: ['/tidb/dev/usage-scenario-downstream-more-columns/'] --- # Migrate Data to a Downstream TiDB Table with More Columns @@ -107,5 +106,5 @@ In such cases, you can use the `binlog-schema` command to set a table schema for {{< copyable "shell-regular" >}} ``` - tiup dmctl --master-addr ${advertise-addr} query-status resume-task ${task-name} + tiup dmctl --master-addr ${advertise-addr} query-status ${task-name} ``` diff --git a/migration-overview.md b/migration-overview.md index 326eced1fe5c1..e267af83baf3d 100644 --- a/migration-overview.md +++ b/migration-overview.md @@ -1,6 +1,7 @@ --- title: Data Migration Overview summary: Learn the overview of data migration scenarios and the solutions. +aliases: ['/tidb/stable/migrate-full-data-from-mysql'] --- # Data Migration Overview diff --git a/migration-tools.md b/migration-tools.md index 104346177e4e8..cc90a20025046 100644 --- a/migration-tools.md +++ b/migration-tools.md @@ -29,7 +29,7 @@ This document introduces the user scenarios, supported upstreams and downstreams - **User scenario**: Full data import into TiDB - **Upstream (the imported source file)**: - Files exported from Dumpling - - Parquet files exported by Amazon Aurora or Apache Hive + - Parquet files exported by Amazon Aurora, Apache Hive, and Snowflake - CSV files - Data from local disks or Amazon S3 - **Downstream**: TiDB @@ -49,7 +49,7 @@ This document introduces the user scenarios, supported upstreams and downstreams - **Advantages**: - Support the table-filter feature that enables you to filter data easier - Support exporting data to Amazon S3 -- **Limitation**: +- **Recommendation**: - If you want to restore the exported data to a database other than TiDB, it is recommended to use Dumpling. - If you want to restore the exported data to another TiDB cluster, it is recommended to use Backup & Restore (BR). @@ -103,38 +103,49 @@ source ~/.bash_profile ### Step 2. Install components -You can use the following command to see all the available components: +You can use the following command to see the available components: ```shell tiup list ``` -The command output lists all the available components: +The command outputs the available components: ```bash Available components: -Name Owner Description ----- ----- ----------- -bench pingcap Benchmark database with different workloads -br pingcap TiDB/TiKV cluster backup restore tool -cdc pingcap CDC is a change data capture tool for TiDB -client pingcap Client to connect playground -cluster pingcap Deploy a TiDB cluster for production -ctl pingcap TiDB controller suite -dm pingcap Data Migration Platform manager -dmctl pingcap dmctl component of Data Migration Platform -errdoc pingcap Document about TiDB errors -pd-recover pingcap PD Recover is a disaster recovery tool of PD, used to recover the PD cluster which cannot start or provide services normally -playground pingcap Bootstrap a local TiDB cluster for fun -tidb pingcap TiDB is an open source distributed HTAP database compatible with the MySQL protocol -tidb-lightning pingcap TiDB Lightning is a tool used for fast full import of large amounts of data into a TiDB cluster -tiup pingcap TiUP is a command-line component management tool that can help to download and install TiDB platform components to the local system +Name Owner Description +---- ----- ----------- +PCC community A tool used to capture plan changes among different versions of TiDB +bench pingcap Benchmark database with different workloads +br pingcap TiDB/TiKV cluster backup restore tool. +cdc pingcap CDC is a change data capture tool for TiDB +chaosd community An easy-to-use Chaos Engineering tool used to inject failures to a physical node +client pingcap Client to connect playground +cloud pingcap CLI tool to manage TiDB Cloud +cluster pingcap Deploy a TiDB cluster for production +ctl pingcap TiDB controller suite +dm pingcap Data Migration Platform manager +dmctl pingcap dmctl component of Data Migration Platform. +errdoc pingcap Document about TiDB errors +pd-recover pingcap PD Recover is a disaster recovery tool of PD, used to recover the PD cluster which cannot start or provide services normally. +playground pingcap Bootstrap a local TiDB cluster for fun +tidb pingcap TiDB is an open source distributed HTAP database compatible with the MySQL protocol. +tidb-dashboard pingcap TiDB Dashboard is a Web UI for monitoring, diagnosing, and managing the TiDB cluster +tidb-lightning pingcap TiDB Lightning is a tool used for fast full import of large amounts of data into a TiDB cluster +tikv-br pingcap TiKV cluster backup restore tool +tikv-cdc pingcap TiKV-CDC is a change data capture tool for TiKV +tiproxy pingcap TiProxy is a database proxy that is based on TiDB. +tiup pingcap TiUP is a command-line component management tool that can help to download and install TiDB platform components to the local system ``` -Choose the components to install: +Install one or more components that you need. For example: + +```shell +tiup install dm +``` ```shell -tiup install dumpling tidb-lightning +tiup install dm tidb-lightning ``` > **Note:** diff --git a/minimal-deployment-topology.md b/minimal-deployment-topology.md index 986ec1bde8050..8adc020042613 100644 --- a/minimal-deployment-topology.md +++ b/minimal-deployment-topology.md @@ -1,7 +1,6 @@ --- title: Minimal Deployment Topology summary: Learn the minimal deployment topology of TiDB clusters. -aliases: ['/docs/dev/minimal-deployment-topology/'] --- # Minimal Deployment Topology @@ -17,6 +16,10 @@ This document describes the minimal deployment topology of TiDB clusters. | TiKV | 3 | 16 VCore 32 GiB
2 TiB (NVMe SSD) for storage | 10.0.1.7
10.0.1.8
10.0.1.9 | Default port
Global directory configuration | | Monitoring & Grafana | 1 | 4 VCore 8 GiB
500 GiB (SSD) for storage | 10.0.1.10 | Default port
Global directory configuration | +> **Note:** +> +> The IP addresses of the instances are given as examples only. In your actual deployment, replace the IP addresses with your actual IP addresses. + ### Topology templates - [The simple template for the minimal topology](https://github.com/pingcap/docs/blob/master/config-templates/simple-mini.yaml) diff --git a/multi-data-centers-in-one-city-deployment.md b/multi-data-centers-in-one-city-deployment.md index 5f67a00b6040f..456aebb04fbe9 100644 --- a/multi-data-centers-in-one-city-deployment.md +++ b/multi-data-centers-in-one-city-deployment.md @@ -1,7 +1,6 @@ --- title: Multiple Availability Zones in One Region Deployment summary: Learn the deployment solution to multiple availability zones in one region. -aliases: ['/docs/dev/how-to/deploy/geographic-redundancy/overview/','/docs/dev/geo-redundancy-deployment/','/tidb/dev/geo-redundancy-deployment'] --- # Multiple Availability Zones in One Region Deployment diff --git a/mysql-compatibility.md b/mysql-compatibility.md index ce0e99346216f..f05c32323465d 100644 --- a/mysql-compatibility.md +++ b/mysql-compatibility.md @@ -1,7 +1,6 @@ --- title: MySQL Compatibility summary: Learn about the compatibility of TiDB with MySQL, and the unsupported and different features. -aliases: ['/docs/dev/mysql-compatibility/','/docs/dev/reference/mysql-compatibility/'] --- # MySQL Compatibility @@ -72,6 +71,7 @@ You can try out TiDB features on [TiDB Playground](https://play.tidbcloud.com/?u + Descending Index [#2519](https://github.com/pingcap/tidb/issues/2519) + `SKIP LOCKED` syntax [#18207](https://github.com/pingcap/tidb/issues/18207) + Lateral derived tables [#40328](https://github.com/pingcap/tidb/issues/40328) ++ JOIN ON subquery [#11414](https://github.com/pingcap/tidb/issues/11414) ## Differences from MySQL @@ -137,13 +137,13 @@ As shown, because of the shared allocator, the `id` increments by 2 each time. T -TiDB utilizes a combination of [Prometheus and Grafana](/tidb-monitoring-api.md) for storing and querying performance monitoring metrics. In TiDB, performance schema tables do not return any results. +TiDB utilizes a combination of [Prometheus and Grafana](/tidb-monitoring-api.md) for storing and querying performance monitoring metrics. In TiDB, most [performance schema tables](/performance-schema/performance-schema.md) do not return any results. -To check performance metrics in TiDB Cloud, you can either check the cluster overview page on the TiDB Cloud console or use [third-party monitoring integrations](/tidb-cloud/third-party-monitoring-integrations.md). Performance schema tables return empty results in TiDB. +To check performance metrics in TiDB Cloud, you can either check the cluster overview page in the TiDB Cloud console or use [third-party monitoring integrations](/tidb-cloud/third-party-monitoring-integrations.md). Most [performance schema tables](/performance-schema/performance-schema.md) return empty results in TiDB. @@ -157,9 +157,7 @@ For more information, refer to [Understand the Query Execution Plan](/explain-ov ### Built-in functions -TiDB supports most of the built-in functions in MySQL, but not all. You can use the statement `SHOW BUILTINS` to get a list of the available functions. - -For more information, refer to the [TiDB SQL Grammar](https://pingcap.github.io/sqlgram/#functioncallkeyword). +TiDB supports most of the built-in functions in MySQL, but not all. You can use the statement [`SHOW BUILTINS`](/sql-statements/sql-statement-show-builtins.md) to get a list of the available functions. ### DDL operations @@ -239,16 +237,16 @@ TiDB supports most [SQL modes](/sql-mode.md): TiDB has default differences when compared with MySQL 5.7 and MySQL 8.0: - Default character set: - - TiDB’s default value is `utf8mb4`. - - MySQL 5.7’s default value is `latin1`. - - MySQL 8.0’s default value is `utf8mb4`. + - TiDB's default value is `utf8mb4`. + - MySQL 5.7's default value is `latin1`. + - MySQL 8.0's default value is `utf8mb4`. - Default collation: - - TiDB’s default collation is `utf8mb4_bin`. - - MySQL 5.7’s default collation is `utf8mb4_general_ci`. - - MySQL 8.0’s default collation is `utf8mb4_0900_ai_ci`. + - TiDB's default collation is `utf8mb4_bin`. + - MySQL 5.7's default collation is `utf8mb4_general_ci`. + - MySQL 8.0's default collation is `utf8mb4_0900_ai_ci`. - Default SQL mode: - - TiDB’s default SQL mode includes these modes: `ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION`. - - MySQL’s default SQL mode: + - TiDB's default SQL mode includes these modes: `ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION`. + - MySQL's default SQL mode: - The default SQL mode in MySQL 5.7 is the same as TiDB. - The default SQL mode in MySQL 8.0 includes these modes: `ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION`. - Default value of `lower_case_table_names`: diff --git a/mysql-schema/mysql-schema-user.md b/mysql-schema/mysql-schema-user.md new file mode 100644 index 0000000000000..175d36b3fcfb6 --- /dev/null +++ b/mysql-schema/mysql-schema-user.md @@ -0,0 +1,110 @@ +--- +title: mysql.user +summary: Learn about the `user` table in the `mysql` schema. +--- + +# `mysql.user` + +The `mysql.user` table provides information about user accounts and their privileges. + +To view the structure of `mysql.user`, use the following SQL statement: + +```sql +DESC mysql.user; +``` + +The output is as follows: + +``` ++------------------------+----------------------+------+------+-------------------+-------+ +| Field | Type | Null | Key | Default | Extra | ++------------------------+----------------------+------+------+-------------------+-------+ +| Host | char(255) | NO | PRI | NULL | | +| User | char(32) | NO | PRI | NULL | | +| authentication_string | text | YES | | NULL | | +| plugin | char(64) | YES | | NULL | | +| Select_priv | enum('N','Y') | NO | | N | | +| Insert_priv | enum('N','Y') | NO | | N | | +| Update_priv | enum('N','Y') | NO | | N | | +| Delete_priv | enum('N','Y') | NO | | N | | +| Create_priv | enum('N','Y') | NO | | N | | +| Drop_priv | enum('N','Y') | NO | | N | | +| Process_priv | enum('N','Y') | NO | | N | | +| Grant_priv | enum('N','Y') | NO | | N | | +| References_priv | enum('N','Y') | NO | | N | | +| Alter_priv | enum('N','Y') | NO | | N | | +| Show_db_priv | enum('N','Y') | NO | | N | | +| Super_priv | enum('N','Y') | NO | | N | | +| Create_tmp_table_priv | enum('N','Y') | NO | | N | | +| Lock_tables_priv | enum('N','Y') | NO | | N | | +| Execute_priv | enum('N','Y') | NO | | N | | +| Create_view_priv | enum('N','Y') | NO | | N | | +| Show_view_priv | enum('N','Y') | NO | | N | | +| Create_routine_priv | enum('N','Y') | NO | | N | | +| Alter_routine_priv | enum('N','Y') | NO | | N | | +| Index_priv | enum('N','Y') | NO | | N | | +| Create_user_priv | enum('N','Y') | NO | | N | | +| Event_priv | enum('N','Y') | NO | | N | | +| Repl_slave_priv | enum('N','Y') | NO | | N | | +| Repl_client_priv | enum('N','Y') | NO | | N | | +| Trigger_priv | enum('N','Y') | NO | | N | | +| Create_role_priv | enum('N','Y') | NO | | N | | +| Drop_role_priv | enum('N','Y') | NO | | N | | +| Account_locked | enum('N','Y') | NO | | N | | +| Shutdown_priv | enum('N','Y') | NO | | N | | +| Reload_priv | enum('N','Y') | NO | | N | | +| FILE_priv | enum('N','Y') | NO | | N | | +| Config_priv | enum('N','Y') | NO | | N | | +| Create_Tablespace_Priv | enum('N','Y') | NO | | N | | +| Password_reuse_history | smallint(5) unsigned | YES | | NULL | | +| Password_reuse_time | smallint(5) unsigned | YES | | NULL | | +| User_attributes | json | YES | | NULL | | +| Token_issuer | varchar(255) | YES | | NULL | | +| Password_expired | enum('N','Y') | NO | | N | | +| Password_last_changed | timestamp | YES | | CURRENT_TIMESTAMP | | +| Password_lifetime | smallint(5) unsigned | YES | | NULL | | ++------------------------+----------------------+------+------+-------------------+-------+ +44 rows in set (0.00 sec) +``` + +The `mysql.user` table contains several fields that can be categorized into three groups: + + + +* Scope: + * `Host`: specifies the hostname of a TiDB account. + * `User`: specifies the username of a TiDB account. +* Privilege: + + The fields ending with `_priv` or `_Priv` define the permissions granted to a user account. For example, `Select_priv` means that the user has global `Select` privilege. For more information, see [Privileges required for TiDB operations](/privilege-management.md#privileges-required-for-tidb-operations). + +* Security: + * `authentication_string` and `plugin`: `authentication_string` stores the credentials for the user account. The credentials are interpreted based on the authentication plugin specified in the `plugin` field. + * `Account_locked`: indicates whether the user account is locked. + * `Password_reuse_history` and `Password_reuse_time`: used for [Password reuse policy](/password-management.md#password-reuse-policy). + * `User_attributes`: provides information about user comments and user attributes. + * `Token_issuer`: used for the [`tidb_auth_token`](/security-compatibility-with-mysql.md#tidb_auth_token) authentication plugin. + * `Password_expired`, `Password_last_changed`, and `Password_lifetime`: used for [Password expiration policy](/password-management.md#password-expiration-policy). + + + + + +* Scope: + * `Host`: specifies the hostname of a TiDB account. + * `User`: specifies the username of a TiDB account. +* Privilege: + + The fields ending with `_priv` or `_Priv` define the permissions granted to a user account. For example, `Select_priv` means that the user has global `Select` privilege. For more information, see [Privileges required for TiDB operations](https://docs.pingcap.com/tidb/stable/privilege-management#privileges-required-for-tidb-operations). + +* Security: + * `authentication_string` and `plugin`: `authentication_string` stores the credentials for the user account. The credentials are interpreted based on the authentication plugin specified in the `plugin` field. + * `Account_locked`: indicates whether the user account is locked. + * `Password_reuse_history` and `Password_reuse_time`: used for [Password reuse policy](https://docs.pingcap.com/tidb/stable/password-management#password-reuse-policy). + * `User_attributes`: provides information about user comments and user attributes. + * `Token_issuer`: used for the [`tidb_auth_token`](https://docs.pingcap.com/tidb/stable/security-compatibility-with-mysql#tidb_auth_token) authentication plugin. + * `Password_expired`, `Password_last_changed`, and `Password_lifetime`: used for [Password expiration policy](https://docs.pingcap.com/tidb/stable/password-management#password-expiration-policy). + + + +Although most of the fields in the TiDB `mysql.user` table also exist in the MySQL `mysql.user` table, the `Token_issuer` field is specific to TiDB. \ No newline at end of file diff --git a/mysql-schema.md b/mysql-schema/mysql-schema.md similarity index 72% rename from mysql-schema.md rename to mysql-schema/mysql-schema.md index 02d95816d31e5..66cfde5f464a4 100644 --- a/mysql-schema.md +++ b/mysql-schema/mysql-schema.md @@ -1,18 +1,21 @@ --- title: mysql Schema summary: Learn about the TiDB system tables. -aliases: ['/docs/dev/system-tables/system-table-overview/','/docs/dev/reference/system-databases/mysql/','/tidb/dev/system-table-overview/'] --- # `mysql` Schema The `mysql` schema contains TiDB system tables. The design is similar to the `mysql` schema in MySQL, where tables such as `mysql.user` can be edited directly. It also contains a number of tables which are extensions to MySQL. +> **Note:** +> +> In most scenarios, it is not recommended to change the content of system tables directly using `INSERT`, `UPDATE`, or `DELETE`. Instead, use statements such as [`CREATE USER`](/sql-statements/sql-statement-create-user.md), [`ALTER USER`](/sql-statements/sql-statement-alter-user.md), [`DROP USER`](/sql-statements/sql-statement-drop-user.md), [`GRANT`](/sql-statements/sql-statement-grant-privileges.md), [`REVOKE`](/sql-statements/sql-statement-revoke-privileges.md), and [`SHOW CREATE USER`](/sql-statements/sql-statement-show-create-user.md) to manage users and privileges. If direct modification of system tables is unavoidable, use [`FLUSH PRIVILEGES`](/sql-statements/sql-statement-flush-privileges.md) to make the changes take effect. + ## Grant system tables These system tables contain grant information about user accounts and their privileges: -- `user`: user accounts, global privileges, and other non-privilege columns +- [`user`](/mysql-schema/mysql-schema-user.md): user accounts, global privileges, and other non-privilege columns - `db`: database-level privileges - `tables_priv`: table-level privileges - `columns_priv`: column-level privileges @@ -44,9 +47,11 @@ Currently, the `help_topic` is NULL. - `stats_extended`: extended statistics, such as the order correlation between columns - `stats_feedback`: the query feedback of statistics - `stats_fm_sketch`: the FMSketch distribution of the histogram of the statistics column +- `stats_table_locked`: information about the locked statistics +- `stats_meta_history`: the meta information in the historical statistics +- `stats_history`: the other information in the historical statistics - `analyze_options`: the default `analyze` options for each table - `column_stats_usage`: the usage of column statistics -- `schema_index_usage`: the usage of indexes - `analyze_jobs`: the ongoing statistics collection tasks and the history task records within the last 7 days ## Execution plan-related system tables @@ -54,11 +59,16 @@ Currently, the `help_topic` is NULL. - `bind_info`: the binding information of execution plans - `capture_plan_baselines_blacklist`: the blocklist for the automatic binding of the execution plan +## System tables related to PLAN REPLAYER + +- `plan_replayer_status`: the [`PLAN REPLAYER CAPTURE`](https://docs.pingcap.com/tidb/stable/sql-plan-replayer#use-plan-replayer-capture) tasks registered by the user +- `plan_replayer_task`: the results of [`PLAN REPLAYER CAPTURE`](https://docs.pingcap.com/tidb/stable/sql-plan-replayer#use-plan-replayer-capture) tasks + ## GC worker system tables > **Note:** > -> The GC worker system tables are only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> The GC worker system tables are only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). - `gc_delete_range`: the KV range to be deleted - `gc_delete_range_done`: the deleted KV range @@ -69,10 +79,6 @@ Currently, the `help_topic` is NULL. ## TTL related system tables -> **Note:** - -> The TTL related system tables are not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. - * `tidb_ttl_table_status`: the previously executed TTL job and ongoing TTL job for all TTL tables * `tidb_ttl_task`: the current ongoing TTL subtasks * `tidb_ttl_job_history`: the execution history of TTL tasks in the last 90 days @@ -91,7 +97,7 @@ Currently, the `help_topic` is NULL. ## System tables related to DDL statements * `tidb_ddl_history`: the history records of DDL statements -* `tidb_ddl_jobs`: the metadata of DDL statements that are currently being executed by TiDB +* `tidb_ddl_job`: the metadata of DDL statements that are currently being executed by TiDB * `tidb_ddl_reorg`: the metadata of physical DDL statements (such as adding indexes) that are currently being executed by TiDB ## System tables related to TiDB Distributed eXecution Framework (DXF) @@ -112,13 +118,14 @@ Currently, the `help_topic` is NULL. > **Note:** > -> The `tidb`, `expr_pushdown_blacklist`, `opt_rule_blacklist`, `table_cache_meta`, `tidb_import_jobs`, and `tidb_timers` system tables are only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> The `tidb`, `expr_pushdown_blacklist`, `opt_rule_blacklist`, `table_cache_meta`, `tidb_import_jobs`, and `tidb_timers` system tables are only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). - `GLOBAL_VARIABLES`: global system variable table - `expr_pushdown_blacklist`: the blocklist for expression pushdown - `opt_rule_blacklist`: the blocklist for logical optimization rules - `tidb_import_jobs`: the job information of [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) - `tidb_timers`: the metadata of internal timers +- `advisory_locks`: information related to [Locking functions](/functions-and-operators/locking-functions.md) diff --git a/non-transactional-dml.md b/non-transactional-dml.md index 921e6642dcae3..248d9077d23db 100644 --- a/non-transactional-dml.md +++ b/non-transactional-dml.md @@ -335,7 +335,7 @@ This error occurs when the `WHERE` clause concatenated in a query involves table ```sql BATCH ON test.t2.id LIMIT 1 INSERT INTO t -SELECT t2.id, t2.v, t3. FROM t2, t3 WHERE t2.id = t3.id +SELECT t2.id, t2.v, t3.id FROM t2, t3 WHERE t2.id = t3.id ``` ```sql @@ -347,7 +347,7 @@ If the error occurs, you can print the query statement for confirmation by using ```sql BATCH ON test.t2.id LIMIT 1 DRY RUN QUERY INSERT INTO t -SELECT t2.id, t2.v, t3. FROM t2, t3 WHERE t2.id = t3.id +SELECT t2.id, t2.v, t3.id FROM t2, t3 WHERE t2.id = t3.id ``` To avoid the error, you can move the condition related to other tables in the `WHERE` clause to the `ON` condition in the `JOIN` clause. For example: @@ -355,7 +355,7 @@ To avoid the error, you can move the condition related to other tables in the `W ```sql BATCH ON test.t2.id LIMIT 1 INSERT INTO t -SELECT t2.id, t2.v, t3. FROM t2 JOIN t3 ON t2.id=t3.id +SELECT t2.id, t2.v, t3.id FROM t2 JOIN t3 ON t2.id = t3.id ``` ``` diff --git a/optimistic-transaction.md b/optimistic-transaction.md index f7e4b1fe558c1..4c98acc8c4d24 100644 --- a/optimistic-transaction.md +++ b/optimistic-transaction.md @@ -1,7 +1,6 @@ --- title: TiDB Optimistic Transaction Model summary: Learn the optimistic transaction model in TiDB. -aliases: ['/docs/dev/optimistic-transaction/','/docs/dev/reference/transactions/transaction-optimistic/','/docs/dev/reference/transactions/transaction-model/'] --- # TiDB Optimistic Transaction Model diff --git a/optimizer-fix-controls.md b/optimizer-fix-controls.md index 0fc68bc3fce56..420c877ccd69b 100644 --- a/optimizer-fix-controls.md +++ b/optimizer-fix-controls.md @@ -71,9 +71,31 @@ SET SESSION tidb_opt_fix_control = '44262:ON,44389:ON'; - This variable sets the threshold for the optimizer's heuristic strategy to select access paths. If the estimated rows for an access path (such as `Index_A`) is much smaller than that of other access paths (default `1000` times), the optimizer skips the cost comparison and directly selects `Index_A`. - `0` means to disable this heuristic strategy. +### [`45798`](https://github.com/pingcap/tidb/issues/45798) New in v7.5.0 + +- Default value: `ON` +- Possible values: `ON`, `OFF` +- This variable controls whether Plan Cache is allowed to cache execution plans that access [generated columns](/generated-columns.md). + +### [`46177`](https://github.com/pingcap/tidb/issues/46177) New in v6.5.6, v7.1.3 and v7.5.0 + +- Default value: `OFF` +- Possible values: `ON`, `OFF` +- This variable controls whether the optimizer explores enforced plans during query optimization after finding an unenforced plan. + ### [`52869`](https://github.com/pingcap/tidb/issues/52869) New in v8.1.0 - Default value: `OFF` - Possible values: `ON`, `OFF` - As stated in the **Note** of [Explain Statements Using Index Merge](/explain-index-merge.md#examples), if the optimizer can choose the single index scan method (other than full table scan) for a query plan, the optimizer will not automatically use index merge. - You can remove this limitation by enabling this fix control. Removing this limitation enables the optimizer to choose index merge automatically in more queries, but might cause the optimizer to ignore the optimal execution plans. Therefore, it is recommended to conduct sufficient tests on actual use cases before removing this limitation to make sure that it will not cause performance regressions. + +### [`56318`](https://github.com/pingcap/tidb/issues/56318) + +> **Note:** +> +> This is only available for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). + +- Default value: `ON` +- Possible values: `ON`, `OFF` +- This variable controls whether to avoid calculating the heavy expression used in the `ORDER BY` statement twice. diff --git a/optimizer-hints.md b/optimizer-hints.md index 369020b628aae..d90901f89f920 100644 --- a/optimizer-hints.md +++ b/optimizer-hints.md @@ -1,7 +1,6 @@ --- title: Optimizer Hints summary: Use Optimizer Hints to influence query execution plans -aliases: ['/docs/dev/optimizer-hints/','/docs/dev/reference/performance/optimizer-hints/'] --- # Optimizer Hints @@ -139,6 +138,10 @@ SELECT /*+ NO_MERGE_JOIN(t1, t2) */ * FROM t1, t2 WHERE t1.id = t2.id; ### INL_JOIN(t1_name [, tl_name ...]) +> **Note:** +> +> In some cases, the `INL_JOIN` hint might not take effect. For more information, see [`INL_JOIN` hint does not take effect](#inl_join-hint-does-not-take-effect). + The `INL_JOIN(t1_name [, tl_name ...])` hint tells the optimizer to use the index nested loop join algorithm for the given table(s). This algorithm might consume less system resources and take shorter processing time in some scenarios and might produce an opposite result in other scenarios. If the result set is less than 10,000 rows after the outer table is filtered by the `WHERE` condition, it is recommended to use this hint. For example: {{< copyable "sql" >}} @@ -525,7 +528,7 @@ SELECT /*+ LIMIT_TO_COP() */ * FROM t WHERE a = 1 AND b > 10 ORDER BY c LIMIT 1; ### READ_FROM_STORAGE(TIFLASH[t1_name [, tl_name ...]], TIKV[t2_name [, tl_name ...]]) -The `READ_FROM_STORAGE(TIFLASH[t1_name [, tl_name ...]], TIKV[t2_name [, tl_name ...]])` hint tells the optimizer to read specific table(s) from specific storage engine(s). Currently, this hint supports two storage engine parameters - `TIKV` and `TIFLASH`. If a table has an alias, use the alias as the parameter of `READ_FROM_STORAGE()`; if the table does not has an alias, use the table's original name as the parameter. For example: +The `READ_FROM_STORAGE(TIFLASH[t1_name [, tl_name ...]], TIKV[t2_name [, tl_name ...]])` hint tells the optimizer to read specific table(s) from specific storage engine(s). Currently, this hint supports two storage engine parameters - `TIKV` and `TIFLASH`. If a table has an alias, use the alias as the parameter of `READ_FROM_STORAGE()`; if the table does not have an alias, use the table's original name as the parameter. For example: {{< copyable "sql" >}} @@ -930,9 +933,76 @@ The warning is as follows: In this case, you need to place the hint directly after the `SELECT` keyword. For more details, see the [Syntax](#syntax) section. -### INL_JOIN hint does not take effect due to collation incompatibility +### `INL_JOIN` hint does not take effect + +#### `INL_JOIN` hint does not take effect when built-in functions are used on columns for joining tables + +In some cases, if you use a built-in function on a column that joins tables, the optimizer might fail to choose the `IndexJoin` plan, resulting in the `INL_JOIN` hint not taking effect either. + +For example, the following query uses the built-in function `substr` on the column `tname` that joins tables: + +```sql +CREATE TABLE t1 (id varchar(10) primary key, tname varchar(10)); +CREATE TABLE t2 (id varchar(10) primary key, tname varchar(10)); +EXPLAIN SELECT /*+ INL_JOIN(t1, t2) */ * FROM t1, t2 WHERE t1.id=t2.id and SUBSTR(t1.tname,1,2)=SUBSTR(t2.tname,1,2); +``` + +The execution plan is as follows: + +```sql ++------------------------------+----------+-----------+---------------+-----------------------------------------------------------------------+ +| id | estRows | task | access object | operator info | ++------------------------------+----------+-----------+---------------+-----------------------------------------------------------------------+ +| HashJoin_12 | 12500.00 | root | | inner join, equal:[eq(test.t1.id, test.t2.id) eq(Column#5, Column#6)] | +| ├─Projection_17(Build) | 10000.00 | root | | test.t2.id, test.t2.tname, substr(test.t2.tname, 1, 2)->Column#6 | +| │ └─TableReader_19 | 10000.00 | root | | data:TableFullScan_18 | +| │ └─TableFullScan_18 | 10000.00 | cop[tikv] | table:t2 | keep order:false, stats:pseudo | +| └─Projection_14(Probe) | 10000.00 | root | | test.t1.id, test.t1.tname, substr(test.t1.tname, 1, 2)->Column#5 | +| └─TableReader_16 | 10000.00 | root | | data:TableFullScan_15 | +| └─TableFullScan_15 | 10000.00 | cop[tikv] | table:t1 | keep order:false, stats:pseudo | ++------------------------------+----------+-----------+---------------+-----------------------------------------------------------------------+ +7 rows in set, 1 warning (0.01 sec) +``` + +```sql +SHOW WARNINGS; +``` + +``` ++---------+------+------------------------------------------------------------------------------------+ +| Level | Code | Message | ++---------+------+------------------------------------------------------------------------------------+ +| Warning | 1815 | Optimizer Hint /*+ INL_JOIN(t1, t2) */ or /*+ TIDB_INLJ(t1, t2) */ is inapplicable | ++---------+------+------------------------------------------------------------------------------------+ +1 row in set (0.00 sec) +``` + +As you can see from the preceding example, the `INL_JOIN` hint does not take effect. This is due to a limitation of the optimizer that prevents using the `Projection` or `Selection` operator as the probe side of `IndexJoin`. + +Starting from TiDB v8.0.0, you can avoid this issue by setting [`tidb_enable_inl_join_inner_multi_pattern`](/system-variables.md#tidb_enable_inl_join_inner_multi_pattern-new-in-v700) to `ON`. + +```sql +SET @@tidb_enable_inl_join_inner_multi_pattern=ON; +Query OK, 0 rows affected (0.00 sec) + +EXPLAIN SELECT /*+ INL_JOIN(t1, t2) */ * FROM t1, t2 WHERE t1.id=t2.id AND SUBSTR(t1.tname,1,2)=SUBSTR(t2.tname,1,2); ++------------------------------+--------------+-----------+---------------+--------------------------------------------------------------------------------------------------------------------------------------------+ +| id | estRows | task | access object | operator info | ++------------------------------+--------------+-----------+---------------+--------------------------------------------------------------------------------------------------------------------------------------------+ +| IndexJoin_18 | 12500.00 | root | | inner join, inner:Projection_14, outer key:test.t1.id, inner key:test.t2.id, equal cond:eq(Column#5, Column#6), eq(test.t1.id, test.t2.id) | +| ├─Projection_32(Build) | 10000.00 | root | | test.t1.id, test.t1.tname, substr(test.t1.tname, 1, 2)->Column#5 | +| │ └─TableReader_34 | 10000.00 | root | | data:TableFullScan_33 | +| │ └─TableFullScan_33 | 10000.00 | cop[tikv] | table:t1 | keep order:false, stats:pseudo | +| └─Projection_14(Probe) | 100000000.00 | root | | test.t2.id, test.t2.tname, substr(test.t2.tname, 1, 2)->Column#6 | +| └─TableReader_13 | 10000.00 | root | | data:TableRangeScan_12 | +| └─TableRangeScan_12 | 10000.00 | cop[tikv] | table:t2 | range: decided by [eq(test.t2.id, test.t1.id)], keep order:false, stats:pseudo | ++------------------------------+--------------+-----------+---------------+--------------------------------------------------------------------------------------------------------------------------------------------+ +7 rows in set (0.00 sec) +``` + +#### `INL_JOIN`, `INL_HASH_JOIN`, and `INL_MERGE_JOIN` hints do not take effect due to collation incompatibility -When the collation of the join key is incompatible between two tables, the `IndexJoin` operator cannot be utilized to execute the query. In this case, the [`INL_JOIN` hint](#inl_joint1_name--tl_name-) does not take effect. For example: +When the collation of the join key is incompatible between two tables, the `IndexJoin` operator cannot be utilized to execute the query. In this case, the [`INL_JOIN`](#inl_joint1_name--tl_name-), [`INL_HASH_JOIN`](#inl_hash_join), and [`INL_MERGE_JOIN`](#inl_merge_join) hints do not take effect. For example: ```sql CREATE TABLE t1 (k varchar(8), key(k)) COLLATE=utf8mb4_general_ci; @@ -967,7 +1037,7 @@ SHOW WARNINGS; 1 row in set (0.00 sec) ``` -### `INL_JOIN` hint does not take effect because of join order +#### `INL_JOIN` hint does not take effect due to join order The [`INL_JOIN(t1, t2)`](#inl_joint1_name--tl_name-) or `TIDB_INLJ(t1, t2)` hint semantically instructs `t1` and `t2` to act as inner tables in an `IndexJoin` operator to join with other tables, rather than directly joining them using an `IndexJoin` operator. For example: diff --git a/overview.md b/overview.md index 96c17e9eb245d..8d8b582b1c458 100644 --- a/overview.md +++ b/overview.md @@ -1,10 +1,9 @@ --- -title: TiDB Introduction +title: What is TiDB Self-Managed summary: Learn about the key features and usage scenarios of TiDB. -aliases: ['/docs/dev/key-features/','/tidb/dev/key-features','/docs/dev/overview/'] --- -# TiDB Introduction +# What is TiDB Self-Managed -[TiDB](https://github.com/pingcap/tidb) (/’taɪdiːbi:/, "Ti" stands for Titanium) is an open-source distributed SQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. It is MySQL compatible and features horizontal scalability, strong consistency, and high availability. The goal of TiDB is to provide users with a one-stop database solution that covers OLTP (Online Transactional Processing), OLAP (Online Analytical Processing), and HTAP services. TiDB is suitable for various use cases that require high availability and strong consistency with large-scale data. +[TiDB](https://github.com/pingcap/tidb) (/'taɪdiːbi:/, "Ti" stands for Titanium) is an open-source distributed SQL database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. It is MySQL compatible and features horizontal scalability, strong consistency, and high availability. The goal of TiDB is to provide users with a one-stop database solution that covers OLTP (Online Transactional Processing), OLAP (Online Analytical Processing), and HTAP services. TiDB is suitable for various use cases that require high availability and strong consistency with large-scale data. + +TiDB Self-Managed is a product option of TiDB, where users or organizations can deploy and manage TiDB on their own infrastructure with complete flexibility. With TiDB Self-Managed, you can enjoy the power of open source, distributed SQL while retaining full control over your environment. The following video introduces key features of TiDB. diff --git a/package-lock.json b/package-lock.json index fd224f072eb07..2e983b5f14b33 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,9 @@ "micromark-extension-mdxjs": "^1.0.0", "octokit": "^3.1.0", "unist-util-visit": "^4.1.0" + }, + "devDependencies": { + "prettier": "3.3.3" } }, "node_modules/@octokit/app": { @@ -1568,6 +1571,22 @@ "is-hexadecimal": "^2.0.0" } }, + "node_modules/prettier": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz", + "integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, "node_modules/proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", @@ -3076,6 +3095,12 @@ "is-hexadecimal": "^2.0.0" } }, + "prettier": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.3.3.tgz", + "integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==", + "dev": true + }, "proxy-from-env": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", diff --git a/package.json b/package.json index 7367d12460c5a..92ae96ed20d4f 100644 --- a/package.json +++ b/package.json @@ -17,5 +17,8 @@ "micromark-extension-mdxjs": "^1.0.0", "octokit": "^3.1.0", "unist-util-visit": "^4.1.0" + }, + "devDependencies": { + "prettier": "3.3.3" } } diff --git a/partitioned-table.md b/partitioned-table.md index 1a70299593a64..68ce664f6ac08 100644 --- a/partitioned-table.md +++ b/partitioned-table.md @@ -1,7 +1,6 @@ --- title: Partitioning summary: Learn how to use partitioning in TiDB. -aliases: ['/docs/dev/partitioned-table/','/docs/dev/reference/sql/partitioning/'] --- # Partitioning @@ -169,6 +168,23 @@ Range partitioning is particularly useful when one or more of the following cond Range COLUMNS partitioning is a variant of Range partitioning. You can use one or more columns as partitioning keys. The data types of partition columns can be integer, string (`CHAR` or `VARCHAR`), `DATE`, and `DATETIME`. Any expressions, such as non-COLUMNS partitioning, are not supported. +Like Range partitioning, Range COLUMNS partitioning also requires the partition ranges to be strictly increasing. The partition definition in the following example is not supported: + +```sql +CREATE TABLE t( + a int, + b datetime, + c varchar(8) +) PARTITION BY RANGE COLUMNS(`c`,`b`) +(PARTITION `p20240520A` VALUES LESS THAN ('A','2024-05-20 00:00:00'), + PARTITION `p20240520Z` VALUES LESS THAN ('Z','2024-05-20 00:00:00'), + PARTITION `p20240521A` VALUES LESS THAN ('A','2024-05-21 00:00:00')); +``` + +``` +Error 1493 (HY000): VALUES LESS THAN value must be strictly increasing for each partition +``` + Suppose that you want to partition by name, and drop old and invalid data, then you can create a table as follows: ```sql @@ -1451,6 +1467,12 @@ Partition selection is supported for all types of table partitioning, including This section introduces some restrictions and limitations on partitioned tables in TiDB. +- Using the [`ALTER TABLE ... CHANGE COLUMN`](/sql-statements/sql-statement-change-column.md) statement to change column types of partitioned tables is not supported. +- Using the [`ALTER TABLE ... CACHE`](/cached-tables.md) statement to set partitioned tables to cached tables is not supported. +- [Temporary tables](/temporary-tables.md) in TiDB are **NOT** compatible with partitioned tables. +- Creating a [foreign key](/foreign-key.md) on a partitioned table is not supported. +- The [`ORDER_INDEX(t1_name, idx1_name [, idx2_name ...])`](/optimizer-hints.md#order_indext1_name-idx1_name--idx2_name-) hint does not work for partitioned tables and their related indexes, because indexes on partitioned tables cannot be read in order. + ### Partitioning keys, primary keys and unique keys This section discusses the relationship of partitioning keys with primary keys and unique keys. The rule governing this relationship can be expressed as follows: **Every unique key on the table must use every column in the table's partitioning expression**. This also includes the table's primary key, because it is by definition a unique key. @@ -1696,8 +1718,6 @@ YEARWEEK() Currently, TiDB supports Range partitioning, Range COLUMNS partitioning, List partitioning, List COLUMNS partitioning, Hash partitioning, and Key partitioning. Other partitioning types that are available in MySQL are not supported yet in TiDB. -With regard to partition management, any operation that requires moving data in the bottom implementation is not supported currently, including but not limited to: adjust the number of partitions in a Hash partitioned table, modify the Range of a Range partitioned table, and merge partitions. - For the unsupported partitioning types, when you create a table in TiDB, the partitioning information is ignored and the table is created in the regular form with a warning reported. The `LOAD DATA` syntax does not support partition selection currently in TiDB. diff --git a/password-management.md b/password-management.md index 12df3b05d02ec..2d839d8989fd2 100644 --- a/password-management.md +++ b/password-management.md @@ -33,7 +33,7 @@ Password complexity check is disabled by default in TiDB. By configuring system The password complexity policy has the following features: - For SQL statements that set user passwords in plaintext (including `CREATE USER`, `ALTER USER`, and `SET PASSWORD`), TiDB checks the passwords against the password complexity policy. If a password does not meet the requirements, the password is rejected. -- You can use the SQL function [`VALIDATE_PASSWORD_STRENGTH()`](https://dev.mysql.com/doc/refman/8.0/en/encryption-functions.html#function_validate-password-strength) to validate the password strength. +- You can use the SQL function [`VALIDATE_PASSWORD_STRENGTH()`](/functions-and-operators/encryption-and-compression-functions.md#validate_password_strength) to validate the password strength. > **Note:** > @@ -136,13 +136,13 @@ ERROR 1819 (HY000): Require Password Length: 8 ### Password strength validation function -To check the password strength, you can use the `VALIDATE_PASSWORD_STRENGTH()` function. This function accepts a password argument and returns an integer from 0 (weak) to 100 (strong). +To check the password strength, you can use the [`VALIDATE_PASSWORD_STRENGTH()`](/functions-and-operators/encryption-and-compression-functions.md#validate_password_strength) function. This function accepts a password argument and returns an integer from 0 (weak) to 100 (strong). > **Note:** > > This function evaluates the password strength based on the current password complexity policy. If the password complexity policy is changed, the same password might get different evaluation results. -The following example shows how to use the `VALIDATE_PASSWORD_STRENGTH()` function: +The following example shows how to use the [`VALIDATE_PASSWORD_STRENGTH()`](/functions-and-operators/encryption-and-compression-functions.md#validate_password_strength) function: ```sql mysql> SELECT VALIDATE_PASSWORD_STRENGTH('weak'); diff --git a/pd-configuration-file.md b/pd-configuration-file.md index f727ce33ba9b5..3f9f0a02f6827 100644 --- a/pd-configuration-file.md +++ b/pd-configuration-file.md @@ -1,14 +1,13 @@ --- title: PD Configuration File summary: Learn the PD configuration file. -aliases: ['/docs/dev/pd-configuration-file/','/docs/dev/reference/configuration/pd-server/configuration-file/'] --- # PD Configuration File -The PD configuration file supports more options than command-line parameters. You can find the default configuration file [here](https://github.com/pingcap/pd/blob/master/conf/config.toml). +The PD configuration file supports more options than command-line parameters. You can find the default configuration file [here](https://github.com/pingcap/pd/blob/release-8.1/conf/config.toml). This document only describes parameters that are not included in command-line parameters. Check [here](/command-line-flags-for-pd-configuration.md) for the command line parameters. @@ -97,6 +96,22 @@ This document only describes parameters that are not included in command-line pa + The time interval for automatic compaction of the meta-information database when `auto-compaction-retention` is `periodic`. When the compaction mode is set to `revision`, this parameter indicates the version number for the automatic compaction. + Default value: 1h +### `tick-interval` + ++ Equivalent to the `heartbeat-interval` configuration item of etcd. It controls the Raft heartbeat interval between embedded etcd instances in different PD nodes. A smaller value accelerates failure detection but increases network load. ++ Default value: `500ms` + +### `election-interval` + ++ Equivalent to the `election-timeout` configuration item of etcd. It controls the election timeout for embedded etcd instances in PD nodes. If an etcd instance does not receive a valid heartbeat from other etcd instances within this period, it initiates a Raft election. ++ Default value: `3000ms` ++ This value must be at least five times the [`tick-interval`](#tick-interval). For example, if `tick-interval` is `500ms`, `election-interval` must be greater than or equal to `2500ms`. + +### `enable-prevote` + ++ Equivalent to the `pre-vote` configuration item of etcd. It controls whether the embedded etcd in the PD node enables Raft pre-vote. When enabled, etcd performs an additional election phase to check whether enough votes can be obtained to win the election, minimizing service disruption. ++ Default value: `true` + ### `force-new-cluster` + Determines whether to force PD to start as a new cluster and modify the number of Raft members to `1` @@ -258,6 +273,13 @@ Configuration items related to monitoring Configuration items related to scheduling +> **Note:** +> +> To modify these PD configuration items related to `schedule`, choose one of the following methods based on your cluster status: +> +> - For clusters to be newly deployed, you can modify the PD configuration file directly. +> - For existing clusters, use the command-line tool [PD Control](/pd-control.md) to make changes instead. Direct modifications to these PD configuration items related to `schedule` in the configuration file do not take effect on existing clusters. + ### `max-merge-region-size` + Controls the size limit of `Region Merge`. When the Region size is greater than the specified value, PD does not merge the Region with the adjacent Regions. @@ -400,6 +422,16 @@ Configuration items related to scheduling + Specifies how many days the hot Region information is retained. + Default value: `7` +### `enable-heartbeat-breakdown-metrics` New in v8.0.0 + ++ Controls whether to enable breakdown metrics for Region heartbeats. These metrics measure the time consumed in each stage of Region heartbeat processing, facilitating analysis through monitoring. ++ Default value: `true` + +### `enable-heartbeat-concurrent-runner` New in v8.0.0 + ++ Controls whether to enable asynchronous concurrent processing for Region heartbeats. When enabled, an independent executor handles Region heartbeat requests asynchronously and concurrently, which can improve heartbeat processing throughput and reduce latency. ++ Default value: `true` + ## `replication` Configuration items related to replicas @@ -432,16 +464,20 @@ Configuration items related to replicas + Default value: `true` + See [Placement Rules](/configure-placement-rules.md). -## `label-property` +## `label-property` (deprecated) -Configuration items related to labels +Configuration items related to labels, which only support the `reject-leader` type. + +> **Note:** +> +> Starting from v5.2, the configuration items related to labels are deprecated. It is recommended to use [Placement Rules](/configure-placement-rules.md#scenario-2-place-five-replicas-in-three-data-centers-in-the-proportion-of-221-and-the-leader-should-not-be-in-the-third-data-center) to configure the replica policy. -### `key` +### `key` (deprecated) + The label key for the store that rejected the Leader + Default value: `""` -### `value` +### `value` (deprecated) + The label value for the store that rejected the Leader + Default value: `""` @@ -450,6 +486,12 @@ Configuration items related to labels Configuration items related to the [TiDB Dashboard](/dashboard/dashboard-intro.md) built in PD. +### `disable-custom-prom-addr` + ++ Whether to disable configuring a custom Prometheus data source address in [TiDB Dashboard](/dashboard/dashboard-intro.md). ++ Default value: `false` ++ When it is set to `true`, if you configure a custom Prometheus data source address in TiDB Dashboard, TiDB Dashboard reports an error. + ### `tidb-cacert-path` + The path of the root CA certificate file. You can configure this path when you connect to TiDB's SQL services using TLS. @@ -473,15 +515,18 @@ Configuration items related to the [TiDB Dashboard](/dashboard/dashboard-intro.m ### `enable-telemetry` -+ Determines whether to enable the telemetry collection feature in TiDB Dashboard. +> **Warning:** +> +> Starting from v8.1.0, the telemetry feature in TiDB Dashboard is removed, and this configuration item is no longer functional. It is retained solely for compatibility with earlier versions. + ++ Before v8.1.0, this configuration item controls whether to enable telemetry collection in TiDB Dashboard. + Default value: `false` -+ See [Telemetry](/telemetry.md) for details. ## `replication-mode` Configuration items related to the replication mode of all Regions. See [Enable the DR Auto-Sync mode](/two-data-centers-in-one-city-deployment.md#enable-the-dr-auto-sync-mode) for details. -## Controller +## controller This section describes the configuration items that are built into PD for [Resource Control](/tidb-resource-control.md). @@ -498,7 +543,7 @@ The following are the configuration items about the [Request Unit (RU)](/tidb-re #### `read-base-cost` + Basis factor for conversion from a read request to RU -+ Default value: 0.25 ++ Default value: 0.125 #### `write-base-cost` @@ -521,4 +566,4 @@ The following are the configuration items about the [Request Unit (RU)](/tidb-re + Basis factor for conversion from CPU to RU + Default value: 1/3 -+ 1 RU = 3 millisecond CPU time ++ 1 RU = 3 millisecond CPU time \ No newline at end of file diff --git a/pd-control.md b/pd-control.md index 6eba08566eb3a..e283e35d801af 100644 --- a/pd-control.md +++ b/pd-control.md @@ -1,7 +1,6 @@ --- title: PD Control User Guide summary: Use PD Control to obtain the state information of a cluster and tune a cluster. -aliases: ['/docs/dev/pd-control/','/docs/dev/reference/tools/pd-control/'] --- # PD Control User Guide @@ -29,7 +28,7 @@ To obtain `pd-ctl` of the latest version, download the TiDB server installation > **Note:** > -> `{version}` in the link indicates the version number of TiDB. For example, the download link for `v8.0.0` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-server-v8.0.0-linux-amd64.tar.gz`. +> `{version}` in the link indicates the version number of TiDB. For example, the download link for `v8.1.2` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-server-v8.1.2-linux-amd64.tar.gz`. ### Compile from source code @@ -333,7 +332,7 @@ Usage: - `store-limit-mode` is used to control the mode of limiting the store speed. The optional modes are `auto` and `manual`. In `auto` mode, the stores are automatically balanced according to the load (deprecated). -- `store-limit-version` controls the version of the store limit formula. In v1 mode, you can manually modify the `store limit` to limit the scheduling speed of a single TiKV. The v2 mode is an experimental feature. In v2 mode, you do not need to manually set the `store limit` value, as PD dynamically adjusts it based on the capability of TiKV snapshots. For more details, refer to [Principles of store limit v2](/configure-store-limit.md#principles-of-store-limit-v2). +- `store-limit-version` controls the version of the store limit formula. In v1 mode, you can manually modify the `store limit` to limit the scheduling speed of a single TiKV. The v2 mode is an experimental feature. It is not recommended that you use it in the production environment. In v2 mode, you do not need to manually set the `store limit` value, as PD dynamically adjusts it based on the capability of TiKV snapshots. For more details, refer to [Principles of store limit v2](/configure-store-limit.md#principles-of-store-limit-v2). ```bash config set store-limit-version v2 // using store limit v2 @@ -800,20 +799,20 @@ Use this command to view and control the scheduling policy. Usage: ```bash ->> scheduler show // Display all created schedulers ->> scheduler add grant-leader-scheduler 1 // Schedule all the leaders of the Regions on store 1 to store 1 ->> scheduler add evict-leader-scheduler 1 // Move all the Region leaders on store 1 out ->> scheduler config evict-leader-scheduler // Display the stores in which the scheduler is located since v4.0.0 ->> scheduler add shuffle-leader-scheduler // Randomly exchange the leader on different stores ->> scheduler add shuffle-region-scheduler // Randomly scheduling the Regions on different stores ->> scheduler add evict-slow-store-scheduler // When there is one and only one slow store, evict all Region leaders of that store ->> scheduler remove grant-leader-scheduler-1 // Remove the corresponding scheduler, and `-1` corresponds to the store ID ->> scheduler pause balance-region-scheduler 10 // Pause the balance-region scheduler for 10 seconds ->> scheduler pause all 10 // Pause all schedulers for 10 seconds ->> scheduler resume balance-region-scheduler // Continue to run the balance-region scheduler ->> scheduler resume all // Continue to run all schedulers ->> scheduler config balance-hot-region-scheduler // Display the configuration of the balance-hot-region scheduler ->> scheduler describe balance-region-scheduler // Display the running state and related diagnostic information of the balance-region scheduler +>> scheduler show // Display all created schedulers +>> scheduler add grant-leader-scheduler 1 // Schedule all the leaders of the Regions on store 1 to store 1 +>> scheduler add evict-leader-scheduler 1 // Move all the Region leaders on store 1 out +>> scheduler config evict-leader-scheduler // Display the stores in which the scheduler is located since v4.0.0 +>> scheduler config evict-leader-scheduler add-store 2 // Add leader eviction scheduling for store 2 +>> scheduler config evict-leader-scheduler delete-store 2 // Remove leader eviction scheduling for store 2 +>> scheduler add evict-slow-store-scheduler // When there is one and only one slow store, evict all Region leaders of that store +>> scheduler remove grant-leader-scheduler-1 // Remove the corresponding scheduler, and `-1` corresponds to the store ID +>> scheduler pause balance-region-scheduler 10 // Pause the balance-region scheduler for 10 seconds +>> scheduler pause all 10 // Pause all schedulers for 10 seconds +>> scheduler resume balance-region-scheduler // Continue to run the balance-region scheduler +>> scheduler resume all // Continue to run all schedulers +>> scheduler config balance-hot-region-scheduler // Display the configuration of the balance-hot-region scheduler +>> scheduler describe balance-region-scheduler // Display the running state and related diagnostic information of the balance-region scheduler ``` ### `scheduler describe balance-region-scheduler` @@ -959,6 +958,24 @@ Usage: scheduler config balance-hot-region-scheduler set enable-for-tiflash true ``` +### `scheduler config evict-leader-scheduler` + +Use this command to view and manage the configuration of the `evict-leader-scheduler`. + +- When an `evict-leader-scheduler` already exists, use the `add-store` subcommand to add leader eviction scheduling for the specified store: + + ```bash + scheduler config evict-leader-scheduler add-store 2 // Add leader eviction scheduling for store 2 + ``` + +- When an `evict-leader-scheduler` already exists, use the `delete-store` subcommand to remove leader eviction scheduling for the specified store: + + ```bash + scheduler config evict-leader-scheduler delete-store 2 // Remove leader eviction scheduling for store 2 + ``` + + If all store configurations of an `evict-leader-scheduler` are removed, the scheduler itself is automatically removed. + ### `service-gc-safepoint` Use this command to query the current GC safepoint and service GC safepoint. The output is as follows: diff --git a/pd-recover.md b/pd-recover.md index 24b3bd3049d80..16f7aed026f0b 100644 --- a/pd-recover.md +++ b/pd-recover.md @@ -1,7 +1,6 @@ --- title: PD Recover User Guide summary: Use PD Recover to recover a PD cluster which cannot start or provide services normally. -aliases: ['/docs/dev/pd-recover/','/docs/dev/reference/tools/pd-recover/'] --- # PD Recover User Guide @@ -145,7 +144,7 @@ Before deploying a new PD cluster, you need to stop the existing PD cluster and ### Step 4: Use pd-recover -You only need to run `pd-recover` on one PD node. +You only need to run `pd-recover` on one PD node. Note that to avoid reallocation, it is recommended to set the `-alloc-id` parameter to a value larger than the allocated ID. For example, if the maximum allocated ID obtained from monitoring or logs is `9000`, it is recommended to pass `10000` or a larger value to the `-alloc-id` parameter. {{< copyable "shell-regular" >}} diff --git a/performance-schema/performance-schema-session-connect-attrs.md b/performance-schema/performance-schema-session-connect-attrs.md index e1a758d7d2b33..5455876cef96f 100644 --- a/performance-schema/performance-schema-session-connect-attrs.md +++ b/performance-schema/performance-schema-session-connect-attrs.md @@ -52,7 +52,7 @@ TABLE SESSION_CONNECT_ATTRS; | PROCESSLIST_ID | ATTR_NAME | ATTR_VALUE | ORDINAL_POSITION | +----------------+-----------------+------------+------------------+ | 2097154 | _client_name | libmysql | 0 | -| 2097154 | _client_version | 8.1.0 | 1 | +| 2097154 | _client_version | 8.1.2 | 1 | | 2097154 | _os | Linux | 2 | | 2097154 | _pid | 1299203 | 3 | | 2097154 | _platform | x86_64 | 4 | diff --git a/performance-tuning-methods.md b/performance-tuning-methods.md index af4573457d918..b4ca5a72d96a8 100644 --- a/performance-tuning-methods.md +++ b/performance-tuning-methods.md @@ -90,7 +90,7 @@ The diagrams of database time breakdown and execution time overview present both > **Note:** > - > It is normal that the total KV request time is greater than the execute time. Because the TiDB executor may send KV requests to multiple TiKVs concurrently, causing the total KV request wait time to be greater than the execute time. In the preceding TPC-C workload, TiDB sends `Prewrite` and `Commit` requests concurrently to multiple TiKVs when a transaction is committed. Therefore, the total time for `Prewrite`, `Commit`, and `PessimisticsLock` requests in this example is obviously longer than the execute time. + > It is normal that the total KV request time is greater than the execute time. Because the TiDB executor may send KV requests to multiple TiKVs concurrently, causing the total KV request wait time to be greater than the execute time. In the preceding TPC-C workload, TiDB sends `Prewrite` and `Commit` requests concurrently to multiple TiKVs when a transaction is committed. Therefore, the total time for `Prewrite`, `Commit`, and `PessimisticLock` requests in this example is obviously longer than the execute time. > > - The `execute` time may also be significantly greater than the total time of the KV request plus the `tso_wait` time. This means that the SQL execution time is spent mostly inside the TiDB executor. Here are two common examples: > @@ -170,10 +170,10 @@ In this workload, `Commit QPS` = `Rollback QPS` = `Select QPS`. The application **Example 3: Prepared plan cache unavailable with prepared statement enabled for OLTP workload** -`StmtPreare` times = `StmtExecute` times = `StmtClose` times ~= `StmtFetch` times. The application uses the prepare > execute > fetch > close loop. To prevent prepared statement object leak, many application frameworks call `close` after the `execute` phase. This creates two problems. +`StmtPrepare` times = `StmtExecute` times = `StmtClose` times ~= `StmtFetch` times. The application uses the prepare > execute > fetch > close loop. To prevent prepared statement object leak, many application frameworks call `close` after the `execute` phase. This creates two problems. - A SQL execution requires four commands and four network round trips. -- Queries Using Plan Cache OPS is 0, indicating zero hit of prepared plan cache. The `StmtClose` command clears cached execution plans by default and the next `StmtPreare` command needs to generate the execution plan again. +- Queries Using Plan Cache OPS is 0, indicating zero hit of prepared plan cache. The `StmtClose` command clears cached execution plans by default and the next `StmtPrepare` command needs to generate the execution plan again. > **Note:** > @@ -204,7 +204,7 @@ The number of `StmtPrepare` commands per second is much greater than that of `St In this TPC-C workload: -- The total number of KV requests per second is 79,700. The top request types are `Prewrite`, `Commit`, `PessimisticsLock`, and `BatchGet` in order of number of requests. +- The total number of KV requests per second is 79,700. The top request types are `Prewrite`, `Commit`, `PessimisticLock`, and `BatchGet` in order of number of requests. - Most of the KV processing time is spent on `Commit-external_Commit` and `Prewrite-external_Commit`, which indicates that the most time-consuming KV requests are `Commit` and `Prewrite` from external commit statements. **Example 2: Analyze workload** @@ -279,7 +279,7 @@ In this workload: - The average latency and P99 latency of all SQL statements are 10.8 ms and 84.1 ms, respectively. - The average connection idle time in transactions `avg-in-txn` is 9.4 ms. -- The total number of connections to the cluster is 3,700, and the number of connections to each TiDB node is 1,800. The average number of active connections is 40.3, which indicates that most of the connections are idle. The average number of `disonnnection/s` is 55.8, which indicates that the application is connecting and disconnecting frequently. The behavior of short connections will have a certain impact on TiDB resources and response time. +- The total number of connections to the cluster is 3,700, and the number of connections to each TiDB node is 1,800. The average number of active connections is 40.3, which indicates that most of the connections are idle. The average number of `disconnection/s` is 55.8, which indicates that the application is connecting and disconnecting frequently. The behavior of short connections will have a certain impact on TiDB resources and response time. **Example 2: TiDB is the bottleneck of user response time** @@ -393,7 +393,7 @@ TiKV processes a write request in the following procedure: Raftstore consists of a `Store` thread and an `Apply` thread: - The `Store` thread processes Raft messages and new `proposals`. When a new `proposals` is received, the `Store` thread of the leader node writes to the local Raft DB and copies the message to multiple follower nodes. When this `proposals` is successfully persisted in most instances, the `proposals` is successfully committed. - - The `Apply` thread writes the committed `proposals` to the KV DB. When the content is successfully written to the KV DB, the `Apply` thread notifies externally that the write request has completed. + - The `Apply` thread writes the committed `proposals` to the KV DB. When the data is successfully written to the KV DB, the `Apply` thread notifies externally that the write request has completed. ![TiKV Write](/media/performance/store_apply.png) @@ -489,6 +489,6 @@ For the `Store` thread, `Commit Log Duration` is obviously higher than `Apply Lo ## If my TiDB version is earlier than v6.1.0, what should I do to use the Performance Overview dashboard? -Starting from v6.1.0, Grafana has a built-in Performance Overview dashboard by default. This dashboard is compatible with TiDB v4.x and v5.x versions. If your TiDB is earlier than v6.1.0, you need to manually import [`performance_overview.json`](https://github.com/pingcap/tidb/blob/master/pkg/metrics/grafana/performance_overview.json), as shown in the following figure: +Starting from v6.1.0, Grafana has a built-in Performance Overview dashboard by default. This dashboard is compatible with TiDB v4.x and v5.x versions. If your TiDB is earlier than v6.1.0, you need to manually import [`performance_overview.json`](https://github.com/pingcap/tidb/blob/release-8.1/pkg/metrics/grafana/performance_overview.json), as shown in the following figure: ![Store](/media/performance/import_dashboard.png) diff --git a/performance-tuning-practices.md b/performance-tuning-practices.md index 13a52c0db9730..1704b228e6f9d 100644 --- a/performance-tuning-practices.md +++ b/performance-tuning-practices.md @@ -413,13 +413,13 @@ The following table lists the performance of seven different scenarios. | Metrics | Scenario 1 | Scenario 2 | Scenario 3 | Scenario 4 | Scenario 5 | Scenario 6 | Scenario 7 | Comparing Scenario 5 with Scenario 2 (%) | Comparing Scenario 7 with Scenario 3 (%) | | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | -| query duration | 479μs | 1120μs | 528μs | 426μs |690μs | 533μs | 313μs | -38% | -51% | +| query duration | 479μs | 1120μs | 528μs | 426μs |690μs | 533μs | 313μs | -38% | -41% | | QPS | 56.3k | 24.2k | 19.7k | 22.1k | 30.9k | 34.9k | 40.9k | +28% | +108% | In these scenarios, Scenario 2 is a common scenario where applications use the Query interface, and Scenario 5 is an ideal scenario where applications use the Prepared Statement interface. -- Comparing Scenario 2 with Scenario 5, you can see that by using best practices for Java application development and caching Prepared Statement objects on the client side, each SQL statement requires only one command and database interaction to hit the execution plan cache, which results in a 38% drop in query latency and a 28% increase in QPS, while the average TiDB CPU utilization drops from 936% to 577%. -- Comparing Scenario 2 with Scenario 7, you can see that with the latest TiDB optimization features such as RC Read and small table cache on top of Scenario 5, latency is reduced by 51% and QPS is increased by 108%, while the average TiDB CPU utilization drops from 936% to 478%. +- Comparing Scenario 5 with Scenario 2, you can see that by using best practices for Java application development and caching Prepared Statement objects on the client side, each SQL statement requires only one command and database interaction to hit the execution plan cache, which results in a 38% drop in query latency and a 28% increase in QPS, while the average TiDB CPU utilization drops from 936% to 577%. +- Comparing Scenario 7 with Scenario 3, you can see that with the latest TiDB optimization features such as RC Read and small table cache on top of Scenario 5, latency is reduced by 41% and QPS is increased by 108%, while the average TiDB CPU utilization drops from 936% to 478%. By comparing the performance of each scenario, we can draw the following conclusions: @@ -427,7 +427,7 @@ By comparing the performance of each scenario, we can draw the following conclus - TiDB is compatible with different commands of the MySQL protocol. When using the Prepared Statement interface and setting the following JDBC connection parameters, the application can achieve its best performance: ``` - useServerPrepStmts=true&cachePrepStmts=true&prepStmtCacheSize=1000&prepStmtCacheSqlLimit=20480&useConfigs= maxPerformance + useServerPrepStmts=true&cachePrepStmts=true&prepStmtCacheSize=1000&prepStmtCacheSqlLimit=20480&useConfigs=maxPerformance ``` - It is recommended that you use TiDB Dashboard (for example, the Top SQL feature and Continuous Profiling feature) and Performance Overview dashboard for performance analysis and tuning. diff --git a/pessimistic-transaction.md b/pessimistic-transaction.md index b759d2af95854..fb25b2b5522c1 100644 --- a/pessimistic-transaction.md +++ b/pessimistic-transaction.md @@ -1,7 +1,6 @@ --- title: TiDB Pessimistic Transaction Mode summary: Learn the pessimistic transaction mode in TiDB. -aliases: ['/docs/dev/pessimistic-transaction/','/docs/dev/reference/transactions/transaction-pessimistic/'] --- # TiDB Pessimistic Transaction Mode diff --git a/placement-rules-in-sql.md b/placement-rules-in-sql.md index 75d234c5c9c01..4db46c363d1d0 100644 --- a/placement-rules-in-sql.md +++ b/placement-rules-in-sql.md @@ -15,7 +15,7 @@ This feature can fulfill the following use cases: > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## Overview @@ -30,7 +30,7 @@ With the Placement Rules in SQL feature, you can [create placement policies](#cr > **Tip:** > -> The implementation of *Placement Rules in SQL* relies on the *placement rules feature* of PD. For details, refer to [Configure Placement Rules](https://docs.pingcap.com/zh/tidb/stable/configure-placement-rules). In the context of Placement Rules in SQL, *placement rules* might refer to *placement policies* attached to other objects, or to rules that are sent from TiDB to PD. +> The implementation of *Placement Rules in SQL* relies on the *placement rules feature* of PD. For details, refer to [Configure Placement Rules](https://docs.pingcap.com/tidb/stable/configure-placement-rules). In the context of Placement Rules in SQL, *placement rules* might refer to *placement policies* attached to other objects, or to rules that are sent from TiDB to PD. ## Limitations @@ -44,7 +44,7 @@ Placement policies rely on the configuration of labels on TiKV nodes. For exampl -When you create a placement policy, TiDB does not check whether the labels specified in the policy exist. Instead, TiDB performs the check when you attach the policy. Therefore, before attaching a placement policy, make sure that each TiKV node is configured with correct labels. The configuration method for a TiDB Self-Hosted cluster is as follows: +When you create a placement policy, TiDB does not check whether the labels specified in the policy exist. Instead, TiDB performs the check when you attach the policy. Therefore, before attaching a placement policy, make sure that each TiKV node is configured with correct labels. The configuration method for a TiDB Self-Managed cluster is as follows: ``` tikv-server --labels region=,zone=,host= @@ -60,13 +60,13 @@ For detailed configuration methods, see the following examples: > **Note:** > -> For TiDB Dedicated clusters, you can skip these label configuration steps because the labels on TiKV nodes in TiDB Dedicated clusters are configured automatically. +> For TiDB Cloud Dedicated clusters, you can skip these label configuration steps because the labels on TiKV nodes in TiDB Cloud Dedicated clusters are configured automatically. -For TiDB Dedicated clusters, labels on TiKV nodes are configured automatically. +For TiDB Cloud Dedicated clusters, labels on TiKV nodes are configured automatically. @@ -231,7 +231,7 @@ You can configure `CONSTRAINTS`, `FOLLOWER_CONSTRAINTS`, and `LEARNER_CONSTRAINT | CONSTRAINTS format | Description | |----------------------------|-----------------------------------------------------------------------------------------------------------| -| List format | If a constraint to be specified applies to all replicas, you can use a key-value list format. Each key starts with `+` or `-`. For example:
  • `[+region=us-east-1]` means placing data on nodes that have a `region` label as `us-east-1`.
  • `[+region=us-east-1,-type=fault]` means placing data on nodes that have a `region` label as `us-east-1` but do not have a `type` label as `fault`.

| +| List format | If a constraint to be specified applies to all replicas, you can use a key-value list format. Each key starts with `+` or `-`. For example:
  • `[+region=us-east-1]` means placing data on nodes that have a `region` label as `us-east-1`.
  • `[+region=us-east-1,-type=fault]` means placing data on nodes that have a `region` label as `us-east-1` but do not have a `type` label as `fault`.

| | Dictionary format | If you need to specify different numbers of replicas for different constraints, you can use the dictionary format. For example:
  • `FOLLOWER_CONSTRAINTS="{+region=us-east-1: 1,+region=us-east-2: 1,+region=us-west-1: 1}";` means placing one Follower in `us-east-1`, one Follower in `us-east-2`, and one Follower in `us-west-1`.
  • `FOLLOWER_CONSTRAINTS='{"+region=us-east-1,+type=scale-node": 1,"+region=us-west-1": 1}';` means placing one Follower on a node that is located in the `us-east-1` region and has the `type` label as `scale-node`, and one Follower in `us-west-1`.
The dictionary format supports each key starting with `+` or `-` and allows you to configure the special `#evict-leader` attribute. For example, `FOLLOWER_CONSTRAINTS='{"+region=us-east-1":1, "+region=us-east-2": 2, "+region=us-west-1,#evict-leader": 1}'` means that the Leaders elected in `us-west-1` will be evicted as much as possible during disaster recovery.| > **Note:** diff --git a/post-installation-check.md b/post-installation-check.md index 176e7d8f25f02..b635876c482ad 100644 --- a/post-installation-check.md +++ b/post-installation-check.md @@ -1,7 +1,6 @@ --- title: Check Cluster Status summary: Learn how to check the running status of the TiDB cluster. -aliases: ['/docs/dev/post-installation-check/'] --- # Check Cluster Status @@ -63,7 +62,7 @@ The following information indicates successful login: ```sql Welcome to the MySQL monitor. Commands end with ; or \g. Your MySQL connection id is 3 -Server version: 8.0.11-TiDB-v8.0.0 TiDB Server (Apache License 2.0) Community Edition, MySQL 8.0 compatible +Server version: 8.0.11-TiDB-v8.1.2 TiDB Server (Apache License 2.0) Community Edition, MySQL 8.0 compatible Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. Oracle is a registered trademark of Oracle Corporation and/or its affiliates. Other names may be trademarks of their respective diff --git a/predicate-push-down.md b/predicate-push-down.md index dbe6622d90101..de18a6981c069 100644 --- a/predicate-push-down.md +++ b/predicate-push-down.md @@ -1,7 +1,6 @@ --- title: Predicates Push Down summary: Introduce one of the TiDB's logic optimization rules—Predicate Push Down (PPD). -aliases: ['/tidb/dev/predicates-push-down'] --- # Predicates Push Down (PPD) diff --git a/privilege-management.md b/privilege-management.md index 4bdb158a18bf8..77c0172412f85 100644 --- a/privilege-management.md +++ b/privilege-management.md @@ -1,7 +1,6 @@ --- title: Privilege Management summary: Learn how to manage the privilege. -aliases: ['/docs/dev/privilege-management/','/docs/dev/reference/security/privilege-system/'] --- # Privilege Management @@ -345,7 +344,7 @@ Requires the `INSERT` and `SELECT` privileges for the table. `SHOW GRANTS` requires the `SELECT` privilege to the `mysql` database. If the target user is current user, `SHOW GRANTS` does not require any privilege. -`SHOW PROCESSLIST` requires the `SUPER` privilege to show connections belonging to other users. +`SHOW PROCESSLIST` requires the `PROCESS` privilege to show connections belonging to other users. `SHOW IMPORT JOB` requires the `SUPER` privilege to show connections belonging to other users. Otherwise, it only shows jobs created by the current user. @@ -417,7 +416,7 @@ Requires `SUPER` or `RESOURCE_GROUP_ADMIN` privilege. ### Privilege table -The following system tables are special because all the privilege-related data is stored in them: +The following [`mysql` system tables](/mysql-schema/mysql-schema.md) are special because all the privilege-related data is stored in them: - `mysql.user` (user account, global privilege) - `mysql.db` (database-level privilege) diff --git a/production-deployment-using-tiup.md b/production-deployment-using-tiup.md index e56f3934bda39..6d7eca04240e7 100644 --- a/production-deployment-using-tiup.md +++ b/production-deployment-using-tiup.md @@ -1,7 +1,6 @@ --- title: Deploy a TiDB Cluster Using TiUP summary: Learn how to easily deploy a TiDB cluster using TiUP. -aliases: ['/docs/dev/production-deployment-using-tiup/','/docs/dev/how-to/deploy/orchestrated/tiup/','/docs/dev/tiflash/deploy-tiflash/','/docs/dev/reference/tiflash/deploy/','/tidb/dev/deploy-tidb-from-dbdeployer/','/docs/dev/deploy-tidb-from-dbdeployer/','/docs/dev/how-to/get-started/deploy-tidb-from-dbdeployer/','/tidb/dev/deploy-tidb-from-homebrew/','/docs/dev/deploy-tidb-from-homebrew/','/docs/dev/how-to/get-started/deploy-tidb-from-homebrew/','/tidb/dev/production-offline-deployment-using-tiup','/docs/dev/production-offline-deployment-using-tiup/','/tidb/dev/deploy-tidb-from-binary','/tidb/dev/production-deployment-from-binary-tarball','/tidb/dev/test-deployment-from-binary-tarball','/tidb/dev/deploy-test-cluster-using-docker-compose','/tidb/dev/test-deployment-using-docker'] --- # Deploy a TiDB Cluster Using TiUP @@ -23,6 +22,10 @@ You can deploy TiUP on the control machine in either of the two ways: online dep ### Deploy TiUP online +> **Note:** +> +> If the TiUP environment switches to offline, refer to [Deploy TiUP offline](#deploy-tiup-offline) for deployment. Otherwise, TiUP cannot work properly. + Log in to the control machine using a regular user account (take the `tidb` user as an example). Subsequent TiUP installation and cluster management can be performed by the `tidb` user. 1. Install TiUP by running the following command: @@ -67,7 +70,7 @@ Log in to the control machine using a regular user account (take the `tidb` user tiup update --self && tiup update cluster ``` - If `Update successfully!` is displayed, the TiUP cluster is updated successfully. + If `Updated successfully!` is displayed, the TiUP cluster is updated successfully. 5. Verify the current version of your TiUP cluster: @@ -95,7 +98,7 @@ https://download.pingcap.org/tidb-community-toolkit-{version}-linux-{arch}.tar.g > **Tip:** > -> `{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v8.0.0` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-toolkit-v8.0.0-linux-amd64.tar.gz`. +> `{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v8.1.2` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-toolkit-v8.1.2-linux-amd64.tar.gz`. **Method 2**: Manually pack an offline component package using `tiup mirror clone`. The detailed steps are as follows: @@ -303,10 +306,10 @@ The following examples cover seven common scenarios. You need to modify the conf For more configuration description, see the following configuration examples: -- [TiDB `config.toml.example`](https://github.com/pingcap/tidb/blob/master/pkg/config/config.toml.example) -- [TiKV `config.toml.example`](https://github.com/tikv/tikv/blob/master/etc/config-template.toml) -- [PD `config.toml.example`](https://github.com/pingcap/pd/blob/master/conf/config.toml) -- [TiFlash `config.toml.example`](https://github.com/pingcap/tiflash/blob/master/etc/config-template.toml) +- [TiDB `config.toml.example`](https://github.com/pingcap/tidb/blob/release-8.1/pkg/config/config.toml.example) +- [TiKV `config.toml.example`](https://github.com/tikv/tikv/blob/release-8.1/etc/config-template.toml) +- [PD `config.toml.example`](https://github.com/pingcap/pd/blob/release-8.1/conf/config.toml) +- [TiFlash `config.toml.example`](https://github.com/pingcap/tiflash/blob/release-8.1/etc/config-template.toml) ## Step 4. Run the deployment command @@ -346,13 +349,13 @@ Before you run the `deploy` command, use the `check` and `check --apply` command {{< copyable "shell-regular" >}} ```shell - tiup cluster deploy tidb-test v8.0.0 ./topology.yaml --user root [-p] [-i /home/root/.ssh/gcp_rsa] + tiup cluster deploy tidb-test v8.1.2 ./topology.yaml --user root [-p] [-i /home/root/.ssh/gcp_rsa] ``` In the `tiup cluster deploy` command above: - `tidb-test` is the name of the TiDB cluster to be deployed. -- `v8.0.0` is the version of the TiDB cluster to be deployed. You can see the latest supported versions by running `tiup list tidb`. +- `v8.1.2` is the version of the TiDB cluster to be deployed. You can see the latest supported versions by running `tiup list tidb`. - `topology.yaml` is the initialization configuration file. - `--user root` indicates logging into the target machine as the `root` user to complete the cluster deployment. The `root` user is expected to have `ssh` and `sudo` privileges to the target machine. Alternatively, you can use other users with `ssh` and `sudo` privileges to complete the deployment. - `[-i]` and `[-p]` are optional. If you have configured login to the target machine without password, these parameters are not required. If not, choose one of the two parameters. `[-i]` is the private key of the root user (or other users specified by `--user`) that has access to the target machine. `[-p]` is used to input the user password interactively. diff --git a/quick-start-with-tidb.md b/quick-start-with-tidb.md index ab7d80d333bd3..8148ac21cfc2d 100644 --- a/quick-start-with-tidb.md +++ b/quick-start-with-tidb.md @@ -1,7 +1,6 @@ --- title: Quick Start Guide for the TiDB Database Platform summary: Learn how to quickly get started with the TiDB platform and see if TiDB is the right choice for you. -aliases: ['/docs/dev/quick-start-with-tidb/','/docs/dev/test-deployment-using-docker/'] --- # Quick Start Guide for the TiDB Database Platform @@ -68,6 +67,15 @@ As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB in 3. Start the cluster in the current session: + > **Note:** + > + > - For the playground operated in the following way, after the deployment and testing are finished, TiUP will automatically clean up the cluster data. You will get a new cluster after re-running the command. + > - If you want to persist data on storage, then add the `--tag` flag when you start the cluster. For details, see [Specify a tag when starting the TiDB cluster to store the data](/tiup/tiup-playground.md#specify-a-tag-when-starting-the-tidb-cluster-to-store-the-data). + > + > ```shell + > tiup playground --tag ${tag_name} + > ``` + - To start a TiDB cluster of the latest version with 1 TiDB instance, 1 TiKV instance, 1 PD instance, and 1 TiFlash instance, run the following command: {{< copyable "shell-regular" >}} @@ -81,10 +89,10 @@ As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB in {{< copyable "shell-regular" >}} ```shell - tiup playground v8.0.0 --db 2 --pd 3 --kv 3 + tiup playground v8.1.2 --db 2 --pd 3 --kv 3 ``` - The command downloads a version cluster to the local machine and starts it, such as v8.0.0. To view the latest version, run `tiup list tidb`. + The command downloads a version cluster to the local machine and starts it, such as v8.1.2. To view the latest version, run `tiup list tidb`. This command returns the access methods of the cluster: @@ -100,9 +108,7 @@ As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB in > **Note:** > - > + Since v5.2.0, TiDB supports running `tiup playground` on the machine that uses the Apple M1 chip. - > + For the playground operated in this way, after the test deployment is finished, TiUP will clean up the original cluster data. You will get a new cluster after re-running the command. - > + If you want the data to be persisted on storage, run `tiup --tag playground ...`. For details, refer to the [TiUP Reference](/tiup/tiup-reference.md#-t---tag) guide. + > Starting from v5.2.0, TiDB supports running `tiup playground` on the machine that uses the Apple M1 chip. 4. Start a new session to access TiDB: @@ -189,6 +195,15 @@ As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB in 3. Start the cluster in the current session: + > **Note:** + > + > - For the playground operated in the following way, after the deployment and testing are finished, TiUP will automatically clean up the cluster data. You will get a new cluster after re-running the command. + > - If you want to persist data on storage, then add the `--tag` flag when you start the cluster. For details, see [Specify a tag when starting the TiDB cluster to store the data](/tiup/tiup-playground.md#specify-a-tag-when-starting-the-tidb-cluster-to-store-the-data). + > + > ```shell + > tiup playground --tag ${tag_name} + > ``` + - To start a TiDB cluster of the latest version with 1 TiDB instance, 1 TiKV instance, 1 PD instance, and 1 TiFlash instance, run the following command: {{< copyable "shell-regular" >}} @@ -202,10 +217,10 @@ As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB in {{< copyable "shell-regular" >}} ```shell - tiup playground v8.0.0 --db 2 --pd 3 --kv 3 + tiup playground v8.1.2 --db 2 --pd 3 --kv 3 ``` - The command downloads a version cluster to the local machine and starts it, such as v8.0.0. To view the latest version, run `tiup list tidb`. + The command downloads a version cluster to the local machine and starts it, such as v8.1.2. To view the latest version, run `tiup list tidb`. This command returns the access methods of the cluster: @@ -218,11 +233,6 @@ As a distributed system, a basic TiDB test cluster usually consists of 2 TiDB in To view the Grafana: http://127.0.0.1:3000 ``` - > **Note:** - > - > For the playground operated in this way, after the test deployment is finished, TiUP will clean up the original cluster data. You will get a new cluster after re-running the command. - > If you want the data to be persisted on storage, run `tiup --tag playground ...`. For details, refer to the [TiUP Reference](/tiup/tiup-reference.md#-t---tag) guide. - 4. Start a new session to access TiDB: + Use the TiUP client to connect to TiDB. @@ -297,8 +307,8 @@ The smallest TiDB cluster topology consists of the following instances: Other requirements for the target machine include: -- The `root` user and its password are required -- [Stop the firewall service of the target machine](/check-before-deployment.md#check-and-stop-the-firewall-service-of-target-machines), or open the port needed by the TiDB cluster nodes +- The `root` user and its password are required. +- [Stop the firewall service of the target machine](/check-before-deployment.md#check-the-firewall-service-of-target-machines), or open the ports needed by the TiDB cluster nodes. - Currently, the TiUP cluster supports deploying TiDB on the x86_64 (AMD64) and ARM architectures: - It is recommended to use CentOS 7.3 or later versions on AMD64. @@ -437,7 +447,7 @@ Other requirements for the target machine include: ``` - ``: Set the cluster name - - ``: Set the TiDB cluster version, such as `v8.0.0`. You can see all the supported TiDB versions by running the `tiup list tidb` command + - ``: Set the TiDB cluster version, such as `v8.1.2`. You can see all the supported TiDB versions by running the `tiup list tidb` command - `-p`: Specify the password used to connect to the target machine. > **Note:** diff --git a/read-historical-data.md b/read-historical-data.md index bb532cb7db618..0a75f4fd76c6e 100644 --- a/read-historical-data.md +++ b/read-historical-data.md @@ -1,7 +1,6 @@ --- title: Read Historical Data Using the System Variable `tidb_snapshot` summary: Learn about how TiDB reads data from history versions using the system variable `tidb_snapshot`. -aliases: ['/docs/dev/read-historical-data/','/docs/dev/how-to/get-started/read-historical-data/'] --- # Read Historical Data Using the System Variable `tidb_snapshot` diff --git a/releases/release-1.0-ga.md b/releases/release-1.0-ga.md index 3dc4937fbc221..b97ea2bc7d6e2 100644 --- a/releases/release-1.0-ga.md +++ b/releases/release-1.0-ga.md @@ -1,6 +1,5 @@ --- title: TiDB 1.0 release notes -aliases: ['/docs/dev/releases/release-1.0-ga/','/docs/dev/releases/ga/'] summary: TiDB 1.0 is released with a focus on MySQL compatibility, SQL optimization, stability, and performance. It includes enhancements to the SQL query optimizer, internal data format optimization, and support for various operators. PD now supports read flow based balancing and setting store weight. TiKV has improved coprocessor support and performance, and added a Debug API. Special thanks to enterprises, open source software, and individual contributors for their support. --- diff --git a/releases/release-1.0.1.md b/releases/release-1.0.1.md index 7ff5bc2e67a72..615989a00d889 100644 --- a/releases/release-1.0.1.md +++ b/releases/release-1.0.1.md @@ -1,6 +1,5 @@ --- title: TiDB 1.0.1 Release Notes -aliases: ['/docs/dev/releases/release-1.0.1/','/docs/dev/releases/101/'] summary: TiDB 1.0.1 was released on November 1, 2017. Updates include support for canceling DDL Job, optimizing the `IN` expression, correcting the result type of the `Show` statement, supporting log slow query into a separate log file, and fixing bugs. TiKV now supports flow control with write bytes, reduces Raft allocation, increases coprocessor stack size to 10MB, and removes the useless log from the coprocessor. --- diff --git a/releases/release-1.0.2.md b/releases/release-1.0.2.md index a3116bfa8ae5b..20705c1d1760c 100644 --- a/releases/release-1.0.2.md +++ b/releases/release-1.0.2.md @@ -1,6 +1,5 @@ --- title: TiDB 1.0.2 Release Notes -aliases: ['/docs/dev/releases/release-1.0.2/','/docs/dev/releases/102/'] summary: TiDB 1.0.2 was released on November 13, 2017. Updates include optimized cost estimation for index point query, support for Alter Table Add Column syntax, and improved query optimization. Placement Driver (PD) scheduling stability was enhanced, and TiKV now supports table splitting and limits key length to 4 KB. Other improvements include more accurate read traffic statistics and bug fixes for LIKE behavior and do_div_mod bug. --- diff --git a/releases/release-1.0.3.md b/releases/release-1.0.3.md index 29396839835c2..064e38ba5b4f1 100644 --- a/releases/release-1.0.3.md +++ b/releases/release-1.0.3.md @@ -1,6 +1,5 @@ --- title: TiDB 1.0.3 Release Notes -aliases: ['/docs/dev/releases/release-1.0.3/','/docs/dev/releases/103/'] summary: TiDB 1.0.3 was released on November 28, 2017. Updates include performance optimization, new configuration options, and bug fixes. PD now supports adding more schedulers using API, and TiKV has fixed deadlock and leader value issues. To upgrade from 1.0.2 to 1.0.3, follow the rolling upgrade order of PD, TiKV, and TiDB. --- diff --git a/releases/release-1.0.4.md b/releases/release-1.0.4.md index 555dd7e3729a4..522c2039c96b3 100644 --- a/releases/release-1.0.4.md +++ b/releases/release-1.0.4.md @@ -1,6 +1,5 @@ --- title: TiDB 1.0.4 Release Notes -aliases: ['/docs/dev/releases/release-1.0.4/','/docs/dev/releases/104/'] summary: TiDB 1.0.4 was released on December 11, 2017. Updates include speed improvements, performance enhancements, and fixes for potential issues in TiDB and TiKV. To upgrade from 1.0.3 to 1.0.4, follow the rolling upgrade order of PD, TiKV, and TiDB. --- diff --git a/releases/release-1.0.5.md b/releases/release-1.0.5.md index 3873b9d8ae19f..5affec8ed2a50 100644 --- a/releases/release-1.0.5.md +++ b/releases/release-1.0.5.md @@ -1,6 +1,5 @@ --- title: TiDB 1.0.5 Release Notes -aliases: ['/docs/dev/releases/release-1.0.5/','/docs/dev/releases/105/'] summary: TiDB 1.0.5 was released on December 26, 2017. Updates include adding max value for Auto_Increment ID, fixing goroutine leak, supporting output of slow queries to separate file, loading TimeZone variable from TiKV, and more. PD fixes include balancing leaders and potential panic during bootstrapping. TiKV fixes slow CPU ID retrieval and supports dynamic-level-bytes parameter. Upgrade order is PD -> TiKV -> TiDB. --- diff --git a/releases/release-1.0.6.md b/releases/release-1.0.6.md index 4d6989c4e4544..a794b639cbf31 100644 --- a/releases/release-1.0.6.md +++ b/releases/release-1.0.6.md @@ -1,6 +1,5 @@ --- title: TiDB 1.0.6 Release Notes -aliases: ['/docs/dev/releases/release-1.0.6/','/docs/dev/releases/106/'] summary: TiDB 1.0.6 was released on January 08, 2018. Updates include support for Alter Table Auto_Increment syntax, fixing bugs in Cost Based computation and Null Json issue, and support for extension syntax to shard implicit row ID. Other updates include fixing potential DDL issue, considering timezone setting in certain functions, and support for SEPARATOR syntax in GROUP_CONCAT function. PD fixed store selection problem of hot-region scheduler. To upgrade from 1.0.5 to 1.0.6, follow the rolling upgrade order of PD, TiKV, TiDB. --- diff --git a/releases/release-1.0.7.md b/releases/release-1.0.7.md index 6ba4487c8404a..84e15126d0eb9 100644 --- a/releases/release-1.0.7.md +++ b/releases/release-1.0.7.md @@ -1,6 +1,5 @@ --- title: TiDB 1.0.7 Release Notes -aliases: ['/docs/dev/releases/release-1.0.7/','/docs/dev/releases/107/'] summary: TiDB 1.0.7 is released with various updates including optimization of commands, fixing data race and resource leak issues, adding session variable for log query control, and improving stability of test results. PD and TiKV also have updates to fix scheduling loss issues, compatibility issues, and add support for table scan and remote mode in tikv-ctl. To upgrade from 1.0.6 to 1.0.7, follow the rolling upgrade order of PD, TiKV, and TiDB. --- diff --git a/releases/release-1.0.8.md b/releases/release-1.0.8.md index 94bf7fe008d16..c4bbfed655891 100644 --- a/releases/release-1.0.8.md +++ b/releases/release-1.0.8.md @@ -1,6 +1,5 @@ --- title: TiDB 1.0.8 Release Notes -aliases: ['/docs/dev/releases/release-1.0.8/','/docs/dev/releases/108/'] summary: TiDB 1.0.8 is released with updates including fixes for various issues, performance optimizations, and stability improvements. PD and TiKV also have updates related to reducing lock overheat, fixing leader selection issues, and improving starting speed. To upgrade, follow the order of PD -> TiKV -> TiDB. --- diff --git a/releases/release-1.1-alpha.md b/releases/release-1.1-alpha.md index 7c0bd5095a48d..1eed9eefc6123 100644 --- a/releases/release-1.1-alpha.md +++ b/releases/release-1.1-alpha.md @@ -1,6 +1,5 @@ --- title: TiDB 1.1 Alpha Release Notes -aliases: ['/docs/dev/releases/release-1.1-alpha/','/docs/dev/releases/11alpha/'] summary: TiDB 1.1 Alpha, released on January 19, 2018, brings significant improvements in MySQL compatibility, SQL optimization, stability, and performance. Key updates include enhanced SQL parser, query optimizer, and executor, as well as server support for the PROXY protocol. PD now offers more APIs, TLS support, and improved scheduling, while TiKV introduces Raft learner support, TLS, and performance optimizations. Additionally, it enhances data recovery tools and improves flow control mechanisms. --- diff --git a/releases/release-1.1-beta.md b/releases/release-1.1-beta.md index d224ad60e848c..7574880b58532 100644 --- a/releases/release-1.1-beta.md +++ b/releases/release-1.1-beta.md @@ -1,6 +1,5 @@ --- title: TiDB 1.1 Beta Release Notes -aliases: ['/docs/dev/releases/release-1.1-beta/','/docs/dev/releases/11beta/'] summary: TiDB 1.1 Beta, released on February 24, 2018, brings significant improvements in MySQL compatibility, SQL optimization, stability, and performance. Key updates include more monitoring metrics, enhanced MySQL syntax compatibility, improved query optimization, and stability fixes. PD introduces new debug interfaces, priority settings, and performance optimizations. TiKV adds support for resolving locks in batches, GC concurrency, and more recovery operations in `tikv-ctl`. Overall, TiDB 1.1 Beta shows great improvement in test results and stability. --- diff --git a/releases/release-2.0-ga.md b/releases/release-2.0-ga.md index f56693e4185a1..fd482ba357f16 100644 --- a/releases/release-2.0-ga.md +++ b/releases/release-2.0-ga.md @@ -1,6 +1,5 @@ --- title: TiDB 2.0 Release Notes -aliases: ['/docs/dev/releases/release-2.0-ga/','/docs/dev/releases/2.0ga/'] summary: TiDB 2.0 GA release on April 27, 2018, brings improvements in MySQL compatibility, SQL optimizer, executor, and stability. Key updates include compact data structure for memory usage reduction, Stream Aggregation operator for empty GROUP BY clause, and support for more MySQL syntaxes. TiKV features include `Region Merge`, `Raw DeleteRange` API, and improved read performance using `ReadPool`. TiSpark 1.0 GA provides distributed computing of TiDB data using Apache Spark, with support for gRPC communication framework, calculation pushdown, index related support, cost-based optimization, and multiple Spark interfaces. --- diff --git a/releases/release-2.0-rc.1.md b/releases/release-2.0-rc.1.md index de346b63c53c0..3e751aff0c482 100644 --- a/releases/release-2.0-rc.1.md +++ b/releases/release-2.0-rc.1.md @@ -1,6 +1,5 @@ --- title: TiDB 2.0 RC1 Release Notes -aliases: ['/docs/dev/releases/release-2.0-rc.1/','/docs/dev/releases/2rc1/'] summary: TiDB 2.0 RC1, released on March 9, 2018, brings improvements in MySQL compatibility, SQL optimization, and stability. Key updates include memory usage limitation for SQL statements, Stream Aggregate operator support, configuration file validation, and HTTP API for configuration information. TiDB also enhances MySQL syntax compatibility, optimizer, and Boolean field length. PD sees logic and performance optimizations, while TiKV fixes gRPC call and adds gRPC APIs for metrics. Additionally, TiKV checks SSD usage, optimizes read performance, and improves metrics usage. --- diff --git a/releases/release-2.0-rc.3.md b/releases/release-2.0-rc.3.md index d16cf41499d94..790bdd6c134eb 100644 --- a/releases/release-2.0-rc.3.md +++ b/releases/release-2.0-rc.3.md @@ -1,6 +1,5 @@ --- title: TiDB 2.0 RC3 Release Notes -aliases: ['/docs/dev/releases/release-2.0-rc.3/','/docs/dev/releases/2rc3/'] summary: TiDB 2.0 RC3, released on March 23, 2018, brings improvements in MySQL compatibility, SQL optimization, and stability. It includes fixes for various issues, optimizations for execution speed, memory control, and DDL job management. PD now supports Region Merge and has optimizations for leader balance and abnormal Regions. TiKV also supports Region Merge, Raft snapshot process, and streaming in Coprocessor, with various improvements in space management and data recovery. --- diff --git a/releases/release-2.0-rc.4.md b/releases/release-2.0-rc.4.md index 4a0f2af4f7674..74ef45ec37f91 100644 --- a/releases/release-2.0-rc.4.md +++ b/releases/release-2.0-rc.4.md @@ -1,6 +1,5 @@ --- title: TiDB 2.0 RC4 Release Notes -aliases: ['/docs/dev/releases/release-2.0-rc.4/','/docs/dev/releases/2rc4/'] summary: TiDB 2.0 RC4, released on March 30, 2018, brings improvements in MySQL compatibility, SQL optimization, and stability. Key updates include support for various syntax, bug fixes, and performance optimizations in TiDB, PD, and TiKV. Notable changes include manual Region splitting in PD, memory usage limitation in TiKV, and support for data pattern import. Overall, the release focuses on enhancing functionality and addressing performance issues. --- diff --git a/releases/release-2.0-rc.5.md b/releases/release-2.0-rc.5.md index 2f4f938a2ce4f..8e5d249bf48e3 100644 --- a/releases/release-2.0-rc.5.md +++ b/releases/release-2.0-rc.5.md @@ -1,6 +1,5 @@ --- title: TiDB 2.0 RC5 Release Notes -aliases: ['/docs/dev/releases/release-2.0-rc.5/','/docs/dev/releases/2rc5/'] summary: TiDB 2.0 RC5 was released on April 17, 2018, with improvements in MySQL compatibility, SQL optimization, and stability. Fixes and optimizations were made to TiDB, PD, and TiKV components, including support for Raft Learner, reducing scheduling overhead, and adding new batch operations. The release also addressed issues related to memory usage, error reporting, and configuration adjustments. --- diff --git a/releases/release-2.0.1.md b/releases/release-2.0.1.md index b8242d7eaca04..a7e8c2e914dc3 100644 --- a/releases/release-2.0.1.md +++ b/releases/release-2.0.1.md @@ -1,6 +1,5 @@ --- title: TiDB 2.0.1 Release Notes -aliases: ['/docs/dev/releases/release-2.0.1/','/docs/dev/releases/201/'] summary: TiDB 2.0.1 was released on May 16, 2018, with improvements in MySQL compatibility and system stability. Updates include real-time progress for 'Add Index', a new session variable for automatic statistics update, bug fixes, compatibility improvements, and behavior changes. PD added a new scheduler, optimized region balancing, and fixed various issues. TiKV fixed issues related to reading, thread calls, raftstore blocking, and split causing dirty read. Overall, the release focuses on enhancing performance, stability, and compatibility. --- @@ -23,7 +22,7 @@ On May 16, 2018, TiDB 2.0.1 is released. Compared with TiDB 2.0.0 (GA), this rel - Add an option for TiDB to control the behaviour of Binlog failure - Refactor the `Coprocessor` slow log to distinguish between the scenario of tasks with long processing time and long waiting time - Log nothing when meeting MySQL protocol handshake error, to avoid too many logs caused by the load balancer Keep Alive mechanism -- Refine the “Out of range value for column” error message +- Refine the "Out of range value for column" error message - Fix a bug when there is a subquery in an `Update` statement - Change the behaviour of handling `SIGTERM`, and do not wait for all queries to terminate anymore diff --git a/releases/release-2.0.10.md b/releases/release-2.0.10.md index 637062327b381..a6f02f70a8e92 100644 --- a/releases/release-2.0.10.md +++ b/releases/release-2.0.10.md @@ -1,6 +1,5 @@ --- title: TiDB 2.0.10 Release Notes -aliases: ['/docs/dev/releases/release-2.0.10/','/docs/dev/releases/2.0.10/'] summary: TiDB 2.0.10 and TiDB Ansible 2.0.10 were released on December 18, 2018. The release includes improvements in system compatibility and stability. Fixes include issues with DDL jobs, ORDER BY and UNION clauses, UNCOMPRESS function, ANSI_QUOTES SQL_MODE, select results, and more. PD fixes a possible RaftCluster deadlock issue, while TiKV optimizes leader transfer and fixes redundant Region heartbeats. --- diff --git a/releases/release-2.0.11.md b/releases/release-2.0.11.md index 401f4d313c4a5..2e49ec415e332 100644 --- a/releases/release-2.0.11.md +++ b/releases/release-2.0.11.md @@ -1,6 +1,5 @@ --- title: TiDB 2.0.11 Release Notes -aliases: ['/docs/dev/releases/release-2.0.11/','/docs/dev/releases/2.0.11/'] summary: TiDB 2.0.11 and TiDB Ansible 2.0.11 were released on January 3, 2019. The release includes improvements in system compatibility and stability. Fixes include handling errors when PD is in an abnormal condition, compatibility issues with MySQL, error message reporting, prefix index range, and panic issues with the `UPDATE` statement. TiKV also fixed two issues related to Region merge. --- diff --git a/releases/release-2.0.2.md b/releases/release-2.0.2.md index 381efa9a21047..63e99a3ac4be1 100644 --- a/releases/release-2.0.2.md +++ b/releases/release-2.0.2.md @@ -1,6 +1,5 @@ --- title: TiDB 2.0.2 Release Notes -aliases: ['/docs/dev/releases/release-2.0.2/','/docs/dev/releases/202/'] summary: TiDB 2.0.2 was released on May 21, 2018, with improvements in system stability. The release includes fixes for Decimal division expression, support for `USE INDEX` syntax in `Delete` statement, and timeout mechanism for writing Binlog in TiDB. PD now filters disconnected nodes in balance leader scheduler, modifies transfer leader operator timeout, and fixes scheduling issues. TiKV fixes Raft log printing, supports configuring gRPC parameters, leader election timeout range, and resolves snapshot intermediate file deletion issue. --- diff --git a/releases/release-2.0.3.md b/releases/release-2.0.3.md index 5d7de183d7401..dd8c8216d02d3 100644 --- a/releases/release-2.0.3.md +++ b/releases/release-2.0.3.md @@ -1,6 +1,5 @@ --- title: TiDB 2.0.3 Release Notes -aliases: ['/docs/dev/releases/release-2.0.3/','/docs/dev/releases/203/'] summary: TiDB 2.0.3 was released on June 1, 2018, with improvements in system compatibility and stability. It includes various fixes and optimizations for TiDB, PD, and TiKV. Some highlights are support for modifying log level online, fixing issues with unique index and `ON DUPLICATE KEY UPDATE`, and addressing panic issues in specific conditions. --- diff --git a/releases/release-2.0.4.md b/releases/release-2.0.4.md index cbb3278e1248d..705a6ca847240 100644 --- a/releases/release-2.0.4.md +++ b/releases/release-2.0.4.md @@ -1,6 +1,5 @@ --- title: TiDB 2.0.4 Release Notes -aliases: ['/docs/dev/releases/release-2.0.4/','/docs/dev/releases/204/'] summary: TiDB 2.0.4 was released on June 15, 2018, with improvements in system compatibility and stability. It includes various enhancements and fixes for TiDB, PD, and TiKV. Some highlights for TiDB are support for `ALTER TABLE t DROP COLUMN a CASCADE` syntax, refining statement type display, and fixing issues related to data conversion and result order. PD now has improved behavior for the `max-pending-peer-count` argument, while TiKV includes the addition of the RocksDB `PerfContext` interface and fixes for slow `reverse-seek` and crash issues. --- diff --git a/releases/release-2.0.5.md b/releases/release-2.0.5.md index 1d08895251b11..2ebfa0667d0f1 100644 --- a/releases/release-2.0.5.md +++ b/releases/release-2.0.5.md @@ -1,6 +1,5 @@ --- title: TiDB 2.0.5 Release Notes -aliases: ['/docs/dev/releases/release-2.0.5/','/docs/dev/releases/205/'] summary: TiDB 2.0.5 was released on July 6, 2018, with improvements in system compatibility and stability. New features include the `tidb_disable_txn_auto_retry` system variable. Bug fixes address issues with user login, data insertion, and command compatibility. PD and TiKV also received fixes for various issues. --- diff --git a/releases/release-2.0.6.md b/releases/release-2.0.6.md index 9dbb73443f984..d51b6be7fe737 100644 --- a/releases/release-2.0.6.md +++ b/releases/release-2.0.6.md @@ -1,6 +1,5 @@ --- title: TiDB 2.0.6 Release Notes -aliases: ['/docs/dev/releases/release-2.0.6/','/docs/dev/releases/206/'] summary: TiDB 2.0.6 was released on August 6, 2018, with improvements in system compatibility and stability. The release includes various improvements and bug fixes for TiDB and TiKV. Some notable improvements include reducing transaction conflicts, improving row count estimation accuracy, and adding a recover mechanism for panics during the execution of `ANALYZE TABLE`. Bug fixes address issues such as incompatible `DROP USER` statement behavior, OOM errors for `INSERT`/`LOAD DATA` statements, and incorrect results for prefix index and `DECIMAL` operations. TiKV also sees improvements in scheduler slots, rollback transaction records, and RocksDB log file management, along with a fix for a crash issue during data type conversion. --- diff --git a/releases/release-2.0.7.md b/releases/release-2.0.7.md index f21cb531a0379..b2491ad4be665 100644 --- a/releases/release-2.0.7.md +++ b/releases/release-2.0.7.md @@ -1,6 +1,5 @@ --- title: TiDB 2.0.7 Release Notes -aliases: ['/docs/dev/releases/release-2.0.7/','/docs/dev/releases/207/'] summary: TiDB 2.0.7 was released on September 7, 2018, with improvements in system compatibility and stability. New features include the addition of the `PROCESSLIST` table in `information_schema`. Bug fixes address issues with index usage, join output, and query conditions. TiKV now opens the `dynamic-level-bytes` parameter by default to reduce space amplification, and updates approximate size and keys count after region merging. --- diff --git a/releases/release-2.0.8.md b/releases/release-2.0.8.md index 9df2dcec3a6fa..61d7e5b014543 100644 --- a/releases/release-2.0.8.md +++ b/releases/release-2.0.8.md @@ -1,6 +1,5 @@ --- title: TiDB 2.0.8 Release Notes -aliases: ['/docs/dev/releases/release-2.0.8/','/docs/dev/releases/208/'] summary: TiDB 2.0.8 was released on October 16, 2018, with improvements in system compatibility and stability. The release includes various bug fixes for TiDB and TiKV, addressing issues related to AUTO-ID, etcd session recovery, time zone handling, memory leaks, and join conversions. The TiKV bug fix resolves the increasing memory consumption by Raftstore EntryCache when a node goes down. --- diff --git a/releases/release-2.0.9.md b/releases/release-2.0.9.md index 0ce6b2a3b30bb..6c25dec981390 100644 --- a/releases/release-2.0.9.md +++ b/releases/release-2.0.9.md @@ -1,6 +1,5 @@ --- title: TiDB 2.0.9 Release Notes -aliases: ['/docs/dev/releases/release-2.0.9/','/docs/dev/releases/209/'] summary: TiDB 2.0.9 was released on November 19, 2018, with significant improvements in system compatibility and stability. The release includes fixes for various issues, such as empty statistics histogram, panic issue with UNION ALL statement, stack overflow issue, and support for specifying utf8mb4 character set. PD and TiKV also received fixes for issues related to server startup failure and interface limits. --- diff --git a/releases/release-2.1-beta.md b/releases/release-2.1-beta.md index fe15d411820c6..03bc7c8756287 100644 --- a/releases/release-2.1-beta.md +++ b/releases/release-2.1-beta.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1 Beta Release Notes -aliases: ['/docs/dev/releases/release-2.1-beta/','/docs/dev/releases/21beta/'] summary: TiDB 2.1 Beta release includes improvements in stability, SQL optimizer, statistics, and execution engine. It supports more MySQL syntax, decreases memory usage, and optimizes DDL and DML statements. PD enables Raft PreVote, optimizes scheduler issues, and adds metrics. TiKV upgrades Rust, adds metrics, and improves performance. Compatibility notes include not supporting rollback to v2.0.x and enabling raft learner by default in the new version. --- diff --git a/releases/release-2.1-ga.md b/releases/release-2.1-ga.md index db3b5629a9bf1..cb0d2a682f608 100644 --- a/releases/release-2.1-ga.md +++ b/releases/release-2.1-ga.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1 GA Release Notes -aliases: ['/docs/dev/releases/release-2.1-ga/','/docs/dev/releases/2.1ga/'] summary: TiDB 2.1 GA was released on November 30, 2018, with significant improvements in stability, performance, compatibility, and usability. The release includes optimizations in SQL optimizer, SQL executor, statistics, expressions, server, DDL, compatibility, Placement Driver (PD), TiKV, and tools. It also introduces TiDB Lightning for fast full data import and supports new TiDB Binlog. However, TiDB 2.1 does not support downgrading to v2.0.x or earlier due to the adoption of the new storage engine. Additionally, parallel DDL is enabled in TiDB 2.1, so clusters with TiDB version earlier than 2.0.1 cannot upgrade to 2.1 using rolling update. If upgrading from TiDB 2.0.6 or earlier to TiDB 2.1, ongoing DDL operations may slow down the upgrading process. --- @@ -80,7 +79,7 @@ On November 30, 2018, TiDB 2.1 GA is released. See the following updates in this - Support Server Side Cursor - + Add the [HTTP API](https://github.com/pingcap/tidb/blob/master/docs/tidb_http_api.md) + + Add the [HTTP API](https://github.com/pingcap/tidb/blob/release-2.1/docs/tidb_http_api.md) - Scatter the distribution of table Regions in the TiKV cluster @@ -116,7 +115,7 @@ On November 30, 2018, TiDB 2.1 GA is released. See the following updates in this - Add the table name and database name in the output information of `admin show ddl jobs` - - [Support using the `ddl/owner/resign` HTTP interface to release the DDL owner and start electing a new DDL owner](https://github.com/pingcap/tidb/blob/master/docs/tidb_http_api.md) + - [Support using the `ddl/owner/resign` HTTP interface to release the DDL owner and start electing a new DDL owner](https://github.com/pingcap/tidb/blob/release-2.1/docs/tidb_http_api.md) + Compatibility diff --git a/releases/release-2.1-rc.1.md b/releases/release-2.1-rc.1.md index b963f78f5f331..5b8db24d68d04 100644 --- a/releases/release-2.1-rc.1.md +++ b/releases/release-2.1-rc.1.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1 RC1 Release Notes -aliases: ['/docs/dev/releases/release-2.1-rc.1/','/docs/dev/releases/21rc1/'] summary: TiDB 2.1 RC1 was released on August 24, 2018, with improvements in stability, SQL optimizer, statistics, and execution engine. The release includes fixes for various issues in SQL optimizer and execution engine. PD introduces version control, rolling update, and region merge features. TiKV supports batch split and row-based region splitting for improved efficiency. Overall, the release focuses on performance optimization and bug fixes. --- diff --git a/releases/release-2.1-rc.2.md b/releases/release-2.1-rc.2.md index 51e08ec5132bd..988a8a0edf34a 100644 --- a/releases/release-2.1-rc.2.md +++ b/releases/release-2.1-rc.2.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1 RC2 Release Notes -aliases: ['/docs/dev/releases/release-2.1-rc.2/','/docs/dev/releases/21rc2/'] summary: TiDB 2.1 RC2 was released on September 14, 2018, with improvements in stability, SQL optimizer, statistics, and execution engine. The release includes enhancements to SQL optimizer, SQL execution engine, statistics, server, compatibility, expressions, DML, DDL, TiKV Go Client, and Table Partition. PD features, improvements, and bug fixes are also included. TiKV performance, improvements, and bug fixes are part of the release as well. --- @@ -23,7 +22,7 @@ On September 14, 2018, TiDB 2.1 RC2 is released. Compared with TiDB 2.1 RC1, thi * Optimize the performance of Hash aggregate operators [#7541](https://github.com/pingcap/tidb/pull/7541) * Optimize the performance of Join operators [#7493](https://github.com/pingcap/tidb/pull/7493), [#7433](https://github.com/pingcap/tidb/pull/7433) * Fix the issue that the result of `UPDATE JOIN` is incorrect when the Join order is changed [#7571](https://github.com/pingcap/tidb/pull/7571) - * Improve the performance of Chunk’s iterator [#7585](https://github.com/pingcap/tidb/pull/7585) + * Improve the performance of Chunk's iterator [#7585](https://github.com/pingcap/tidb/pull/7585) * Statistics * Fix the issue that the auto Analyze work repeatedly analyzes the statistics [#7550](https://github.com/pingcap/tidb/pull/7550) * Fix the statistics update error that occurs when there is no statistics change [#7530](https://github.com/pingcap/tidb/pull/7530) @@ -46,7 +45,7 @@ On September 14, 2018, TiDB 2.1 RC2 is released. Compared with TiDB 2.1 RC1, thi * Use different labels to filter internal SQL and user SQL in monitoring metrics [#7631](https://github.com/pingcap/tidb/pull/7631) * Store the top 30 slow queries in the last week to the TiDB server [#7646](https://github.com/pingcap/tidb/pull/7646) * Put forward a proposal of setting the global system time zone for the TiDB cluster [#7656](https://github.com/pingcap/tidb/pull/7656) - * Enrich the error message of “GC life time is shorter than transaction duration” [#7658](https://github.com/pingcap/tidb/pull/7658) + * Enrich the error message of "GC life time is shorter than transaction duration" [#7658](https://github.com/pingcap/tidb/pull/7658) * Set the global system time zone when starting the TiDB cluster [#7638](https://github.com/pingcap/tidb/pull/7638) * Compatibility * Add the unsigned flag for the `Year` type [#7542](https://github.com/pingcap/tidb/pull/7542) diff --git a/releases/release-2.1-rc.3.md b/releases/release-2.1-rc.3.md index 0f17308bb4958..c6f8ca6e9bb5a 100644 --- a/releases/release-2.1-rc.3.md +++ b/releases/release-2.1-rc.3.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1 RC3 Release Notes -aliases: ['/docs/dev/releases/release-2.1-rc.3/','/docs/dev/releases/21rc3/'] summary: TiDB 2.1 RC3 was released on September 29, 2018, with improvements in stability, compatibility, SQL optimizer, and execution engine. The release includes fixes and enhancements for SQL optimizer, execution engine, server, compatibility, expressions, DML, DDL, and PD. TiKV also received performance optimizations, new features, and bug fixes. --- diff --git a/releases/release-2.1-rc.4.md b/releases/release-2.1-rc.4.md index ead55abe40716..355f398c58f6f 100644 --- a/releases/release-2.1-rc.4.md +++ b/releases/release-2.1-rc.4.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1 RC4 Release Notes -aliases: ['/docs/dev/releases/release-2.1-rc.4/','/docs/dev/releases/21rc4/'] summary: TiDB 2.1 RC4 was released on October 23, 2018, with improvements in stability, SQL optimizer, statistics information, and execution engine. Fixes include issues with SQL optimizer, execution engine, statistics, server, compatibility, expressions, and DDL. PD fixes issues with tombstone TiKV, data race, PD server getting stuck, and leader switching. TiKV optimizes RocksDB Write stall issue, adds raftstore tick metrics, and upgrades RocksDB and grpcio. --- @@ -29,7 +28,7 @@ On October 23, 2018, TiDB 2.1 RC4 is released. Compared with TiDB 2.1 RC3, this - Refactor Latch to avoid misjudgment of transaction conflicts and improve the execution performance of concurrent transactions [#7711](https://github.com/pingcap/tidb/pull/7711) - Fix the panic issue caused by collecting slow queries in some cases [#7874](https://github.com/pingcap/tidb/pull/7847) - Fix the panic issue when `ESCAPED BY` is an empty string in the `LOAD DATA` statement [#8005](https://github.com/pingcap/tidb/pull/8005) - - Complete the “coprocessor error” log information [#8006](https://github.com/pingcap/tidb/pull/8006) + - Complete the "coprocessor error" log information [#8006](https://github.com/pingcap/tidb/pull/8006) + Compatibility - Set the `Command` field of the `SHOW PROCESSLIST` result to `Sleep` when the query is empty [#7839](https://github.com/pingcap/tidb/pull/7839) + Expressions @@ -52,4 +51,4 @@ On October 23, 2018, TiDB 2.1 RC4 is released. Compared with TiDB 2.1 RC3, this - Optimize the RocksDB Write stall issue caused by applying snapshots [#3606](https://github.com/tikv/tikv/pull/3606) - Add raftstore `tick` metrics [#3657](https://github.com/tikv/tikv/pull/3657) - Upgrade RocksDB and fix the Write block issue and that the source file might be damaged by the Write operation when performing `IngestExternalFile` [#3661](https://github.com/tikv/tikv/pull/3661) -- Upgrade grpcio and fix the issue that “too many pings” is wrongly reported [#3650](https://github.com/tikv/tikv/pull/3650) \ No newline at end of file +- Upgrade grpcio and fix the issue that "too many pings" is wrongly reported [#3650](https://github.com/tikv/tikv/pull/3650) \ No newline at end of file diff --git a/releases/release-2.1-rc.5.md b/releases/release-2.1-rc.5.md index 46d91d91c955d..3886d20f3f717 100644 --- a/releases/release-2.1-rc.5.md +++ b/releases/release-2.1-rc.5.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1 RC5 Release Notes -aliases: ['/docs/dev/releases/release-2.1-rc.5/','/docs/dev/releases/21rc5/'] summary: TiDB 2.1 RC5 was released on November 12, 2018, with improvements in stability, SQL optimizer, statistics, and execution engine. Fixes include issues with IndexReader, IndexScan Prepared statement, Union statement, and JSON data conversion. Server improvements include log readability, table data retrieval, and environment variable additions. PD fixes issues related to Region key reading, `regions/check` API, PD restart join, and event loss. TiKV improves error messages, adds panic mark file, downgrades grpcio, and adds an upper limit to the `kv_scan` interface. Tools now support the TiDB-Binlog cluster. --- diff --git a/releases/release-2.1.1.md b/releases/release-2.1.1.md index f76a6de6c7a5b..ef7d195c2f55f 100644 --- a/releases/release-2.1.1.md +++ b/releases/release-2.1.1.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.1 Release Notes -aliases: ['/docs/dev/releases/release-2.1.1/','/docs/dev/releases/2.1.1/'] summary: TiDB 2.1.1 was released on December 12, 2018, with improvements in stability, SQL optimizer, statistics information, and execution engine. Fixes include round error of negative date, uncompress function data length check, and transaction retries. Default character set and collation of tables changed to utf8mb4. PD and TiKV also received various fixes and optimizations. Lightning tool optimized analyze mechanism and added support for storing checkpoint information locally. TiDB Binlog fixed output bug of pb files for tables with only primary key column. --- diff --git a/releases/release-2.1.10.md b/releases/release-2.1.10.md index 0ee23d4731f06..c37726248bbf8 100644 --- a/releases/release-2.1.10.md +++ b/releases/release-2.1.10.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.10 Release Notes -aliases: ['/docs/dev/releases/release-2.1.10/','/docs/dev/releases/2.1.10/'] summary: TiDB 2.1.10 was released on May 22, 2019, with various bug fixes and improvements. The release includes fixes for issues related to table schema, read results, generated columns, datetime functions, slow logs, and more. Additionally, improvements were made to TiKV and tools like TiDB Lightning and TiDB Binlog. The TiDB Ansible version 2.1.10 also received updates. --- diff --git a/releases/release-2.1.11.md b/releases/release-2.1.11.md index b78417e779379..77984c4f5d4a8 100644 --- a/releases/release-2.1.11.md +++ b/releases/release-2.1.11.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.11 Release Notes -aliases: ['/docs/dev/releases/release-2.1.11/','/docs/dev/releases/2.1.11/'] summary: TiDB 2.1.11 was released on June 03, 2019. It includes fixes for various issues in TiDB, PD, TiKV, and Tools. Some highlights are the fix for incorrect schema in delete from join, calculation errors of unix_timestamp(), and the addition of Drainer parameters in TiDB Ansible. --- diff --git a/releases/release-2.1.12.md b/releases/release-2.1.12.md index 85b3ac1f5570c..34510d7f1bd72 100644 --- a/releases/release-2.1.12.md +++ b/releases/release-2.1.12.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.12 Release Notes -aliases: ['/docs/dev/releases/release-2.1.12/','/docs/dev/releases/2.1.12/'] summary: TiDB 2.1.12 was released on June 13, 2019. It includes various bug fixes and improvements, such as fixing issues with data type mismatches, charset altering, and GRANT operations. The release also improves compatibility with MySQL and addresses issues with functions, data conversion, and error reporting. Additionally, PD and TiKV have also been updated to fix issues related to leader election and data availability during leader transfer and power failure. --- diff --git a/releases/release-2.1.13.md b/releases/release-2.1.13.md index 30c49ad2f60f3..10ff3b4629341 100644 --- a/releases/release-2.1.13.md +++ b/releases/release-2.1.13.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.13 Release Notes -aliases: ['/docs/dev/releases/release-2.1.13/','/docs/dev/releases/2.1.13/'] summary: TiDB 2.1.13 was released on June 21, 2019. It includes features to scatter row IDs, optimize DDL metadata lifetime, fix OOM issue, update statistics, support Region presplit, improve MySQL compatibility, and fix estimation issues. TiKV fixes incomplete snapshots and adds a feature to check the validity of the block-size configuration. TiDB Binlog fixes wrong offset and adds advertise-addr configuration in Drainer. --- diff --git a/releases/release-2.1.14.md b/releases/release-2.1.14.md index ae975f6dead25..c992ff3b577d8 100644 --- a/releases/release-2.1.14.md +++ b/releases/release-2.1.14.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.14 Release Notes -aliases: ['/docs/dev/releases/release-2.1.14/','/docs/dev/releases/2.1.14/'] summary: TiDB 2.1.14 was released on July 04, 2019. It includes various bug fixes and improvements, such as fixing wrong query results, adding new system variables, optimizing memory usage, and adding new configuration items for TiDB Binlog and TiDB Ansible. Additionally, there are optimizations for TiKV and PD. --- diff --git a/releases/release-2.1.15.md b/releases/release-2.1.15.md index deb8a2f4341f8..ff4f5286eee57 100644 --- a/releases/release-2.1.15.md +++ b/releases/release-2.1.15.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.15 Release Notes -aliases: ['/docs/dev/releases/release-2.1.15/','/docs/dev/releases/2.1.15/'] summary: TiDB 2.1.15 was released on July 16, 2019. It includes various bug fixes and improvements, such as fixing issues with functions like DATE_ADD and INSERT, adding new SQL statements like SHOW TABLE REGIONS, and enhancing the Audit plugin. TiKV and PD also received updates to unify log formats and improve accuracy. Additionally, there were optimizations made to TiDB Binlog and TiDB Lightning, and new monitoring items added to TiDB Ansible. --- diff --git a/releases/release-2.1.16.md b/releases/release-2.1.16.md index 10f2b55d1a99d..8cd44c99ff50b 100644 --- a/releases/release-2.1.16.md +++ b/releases/release-2.1.16.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.16 Release Notes -aliases: ['/docs/dev/releases/release-2.1.16/','/docs/dev/releases/2.1.16/'] summary: TiDB 2.1.16 was released on August 15, 2019. It includes various fixes and improvements to the SQL optimizer, SQL execution engine, server, DDL, TiKV, TiDB Binlog, TiDB Lightning, and TiDB Ansible. Some notable changes include support for subqueries within SHOW statements, fixing issues with DATE_ADD function, and adding configuration items in Drainer for TiDB Binlog. --- @@ -54,7 +53,7 @@ TiDB Ansible version: 2.1.16 + TiDB Binlog - Add the `ignore-txn-commit-ts` configuration item in Drainer to skip executing some statements in a transaction [#697](https://github.com/pingcap/tidb-binlog/pull/697) - Add the configuration item check on startup, which stops Pump and Drainer from running and returns an error message when meeting invalid configuration items [#708](https://github.com/pingcap/tidb-binlog/pull/708) - - Add the `node-id` configuration in Drainer to specify Drainer’s node ID [#706](https://github.com/pingcap/tidb-binlog/pull/706) + - Add the `node-id` configuration in Drainer to specify Drainer's node ID [#706](https://github.com/pingcap/tidb-binlog/pull/706) + TiDB Lightning - Fix the issue that `tikv_gc_life_time` fails to be changed back to its original value when 2 checksums are running at the same time [#224](https://github.com/pingcap/tidb-lightning/pull/224) diff --git a/releases/release-2.1.17.md b/releases/release-2.1.17.md index c2ce2a99b1a6f..d63dac1707e0d 100644 --- a/releases/release-2.1.17.md +++ b/releases/release-2.1.17.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.17 Release Notes -aliases: ['/docs/dev/releases/release-2.1.17/','/docs/dev/releases/2.1.17/'] summary: "TiDB 2.1.17 Release Notes: New features include `WHERE` clause in `SHOW TABLE REGIONS`, `config-check` feature in TiKV and PD, `remove-tombstone` command in pd-ctl, and `worker-count` and `txn-batch` configuration items in Reparo. Improvements in PD’s scheduling process and TiKV’s starting process. Changed behaviors in TiDB slow query logs and configuration files. Fixes and optimizations in SQL Optimizer, SQL Execution Engine, Server, DDL, Monitor, TiKV, PD, TiDB Binlog, TiDB Lightning, and TiDB Ansible." --- @@ -13,19 +12,19 @@ TiDB version: 2.1.17 TiDB Ansible version: 2.1.17 + New features - - Add the `WHERE` clause in TiDB’s `SHOW TABLE REGIONS` syntax + - Add the `WHERE` clause in TiDB's `SHOW TABLE REGIONS` syntax - Add the `config-check` feature in TiKV and PD to check the configuration items - Add the `remove-tombstone` command in pd-ctl to clear tombstone store records - Add the `worker-count` and `txn-batch` configuration items in Reparo to control the recovery speed + Improvements - - Optimize PD’s scheduling process by supporting actively pushing operators - - Optimize TiKV’s starting process to reduce jitters caused by restarting nodes + - Optimize PD's scheduling process by supporting actively pushing operators + - Optimize TiKV's starting process to reduce jitters caused by restarting nodes + Changed behaviors - Change `start ts` in TiDB slow query logs from the last retry time to the first execution time - Replace the `Index_ids` field in TiDB slow query logs with the `Index_names` field to improve the usability of slow query logs - - Add the `split-region-max-num` parameter in TiDB’s configuration files to modify the maximum number of Regions allowed by the `SPLIT TABLE` syntax, which is increased from 1,000 to 10,000 in the default configuration + - Add the `split-region-max-num` parameter in TiDB's configuration files to modify the maximum number of Regions allowed by the `SPLIT TABLE` syntax, which is increased from 1,000 to 10,000 in the default configuration ## TiDB @@ -41,13 +40,13 @@ TiDB Ansible version: 2.1.17 - Fix the issue that the result returned by the `unaryMinus` function is incompatible with MySQL, caused by the non-decimal result when the integer result overflows [#11990](https://github.com/pingcap/tidb/pull/11990) - Fix the issue that `last_insert_id()` might be incorrect, caused by the counting order when the `LOAD DATA` statement is being executed [#11994](https://github.com/pingcap/tidb/pull/11994) - Fix the issue that `last_insert_id()` might be incorrect when the user writes auto-increment column data in an explicit-implicit mixed way [#12001](https://github.com/pingcap/tidb/pull/12001) - - Fix an over-quoted bug for the `JSON_UNQUOTE` function: only values enclosed by double quote marks (`"`) should be unquoted. For example, the result of “`SELECT JSON_UNQUOTE("\\\\")`” should be “`\\`” (not changed) [#12096](https://github.com/pingcap/tidb/pull/12096) + - Fix an over-quoted bug for the `JSON_UNQUOTE` function: only values enclosed by double quote marks (`"`) should be unquoted. For example, the result of "`SELECT JSON_UNQUOTE("\\\\")`" should be "`\\`" (not changed) [#12096](https://github.com/pingcap/tidb/pull/12096) + Server - Change `start ts` recorded in slow query logs from the last retry time to the first execution time when retrying TiDB transactions [#11878](https://github.com/pingcap/tidb/pull/11878) - Add the number of keys of a transaction in `LockResolver` to avoid the scan operation on the whole Region and reduce costs of resolving locking when the number of keys is reduced [#11889](https://github.com/pingcap/tidb/pull/11889) - Fix the issue that the `succ` field value might be incorrect in slow query logs [#11886](https://github.com/pingcap/tidb/pull/11886) - Replace the `Index_ids` field in slow query logs with the `Index_names` field to improve the usability of slow query logs [#12063](https://github.com/pingcap/tidb/pull/12063) - - Fix the connection break issue caused by TiDB parsing `-` into EOF Error when `Duration` contains `-` (like `select time(‘--’)`) [#11910](https://github.com/pingcap/tidb/pull/11910) + - Fix the connection break issue caused by TiDB parsing `-` into EOF Error when `Duration` contains `-` (like `select time(‘--')`) [#11910](https://github.com/pingcap/tidb/pull/11910) - Remove an invalid Region from `RegionCache` more quickly to reduce the number of requests sent to this Region [#11931](https://github.com/pingcap/tidb/pull/11931) - Fix the connection break issue caused by incorrectly handling the OOM panic issue when `oom-action = "cancel"` and OOM occurs in the `Insert Into … Select` syntax [#12126](https://github.com/pingcap/tidb/pull/12126) + DDL diff --git a/releases/release-2.1.18.md b/releases/release-2.1.18.md index f1b9d893447c9..ba617821ee580 100644 --- a/releases/release-2.1.18.md +++ b/releases/release-2.1.18.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.18 Release Notes -aliases: ['/docs/dev/releases/release-2.1.18/','/docs/dev/releases/2.1.18/'] summary: TiDB 2.1.18 was released on November 4, 2019. The release includes various fixes and optimizations for SQL optimizer, SQL engine, server, DDL, monitor, and tools. Some notable improvements include support for using parameters in ORDER BY, GROUP BY, and LIMIT OFFSET, and adding new metrics for monitoring Add Index operation progress. The TiDB Ansible version 2.1.18 also includes updates and new monitoring items for TiDB Binlog. --- @@ -36,7 +35,7 @@ TiDB Ansible version: 2.1.18 - Fix the issue that no error is reported when `AUTO_INCREMENT` incorrectly allocates `MAX int64` and `MAX uint64` [#12210](https://github.com/pingcap/tidb/pull/12210) - Fix the issue that the slow query logs are not recorded when the log level is `ERROR` [#12373](https://github.com/pingcap/tidb/pull/12373) - Adjust the number of times that TiDB caches schema changes and corresponding changed table information from 100 to 1024, and support modification by using the `tidb_max_delta_schema_count` system variable [#12515](https://github.com/pingcap/tidb/pull/12515) - - Change the query start time from the point of "starting to execute" to “starting to compile” to make SQL statistics more accurate [#12638](https://github.com/pingcap/tidb/pull/12638) + - Change the query start time from the point of "starting to execute" to "starting to compile" to make SQL statistics more accurate [#12638](https://github.com/pingcap/tidb/pull/12638) - Add the record of `set session autocommit` in TiDB logs [#12568](https://github.com/pingcap/tidb/pull/12568) - Record SQL query start time in `SessionVars` to prevent it from being reset during plan execution [#12676](https://github.com/pingcap/tidb/pull/12676) - Support `?` placeholder in `ORDER BY`, `GROUP BY` and `LIMIT OFFSET` [#12514](https://github.com/pingcap/tidb/pull/12514) @@ -70,7 +69,7 @@ TiDB Ansible version: 2.1.18 ## TiDB Ansible -- Add two monitoring items "queue size" and “query histogram” for TiDB Binlog [#952](https://github.com/pingcap/tidb-ansible/pull/952) +- Add two monitoring items "queue size" and "query histogram" for TiDB Binlog [#952](https://github.com/pingcap/tidb-ansible/pull/952) - Update TiDB alerting rules [#961](https://github.com/pingcap/tidb-ansible/pull/961) - Check the configuration file before the deployment and upgrade [#973](https://github.com/pingcap/tidb-ansible/pull/973) - Add a new metric to monitor index speed in TiDB [#987](https://github.com/pingcap/tidb-ansible/pull/987) diff --git a/releases/release-2.1.19.md b/releases/release-2.1.19.md index 8b20ccc3fce86..2e51325667e41 100644 --- a/releases/release-2.1.19.md +++ b/releases/release-2.1.19.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.19 Release Notes -aliases: ['/docs/dev/releases/release-2.1.19/','/docs/dev/releases/2.1.19/'] summary: TiDB 2.1.19 was released on December 27, 2019. It includes various fixes and optimizations for SQL optimizer, SQL execution engine, server, DDL, TiKV, PD, and TiDB Ansible. Some notable fixes include resolving incorrect query results, memory overhead reduction, and fixing issues related to timezone, data duplication, and panic occurrences. The release also includes upgrades and optimizations for TiDB Binlog and TiDB Ansible. --- @@ -59,7 +58,7 @@ TiDB Ansible version: 2.1.19 - Add the `server-version` configuration item to control and modify the version of TiDB server [#13904](https://github.com/pingcap/tidb/pull/13904) - Fix the issue that the binary `plugin` compiled with Go1.13 does not run normally [#13527](https://github.com/pingcap/tidb/pull/13527) + DDL - - Use the table’s `COLLATE` instead of the system’s default charset in the column when a table is created and the table contains `COLLATE` [#13190](https://github.com/pingcap/tidb/pull/13190) + - Use the table's `COLLATE` instead of the system's default charset in the column when a table is created and the table contains `COLLATE` [#13190](https://github.com/pingcap/tidb/pull/13190) - Limit the length of the index name when creating a table [#13311](https://github.com/pingcap/tidb/pull/13311) - Fix the issue that the length of the table name is not checked when renaming a table [#13345](https://github.com/pingcap/tidb/pull/13345) - Check the width range of the `BIT` column [#13511](https://github.com/pingcap/tidb/pull/13511) @@ -85,7 +84,7 @@ TiDB Ansible version: 2.1.19 - Fix the panic occurred because of the `nil` value in the Drainer filter configuration [#802](https://github.com/pingcap/tidb-binlog/pull/802) - Optimize the `Graceful` exit of Pump [#825](https://github.com/pingcap/tidb-binlog/pull/825) - Add more detailed monitoring metrics when Pump writes binlog data [#830](https://github.com/pingcap/tidb-binlog/pull/830) - - Optimize Drainer’s logic to refresh table information after Drainer has executed a DDL operation [#836](https://github.com/pingcap/tidb-binlog/pull/836) + - Optimize Drainer's logic to refresh table information after Drainer has executed a DDL operation [#836](https://github.com/pingcap/tidb-binlog/pull/836) - Fix the issue that the commit binlog of a DDL operation is ignored when Pump does not receive this binlog [#855](https://github.com/pingcap/tidb-binlog/pull/855) ## TiDB Ansible diff --git a/releases/release-2.1.2.md b/releases/release-2.1.2.md index 5c1555236e222..41014e1feabef 100644 --- a/releases/release-2.1.2.md +++ b/releases/release-2.1.2.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.2 Release Notes -aliases: ['/docs/dev/releases/release-2.1.2/','/docs/dev/releases/2.1.2/'] summary: TiDB 2.1.2 and TiDB Ansible 2.1.2 were released on December 22, 2018. The release includes improvements in system compatibility and stability. Key updates include compatibility with TiDB Binlog of the Kafka version, improved exit mechanism during rolling updates, and fixes for various issues. PD and TiKV also received updates, such as fixing Region merge issues and support for configuration format in the unit of 'DAY'. Additionally, TiDB Lightning and TiDB Binlog were updated to support new features and eliminate bottlenecks. --- diff --git a/releases/release-2.1.3.md b/releases/release-2.1.3.md index 6d80ceb57bf92..21f8529051d2d 100644 --- a/releases/release-2.1.3.md +++ b/releases/release-2.1.3.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.3 Release Notes -aliases: ['/docs/dev/releases/release-2.1.3/','/docs/dev/releases/2.1.3/'] summary: TiDB 2.1.3 and TiDB Ansible 2.1.3 are released with improvements in system stability, SQL optimizer, statistics, and execution engine. Fixes include issues with Prepared Plan Cache, Range computing, `CAST(str AS TIME(N))`, Generated Column, statistics histogram, `Sort Merge Join`, and more. Other improvements include support for Range for `_tidb_rowid` construction queries, `ALLOW_INVALID_DATES` SQL mode, and more. PD and TiKV also have fixes and improvements. TiDB Binlog fixes issues with the Pump client log and data inconsistency caused by unique key containing NULL value. --- diff --git a/releases/release-2.1.4.md b/releases/release-2.1.4.md index 7e3f2d0ce0494..0e1eb3bf59992 100644 --- a/releases/release-2.1.4.md +++ b/releases/release-2.1.4.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.4 Release Notes -aliases: ['/docs/dev/releases/release-2.1.4/','/docs/dev/releases/2.1.4/'] summary: TiDB 2.1.4 and TiDB Ansible 2.1.4 were released on February 15, 2019. The release includes improvements in stability, SQL optimizer, statistics, and execution engine. Fixes include issues with the SQL optimizer/executor, server, DDL, and TiKV. Lightning tool optimizations include memory usage, chunk separation removal, I/O concurrency limitation, batch data import support, and auto compactions in TiKV import mode. Additionally, support for disabling TiKV periodic Level-1 compaction parameter and limiting the number of import engines is added. Sync-diff-inspector now supports splitting chunks using TiDB statistics. --- @@ -18,7 +17,7 @@ On February 15, 2019, TiDB 2.1.4 is released. The corresponding TiDB Ansible 2.1 - Fix the issue that the `VALUES` function does not handle the ENUM type correctly [#9280](https://github.com/pingcap/tidb/pull/9280) - Fix the wrong result issue of `DATE_ADD`/`DATE_SUB` in some cases [#9284](https://github.com/pingcap/tidb/pull/9284) + Server - - Optimize the “reload privilege success” log and change it to the DEBUG level [#9274](https://github.com/pingcap/tidb/pull/9274) + - Optimize the "reload privilege success" log and change it to the DEBUG level [#9274](https://github.com/pingcap/tidb/pull/9274) + DDL - Change `tidb_ddl_reorg_worker_cnt` and `tidb_ddl_reorg_batch_size` to global variables [#9134](https://github.com/pingcap/tidb/pull/9134) - Fix the bug caused by adding an index to a generated column in some abnormal conditions [#9289](https://github.com/pingcap/tidb/pull/9289) diff --git a/releases/release-2.1.5.md b/releases/release-2.1.5.md index 00f0b7b911806..f4319bb9744eb 100644 --- a/releases/release-2.1.5.md +++ b/releases/release-2.1.5.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.5 Release Notes -aliases: ['/docs/dev/releases/release-2.1.5/','/docs/dev/releases/2.1.5/'] summary: TiDB 2.1.5 and TiDB Ansible 2.1.5 are released on February 28, 2019. The release improves stability, SQL optimizer, statistics, and execution engine. Fixes include issues with sorting, data overflow, and SQL query results. New features include system variables, HTTP API, and detailed error messages. PD now has an option to exclude Tombstone stores, and TiKV fixes issues with data import, errors, and panic caused by Region merge. Tools like Lightning and TiDB Binlog also receive updates. --- diff --git a/releases/release-2.1.6.md b/releases/release-2.1.6.md index 785d08957bcab..e93985930ab38 100644 --- a/releases/release-2.1.6.md +++ b/releases/release-2.1.6.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.6 Release Notes -aliases: ['/docs/dev/releases/release-2.1.6/','/docs/dev/releases/2.1.6/'] summary: TiDB 2.1.6 and TiDB Ansible 2.1.6 were released on March 15, 2019. The release includes improvements in stability, SQL optimizer, statistics, and execution engine. Fixes and enhancements were made in SQL optimizer/executor, server, DDL, TiKV, and Tools. Notable changes include support for log_bin variable, sanity check for transactions, and fixing import failure due to non-alphanumeric characters in schema names. --- diff --git a/releases/release-2.1.7.md b/releases/release-2.1.7.md index 85b6fe9f78351..b49c0b711f5db 100644 --- a/releases/release-2.1.7.md +++ b/releases/release-2.1.7.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.7 Release Notes -aliases: ['/docs/dev/releases/release-2.1.7/','/docs/dev/releases/2.1.7/'] summary: TiDB 2.1.7 was released on March 28, 2019. It includes various bug fixes, compatibility improvements, and new features such as support for subqueries in the `DO` statement, plugin framework, and checking binlog and Pump/Drainer status using SQL statements. PD also fixed an issue related to transferring leader step in balance-region. Additionally, the default retention time of Prometheus monitoring data in TiDB Ansible has been changed to 30d. --- diff --git a/releases/release-2.1.8.md b/releases/release-2.1.8.md index a2c95a1a5fd29..cc2f8921636d8 100644 --- a/releases/release-2.1.8.md +++ b/releases/release-2.1.8.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.8 Release Notes -aliases: ['/docs/dev/releases/release-2.1.8/','/docs/dev/releases/2.1.8/'] summary: TiDB 2.1.8 was released on April 12, 2019. It includes various bug fixes and improvements for TiDB, PD, TiKV, Tools, and TiDB Ansible. Some notable fixes include compatibility issues with MySQL, inaccurate statistics estimation, and performance improvements. The release also adds new configuration items and features for TiDB Binlog Pump and Drainer. Additionally, TiDB Ansible now has version limits for the operating system and rolling updates. --- diff --git a/releases/release-2.1.9.md b/releases/release-2.1.9.md index ddccf0eb0935b..9cdd2d125de9e 100644 --- a/releases/release-2.1.9.md +++ b/releases/release-2.1.9.md @@ -1,6 +1,5 @@ --- title: TiDB 2.1.9 Release Notes -aliases: ['/docs/dev/releases/release-2.1.9/','/docs/dev/releases/2.1.9/'] summary: TiDB 2.1.9 was released on May 6, 2019. It includes various bug fixes and improvements, such as fixing compatibility issues, privilege check problems, and wrong result issues. The release also includes improvements to slow query logs and support for controlling the number of rows returned by operators. Additionally, there are updates to PD, TiKV, TiDB Binlog, TiDB Lightning, and sync-diff-inspector. TiDB Ansible has also been updated with documentation links and parameter removal. --- diff --git a/releases/release-3.0-beta.md b/releases/release-3.0-beta.md index 2aa292008af59..db0cf08d6dad5 100644 --- a/releases/release-3.0-beta.md +++ b/releases/release-3.0-beta.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0 Beta Release Notes -aliases: ['/docs/dev/releases/release-3.0-beta/','/docs/dev/releases/3.0beta/'] summary: TiDB 3.0 Beta, released on January 19, 2019, focuses on stability, SQL optimizer, statistics, and execution engine. New features include support for views, window functions, range partitioning, and hash partitioning. The SQL optimizer has been enhanced with various optimizations, including support for index join in transactions, constant propagation optimization, and support for subqueries in the DO statement. The SQL executor has also been optimized for better performance. Privilege management, server, compatibility, and DDL have all been improved. TiDB Lightning now supports batch import for a single table, while PD and TiKV have also received various enhancements and new features. --- @@ -70,7 +69,7 @@ On January 19, 2019, TiDB 3.0 Beta is released. The corresponding TiDB Ansible 3 - Support the MySQL 320 handshake protocol [#8812](https://github.com/pingcap/tidb/pull/8812) - Support using the unsigned `bigint` column as the auto-increment column [#8181](https://github.com/pingcap/tidb/pull/8181) - Support the `SHOW CREATE DATABASE IF NOT EXISTS` syntax [#8926](https://github.com/pingcap/tidb/pull/8926) - - Abandon the predicate pushdown operation when the filtering condition contains a user variable to improve the compatibility with MySQL’s behavior of using user variables to mock the Window Function behavior [#8412](https://github.com/pingcap/tidb/pull/8412) + - Abandon the predicate pushdown operation when the filtering condition contains a user variable to improve the compatibility with MySQL's behavior of using user variables to mock the Window Function behavior [#8412](https://github.com/pingcap/tidb/pull/8412) + DDL - Support fast recovery of mistakenly deleted tables [#7937](https://github.com/pingcap/tidb/pull/7937) - Support adjusting the number of concurrencies of `ADD INDEX` dynamically [#8295](https://github.com/pingcap/tidb/pull/8295) diff --git a/releases/release-3.0-ga.md b/releases/release-3.0-ga.md index 39cdf517dcb98..78d5cdcdd219e 100644 --- a/releases/release-3.0-ga.md +++ b/releases/release-3.0-ga.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0 GA Release Notes -aliases: ['/docs/dev/releases/release-3.0-ga/','/docs/dev/releases/3.0-ga/'] summary: TiDB 3.0 GA was released on June 28, 2019, with improved stability, usability, and performance. New features include Window Functions, Views, partitioned tables, and the plugin framework. The SQL Optimizer has been optimized for better performance, and DDL now supports fast recovery of mistakenly deleted tables. TiKV now supports distributed GC, multi-thread Raftstore, and batch receiving and sending Raft messages. Tools like TiDB Lightning and TiDB Binlog have also been enhanced with new features and performance improvements. The TiDB Ansible has been upgraded to support deployment and operations for TiDB Lightning, and to optimize monitoring components. --- @@ -115,7 +114,7 @@ On June 28, 2019, TiDB 3.0 GA is released. The corresponding TiDB Ansible versio - Support manifesting unsigned BIGINT columns as auto-increment columns - Support the `SHOW CREATE DATABASE IF NOT EXISTS` syntax - Optimize the fault tolerance of `load data` for CSV files - - Abandon the predicate pushdown operation when the filtering condition contains a user variable to improve the compatibility with MySQL’s behavior of using user variables to simulate Window Functions + - Abandon the predicate pushdown operation when the filtering condition contains a user variable to improve the compatibility with MySQL's behavior of using user variables to simulate Window Functions ## PD @@ -190,7 +189,7 @@ On June 28, 2019, TiDB 3.0 GA is released. The corresponding TiDB Ansible versio - Add the `GetMvccByEncodeKey` function in Pump to speed up querying the transaction status - Support compressing communication data among components to reduce network resource consumption - Add the Arbiter tool that supports reading binlog from Kafka and replicate the data into MySQL - - Support filtering out files that don’t require replication via Reparo + - Support filtering out files that don't require replication via Reparo - Support replicating generated columns - Add the `syncer.sql-mode` configuration item to support using different sql-modes to parse DDL queries - Add the `syncer.ignore-table` configuration item to support filtering tables not to be replicated diff --git a/releases/release-3.0.0-beta.1.md b/releases/release-3.0.0-beta.1.md index 9ccb5a48066b4..52ec364c74509 100644 --- a/releases/release-3.0.0-beta.1.md +++ b/releases/release-3.0.0-beta.1.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.0 Beta.1 Release Notes -aliases: ['/docs/dev/releases/release-3.0.0-beta.1/','/docs/dev/releases/3.0.0-beta.1/'] summary: TiDB 3.0.0 Beta.1 was released on March 26, 2019, with improved stability, usability, features, SQL optimizer, statistics, and execution engine. The release includes support for various SQL functions, privilege management, server enhancements, DDL improvements, and PD and TiKV optimizations. Tools like TiDB Binlog, Lightning, and data replication comparison tool have also been updated with new features and improvements. --- @@ -59,7 +58,7 @@ On March 26, 2019, TiDB 3.0.0 Beta.1 is released. The corresponding TiDB Ansible - Add the `high_error_rate_feedback_total` monitoring item to record the difference between the actual data volume and the estimated data volume based on statistics [#9209](https://github.com/pingcap/tidb/pull/9209) - Add the QPS monitoring item in the database dimension, which can be enabled by using a configuration item [#9151](https://github.com/pingcap/tidb/pull/9151) + DDL - - Add the `ddl_error_count_limit` global variable (“512” by default) to limit the number of DDL task retries (If this number exceeds the limit, the DDL task is canceled) [#9295](https://github.com/pingcap/tidb/pull/9295) + - Add the `ddl_error_count_limit` global variable ("512" by default) to limit the number of DDL task retries (If this number exceeds the limit, the DDL task is canceled) [#9295](https://github.com/pingcap/tidb/pull/9295) - Support ALTER ALGORITHM `INPLACE`/`INSTANT` [#8811](https://github.com/pingcap/tidb/pull/8811) - Support the `SHOW CREATE VIEW` statement [#9309](https://github.com/pingcap/tidb/pull/9309) - Support the `SHOW CREATE USER` statement [#9240](https://github.com/pingcap/tidb/pull/9240) diff --git a/releases/release-3.0.0-rc.1.md b/releases/release-3.0.0-rc.1.md index d024c04f0dcb8..f02cb4b02188c 100644 --- a/releases/release-3.0.0-rc.1.md +++ b/releases/release-3.0.0-rc.1.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.0-rc.1 Release Notes -aliases: ['/docs/dev/releases/release-3.0.0-rc.1/','/docs/dev/releases/3.0.0-rc.1/'] summary: TiDB 3.0.0-rc.1 was released on May 10, 2019, with improved stability, usability, features, SQL optimizer, statistics, and execution engine. The release includes enhancements to the SQL optimizer, execution engine, server, DDL, PD, TiKV, TiDB Binlog, Lightning, sync-diff-inspector, and TiDB Ansible. Notable improvements include support for SQL Plan Management, memory usage tracking, and control in the execution engine, and the addition of the `pre_split_regions` option for `CREATE TABLE` statements in DDL. The release also includes various bug fixes and performance optimizations. --- @@ -65,7 +64,7 @@ On May 10, 2019, TiDB 3.0.0-rc.1 is released. The corresponding TiDB Ansible ver + Upgrade ETCD [#1452](https://github.com/pingcap/pd/pull/1452) - Unify the log format of etcd and PD server - Fix the issue of failing to elect Leader by PreVote - - Support fast dropping the “propose” and “read” requests that are to fail to avoid blocking the subsequent requests + - Support fast dropping the "propose" and "read" requests that are to fail to avoid blocking the subsequent requests - Fix the deadlock issue of Lease + Fix the issue that a hot store makes incorrect statistics of keys [#1487](https://github.com/pingcap/pd/pull/1487) + Support forcibly rebuilding a PD cluster from a single PD node [#1485](https://github.com/pingcap/pd/pull/1485) @@ -123,7 +122,7 @@ On May 10, 2019, TiDB 3.0.0-rc.1 is released. The corresponding TiDB Ansible ver - Fix the bug in Importer that import success may still be returned even if part of the SST failed to be imported [#4566](https://github.com/tikv/tikv/pull/4566) - Support speed limit in Importer when uploading SST to TiKV [#4412](https://github.com/tikv/tikv/pull/4412) - Support importing tables by size to reduce impacts on the cluster brought by Checksum and Analyze for big tables, and improve the success rate for Checksum and Analyze [#156](https://github.com/pingcap/tidb-lightning/pull/156) - - Improve Lightning’s SQL encoding performance by 50% by directly parsing data source file as types.Datum of TiDB and saving extra parsing overhead from the KV encoder [#145](https://github.com/pingcap/tidb-lightning/pull/145) + - Improve Lightning's SQL encoding performance by 50% by directly parsing data source file as types.Datum of TiDB and saving extra parsing overhead from the KV encoder [#145](https://github.com/pingcap/tidb-lightning/pull/145) - Change log format to [Unified Log Format](https://github.com/tikv/rfcs/blob/master/text/0018-unified-log-format.md) [#162](https://github.com/pingcap/tidb-lightning/pull/162) - Add some command line options for use when the configuration file is missing [#157](https://github.com/pingcap/tidb-lightning/pull/157) diff --git a/releases/release-3.0.0-rc.2.md b/releases/release-3.0.0-rc.2.md index 27ec05bad3ce4..dd1a79cd9c7c5 100644 --- a/releases/release-3.0.0-rc.2.md +++ b/releases/release-3.0.0-rc.2.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.0-rc.2 Release Notes -aliases: ['/docs/dev/releases/release-3.0.0-rc.2/','/docs/dev/releases/3.0.0-rc.2/'] summary: TiDB 3.0.0-rc.2 was released on May 28, 2019, with improvements in stability, usability, features, SQL optimizer, statistics, and execution engine. The release includes enhancements to the SQL optimizer, execution engine, server, DDL, PD, TiKV, and tools like TiDB Binlog and TiDB Lightning. Some notable improvements include support for Index Join in more scenarios, handling virtual columns properly, and adding a metric to track data replication downstream. --- diff --git a/releases/release-3.0.0-rc.3.md b/releases/release-3.0.0-rc.3.md index baff62964ea59..cf042b59a6d4e 100644 --- a/releases/release-3.0.0-rc.3.md +++ b/releases/release-3.0.0-rc.3.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.0-rc.3 Release Notes -aliases: ['/docs/dev/releases/release-3.0.0-rc.3/','/docs/dev/releases/3.0.0-rc.3/'] summary: TiDB 3.0.0-rc.3 was released on June 21, 2019, with improvements in stability, usability, features, SQL optimizer, statistics, and execution engine. Fixes and new features were added to TiDB, PD, TiKV, and TiDB Ansible. Notable improvements include automatic loading statistics, manual splitting of table and index regions, and support for pessimistic transactions in TiKV. --- diff --git a/releases/release-3.0.1.md b/releases/release-3.0.1.md index 7601cab5971b5..c643b82bb0e89 100644 --- a/releases/release-3.0.1.md +++ b/releases/release-3.0.1.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.1 Release Notes -aliases: ['/docs/dev/releases/release-3.0.1/','/docs/dev/releases/3.0.1/'] summary: "TiDB 3.0.1 Release Notes July 16, 2019. TiDB version 3.0.1. Add support for MAX_EXECUTION_TIME feature. Support auto-adjustment of incremental gap for auto-increment IDs. Add ADMIN PLUGINS ENABLE/DISABLE SQL statement. Prohibit Window Functions from being cached in Prepare Plan Cache. Fix various bugs and issues. TiKV: Add statistics of blob file size. Fix core dump issue. PD: Add enable-grpc-gateway configuration option. Optimize hot Region scheduling strategy. Tools: TiDB Binlog - Optimize Pump GC strategy. TiDB Lightning - Fix import error. TiDB Ansible - Add precheck feature, update monitoring information." --- @@ -39,7 +38,7 @@ TiDB Ansible version: 3.0.1 + Fix the issue that the `DB` and `INFO` columns shown by the `SHOW PROCESSLIST` command are incompatible with MySQL [#11003](https://github.com/pingcap/tidb/pull/11003) + Fix the system panic issue caused by the `FLUSH PRIVILEGES` statement when `skip-grant-table=true` is configured [#11027](https://github.com/pingcap/tidb/pull/11027) + Fix the issue that the primary key statistics collected by `FAST ANALYZE` are not correct when the table primary key is an `UNSIGNED` integer [#11099](https://github.com/pingcap/tidb/pull/11099) -+ Fix the issue that the “invalid key” error is reported by the `FAST ANALYZE` statement in some cases [#11098](https://github.com/pingcap/tidb/pull/11098) ++ Fix the issue that the "invalid key" error is reported by the `FAST ANALYZE` statement in some cases [#11098](https://github.com/pingcap/tidb/pull/11098) + Fix the issue that the precision shown by the `SHOW CREATE TABLE` statement is incomplete when `CURRENT_TIMESTAMP` is used as the default value of the column and the float precision is specified [#11088](https://github.com/pingcap/tidb/pull/11088) + Fix the issue that the function name is not in lowercase when window functions report an error to make it compatible with MySQL [#11118](https://github.com/pingcap/tidb/pull/11118) + Fix the issue that TiDB fails to connect to TiKV and thus cannot provide service after the background thread of TiKV Client Batch gRPC panics [#11101](https://github.com/pingcap/tidb/pull/11101) diff --git a/releases/release-3.0.10.md b/releases/release-3.0.10.md index bfe5818569f0d..2a8139e1489e6 100644 --- a/releases/release-3.0.10.md +++ b/releases/release-3.0.10.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.10 Release Notes -aliases: ['/docs/dev/releases/release-3.0.10/','/docs/dev/releases/3.0.10/'] summary: TiDB 3.0.10 was released on February 20, 2020. It includes various bug fixes and improvements for TiDB, TiKV, PD, and TiDB Ansible. Some notable fixes include wrong Join results, data visibility issues, and system panic problems. TiDB Ansible also added new monitoring items to the dashboard. The release notes recommend using the latest 3.0.x version due to known issues in this release. --- diff --git a/releases/release-3.0.11.md b/releases/release-3.0.11.md index 98bb9f66b5df5..77d470237582e 100644 --- a/releases/release-3.0.11.md +++ b/releases/release-3.0.11.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.11 Release Notes -aliases: ['/docs/dev/releases/release-3.0.11/','/docs/dev/releases/3.0.11/'] summary: TiDB 3.0.11 was released on March 4, 2020. It includes compatibility changes, new features, bug fixes, and updates for TiDB, TiDB Binlog, TiDB Lightning, TiKV, and TiDB Ansible. Some known issues are fixed in new versions, so it is recommended to use the latest 3.0.x version. --- diff --git a/releases/release-3.0.12.md b/releases/release-3.0.12.md index c5a3f92738fe9..8c1953f7c89be 100644 --- a/releases/release-3.0.12.md +++ b/releases/release-3.0.12.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.12 Release Notes -aliases: ['/docs/dev/releases/release-3.0.12/','/docs/dev/releases/3.0.12/'] summary: TiDB 3.0.12 was released on March 16, 2020. It includes compatibility changes, new features, bug fixes, and improvements for TiDB, TiKV, PD, and TiDB Ansible. Some known issues are fixed in new versions, so it is recommended to use the latest 3.0.x version. New features include dynamic loading of replaced certificate files, flow limiting for DDL requests, and support for exiting the TiDB server when binlog write fails. Bug fixes address issues with locking, error message display, decimal point accuracy, and data index inconsistency. Additionally, improvements have been made to TiKV's flow control mechanism and PD's Region information processing. --- diff --git a/releases/release-3.0.13.md b/releases/release-3.0.13.md index 22222276de9dc..befdb9c0e6094 100644 --- a/releases/release-3.0.13.md +++ b/releases/release-3.0.13.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.13 Release Notes -aliases: ['/docs/dev/releases/release-3.0.13/','/docs/dev/releases/3.0.13/'] summary: TiDB 3.0.13 was released on April 22, 2020. The bug fixes include resolving issues with the `INSERT ... ON DUPLICATE KEY UPDATE` statement and fixing the system getting stuck and becoming unavailable during `Region Merge` in TiKV. --- diff --git a/releases/release-3.0.14.md b/releases/release-3.0.14.md index 3af2410a7c682..58258b91dbb6d 100644 --- a/releases/release-3.0.14.md +++ b/releases/release-3.0.14.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.14 Release Notes -aliases: ['/docs/dev/releases/release-3.0.14/','/docs/dev/releases/3.0.14/'] summary: TiDB 3.0.14 was released on May 9, 2020. The release includes compatibility changes, important bug fixes, new features, bug fixes, and improvements for TiDB, TiKV, and Tools. Some of the bug fixes include issues with query results, panic occurrences, and incorrect behavior. New features include enhanced syntax support and improved performance. --- diff --git a/releases/release-3.0.15.md b/releases/release-3.0.15.md index 7c4f2b1023d46..0ba0685dda006 100644 --- a/releases/release-3.0.15.md +++ b/releases/release-3.0.15.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.15 Release Notes -aliases: ['/docs/dev/releases/release-3.0.15/'] summary: TiDB 3.0.15 was released on June 5, 2020. New features include support for admin recover index and admin check index statements on partitioned tables, as well as optimization of memory allocation mechanism. Bug fixes address issues such as incorrect results in PointGet and inconsistent results between TiDB and MySQL when XOR operates on a floating-point number. TiKV fixes issues related to memory defragmentation and gRPC disconnection. --- diff --git a/releases/release-3.0.16.md b/releases/release-3.0.16.md index 84cd337988ed8..a58c0e37a550a 100644 --- a/releases/release-3.0.16.md +++ b/releases/release-3.0.16.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.16 Release Notes -aliases: ['/docs/dev/releases/release-3.0.16/'] summary: TiDB 3.0.16 was released on July 03, 2020. The release includes improvements such as support for 'is null' filter condition, handling of SQL timeout issues, and removal of sensitive information in slow query logs. Bug fixes include resolving data inconsistency issues, fixing panic issues, and addressing errors in JSON comparison and query results. TiKV and PD also received bug fixes for issues related to store heartbeats, peer removal, and error handling. --- diff --git a/releases/release-3.0.2.md b/releases/release-3.0.2.md index 5b42306569a07..140bcd9248172 100644 --- a/releases/release-3.0.2.md +++ b/releases/release-3.0.2.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.2 Release Notes -aliases: ['/docs/dev/releases/release-3.0.2/','/docs/dev/releases/3.0.2/'] summary: TiDB 3.0.2 was released on August 7, 2019, with various fixes and improvements. The release includes fixes for SQL optimizer, SQL execution engine, server, DDL, monitor, TiKV, PD, TiDB Binlog, TiDB Lightning, and TiDB Ansible. Fixes include issues with query plans, query results, error messages, and performance optimizations. --- @@ -15,7 +14,7 @@ TiDB Ansible version: 3.0.2 ## TiDB + SQL Optimizer - - Fix the issue that the “Can’t find column in schema” message is reported when the same table occurs multiple times in a query and logically the query result is always empty [#11247](https://github.com/pingcap/tidb/pull/11247) + - Fix the issue that the "Can't find column in schema" message is reported when the same table occurs multiple times in a query and logically the query result is always empty [#11247](https://github.com/pingcap/tidb/pull/11247) - Fix the issue that the query plan does not meet the expectation caused by the `TIDB_INLJ` hint not working correctly in some cases (like `explain select /*+ TIDB_INLJ(t1) */ t1.b, t2.a from t t1, t t2 where t1.b = t2.a`) [#11362](https://github.com/pingcap/tidb/pull/11362) - Fix the issue that the column name in the query result is wrong in some cases (like `SELECT IF(1,c,c) FROM t`) [#11379](https://github.com/pingcap/tidb/pull/11379) - Fix the issue that some queries like `SELECT 0 LIKE 'a string'` return `TRUE` because the `LIKE` expression is implicitly converted to 0 in some cases [#11411](https://github.com/pingcap/tidb/pull/11411) @@ -51,14 +50,14 @@ TiDB Ansible version: 3.0.2 - Fix the issue that the result might be wrong caused by incorrect type conversion when the `DATE_ADD` function receives `FLOAT`, `DOUBLE` and `DECIMAL` column parameters [#11527](https://github.com/pingcap/tidb/pull/11527) - Fix the wrong result caused by incorrectly processing the sign of the INTERVAL fraction in the `DATE_ADD` function [#11615](https://github.com/pingcap/tidb/pull/11615) - Fix the incorrect query result when Index Lookup Join contains the prefix index caused by `Ranger` not correctly handling the prefix index [#11565](https://github.com/pingcap/tidb/pull/11565) - - Fix the issue that the “Incorrect arguments to NAME_CONST” message is reported if the `NAME_CONST` function is executed when the second parameter of `NAME_CONST` is a negative number [#11268](https://github.com/pingcap/tidb/pull/11268) + - Fix the issue that the "Incorrect arguments to NAME_CONST" message is reported if the `NAME_CONST` function is executed when the second parameter of `NAME_CONST` is a negative number [#11268](https://github.com/pingcap/tidb/pull/11268) - Fix the issue that the result is incompatible with MySQL when an SQL statement involves computing the current time and the value is fetched multiple times; use the same value when fetching the current time for the same SQL statement [#11394](https://github.com/pingcap/tidb/pull/11394) - Fix the issue that `Close` is not called for `ChildExecutor` when the `Close` of `baseExecutor` reports an error. This issue might lead to Goroutine leaks when the `KILL` statements do not take effect and `ChildExecutor` is not closed [#11576](https://github.com/pingcap/tidb/pull/11576) + Server - Fix the issue that the auto-added value is 0 instead of the current timestamp when `LOAD DATA` processes the missing `TIMESTAMP` field in the CSV file [#11250](https://github.com/pingcap/tidb/pull/11250) - Fix issues that the `SHOW CREATE USER` statement does not correctly check related privileges, and `USER` and `HOST` returned by `SHOW CREATE USER CURRENT_USER()` might be wrong [#11229](https://github.com/pingcap/tidb/pull/11229) - Fix the issue that the returned result might be wrong when `executeBatch` is used in JDBC [#11290](https://github.com/pingcap/tidb/pull/11290) - - Reduce printing the log information of the streaming client when changing the TiKV server’s port [#11370](https://github.com/pingcap/tidb/pull/11370) + - Reduce printing the log information of the streaming client when changing the TiKV server's port [#11370](https://github.com/pingcap/tidb/pull/11370) - Optimize the logic of reconnecting the streaming client to the TiKV server so that the streaming client will not be blocked for a long time [#11372](https://github.com/pingcap/tidb/pull/11372) - Add `REGION_ID` in `INFORMATION_SCHEMA.TIDB_HOT_REGIONS` [#11350](https://github.com/pingcap/tidb/pull/11350) - Cancel the timeout duration of obtaining Region information from the PD API to ensure that obtaining Region information will not end in a failure when TiDB API `http://{TiDBIP}:10080/regions/hot` is called due to PD timeout when the number of Regions is large [#11383](https://github.com/pingcap/tidb/pull/11383) @@ -136,7 +135,7 @@ TiDB Lightning - Add descriptions for each monitoring item on the TiDB Dashboard [#854](https://github.com/pingcap/tidb-ansible/pull/854) - Add the TiDB Summary Dashboard to better view the cluster status and troubleshoot issues [#855](https://github.com/pingcap/tidb-ansible/pull/855) - Update the Allocator Stats monitoring item on the TiKV Dashboard [#857](https://github.com/pingcap/tidb-ansible/pull/857) -- Fix the unit error in the Node Exporter’s alerting expression [#860](https://github.com/pingcap/tidb-ansible/pull/860) +- Fix the unit error in the Node Exporter's alerting expression [#860](https://github.com/pingcap/tidb-ansible/pull/860) - Upgrade the TiSpark jar package to v2.1.2 [#862](https://github.com/pingcap/tidb-ansible/pull/862) - Update the descriptions of the Ansible Task feature [#867](https://github.com/pingcap/tidb-ansible/pull/867) - Update the expression of the local reader requests monitoring item on the TiDB Dashboard [#874](https://github.com/pingcap/tidb-ansible/pull/874) diff --git a/releases/release-3.0.3.md b/releases/release-3.0.3.md index 24d281beb9b8b..b604bf2c6c9f9 100644 --- a/releases/release-3.0.3.md +++ b/releases/release-3.0.3.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.3 Release Notes -aliases: ['/docs/dev/releases/release-3.0.3/','/docs/dev/releases/3.0.3/'] summary: TiDB 3.0.3 was released on August 29, 2019. It includes various fixes and updates for SQL optimizer, SQL execution engine, server, DDL, monitor, TiKV, PD, TiDB Binlog, TiDB Lightning, and TiDB Ansible. Notable fixes include issues with incorrect results, type errors, panic occurrences, and permission check errors. The release also optimizes PD operations, removes unsupported Grafana Collector components, and updates TiKV alerting rules. Additionally, TiDB Ansible now supports Spark V2.4.3 and TiSpark V2.1.4. --- @@ -17,7 +16,7 @@ TiDB Ansible version: 3.0.3 + SQL Optimizer - Add the `opt_rule_blacklist` table to disable logic optimization rules such as `aggregation_eliminate` and `column_prune` [#11658](https://github.com/pingcap/tidb/pull/11658) - Fix the issue that incorrect results might be returned for `Index Join` when the join key uses a prefix index or an unsigned index column that is equal to a negative value [#11759](https://github.com/pingcap/tidb/pull/11759) - - Fix the issue that `”` or `\` in the `SELECT` statements of `create … binding ...` might result in parsing errors [#11726](https://github.com/pingcap/tidb/pull/11726) + - Fix the issue that `"` or `\` in the `SELECT` statements of `create … binding ...` might result in parsing errors [#11726](https://github.com/pingcap/tidb/pull/11726) + SQL Execution Engine - Fix the issue that type errors in the return value might occur when the Quote function handles a null value [#11619](https://github.com/pingcap/tidb/pull/11619) - Fix the issue that incorrect results for `ifnull` might be returned when Max/Min is used for type inferring with `NotNullFlag` retained [#11641](https://github.com/pingcap/tidb/pull/11641) diff --git a/releases/release-3.0.4.md b/releases/release-3.0.4.md index 93df3984ed0b0..859b575076691 100644 --- a/releases/release-3.0.4.md +++ b/releases/release-3.0.4.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.4 Release Notes -aliases: ['/docs/dev/releases/release-3.0.4/','/docs/dev/releases/3.0.4/'] summary: TiDB 3.0.4 was released on October 8, 2019, with new features including system table for troubleshooting SQL performance issues, improvements in split performance and reverse scan, and fixed issues related to slow query logs and data replication. The release also includes contributions from the community and updates to TiDB, TiKV, PD, and TiDB Ansible. --- @@ -14,7 +13,7 @@ TiDB Ansible version: 3.0.4 - New features - Add the `performance_schema.events_statements_summary_by_digest` system table to troubleshoot performance issues at the SQL level - - Add the `WHERE` clause in TiDB’s `SHOW TABLE REGIONS` syntax + - Add the `WHERE` clause in TiDB's `SHOW TABLE REGIONS` syntax - Add the `worker-count` and `txn-batch` configuration items in Reparo to control the recovery speed - Improvements - Support batch Region split command and empty split command in TiKV to improve split performance @@ -48,7 +47,7 @@ TiDB Ansible version: 3.0.4 - Fix the issue that when a query contains the `SLEEP` function (for example, `select 1 from (select sleep(1)) t;)`), column pruning causes invalid `sleep(1)` during query [#11953](https://github.com/pingcap/tidb/pull/11953) - Use index scan to lower IO when a query only concerns the number of columns rather than the table data [#12112](https://github.com/pingcap/tidb/pull/12112) - Do not use any index when no index is specified in `use index()` to be compatible with MySQL [#12100](https://github.com/pingcap/tidb/pull/12100) - - Strictly limit the number of `TopN` records in the `CMSketch` statistics to fix the issue that the `ANALYZE` statement fails because the statement count exceeds TiDB’s limit on the size of a transaction [#11914](https://github.com/pingcap/tidb/pull/11914) + - Strictly limit the number of `TopN` records in the `CMSketch` statistics to fix the issue that the `ANALYZE` statement fails because the statement count exceeds TiDB's limit on the size of a transaction [#11914](https://github.com/pingcap/tidb/pull/11914) - Fix the error occurred when converting the subqueries contained in the `Update` statement [#12483](https://github.com/pingcap/tidb/pull/12483) - Optimize execution performance of the `select ... limit ... offset ...` statement by pushing the Limit operator down to the `IndexLookUpReader` execution logic [#12378](https://github.com/pingcap/tidb/pull/12378) - SQL Execution Engine diff --git a/releases/release-3.0.5.md b/releases/release-3.0.5.md index a20f90118e935..5aaf5d428d4f3 100644 --- a/releases/release-3.0.5.md +++ b/releases/release-3.0.5.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.5 Release Notes -aliases: ['/docs/dev/releases/release-3.0.5/','/docs/dev/releases/3.0.5/'] summary: TiDB 3.0.5 was released on October 25, 2019, with various improvements and bug fixes. The release includes enhancements to the SQL optimizer, SQL execution engine, server, DDL, monitor, TiKV, PD, TiDB Binlog, TiDB Lightning, and TiDB Ansible. Improvements include support for boundary checking on Window Functions, fixing issues with index join and outer join, and adding monitoring metrics for various operations. Additionally, TiKV received storage and performance optimizations, while PD saw improvements in storage precision and HTTP request handling. TiDB Ansible also received updates to monitoring metrics and configuration file simplification. --- diff --git a/releases/release-3.0.6.md b/releases/release-3.0.6.md index 4f4e895f3274e..faef48b9f59e5 100644 --- a/releases/release-3.0.6.md +++ b/releases/release-3.0.6.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.6 Release Notes -aliases: ['/docs/dev/releases/release-3.0.6/','/docs/dev/releases/3.0.6/'] summary: TiDB 3.0.6 was released on November 28, 2019, with various fixes and optimizations. The release includes improvements to the SQL optimizer, SQL execution engine, server, DDL, TiKV, PD, TiDB Binlog, and TiDB Lightning. Fixes include issues with window function AST, pushing down `STREAM AGG()`, handling quotes for SQL binding, and more. TiKV improvements include accurate `lock_manager`, support for `innodb_lock_wait_timeout`, and dynamic modification of the GC I/O limit using `tikv-ctl`. PD enhancements include lower client log level and warning log for generating a timestamp. TiDB Binlog and TiDB Lightning also received fixes and improvements. --- @@ -64,7 +63,7 @@ TiDB Ansible version: 3.0.6 - `Write_size` - `Prewrite_region` - `Txn_retry` - - Use the table’s `COLLATE` instead of the system’s default charset in the column when a table is created and the table contains `COLLATE` [#13174](https://github.com/pingcap/tidb/pull/13174) + - Use the table's `COLLATE` instead of the system's default charset in the column when a table is created and the table contains `COLLATE` [#13174](https://github.com/pingcap/tidb/pull/13174) - Limit the length of the index name when creating a table [#13310](https://github.com/pingcap/tidb/pull/13310) - Fix the issue that the table name length is not checked when a table is renamed [#13346](https://github.com/pingcap/tidb/pull/13346) - Add the `alter-primary-key` configuration (disabled by default) to support adding/dropping the primary key in TiDB [#13522](https://github.com/pingcap/tidb/pull/13522) @@ -92,11 +91,11 @@ TiDB Ansible version: 3.0.6 ## Tools + TiDB Binlog - - Obtain the initial replication timestamp from PD when `initial-commit-ts` is set to “-1” in Drainer [#788](https://github.com/pingcap/tidb-binlog/pull/788) - - Decouple Drainer’s `Checkpoint` storage from the downstream and support saving `Checkpoint` in MySQL or local files [#790](https://github.com/pingcap/tidb-binlog/pull/790) + - Obtain the initial replication timestamp from PD when `initial-commit-ts` is set to "-1" in Drainer [#788](https://github.com/pingcap/tidb-binlog/pull/788) + - Decouple Drainer's `Checkpoint` storage from the downstream and support saving `Checkpoint` in MySQL or local files [#790](https://github.com/pingcap/tidb-binlog/pull/790) - Fix the Drainer panic issue caused by using empty values when configuring replication database/table filtering [#801](https://github.com/pingcap/tidb-binlog/pull/801) - Fix the issue that processes get into the deadlock status instead of exiting after a panic occurs because Drainer fails to apply binlog files to the downstream [#807](https://github.com/pingcap/tidb-binlog/pull/807) - - Fix the issue that Pump blocks when it exits because of gRPC’s `GracefulStop` [#817](https://github.com/pingcap/tidb-binlog/pull/817) + - Fix the issue that Pump blocks when it exits because of gRPC's `GracefulStop` [#817](https://github.com/pingcap/tidb-binlog/pull/817) - Fix the issue that Drainer fails when it receives a binlog which misses a column during the execution of a `DROP COLUMN` statement in TiDB (v3.0.6 or later) [#827](https://github.com/pingcap/tidb-binlog/pull/827) + TiDB Lightning - Add the `max-allowed-packet` configuration (64 M by default) for the TiDB backend [#248](https://github.com/pingcap/tidb-lightning/pull/248) diff --git a/releases/release-3.0.7.md b/releases/release-3.0.7.md index f2dc668989b93..bd3eac70137f0 100644 --- a/releases/release-3.0.7.md +++ b/releases/release-3.0.7.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.7 Release Notes -aliases: ['/docs/dev/releases/release-3.0.7/','/docs/dev/releases/3.0.7/'] summary: TiDB 3.0.7 was released on December 4, 2019. It includes fixes for issues related to lock TTL, timezone parsing, result accuracy, data precision, and statistics accuracy. TiKV also received updates to improve deadlock detection and fix a memory leak issue. --- @@ -14,7 +13,7 @@ TiDB Ansible version: 3.0.7 ## TiDB -- Fix the issue that the lock TTL’s value is too large because the TiDB server’s local time is behind PD’s timestamp [#13868](https://github.com/pingcap/tidb/pull/13868) +- Fix the issue that the lock TTL's value is too large because the TiDB server's local time is behind PD's timestamp [#13868](https://github.com/pingcap/tidb/pull/13868) - Fix the issue that the timezone is incorrect after parsing the date from strings using `gotime.Local` [#13793](https://github.com/pingcap/tidb/pull/13793) - Fix the issue that the result might be incorrect because the `binSearch` function does not return an error in the implementation of `builtinIntervalRealSig` [#13767](https://github.com/pingcap/tidb/pull/13767) - Fix the issue that data is incorrect because the precision is lost when an integer is converted to an unsigned floating point or decimal type [#13755](https://github.com/pingcap/tidb/pull/13755) diff --git a/releases/release-3.0.8.md b/releases/release-3.0.8.md index 120a96453b2e4..751bbe58d9adb 100644 --- a/releases/release-3.0.8.md +++ b/releases/release-3.0.8.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.8 Release Notes -aliases: ['/docs/dev/releases/release-3.0.8/','/docs/dev/releases/3.0.8/'] summary: TiDB 3.0.8 was released on December 31, 2019. It includes various fixes and improvements for SQL optimizer, SQL execution engine, DDL, server, transaction, monitor, TiKV, PD, and TiDB Ansible. Notable changes include SQL binding plan fixes, error message optimizations, and support for certificate-based authentication. The default value of `tidb_txn_mode` variable is updated to `"pessimistic"`. PD also received performance optimizations and bug fixes. TiDB Ansible saw various logic optimizations and upgrades. --- @@ -47,10 +46,10 @@ TiDB Ansible version: 3.0.8 - Modify the TiDB behavior from using the current database to reporting the `No database selected` error when the `GRANT` statement does not specify a database name, to keep compatibility with the MySQL behavior [#13784](https://github.com/pingcap/tidb/pull/13784) - Modify the execution privilege for the `REVOKE` statement from `SuperPriv` to `REVOKE` being executable only if the user has the privilege for the corresponding schema, to keep consistency with the MySQL behavior [#13306](https://github.com/pingcap/tidb/pull/13306) - Fix the issue that `GrantPriv` is mistakenly granted to the target user when the `GRANT ALL` syntax does not contain `WITH GRANT OPTION` [#13943](https://github.com/pingcap/tidb/pull/13943) - - Fix the issue that the error message does not contain the cause for the `LOAD DATA` statement’s wrong behavior when `LoadDataInfo` fails to call `addRecord` [#13980](https://github.com/pingcap/tidb/pull/13980) + - Fix the issue that the error message does not contain the cause for the `LOAD DATA` statement's wrong behavior when `LoadDataInfo` fails to call `addRecord` [#13980](https://github.com/pingcap/tidb/pull/13980) - Fix the issue that wrong slow query information is output because multiple SQL statements in a query share the same `StartTime` [#13898](https://github.com/pingcap/tidb/pull/13898) - Fix the issue that the memory might leak when `batchClient` processes a large transaction [#14032](https://github.com/pingcap/tidb/pull/14032) - - Fix the issue that `system_time_zone` is always displayed as `CST` and now TiDB’s `system_time_zone` is obtained from `systemTZ` in the `mysql.tidb` table [#14086](https://github.com/pingcap/tidb/pull/14086) + - Fix the issue that `system_time_zone` is always displayed as `CST` and now TiDB's `system_time_zone` is obtained from `systemTZ` in the `mysql.tidb` table [#14086](https://github.com/pingcap/tidb/pull/14086) - Fix the issue that the `GRANT ALL` syntax does not grant all privileges to the user [#14092](https://github.com/pingcap/tidb/pull/14092) - Fix the issue that the `Priv_create_user` privilege is invalid for `CREATE ROLE` and `DROP ROLE` [#14088](https://github.com/pingcap/tidb/pull/14088) - Modify the error code of `ErrInvalidFieldSize` from `1105(Unknow Error)` to `3013` [#13737](https://github.com/pingcap/tidb/pull/13737) diff --git a/releases/release-3.0.9.md b/releases/release-3.0.9.md index ec784d4aff001..dca02b28a87d3 100644 --- a/releases/release-3.0.9.md +++ b/releases/release-3.0.9.md @@ -1,6 +1,5 @@ --- title: TiDB 3.0.9 Release Notes -aliases: ['/docs/dev/releases/release-3.0.9/','/docs/dev/releases/3.0.9/'] summary: TiDB 3.0.9 was released on January 14, 2020. It includes fixes for known issues and new features. Some improvements were made to Executor, Server, DDL, Planner, TiKV, PD, Tools, and TiDB Ansible. Notable changes include support for system variables, monitoring metrics, and optimizations for transaction execution latency. Additionally, support for using backlash in the location label name and automatically creating directories for TiDB Lightning deployment was added. --- @@ -38,7 +37,7 @@ TiDB Ansible version: 3.0.9 + Raftstore - Speed up the configuration change to speed up the Region scattering [#6421](https://github.com/tikv/tikv/pull/6421) + Transaction - - Add the `tikv_lock_manager_waiter_lifetime_duration`, `tikv_lock_manager_detect_duration`, and `tikv_lock_manager_detect_duration` monitoring metrics to monitor `waiter`s’ lifetime, the time cost of detecting deadlocks, and the status of `Wait` table [#6392](https://github.com/tikv/tikv/pull/6392) + - Add the `tikv_lock_manager_waiter_lifetime_duration`, `tikv_lock_manager_detect_duration`, and `tikv_lock_manager_detect_duration` monitoring metrics to monitor `waiter`s' lifetime, the time cost of detecting deadlocks, and the status of `Wait` table [#6392](https://github.com/tikv/tikv/pull/6392) - Optimize the following configuration items to reduce transaction execution latency caused by changing Region leader or the leader of deadlock detector in extreme situations [#6429](https://github.com/tikv/tikv/pull/6429) - Change the default value of `wait-for-lock-time` from `3s` to `1s` - Change the default value of `wake-up-delay-duration` from `100ms` to `20ms` diff --git a/releases/release-3.1.0-beta.1.md b/releases/release-3.1.0-beta.1.md index b260dbb523b00..c4642b33928b9 100644 --- a/releases/release-3.1.0-beta.1.md +++ b/releases/release-3.1.0-beta.1.md @@ -1,6 +1,5 @@ --- title: TiDB 3.1 Beta.1 Release Notes -aliases: ['/docs/dev/releases/release-3.1.0-beta.1/','/docs/dev/releases/3.1.0-beta.1/'] summary: TiDB 3.1 Beta.1 was released on January 10, 2020. The release includes changes to TiKV, such as renaming backup files and adding incremental backup features. Tools like BR have improved backup progress information and added features for partitioned tables. TiDB Ansible now automatically disables Transparent Huge Pages and adds Grafana monitoring for BR components. Overall, the release focuses on improving backup and restore processes, monitoring, and deployment optimization. --- diff --git a/releases/release-3.1.0-beta.2.md b/releases/release-3.1.0-beta.2.md index 8009fb07498a8..c76e191561151 100644 --- a/releases/release-3.1.0-beta.2.md +++ b/releases/release-3.1.0-beta.2.md @@ -1,6 +1,5 @@ --- title: TiDB 3.1 Beta.2 Release Notes -aliases: ['/docs/dev/releases/release-3.1.0-beta.2/','/docs/dev/releases/3.1.0-beta.2/'] summary: TiDB 3.1 Beta.2 was released on March 9, 2020. It includes compatibility changes, new features, bug fixes, and improvements for TiDB, TiKV, PD Client, Backup, PD, TiFlash, and TiDB Ansible. Some known issues are fixed in new versions, so it is recommended to use the latest 3.1.x version. --- @@ -38,7 +37,7 @@ TiDB Ansible version: 3.1.0-beta.2 + PD Client - Support reporting statistics of local threads to PD [#6605](https://github.com/tikv/tikv/pull/6605) + Backup - - Replace the `RocksIOLimiter` flow control library with Rust’s `async-speed-limit` flow control library to eliminate extra memory copies when backing up a file [#6462](https://github.com/tikv/tikv/pull/6462) + - Replace the `RocksIOLimiter` flow control library with Rust's `async-speed-limit` flow control library to eliminate extra memory copies when backing up a file [#6462](https://github.com/tikv/tikv/pull/6462) + PD - Tolerate backslash in the location label name [#2084](https://github.com/pingcap/pd/pull/2084) + TiFlash diff --git a/releases/release-3.1.0-beta.md b/releases/release-3.1.0-beta.md index 51f9deeb4a6cb..8885c9c55d124 100644 --- a/releases/release-3.1.0-beta.md +++ b/releases/release-3.1.0-beta.md @@ -1,6 +1,5 @@ --- title: TiDB 3.1 Beta Release Notes -aliases: ['/docs/dev/releases/release-3.1.0-beta/','/docs/dev/releases/3.1.0-beta/'] summary: TiDB 3.1 Beta was released on December 20, 2019. It includes SQL Optimizer improvements and supports the Follower Read feature. TiKV now supports distributed backup and restore, as well as the Follower Read feature. PD also supports distributed backup and restore. --- diff --git a/releases/release-3.1.0-ga.md b/releases/release-3.1.0-ga.md index 7ae4a6376a6a2..d2b15102edcf1 100644 --- a/releases/release-3.1.0-ga.md +++ b/releases/release-3.1.0-ga.md @@ -1,6 +1,5 @@ --- title: TiDB 3.1.0 GA Release Notes -aliases: ['/docs/dev/releases/release-3.1.0-ga/','/docs/dev/releases/3.1.0-ga/'] summary: TiDB 3.1.0 GA was released on April 16, 2020. It includes compatibility changes, new features, bug fixes, and improvements for TiDB, TiFlash, TiKV, and tools like Backup & Restore and TiDB Binlog. Notable changes include support for displaying Coprocessor tasks, optimizing hot Region scheduling, and fixing various panic and data loss issues. The release also includes improvements to TiDB Ansible for better monitoring and configuration parameters. --- diff --git a/releases/release-3.1.0-rc.md b/releases/release-3.1.0-rc.md index 6ce3a4bca3f3d..2e416fce5cd12 100644 --- a/releases/release-3.1.0-rc.md +++ b/releases/release-3.1.0-rc.md @@ -1,6 +1,5 @@ --- title: TiDB 3.1 RC Release Notes -aliases: ['/docs/dev/releases/release-3.1.0-rc/','/docs/dev/releases/3.1.0-rc/'] summary: TiDB 3.1 RC was released on April 2, 2020. It includes new features such as improved partition pruning, support for `RECOVER` syntax, and TLS certificate updates. Bug fixes include resolving issues with TiFlash replica, `last_insert_id`, and `Aggregation` pushdown. TiKV now supports TLS authentication and AWS IAM web identity for backup. PD has fixed data race issues and placement rule inconsistencies. Tools like TiDB Lightning and BR have also been optimized and fixed. --- diff --git a/releases/release-3.1.1.md b/releases/release-3.1.1.md index 9c0eb6b472503..eb52237a00cdf 100644 --- a/releases/release-3.1.1.md +++ b/releases/release-3.1.1.md @@ -1,6 +1,5 @@ --- title: TiDB 3.1.1 Release Notes -aliases: ['/docs/dev/releases/release-3.1.1/','/docs/dev/releases/3.1.1/'] summary: TiDB 3.1.1 was released on April 30, 2020. New features include table option for `auto_rand_base` and `Feature ID` comment. Bug fixes include isolation read setting, partition selection syntax, and wrong results from nested queries. TiFlash also received bug fixes and improvements in data reading and storage path modification. Backup & Restore (BR) fixed issues related to table restoration and data insertion. --- diff --git a/releases/release-3.1.2.md b/releases/release-3.1.2.md index 0ddf6d75ca472..79e6eb3f4e491 100644 --- a/releases/release-3.1.2.md +++ b/releases/release-3.1.2.md @@ -1,6 +1,5 @@ --- title: TiDB 3.1.2 Release Notes -aliases: ['/docs/dev/releases/release-3.1.2/'] summary: TiDB 3.1.2 was released on June 4, 2020. Bug fixes include error handling during backup and restoration with S3 and GCS, and a `DefaultNotFound` error during restoration. Tools like Backup & Restore (BR) now automatically retry on poor network, fix restoration failures, data loss issues, and support AWS KMS for server-side encryption with S3 storage. --- @@ -23,6 +22,6 @@ TiDB version: 3.1.2 - Retry automatically when the network is poor to improve stability with S3 and GCS storages [#314](https://github.com/pingcap/br/pull/314) [#7965](https://github.com/tikv/tikv/pull/7965) - Fix a restoration failure that occurs because the Region leader cannot be found when restoring small tables [#303](https://github.com/pingcap/br/pull/303) - - Fix a data loss issue during restoration when a table’s row ID exceeds `2^(63)` [#323](https://github.com/pingcap/br/pull/323) + - Fix a data loss issue during restoration when a table's row ID exceeds `2^(63)` [#323](https://github.com/pingcap/br/pull/323) - Fix the issue that empty databases and tables cannot be restored [#318](https://github.com/pingcap/br/pull/318) - Support using AWS KMS for server-side encryption (SSE) when targeting the S3 storage [#261](https://github.com/pingcap/br/pull/261) diff --git a/releases/release-4.0-ga.md b/releases/release-4.0-ga.md index eab61a6aa0b67..7188ad5697196 100644 --- a/releases/release-4.0-ga.md +++ b/releases/release-4.0-ga.md @@ -1,6 +1,5 @@ --- title: TiDB 4.0 GA Release Notes -aliases: ['/docs/dev/releases/release-4.0-ga/'] summary: TiDB 4.0.0 GA was released on May 28, 2020. This version optimized error messages for large-sized transactions, improved usability of `Changefeed` configuration file, added new configuration items and support for various syntax and functions, fixed multiple bugs and issues in TiKV, TiFlash, PD, and Tools, added new monitoring items and support for various features in PD, and fixed various issues in Backup & Restore (BR) and TiCDC. --- diff --git a/releases/release-4.0.0-beta.1.md b/releases/release-4.0.0-beta.1.md index 18fbea5598b25..a9f6c438a6fb3 100644 --- a/releases/release-4.0.0-beta.1.md +++ b/releases/release-4.0.0-beta.1.md @@ -1,6 +1,5 @@ --- title: TiDB 4.0.0 Beta.1 Release Notes -aliases: ['/docs/dev/releases/release-4.0.0-beta.1/','/docs/dev/releases/4.0.0-beta.1/'] summary: TiDB 4.0.0 Beta.1 was released on February 28, 2020. It includes compatibility changes, new features, and bug fixes. Some highlights include support for SQL performance diagnosis, the `Sequence` function, and TLS support between components. Additionally, TiDB Lightning now has a bug fix for the web interface. --- diff --git a/releases/release-4.0.0-beta.2.md b/releases/release-4.0.0-beta.2.md index 5f57b468aea9f..b3443cf8bada5 100644 --- a/releases/release-4.0.0-beta.2.md +++ b/releases/release-4.0.0-beta.2.md @@ -1,6 +1,5 @@ --- title: TiDB 4.0.0 Beta.2 Release Notes -aliases: ['/docs/dev/releases/release-4.0.0-beta.2/','/docs/dev/releases/4.0.0-beta.2/'] summary: TiDB 4.0.0 Beta.2 was released on March 18, 2020. The new features include support for persisting dynamically updated configurations, bidirectional data replication, TLS configuration, change data capture, and experimental features like incremental backup. Bug fixes address issues with panic, hibernate regions, replication delay, and compatibility. TiDB Ansible now supports injecting node information to etcd and deploying services on the ARM platform. --- diff --git a/releases/release-4.0.0-beta.md b/releases/release-4.0.0-beta.md index 2c2a10fcbc23e..90ce06fb7009c 100644 --- a/releases/release-4.0.0-beta.md +++ b/releases/release-4.0.0-beta.md @@ -1,6 +1,5 @@ --- title: TiDB 4.0 Beta Release Notes -aliases: ['/docs/dev/releases/release-4.0.0-beta/','/docs/dev/releases/4.0.0-beta/'] summary: TiDB version 4.0.0-beta and TiDB Ansible version 4.0.0-beta were released on January 17, 2020. The release includes various improvements such as increased accuracy in calculating the cost of Index Join, support for Table Locks, and optimization of the error code of SQL error messages. TiKV was also upgraded to RocksDB version 6.4.6 and now supports quick backup and restoration. PD now supports optimizing hotspot scheduling and adding Placement Rules feature. TiDB Lightning added a parameter to set the password of the downstream database, and TiDB Ansible now supports deploying and maintaining TiFlash. --- diff --git a/releases/release-4.0.0-rc.1.md b/releases/release-4.0.0-rc.1.md index 52f01fb300927..ff24fc7397f44 100644 --- a/releases/release-4.0.0-rc.1.md +++ b/releases/release-4.0.0-rc.1.md @@ -1,6 +1,5 @@ --- title: TiDB 4.0 RC.1 Release Notes -aliases: ['/docs/dev/releases/release-4.0.0-rc.1/','/docs/dev/releases/4.0.0-rc.1/'] summary: TiDB 4.0 RC.1 was released on April 28, 2020. The release includes compatibility changes, important bug fixes, new features, and bug fixes for TiKV, TiDB, TiFlash, TiCDC, Backup & Restore (BR), and Placement Driver (PD). The bug fixes address issues such as data inconsistency, deadlock, and replication failure. New features include support for sending Coprocessor requests to TiFlash in batches and enabling the load-based split region operation. Additionally, TiFlash now supports pushing down the fromUnixTime and dateFormat functions. --- diff --git a/releases/release-4.0.0-rc.2.md b/releases/release-4.0.0-rc.2.md index 2bb086df641c0..5155e65e2b754 100644 --- a/releases/release-4.0.0-rc.2.md +++ b/releases/release-4.0.0-rc.2.md @@ -1,6 +1,5 @@ --- title: TiDB 4.0 RC.2 Release Notes -aliases: ['/docs/dev/releases/release-4.0.0-rc.2/'] summary: TiDB 4.0 RC.2 was released on May 15, 2020. The release includes compatibility changes, important bug fixes, new features, and bug fixes for TiDB, TiKV, PD, TiFlash, and various tools. Some notable changes include the removal of the size limit for a single transaction when TiDB Binlog is enabled, support for the BACKUP and RESTORE commands, and the addition of encryption-related monitoring metrics in Grafana dashboard. Additionally, there are numerous bug fixes for issues such as wrong partition selection, incorrect index range building, and performance reduction. The release also introduces new features like support for the auto_random option in the CREATE TABLE statement and the ability to manage replication tasks using cdc cli. --- diff --git a/releases/release-4.0.0-rc.md b/releases/release-4.0.0-rc.md index 8ed7c2b47e8f7..a2100bbfb387e 100644 --- a/releases/release-4.0.0-rc.md +++ b/releases/release-4.0.0-rc.md @@ -1,6 +1,5 @@ --- title: TiDB 4.0 RC Release Notes -aliases: ['/docs/dev/releases/release-4.0.0-rc/','/docs/dev/releases/4.0.0-rc/'] summary: TiDB 4.0 RC released on April 8, 2020. It includes compatibility changes, bug fixes, new features, and tools. TiKV supports the `pipelined` feature in pessimistic transactions, improving TPC-C performance by 20%. TiDB adds case-insensitive collation and enhances the `RECOVER TABLE` syntax. TiKV now supports TLS in the HTTP port. PD supports getting default PD configuration information through the HTTP API. Bug fixes include issues with replication, subquery results, and DDL job internal retry. Tools like TiDB Lightning and TiCDC also have bug fixes and new features. --- diff --git a/releases/release-4.0.1.md b/releases/release-4.0.1.md index 3f5d38000c7ff..e2a6363c0d223 100644 --- a/releases/release-4.0.1.md +++ b/releases/release-4.0.1.md @@ -1,6 +1,5 @@ --- title: TiDB 4.0.1 Release Notes -aliases: ['/docs/dev/releases/release-4.0.1/'] summary: TiDB 4.0.1 was released on June 12, 2020. New features include support for custom timeout for PD client and new collation framework in TiFlash. Bug fixes address issues with configuration, monitoring metrics, and store information retrieval. Backup & Restore (BR) now includes a version check to avoid compatibility issues. --- diff --git a/releases/release-4.0.10.md b/releases/release-4.0.10.md index c752fa1e25679..491cb43364d50 100644 --- a/releases/release-4.0.10.md +++ b/releases/release-4.0.10.md @@ -34,7 +34,6 @@ TiDB version: 4.0.10 + TiCDC - - Enable the old value feature for the `maxwell` protocol [#1144](https://github.com/pingcap/tiflow/pull/1144) - Enable the unified sorter feature by default [#1230](https://github.com/pingcap/tiflow/pull/1230) + Dumpling @@ -83,7 +82,6 @@ TiDB version: 4.0.10 + TiCDC - - Fix the `maxwell` protocol issues, including the issue of `base64` data output and the issue of outputting TSO to unix timestamp [#1173](https://github.com/pingcap/tiflow/pull/1173) - Fix a bug that outdated metadata might cause the newly created changefeed abnormal [#1184](https://github.com/pingcap/tiflow/pull/1184) - Fix the issue of creating the receiver on the closed notifier [#1199](https://github.com/pingcap/tiflow/pull/1199) - Fix a bug that the TiCDC owner might consume too much memory in the etcd watch client [#1227](https://github.com/pingcap/tiflow/pull/1227) diff --git a/releases/release-4.0.2.md b/releases/release-4.0.2.md index 1153d96655e9c..51fa62791efb3 100644 --- a/releases/release-4.0.2.md +++ b/releases/release-4.0.2.md @@ -1,6 +1,5 @@ --- title: TiDB 4.0.2 Release Notes -aliases: ['/docs/dev/releases/release-4.0.2/'] summary: TiDB 4.0.2 was released on July 1, 2020. The new version includes compatibility changes, new features, improvements, bug fixes, and new changes. Some highlights include support for new aggregate functions, improvements in query latency, and bug fixes related to execution plan, runtime errors, and data replication. Additionally, there are new features and improvements in TiKV, PD, TiFlash, and Tools. --- diff --git a/releases/release-4.0.6.md b/releases/release-4.0.6.md index fd46dac44a0d3..c15f5f6a050db 100644 --- a/releases/release-4.0.6.md +++ b/releases/release-4.0.6.md @@ -27,8 +27,6 @@ TiDB version: 4.0.6 + TiCDC (GA since v4.0.6) - - Support outputting data in the `maxwell` format [#869](https://github.com/pingcap/tiflow/pull/869) - ## Improvements + TiDB diff --git a/releases/release-4.0.8.md b/releases/release-4.0.8.md index 222f66e65f7df..7d90e89ae0165 100644 --- a/releases/release-4.0.8.md +++ b/releases/release-4.0.8.md @@ -149,7 +149,6 @@ TiDB version: 4.0.8 - Fix the unexpected exit caused by the failure to update the GC safepoint [#979](https://github.com/pingcap/tiflow/pull/979) - Fix the issue that the task status is unexpectedly flushed because of the incorrect mod revision cache [#1017](https://github.com/pingcap/tiflow/pull/1017) - - Fix the unexpected empty Maxwell messages [#978](https://github.com/pingcap/tiflow/pull/978) + TiDB Lightning diff --git a/releases/release-5.0.0.md b/releases/release-5.0.0.md index fa4f630eaa1ff..7bc62299d4fbc 100644 --- a/releases/release-5.0.0.md +++ b/releases/release-5.0.0.md @@ -177,7 +177,7 @@ Currently, the main features that the MPP mode does not support are as follows ( When you are designing table structures or analyzing database behaviors, it is recommended to use the clustered index feature if you find that some columns with primary keys are often grouped and sorted, queries on these columns often return a certain range of data or a small amount of data with different values, and the corresponding data does not cause read or write hotspot issues. -Clustered indexes, also known as _index-organized tables_ in some database management systems, is a storage structure associated with the data of a table. When creating a clustered index, you can specify one or more columns from the table as the keys for the index. TiDB stores these keys in a specific structure, which allows TiDB to quickly and efficiently find the rows associated with the keys, thus improves the performance of querying and writing data. +Clustered indexes is a storage structure associated with the data of a table. Some database management systems refer to clustered index tables as _index-organized tables_. When creating a clustered index, you can specify one or more columns from the table as the keys for the index. TiDB stores these keys in a specific structure, which allows TiDB to quickly and efficiently find the rows associated with the keys, thus improves the performance of querying and writing data. When the clustered index feature is enabled, the TiDB performance improves significantly (for example in the TPC-C tpmC test, the performance of TiDB, with clustered index enabled, improves by 39%) in the following cases: diff --git a/releases/release-5.0.6.md b/releases/release-5.0.6.md index 5f780a20fae3a..a6a1454bbaba7 100644 --- a/releases/release-5.0.6.md +++ b/releases/release-5.0.6.md @@ -151,7 +151,7 @@ TiDB version: 5.0.6 - Fix the issue that Avro sink does not support parsing JSON type columns [#3624](https://github.com/pingcap/tiflow/issues/3624) - Fix the bug that TiCDC reads the incorrect schema snapshot from TiKV when the TiKV owner restarts [#2603](https://github.com/pingcap/tiflow/issues/2603) - Fix the memory leak issue after processing DDLs [#3174](https://github.com/pingcap/ticdc/issues/3174) - - Fix the bug that the `enable-old-value` configuration item is not automatically set to `true` on Canal and Maxwell protocols [#3676](https://github.com/pingcap/tiflow/issues/3676) + - Fix the bug that the `enable-old-value` configuration item is not automatically set to `true` on the Canal protocol [#3676](https://github.com/pingcap/tiflow/issues/3676) - Fix the timezone error that occurs when the `cdc server` command runs on some Red Hat Enterprise Linux releases (such as 6.8 and 6.9) [#3584](https://github.com/pingcap/tiflow/issues/3584) - Fix the issue of the inaccurate `txn_batch_size` monitoring metric for Kafka sink [#3431](https://github.com/pingcap/tiflow/issues/3431) - Fix the issue that `tikv_cdc_min_resolved_ts_no_change_for_1m` keeps alerting when there is no changefeed [#11017](https://github.com/tikv/tikv/issues/11017) diff --git a/releases/release-5.1.4.md b/releases/release-5.1.4.md index 1fc9c2ee415d5..aea49c76ca4e9 100644 --- a/releases/release-5.1.4.md +++ b/releases/release-5.1.4.md @@ -153,7 +153,7 @@ TiDB version: 5.1.4 - Fix the TiCDC panic issue that occurs when manually cleaning the task status in etcd [#2980](https://github.com/pingcap/tiflow/issues/2980) - Fix the issue that the service cannot be started because of a timezone issue in the RHEL release [#3584](https://github.com/pingcap/tiflow/issues/3584) - Fix the issue of overly frequent warnings caused by MySQL sink deadlock [#2706](https://github.com/pingcap/tiflow/issues/2706) - - Fix the bug that the `enable-old-value` configuration item is not automatically set to `true` on Canal and Maxwell protocols [#3676](https://github.com/pingcap/tiflow/issues/3676) + - Fix the bug that the `enable-old-value` configuration item is not automatically set to `true` on the Canal protocol [#3676](https://github.com/pingcap/tiflow/issues/3676) - Fix the issue that Avro sink does not support parsing JSON type columns [#3624](https://github.com/pingcap/tiflow/issues/3624) - Fix the negative value error in the changefeed checkpoint lag [#3010](https://github.com/pingcap/ticdc/issues/3010) - Fix the OOM issue in the container environment [#1798](https://github.com/pingcap/tiflow/issues/1798) diff --git a/releases/release-5.2.0.md b/releases/release-5.2.0.md index 1c86e94ba0845..40f4df5a0374a 100644 --- a/releases/release-5.2.0.md +++ b/releases/release-5.2.0.md @@ -211,7 +211,7 @@ Support running the `tiup playground` command on Mac computers with Apple M1 chi - Support completing the garbage collection automatically for the bindings in the "deleted" status [#26206](https://github.com/pingcap/tidb/pull/26206) - Support showing whether a binding is used for query optimization in the result of `EXPLAIN VERBOSE` [#26930](https://github.com/pingcap/tidb/pull/26930) - Add a new status variation `last_plan_binding_update_time` to view the timestamp corresponding to the binding cache in the current TiDB instance [#26340](https://github.com/pingcap/tidb/pull/26340) - - Support reporting an error when starting binding evolution or running `admin evolve bindings` to ban the baseline evolution (currently disabled in the TiDB Self-Hosted version because it is an experimental feature) affecting other features [#26333](https://github.com/pingcap/tidb/pull/26333) + - Support reporting an error when starting binding evolution or running `admin evolve bindings` to ban the baseline evolution (currently disabled in the TiDB Self-Managed version because it is an experimental feature) affecting other features [#26333](https://github.com/pingcap/tidb/pull/26333) + PD diff --git a/releases/release-5.2.4.md b/releases/release-5.2.4.md index a93247a1088c8..95b87a499a889 100644 --- a/releases/release-5.2.4.md +++ b/releases/release-5.2.4.md @@ -182,7 +182,7 @@ TiDB version: 5.2.4 - Fix the issue that the service cannot be started because of a timezone issue in the RHEL release [#3584](https://github.com/pingcap/tiflow/issues/3584) - Fix the issue that `stopped` changefeeds resume automatically after a cluster upgrade [#3473](https://github.com/pingcap/tiflow/issues/3473) - Fix the issue of overly frequent warnings caused by MySQL sink deadlock [#2706](https://github.com/pingcap/tiflow/issues/2706) - - Fix the bug that the `enable-old-value` configuration item is not automatically set to `true` on Canal and Maxwell protocols [#3676](https://github.com/pingcap/tiflow/issues/3676) + - Fix the bug that the `enable-old-value` configuration item is not automatically set to `true` on the Canal protocol [#3676](https://github.com/pingcap/tiflow/issues/3676) - Fix the issue that Avro sink does not support parsing JSON type columns [#3624](https://github.com/pingcap/tiflow/issues/3624) - Fix the negative value error in the changefeed checkpoint lag [#3010](https://github.com/pingcap/tiflow/issues/3010) - Fix the OOM issue in the container environment [#1798](https://github.com/pingcap/tiflow/issues/1798) diff --git a/releases/release-5.3.0.md b/releases/release-5.3.0.md index 0284ae17d7cf9..d33d15cb11638 100644 --- a/releases/release-5.3.0.md +++ b/releases/release-5.3.0.md @@ -75,7 +75,7 @@ In v5.3, the key new features or improvements are as follows: - The `GRANT ALL ON performance_schema.*` syntax is no longer permitted. If you execute this statement in TiDB, an error occurs. - Fix the issue that auto-analyze is unexpectedly triggered outside the specified time period when new indexes are added before v5.3.0. In v5.3.0, after you set the time period through the `tidb_auto_analyze_start_time` and `tidb_auto_analyze_end_time` variables, auto-analyze is triggered only during this time period. - The default storage directory for plugins is changed from `""` to `/data/deploy/plugin`. -- The DM code is migrated to [the folder "dm" in TiCDC code repository](https://github.com/pingcap/tiflow/tree/master/dm). Now DM follows TiDB in version numbers. Next to v2.0.x, the new DM version is v5.3.0, and you can upgrade from v2.0.x to v5.3.0 without any risk. +- The DM code is migrated to [the folder "dm" in TiCDC code repository](https://github.com/pingcap/tiflow/tree/release-5.3/dm). Now DM follows TiDB in version numbers. Next to v2.0.x, the new DM version is v5.3.0, and you can upgrade from v2.0.x to v5.3.0 without any risk. - The default deployed version of Prometheus is upgraded from v2.8.1 to [v2.27.1](https://github.com/prometheus/prometheus/releases/tag/v2.27.1), which is released in May 2021. This version provides more features and fixes a security issue. Compared with Prometheus v2.8.1, alert time representation in v2.27.1 is changed from Unix timestamp to UTC. For details, refer to [Prometheus commit](https://github.com/prometheus/prometheus/commit/7646cbca328278585be15fa615e22f2a50b47d06) for more details. ## New features diff --git a/releases/release-5.3.1.md b/releases/release-5.3.1.md index 849287628fb14..d586221c5cd40 100644 --- a/releases/release-5.3.1.md +++ b/releases/release-5.3.1.md @@ -143,7 +143,7 @@ TiDB version: 5.3.1 - Fix the issue that `stopped` changefeeds resume automatically after a cluster upgrade [#3473](https://github.com/pingcap/tiflow/issues/3473) - Fix the issue that default values cannot be replicated [#3793](https://github.com/pingcap/tiflow/issues/3793) - Fix the issue of overly frequent warnings caused by MySQL sink deadlock [#2706](https://github.com/pingcap/tiflow/issues/2706) - - Fix the bug that the `enable-old-value` configuration item is not automatically set to `true` on Canal and Maxwell protocols [#3676](https://github.com/pingcap/tiflow/issues/3676) + - Fix the bug that the `enable-old-value` configuration item is not automatically set to `true` on the Canal protocol [#3676](https://github.com/pingcap/tiflow/issues/3676) - Fix the issue that Avro sink does not support parsing JSON type columns [#3624](https://github.com/pingcap/tiflow/issues/3624) - Fix the negative value error in the changefeed checkpoint lag [#3010](https://github.com/pingcap/tiflow/issues/3010) diff --git a/releases/release-6.0.0-dmr.md b/releases/release-6.0.0-dmr.md index 6417a8c20462e..8ceb69dcc9b65 100644 --- a/releases/release-6.0.0-dmr.md +++ b/releases/release-6.0.0-dmr.md @@ -271,7 +271,7 @@ TiDB v6.0.0 is a DMR, and its version is 6.0.0-DMR. TiEM not only provides full lifecycle visual management for TiDB clusters, but also provides one-stop services: parameter management, version upgrades, cluster clone, active-standby cluster switching, data import and export, data replication, and data backup and restore services. TiEM can improve the efficiency of DevOps on TiDB and reduce the DevOps cost for enterprises. - Currently, TiEM is provided in the [TiDB Enterprise](https://en.pingcap.com/tidb-enterprise/) edition only. To get TiEM, contact us via the [TiDB Enterprise](https://en.pingcap.com/tidb-enterprise/) page. + Currently, TiEM is provided in the [TiDB Enterprise](https://www.pingcap.com/tidb-enterprise/) edition only. To get TiEM, contact us via the [TiDB Enterprise](https://www.pingcap.com/tidb-enterprise/) page. - Support customizing configurations of the monitoring components @@ -556,7 +556,7 @@ TiDB v6.0.0 is a DMR, and its version is 6.0.0-DMR. - A `loader..on-duplicate` parameter is added. The default value is `replace`, which means using the new data to replace the existing data. If you want to keep the previous behavior, you can set the value to `error`. This parameter only controls the behavior during the full import phase. - To use DM, you should use the corresponding version of `dmctl` - Due to internal mechanism changes, after upgrading DM to v6.0.0, you should also upgrade `dmctl` to v6.0.0. -- In v5.4 (v5.4 only), TiDB allows incorrect values for some noop system variables. Since v6.0.0, TiDB disallows setting incorrect values for system variables. [#31538](https://github.com/pingcap/tidb/issues/31538) +- For v5.4 and earlier versions, TiDB allows incorrect values for some noop system variables. Starting from v6.0.0, TiDB disallows setting incorrect values for system variables. [#31538](https://github.com/pingcap/tidb/issues/31538) ## Improvements diff --git a/releases/release-6.1.1.md b/releases/release-6.1.1.md index 47aadba78f361..0278613f7d582 100644 --- a/releases/release-6.1.1.md +++ b/releases/release-6.1.1.md @@ -152,16 +152,16 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with- + TiCDC - - Fix the wrong maximum compatible version number [#6039](https://github.com/pingcap/tiflow/issues/6039) @[hi-rustin](https://github.com/hi-rustin) + - Fix the wrong maximum compatible version number [#6039](https://github.com/pingcap/tiflow/issues/6039) @[hi-rustin](https://github.com/Rustin170506) - Fix a bug that may cause the cdc server to panic when it receives an HTTP request before it fully starts [#5639](https://github.com/pingcap/tiflow/issues/5639) @[asddongmen](https://github.com/asddongmen) - Fix the ddl sink panic issue when the changefeed sync-point is enabled [#4934](https://github.com/pingcap/tiflow/issues/4934) @[asddongmen](https://github.com/asddongmen) - Fix the issue that a changefeed is stuck in some scenarios when sync-point is enabled [#6827](https://github.com/pingcap/tiflow/issues/6827) @[hicqu](https://github.com/hicqu) - Fix a bug that changefeed API does not work properly after the cdc server restarts [#5837](https://github.com/pingcap/tiflow/issues/5837) @[asddongmen](https://github.com/asddongmen) - Fix the data race issue in the black hole sink [#6206](https://github.com/pingcap/tiflow/issues/6206) @[asddongmen](https://github.com/asddongmen) - - Fix the TiCDC panic issue when you set `enable-old-value = false` [#6198](https://github.com/pingcap/tiflow/issues/6198) @[hi-rustin](https://github.com/hi-rustin) + - Fix the TiCDC panic issue when you set `enable-old-value = false` [#6198](https://github.com/pingcap/tiflow/issues/6198) @[hi-rustin](https://github.com/Rustin170506) - Fix the data consistency issue when the redo log feature is enabled [#6189](https://github.com/pingcap/tiflow/issues/6189) [#6368](https://github.com/pingcap/tiflow/issues/6368) [#6277](https://github.com/pingcap/tiflow/issues/6277) [#6456](https://github.com/pingcap/tiflow/issues/6456) [#6695](https://github.com/pingcap/tiflow/issues/6695) [#6764](https://github.com/pingcap/tiflow/issues/6764) [#6859](https://github.com/pingcap/tiflow/issues/6859) @[asddongmen](https://github.com/asddongmen) - Fix poor redo log performance by writing redo events asynchronously [#6011](https://github.com/pingcap/tiflow/issues/6011) @[CharlesCheung96](https://github.com/CharlesCheung96) - - Fix the issue that the MySQL sink cannot connect to IPv6 addresses [#6135](https://github.com/pingcap/tiflow/issues/6135) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that the MySQL sink cannot connect to IPv6 addresses [#6135](https://github.com/pingcap/tiflow/issues/6135) @[hi-rustin](https://github.com/Rustin170506) + Backup & Restore (BR) diff --git a/releases/release-6.1.2.md b/releases/release-6.1.2.md index 40c9003a0a083..89a1bb57d12b2 100644 --- a/releases/release-6.1.2.md +++ b/releases/release-6.1.2.md @@ -86,9 +86,9 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with- + TiCDC - Fix the issue that the cdc server might panic if it receives an HTTP request before the cdc server fully starts [#6838](https://github.com/pingcap/tiflow/issues/6838) @[asddongmen](https://github.com/asddongmen) - - Fix the log flooding issue during upgrade [#7235](https://github.com/pingcap/tiflow/issues/7235) @[hi-rustin](https://github.com/hi-rustin) - - Fix the issue that changefeed's redo log files might be deleted by mistake [#6413](https://github.com/pingcap/tiflow/issues/6413) @[hi-rustin](https://github.com/hi-rustin) - - Fix the issue that TiCDC might become unavailable when too many operations in an etcd transaction are committed [#7131](https://github.com/pingcap/tiflow/issues/7131) @[hi-rustin](https://github.com/hi-rustin) + - Fix the log flooding issue during upgrade [#7235](https://github.com/pingcap/tiflow/issues/7235) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that changefeed's redo log files might be deleted by mistake [#6413](https://github.com/pingcap/tiflow/issues/6413) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that TiCDC might become unavailable when too many operations in an etcd transaction are committed [#7131](https://github.com/pingcap/tiflow/issues/7131) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that data inconsistency might occur when non-reentrant DDL statements in redo logs are executed twice [#6927](https://github.com/pingcap/tiflow/issues/6927) @[hicqu](https://github.com/hicqu) + Backup & Restore (BR) diff --git a/releases/release-6.1.4.md b/releases/release-6.1.4.md index 4e452fbdbb06d..2d05f02c3e32b 100644 --- a/releases/release-6.1.4.md +++ b/releases/release-6.1.4.md @@ -74,10 +74,10 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with- + TiCDC - Fix the issue that the checkpoint cannot advance when TiCDC replicates an excessively large number of tables [#8004](https://github.com/pingcap/tiflow/issues/8004) @[asddongmen](https://github.com/asddongmen) - - Fix the issue that `transaction_atomicity` and `protocol` cannot be updated via the configuration file [#7935](https://github.com/pingcap/tiflow/issues/7935) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that `transaction-atomicity` and `protocol` cannot be updated via the configuration file [#7935](https://github.com/pingcap/tiflow/issues/7935) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that TiCDC mistakenly reports an error when the version of TiFlash is later than that of TiCDC [#7744](https://github.com/pingcap/tiflow/issues/7744) @[overvenus](https://github.com/overvenus) - Fix the issue that OOM occurs when TiCDC replicates large transactions [#7913](https://github.com/pingcap/tiflow/issues/7913) @[overvenus](https://github.com/overvenus) - - Fix a bug that the context deadline is exceeded when TiCDC replicates data without splitting large transactions [#7982](https://github.com/pingcap/tiflow/issues/7982) @[hi-rustin](https://github.com/hi-rustin) + - Fix a bug that the context deadline is exceeded when TiCDC replicates data without splitting large transactions [#7982](https://github.com/pingcap/tiflow/issues/7982) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that `sasl-password` in the `changefeed query` result is not masked [#7182](https://github.com/pingcap/tiflow/issues/7182) @[dveeden](https://github.com/dveeden) - Fix the issue that data is lost when a user quickly deletes a replication task and then creates another one with the same task name [#7657](https://github.com/pingcap/tiflow/issues/7657) @[overvenus](https://github.com/overvenus) diff --git a/releases/release-6.1.5.md b/releases/release-6.1.5.md index 7d72630a47681..2f46ea952b272 100644 --- a/releases/release-6.1.5.md +++ b/releases/release-6.1.5.md @@ -15,8 +15,8 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with- - Starting from February 20, 2023, the [telemetry feature](/telemetry.md) is disabled by default in new versions of TiDB and TiDB Dashboard, including v6.1.5, and usage information is not collected and shared with PingCAP. Before upgrading to these versions, if the cluster uses the default telemetry configuration, the telemetry feature is disabled after the upgrade. See [TiDB Release Timeline](/releases/release-timeline.md) for a specific version. - - The default value of the [`tidb_enable_telemetry`](/system-variables.md#tidb_enable_telemetry-new-in-v402) system variable is changed from `ON` to `OFF`. - - The default value of the TiDB [`enable-telemetry`](/tidb-configuration-file.md#enable-telemetry-new-in-v402) configuration item is changed from `true` to `false`. + - The default value of the [`tidb_enable_telemetry`](/system-variables.md#tidb_enable_telemetry-new-in-v402-and-deprecated-in-v810) system variable is changed from `ON` to `OFF`. + - The default value of the TiDB [`enable-telemetry`](/tidb-configuration-file.md#enable-telemetry-new-in-v402-and-deprecated-in-v810) configuration item is changed from `true` to `false`. - The default value of the PD [`enable-telemetry`](/pd-configuration-file.md#enable-telemetry) configuration item is changed from `true` to `false`. - Starting from v1.11.3, the telemetry feature is disabled by default in newly deployed TiUP, and usage information is not collected. If you upgrade from a TiUP version earlier than v1.11.3 to v1.11.3 or a later version, the telemetry feature keeps the same status as before the upgrade. diff --git a/releases/release-6.1.6.md b/releases/release-6.1.6.md index ac2608d13fa85..17725f58ad6e3 100644 --- a/releases/release-6.1.6.md +++ b/releases/release-6.1.6.md @@ -92,7 +92,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with- + TiDB Lightning - - Fix the issue that the conflict resolution logic (`duplicate-resolution`) might lead to inconsistent checksums [#40657](https://github.com/pingcap/tidb/issues/40657) @[gozssky](https://github.com/gozssky) + - Fix the issue that the conflict resolution logic (`duplicate-resolution`) might lead to inconsistent checksums [#40657](https://github.com/pingcap/tidb/issues/40657) @[sleepymole](https://github.com/sleepymole) - Fix the issue that TiDB Lightning panics in the split-region phase [#40934](https://github.com/pingcap/tidb/issues/40934) @[lance6716](https://github.com/lance6716) - Fix the issue that when importing data in Local Backend mode, the target columns do not automatically generate data if the compound primary key of the imported target table has an `auto_random` column and no value for the column is specified in the source data [#41454](https://github.com/pingcap/tidb/issues/41454) @[D3Hunter](https://github.com/D3Hunter) - Fix the issue that TiDB Lightning might incorrectly skip conflict resolution when all but the last TiDB Lightning instance encounters a local duplicate record during a parallel import [#40923](https://github.com/pingcap/tidb/issues/40923) @[lichunzhu](https://github.com/lichunzhu) diff --git a/releases/release-6.1.7.md b/releases/release-6.1.7.md index 2977b24b64f94..1ec99802e66d8 100644 --- a/releases/release-6.1.7.md +++ b/releases/release-6.1.7.md @@ -88,9 +88,9 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.1/quick-start-with- + TiCDC - - Fix the issue that TiCDC cannot create a changefeed with a downstream Kafka-on-Pulsar [#8892](https://github.com/pingcap/tiflow/issues/8892) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that TiCDC cannot create a changefeed with a downstream Kafka-on-Pulsar [#8892](https://github.com/pingcap/tiflow/issues/8892) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that TiCDC cannot automatically recover when PD address or leader fails [#8812](https://github.com/pingcap/tiflow/issues/8812) [#8877](https://github.com/pingcap/tiflow/issues/8877) @[asddongmen](https://github.com/asddongmen) - - Fix the issue that when the downstream is Kafka, TiCDC queries the downstream metadata too frequently and causes excessive workload in the downstream [#8957](https://github.com/pingcap/tiflow/issues/8957) [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that when the downstream is Kafka, TiCDC queries the downstream metadata too frequently and causes excessive workload in the downstream [#8957](https://github.com/pingcap/tiflow/issues/8957) [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that TiCDC gets stuck when PD fails such as network isolation or PD Owner node reboot [#8808](https://github.com/pingcap/tiflow/issues/8808) [#8812](https://github.com/pingcap/tiflow/issues/8812) [#8877](https://github.com/pingcap/tiflow/issues/8877) @[asddongmen](https://github.com/asddongmen) + TiDB Lightning diff --git a/releases/release-6.2.0.md b/releases/release-6.2.0.md index 468ecf8ebdac9..5279f419fac43 100644 --- a/releases/release-6.2.0.md +++ b/releases/release-6.2.0.md @@ -222,7 +222,7 @@ In v6.2.0-DMR, the key new features and improvements are as follows: This feature does not need manual configuration. If your TiDB cluster is v6.1.0 or later versions and TiDB Lightning is v6.2.0 or later versions, the new physical import mode takes effect automatically. - [User document](/tidb-lightning/tidb-lightning-physical-import-mode-usage.md#scope-of-pausing-scheduling-during-import) [#35148](https://github.com/pingcap/tidb/issues/35148) @[gozssky](https://github.com/gozssky) + [User document](/tidb-lightning/tidb-lightning-physical-import-mode-usage.md#scope-of-pausing-scheduling-during-import) [#35148](https://github.com/pingcap/tidb/issues/35148) @[sleepymole](https://github.com/sleepymole) * Refactor the [user documentation of TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md) to make its structure more reasonable and clear. The terms for "backend" is also modified to lower the understanding barrier for new users: @@ -261,7 +261,7 @@ In v6.2.0-DMR, the key new features and improvements are as follows: | [tidb_enable_noop_variables](/system-variables.md#tidb_enable_noop_variables-new-in-v620) | Newly added | This variable controls whether to show `noop` variables in the result of `SHOW [GLOBAL] VARIABLES`. | | [tidb_min_paging_size](/system-variables.md#tidb_min_paging_size-new-in-v620) | Newly added | This variable is used to set the maximum number of rows during the coprocessor paging request process. | | [tidb_txn_commit_batch_size](/system-variables.md#tidb_txn_commit_batch_size-new-in-v620) | Newly added | This variable is used to control the batch size of transaction commit requests that TiDB sends to TiKV. | -| tidb_enable_change_multi_schema | Deleted | This variable is used to control whether multiple columns or indexes can be altered in one `ALTER TABLE` statement. | +| tidb_enable_change_multi_schema | Deleted | This variable is deleted because, starting from v6.2.0, you can alter multiple columns or indexes in one `ALTER TABLE` statement by default. | | [tidb_enable_outer_join_reorder](/system-variables.md#tidb_enable_outer_join_reorder-new-in-v610) | Modified | This variable controls whether the Join Reorder algorithm of TiDB supports Outer Join. In v6.1.0, the default value is `ON`, which means the Join Reorder's support for Outer Join is enabled by default. From v6.2.0, the default value is `OFF`, which means the support is disabled by default. | ### Configuration file parameters diff --git a/releases/release-6.3.0.md b/releases/release-6.3.0.md index ad4b13bd9faf8..5adc414039ab2 100644 --- a/releases/release-6.3.0.md +++ b/releases/release-6.3.0.md @@ -171,7 +171,7 @@ In v6.3.0-DMR, the key new features and improvements are as follows: JSON is a popular data format adopted by a large number of programs. TiDB has introduced the [JSON support](/data-type-json.md) as an experimental feature since an earlier version, compatible with MySQL's JSON data type and some JSON functions. - In TiDB v6.3.0, the JSON data type and functions become GA, which enriches TiDB’s data types, supports using JSON functions in [expression indexes](/sql-statements/sql-statement-create-index.md#expression-index) and [generated-columns](/generated-columns.md), and further improves TiDB’s compatibility with MySQL. + In TiDB v6.3.0, the JSON data type and functions become GA, which enriches TiDB's data types, supports using JSON functions in [expression indexes](/sql-statements/sql-statement-create-index.md#expression-index) and [generated-columns](/generated-columns.md), and further improves TiDB's compatibility with MySQL. ### Backup and restore @@ -317,8 +317,8 @@ Since v6.3.0, TiCDC no longer supports configuring Pulsar sink. [kop](https://gi - Improve TiCDC's compatibility with the concurrent DDL framework introduced in the upstream TiDB [#6506](https://github.com/pingcap/tiflow/issues/6506) @[lance6716](https://github.com/lance6716) - Support logging `start ts` of DML statements when MySQL sink gets an error [#6460](https://github.com/pingcap/tiflow/issues/6460) @[overvenus](https://github.com/overvenus) - Enhance the `api/v1/health` API to return a more accurate health state of a TiCDC cluster [#4757](https://github.com/pingcap/tiflow/issues/4757) @[overvenus](https://github.com/overvenus) - - Implement MQ sink and MySQL sink in the asynchronous mode to improve the sink throughput [#5928](https://github.com/pingcap/tiflow/issues/5928) @[hicqu](https://github.com/hicqu) @[hi-rustin](https://github.com/hi-rustin) - - Delete the deprecated pulsar sink [#7087](https://github.com/pingcap/tiflow/issues/7087) @[hi-rustin](https://github.com/hi-rustin) + - Implement MQ sink and MySQL sink in the asynchronous mode to improve the sink throughput [#5928](https://github.com/pingcap/tiflow/issues/5928) @[hicqu](https://github.com/hicqu) @[hi-rustin](https://github.com/Rustin170506) + - Delete the deprecated pulsar sink [#7087](https://github.com/pingcap/tiflow/issues/7087) @[hi-rustin](https://github.com/Rustin170506) - Improve replication performance by discarding DDL statements that are irrelevant to a changefeed [#6447](https://github.com/pingcap/tiflow/issues/6447) @[asddongmen](https://github.com/asddongmen) + TiDB Data Migration (DM) diff --git a/releases/release-6.4.0.md b/releases/release-6.4.0.md index b9f6eb353a396..1984ea9e849f0 100644 --- a/releases/release-6.4.0.md +++ b/releases/release-6.4.0.md @@ -278,6 +278,7 @@ In v6.4.0-DMR, the key new features and improvements are as follows: | Variable name | Change type | Description | |--------|------------------------------|------| +| [`max_execution_time`](/system-variables.md#max_execution_time) | Modified | Before v6.4.0, this variable takes effect on all types of statements. Starting from v6.4.0, this variable only controls the maximum execution time of read-only statements. | | [`tidb_constraint_check_in_place_pessimistic`](/system-variables.md#tidb_constraint_check_in_place_pessimistic-new-in-v630) | Modified | Removes the GLOBAL scope and allows you to modify the default value using the [`pessimistic-txn.constraint-check-in-place-pessimistic`](/tidb-configuration-file.md#constraint-check-in-place-pessimistic-new-in-v640) configuration item. This variable controls when TiDB checks the unique constraints in pessimistic transactions. | | [`tidb_ddl_flashback_concurrency`](/system-variables.md#tidb_ddl_flashback_concurrency-new-in-v630) | Modified | Takes effect starting from v6.4.0 and controls the concurrency of [`FLASHBACK CLUSTER TO TIMESTAMP`](/sql-statements/sql-statement-flashback-cluster.md). The default value is `64`. | | [`tidb_enable_clustered_index`](/system-variables.md#tidb_enable_clustered_index-new-in-v50) | Modified | Changes the default value from `INT_ONLY` to `ON`, meaning that primary keys are created as clustered indexes by default.| @@ -372,7 +373,7 @@ In v6.4.0-DMR, the key new features and improvements are as follows: + TiCDC - Support replicating the exchange partition DDL statements [#639](https://github.com/pingcap/tiflow/issues/639) @[asddongmen](https://github.com/asddongmen) - - Improve non-batch sending performance for the MQ sink module [#7353](https://github.com/pingcap/tiflow/issues/7353) @[hi-rustin](https://github.com/hi-rustin) + - Improve non-batch sending performance for the MQ sink module [#7353](https://github.com/pingcap/tiflow/issues/7353) @[hi-rustin](https://github.com/Rustin170506) - Improve performance of TiCDC puller when a table has a large number of Regions [#7078](https://github.com/pingcap/tiflow/issues/7078) [#7281](https://github.com/pingcap/tiflow/issues/7281) @[sdojjy](https://github.com/sdojjy) - Support reading historical data in the downstream TiDB by using the `tidb_enable_external_ts_read` variable when Syncpoint is enabled [#7419](https://github.com/pingcap/tiflow/issues/7419) @[asddongmen](https://github.com/asddongmen) - Enable transaction split and disable safeMode by default to improve the replication stability [#7505](https://github.com/pingcap/tiflow/issues/7505) @[asddongmen](https://github.com/asddongmen) @@ -437,9 +438,9 @@ In v6.4.0-DMR, the key new features and improvements are as follows: - Fix the issue that `sasl-password` in the `changefeed query` result is not masked [#7182](https://github.com/pingcap/tiflow/issues/7182) @[dveeden](https://github.com/dveeden) - Fix the issue that TiCDC might become unavailable when too many operations in an etcd transaction are committed [#7131](https://github.com/pingcap/tiflow/issues/7131) @[asddongmen](https://github.com/asddongmen) - Fix the issue that redo logs might be deleted incorrectly [#6413](https://github.com/pingcap/tiflow/issues/6413) @[asddongmen](https://github.com/asddongmen) - - Fix performance regression when replicating wide tables in Kafka Sink V2 [#7344](https://github.com/pingcap/tiflow/issues/7344) @[hi-rustin](https://github.com/hi-rustin) - - Fix the issue that checkpoint ts might be advanced incorrectly [#7274](https://github.com/pingcap/tiflow/issues/7274) @[hi-rustin](https://github.com/hi-rustin) - - Fix the issue that too many logs are printed due to improper log level of the mounter module [#7235](https://github.com/pingcap/tiflow/issues/7235) @[hi-rustin](https://github.com/hi-rustin) + - Fix performance regression when replicating wide tables in Kafka Sink V2 [#7344](https://github.com/pingcap/tiflow/issues/7344) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that checkpoint ts might be advanced incorrectly [#7274](https://github.com/pingcap/tiflow/issues/7274) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that too many logs are printed due to improper log level of the mounter module [#7235](https://github.com/pingcap/tiflow/issues/7235) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that a TiCDC cluster might have two owners [#4051](https://github.com/pingcap/tiflow/issues/4051) @[asddongmen](https://github.com/asddongmen) + TiDB Data Migration (DM) diff --git a/releases/release-6.5.0.md b/releases/release-6.5.0.md index ac8a160c35997..02c501eae4c25 100644 --- a/releases/release-6.5.0.md +++ b/releases/release-6.5.0.md @@ -319,10 +319,12 @@ Compared with TiDB [6.4.0-DMR](/releases/release-6.4.0.md), TiDB 6.5.0 introduce | [`tidb_server_memory_limit`](/system-variables.md#tidb_server_memory_limit-new-in-v640) | Modified | Changes the default value from `0` to `80%`. As the TiDB global memory control becomes GA, this default value change enables the memory control by default and sets the memory limit for a TiDB instance to 80% of the total memory by default. | | [`default_password_lifetime`](/system-variables.md#default_password_lifetime-new-in-v650) | Newly added | Sets the global policy for automatic password expiration to require users to change passwords periodically. The default value `0` indicates that passwords never expire. | | [`disconnect_on_expired_password`](/system-variables.md#disconnect_on_expired_password-new-in-v650) | Newly added | Indicates whether TiDB disconnects the client connection when the password is expired. This variable is read-only. | +| [`tidb_store_batch_size`](/system-variables.md#tidb_store_batch_size) | Newly added | This variable is disabled by default, and the feature controlled by it is not stable yet. It is not recommended to modify this variable in production environments. | | [`password_history`](/system-variables.md#password_history-new-in-v650) | Newly added | This variable is used to establish a password reuse policy that allows TiDB to limit password reuse based on the number of password changes. The default value `0` means disabling the password reuse policy based on the number of password changes. | | [`password_reuse_interval`](/system-variables.md#password_reuse_interval-new-in-v650) | Newly added | This variable is used to establish a password reuse policy that allows TiDB to limit password reuse based on time elapsed. The default value `0` means disabling the password reuse policy based on time elapsed. | | [`tidb_auto_build_stats_concurrency`](/system-variables.md#tidb_auto_build_stats_concurrency-new-in-v650) | Newly added | This variable is used to set the concurrency of executing the automatic update of statistics. The default value is `1`. | | [`tidb_cdc_write_source`](/system-variables.md#tidb_cdc_write_source-new-in-v650) | Newly added | When this variable is set to a value other than 0, data written in this session is considered to be written by TiCDC. This variable can only be modified by TiCDC. Do not manually modify this variable in any case. | +| [`tidb_enable_plan_replayer_capture`](/system-variables.md#tidb_enable_plan_replayer_capture) | Newly added | The feature controlled by this variable is not fully functional in TiDB v6.5.0. Do not change the default value. | | [`tidb_index_merge_intersection_concurrency`](/system-variables.md#tidb_index_merge_intersection_concurrency-new-in-v650) | Newly added | Sets the maximum concurrency for the intersection operations that index merge performs. It is effective only when TiDB accesses partitioned tables in the dynamic pruning mode. | | [`tidb_source_id`](/system-variables.md#tidb_source_id-new-in-v650) | Newly added | This variable is used to configure the different cluster IDs in a [bi-directional replication](/ticdc/ticdc-bidirectional-replication.md) cluster.| | [`tidb_sysproc_scan_concurrency`](/system-variables.md#tidb_sysproc_scan_concurrency-new-in-v650) | Newly added | This variable is used to set the concurrency of scan operations performed when TiDB executes internal SQL statements (such as an automatic update of statistics). The default value is `1`. | @@ -351,7 +353,8 @@ Compared with TiDB [6.4.0-DMR](/releases/release-6.4.0.md), TiDB 6.5.0 introduce | TiDB | [`server-memory-quota`](/tidb-configuration-file.md#server-memory-quota-new-in-v409) | Deprecated | Starting from v6.5.0, this configuration item is deprecated. Instead, use the system variable [`tidb_server_memory_limit`](/system-variables.md#tidb_server_memory_limit-new-in-v640) to manage memory globally. | | TiDB | [`disconnect-on-expired-password`](/tidb-configuration-file.md#disconnect-on-expired-password-new-in-v650) | Newly added | Determines whether TiDB disconnects the client connection when the password is expired. The default value is `true`, which means the client connection is disconnected when the password is expired. | | TiKV | `raw-min-ts-outlier-threshold` | Deleted | This configuration item was deprecated in v6.4.0 and deleted in v6.5.0. | -| TiKV | [`cdc.min-ts-interval`](/tikv-configuration-file.md#min-ts-interval) | Modified | To reduce CDC latency, the default value is changed from `1s` to `200ms`. | +| TiKV | [`raft-engine.bytes-per-sync`](/tikv-configuration-file.md#bytes-per-sync-2) | Deprecated | Starting from v6.5.0, Raft Engine writes logs to disk directly without buffering. Therefore, this configuration item is deprecated and no longer functional. | +| TiKV | [`cdc.min-ts-interval`](/tikv-configuration-file.md#min-ts-interval) | Modified | To reduce CDC latency, the default value is changed from `"1s"` to `"200ms"`. | | TiKV | [`memory-use-ratio`](/tikv-configuration-file.md#memory-use-ratio-new-in-v650) | Newly added | Indicates the ratio of available memory to total system memory in PITR log recovery. | | TiCDC | [`sink.terminator`](/ticdc/ticdc-changefeed-config.md#changefeed-configuration-parameters) | Newly added | Indicates the row terminator, which is used for separating two data change events. The value is empty by default, which means `\r\n` is used. | | TiCDC | [`sink.date-separator`](/ticdc/ticdc-changefeed-config.md#changefeed-configuration-parameters) | Newly added | Indicates the date separator type of the file directory. Value options are `none`, `year`, `month`, and `day`. `none` is the default value and means that the date is not separated. | diff --git a/releases/release-6.5.1.md b/releases/release-6.5.1.md index b0febb93ef432..197a485d7ebad 100644 --- a/releases/release-6.5.1.md +++ b/releases/release-6.5.1.md @@ -15,8 +15,8 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Starting from February 20, 2023, the [telemetry feature](/telemetry.md) is disabled by default in new versions of TiDB and TiDB Dashboard, including v6.5.1, and usage information is not collected and shared with PingCAP. Before upgrading to these versions, if the cluster uses the default telemetry configuration, the telemetry feature is disabled after the upgrade. See [TiDB Release Timeline](/releases/release-timeline.md) for specific versions. - - The default value of the [`tidb_enable_telemetry`](/system-variables.md#tidb_enable_telemetry-new-in-v402) system variable is changed from `ON` to `OFF`. - - The default value of the TiDB [`enable-telemetry`](/tidb-configuration-file.md#enable-telemetry-new-in-v402) configuration item is changed from `true` to `false`. + - The default value of the [`tidb_enable_telemetry`](/system-variables.md#tidb_enable_telemetry-new-in-v402-and-deprecated-in-v810) system variable is changed from `ON` to `OFF`. + - The default value of the TiDB [`enable-telemetry`](/tidb-configuration-file.md#enable-telemetry-new-in-v402-and-deprecated-in-v810) configuration item is changed from `true` to `false`. - The default value of the PD [`enable-telemetry`](/pd-configuration-file.md#enable-telemetry) configuration item is changed from `true` to `false`. - Starting from v1.11.3, the telemetry feature is disabled by default in newly deployed TiUP, and usage information is not collected. If you upgrade from a TiUP version earlier than v1.11.3 to v1.11.3 or a later version, the telemetry feature keeps the same status as before the upgrade. @@ -25,6 +25,8 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - The default value of the TiKV [`advance-ts-interval`](/tikv-configuration-file.md#advance-ts-interval) configuration item is changed from `1s` to `20s`. You can modify this configuration item to reduce the latency and improve the timeliness of the Stale Read data. See [Reduce Stale Read latency](/stale-read.md#reduce-stale-read-latency) for details. +- The default value of the TiKV [`cdc.min-ts-interval`](/tikv-configuration-file.md#min-ts-interval) configuration item is changed from `"200ms"` to `"1s"` to reduce network traffic. + ## Improvements + TiDB @@ -64,7 +66,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- + TiCDC - - Enable pull-based sink to optimize system throughput [#8232](https://github.com/pingcap/tiflow/issues/8232) @[hi-rustin](https://github.com/hi-rustin) + - Enable pull-based sink to optimize system throughput [#8232](https://github.com/pingcap/tiflow/issues/8232) @[hi-rustin](https://github.com/Rustin170506) - Support storing redo logs to GCS-compatible or Azure-compatible object storage [#7987](https://github.com/pingcap/tiflow/issues/7987) @[CharlesCheung96](https://github.com/CharlesCheung96) - Implement MQ sink and MySQL sink in the asynchronous mode to improve the sink throughput [#5928](https://github.com/pingcap/tiflow/issues/5928) @[amyangfei](https://github.com/amyangfei) @[CharlesCheung96](https://github.com/CharlesCheung96) @@ -163,11 +165,11 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix the issue that changefeed might get stuck in special scenarios such as when scaling in or scaling out TiKV or TiCDC nodes [#8174](https://github.com/pingcap/tiflow/issues/8174) @[hicqu](https://github.com/hicqu) - Fix the issue that precheck is not performed on the storage path of redo log [#6335](https://github.com/pingcap/tiflow/issues/6335) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue of insufficient duration that redo log can tolerate for S3 storage failure [#8089](https://github.com/pingcap/tiflow/issues/8089) @[CharlesCheung96](https://github.com/CharlesCheung96) - - Fix the issue that `transaction_atomicity` and `protocol` cannot be updated via the configuration file [#7935](https://github.com/pingcap/tiflow/issues/7935) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that `transaction-atomicity` and `protocol` cannot be updated via the configuration file [#7935](https://github.com/pingcap/tiflow/issues/7935) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that the checkpoint cannot advance when TiCDC replicates an excessively large number of tables [#8004](https://github.com/pingcap/tiflow/issues/8004) @[overvenus](https://github.com/overvenus) - Fix the issue that applying redo log might cause OOM when the replication lag is excessively high [#8085](https://github.com/pingcap/tiflow/issues/8085) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that the performance degrades when redo log is enabled to write meta [#8074](https://github.com/pingcap/tiflow/issues/8074) @[CharlesCheung96](https://github.com/CharlesCheung96) - - Fix a bug that the context deadline is exceeded when TiCDC replicates data without splitting large transactions [#7982](https://github.com/pingcap/tiflow/issues/7982) @[hi-rustin](https://github.com/hi-rustin) + - Fix a bug that the context deadline is exceeded when TiCDC replicates data without splitting large transactions [#7982](https://github.com/pingcap/tiflow/issues/7982) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that pausing a changefeed when PD is abnormal results in incorrect status [#8330](https://github.com/pingcap/tiflow/issues/8330) @[sdojjy](https://github.com/sdojjy) - Fix the data inconsistency that occurs when replicating data to a TiDB or MySQL sink and when `CHARACTER SET` is specified on the column that has the non-null unique index without a primary key [#8420](https://github.com/pingcap/tiflow/issues/8420) @[asddongmen](https://github.com/asddongmen) - Fix the panic issue in table scheduling and blackhole sink [#8024](https://github.com/pingcap/tiflow/issues/8024) [#8142](https://github.com/pingcap/tiflow/issues/8142) @[hicqu](https://github.com/hicqu) @@ -182,6 +184,6 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix the issue that TiDB Lightning prechecks cannot find dirty data left by previously failed imports [#39477](https://github.com/pingcap/tidb/issues/39477) @[dsdashun](https://github.com/dsdashun) - Fix the issue that TiDB Lightning panics in the split-region phase [#40934](https://github.com/pingcap/tidb/issues/40934) @[lance6716](https://github.com/lance6716) - - Fix the issue that the conflict resolution logic (`duplicate-resolution`) might lead to inconsistent checksums [#40657](https://github.com/pingcap/tidb/issues/40657) @[gozssky](https://github.com/gozssky) + - Fix the issue that the conflict resolution logic (`duplicate-resolution`) might lead to inconsistent checksums [#40657](https://github.com/pingcap/tidb/issues/40657) @[sleepymole](https://github.com/sleepymole) - Fix the issue that TiDB Lightning might incorrectly skip conflict resolution when all but the last TiDB Lightning instance encounters a local duplicate record during a parallel import [#40923](https://github.com/pingcap/tidb/issues/40923) @[lichunzhu](https://github.com/lichunzhu) - Fix the issue that when importing data in Local Backend mode, the target columns do not automatically generate data if the compound primary key of the imported target table has an `auto_random` column and no value for the column is specified in the source data [#41454](https://github.com/pingcap/tidb/issues/41454) @[D3Hunter](https://github.com/D3Hunter) diff --git a/releases/release-6.5.10.md b/releases/release-6.5.10.md new file mode 100644 index 0000000000000..9971b6c14ab4d --- /dev/null +++ b/releases/release-6.5.10.md @@ -0,0 +1,148 @@ +--- +title: TiDB 6.5.10 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 6.5.10. +--- + +# TiDB 6.5.10 Release Notes + +Release date: June 20, 2024 + +TiDB version: 6.5.10 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) + +## Compatibility changes + +- In earlier versions, when processing a transaction containing `UPDATE` changes, if the primary key or non-null unique index value is modified in an `UPDATE` event, TiCDC splits this event into `DELETE` and `INSERT` events. Starting from v6.5.10, when using the MySQL sink, TiCDC splits an `UPDATE` event into `DELETE` and `INSERT` events if the transaction `commitTS` for the `UPDATE` change is less than TiCDC `thresholdTS` (which is the current timestamp fetched from PD when TiCDC starts replicating the corresponding table to the downstream). This behavior change addresses the issue of downstream data inconsistencies caused by the potentially incorrect order of `UPDATE` events received by TiCDC, which can lead to an incorrect order of split `DELETE` and `INSERT` events. For more information, see [documentation](https://docs.pingcap.com/tidb/v6.5/ticdc-split-update-behavior#split-update-events-for-mysql-sinks). [#10918](https://github.com/pingcap/tiflow/issues/10918) @[lidezhu](https://github.com/lidezhu) +- Must set the line terminator when using TiDB Lightning `strict-format` to import CSV files [#37338](https://github.com/pingcap/tidb/issues/37338) @[lance6716](https://github.com/lance6716) + +## Improvements + ++ TiDB + + - Improve the MySQL compatibility of expression default values displayed in the output of `SHOW CREATE TABLE` [#52939](https://github.com/pingcap/tidb/issues/52939) @[CbcWestwolf](https://github.com/CbcWestwolf) + - Remove stores without Regions during MPP load balancing [#52313](https://github.com/pingcap/tidb/issues/52313) @[xzhangxian1008](https://github.com/xzhangxian1008) + ++ TiKV + + - Accelerate the shutdown speed of TiKV [#16680](https://github.com/tikv/tikv/issues/16680) @[LykxSassinator](https://github.com/LykxSassinator) + - Add monitoring metrics for the queue time for processing CDC events to facilitate troubleshooting downstream CDC event latency issues [#16282](https://github.com/tikv/tikv/issues/16282) @[hicqu](https://github.com/hicqu) + ++ Tools + + + Backup & Restore (BR) + + - BR cleans up empty SST files during data recovery [#16005](https://github.com/tikv/tikv/issues/16005) @[Leavrth](https://github.com/Leavrth) + - Increase the number of retries for failures caused by DNS errors [#53029](https://github.com/pingcap/tidb/issues/53029) @[YuJuncen](https://github.com/YuJuncen) + - Increase the number of retries for failures caused by the absence of a leader in a Region [#54017](https://github.com/pingcap/tidb/issues/54017) @[Leavrth](https://github.com/Leavrth) + + + TiCDC + + - Support directly outputting raw events when the downstream is a Message Queue (MQ) or cloud storage [#11211](https://github.com/pingcap/tiflow/issues/11211) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Improve memory stability during data recovery using redo logs to reduce the probability of OOM [#10900](https://github.com/pingcap/tiflow/issues/10900) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Significantly improve the stability of data replication in transaction conflict scenarios, with up to 10 times performance improvement [#10896](https://github.com/pingcap/tiflow/issues/10896) @[CharlesCheung96](https://github.com/CharlesCheung96) + +## Bug fixes + ++ TiDB + + - Fix the issue that querying metadata during statistics initialization might cause OOM [#52219](https://github.com/pingcap/tidb/issues/52219) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that for tables containing auto-increment columns with `AUTO_ID_CACHE=1`, setting `auto_increment_increment` and `auto_increment_offset` system variables to non-default values might cause incorrect auto-increment ID allocation [#52622](https://github.com/pingcap/tidb/issues/52622) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that restoring a table with `AUTO_ID_CACHE=1` using the `RESTORE` statement might cause a `Duplicate entry` error [#52680](https://github.com/pingcap/tidb/issues/52680) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the `STATE` field in the `INFORMATION_SCHEMA.TIDB_TRX` table is empty due to the `size` of the `STATE` field not being defined [#53026](https://github.com/pingcap/tidb/issues/53026) @[cfzjywxk](https://github.com/cfzjywxk) + - Fix the issue that TiDB does not create corresponding statistics metadata (`stats_meta`) when creating a table with foreign keys [#53652](https://github.com/pingcap/tidb/issues/53652) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the statistics synchronous loading mechanism might fail unexpectedly under high query concurrency [#52294](https://github.com/pingcap/tidb/issues/52294) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `Distinct_count` information in GlobalStats might be incorrect [#53752](https://github.com/pingcap/tidb/issues/53752) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the histogram and TopN in the primary key column statistics are not loaded after restarting TiDB [#37548](https://github.com/pingcap/tidb/issues/37548) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that certain filter conditions in queries might cause the planner module to report an `invalid memory address or nil pointer dereference` error [#53582](https://github.com/pingcap/tidb/issues/53582) [#53580](https://github.com/pingcap/tidb/issues/53580) [#53594](https://github.com/pingcap/tidb/issues/53594) [#53603](https://github.com/pingcap/tidb/issues/53603) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that `PREPARE`/`EXECUTE` statements with the `CONV` expression containing a `?` argument might result in incorrect query results when executed multiple times [#53505](https://github.com/pingcap/tidb/issues/53505) @[qw4990](https://github.com/qw4990) + - Fix the issue of incorrect WARNINGS information when using Optimizer Hints [#53767](https://github.com/pingcap/tidb/issues/53767) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the query latency of stale reads increases, caused by information schema cache misses [#53428](https://github.com/pingcap/tidb/issues/53428) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that DDL statements incorrectly use etcd and cause tasks to queue up [#52335](https://github.com/pingcap/tidb/issues/52335) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that internal columns are not renamed when executing `RENAME INDEX` to rename expression indexes [#51431](https://github.com/pingcap/tidb/issues/51431) @[ywqzzy](https://github.com/ywqzzy) + - Fix the issue that executing `CREATE OR REPLACE VIEW` concurrently might result in the `table doesn't exist` error [#53673](https://github.com/pingcap/tidb/issues/53673) @[tangenta](https://github.com/tangenta) + - Fix the issue that TiDB might panic when the JOIN condition contains an implicit type conversion [#46556](https://github.com/pingcap/tidb/issues/46556) @[qw4990](https://github.com/qw4990) + - Fix the issue that DDL operations get stuck due to network problems [#47060](https://github.com/pingcap/tidb/issues/47060) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that IndexJoin produces duplicate rows when calculating hash values in the Left Outer Anti Semi type [#52902](https://github.com/pingcap/tidb/issues/52902) @[yibin87](https://github.com/yibin87) + - Fix the issue that subqueries included in the `ALL` function might cause incorrect results [#52755](https://github.com/pingcap/tidb/issues/52755) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `TIMESTAMPADD()` function returns incorrect results [#41052](https://github.com/pingcap/tidb/issues/41052) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that TiDB might crash when `tidb_mem_quota_analyze` is enabled and the memory used by updating statistics exceeds the limit [#52601](https://github.com/pingcap/tidb/issues/52601) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that subqueries in an `UPDATE` list might cause TiDB to panic [#52687](https://github.com/pingcap/tidb/issues/52687) @[winoros](https://github.com/winoros) + - Fix the overflow issue of the `Longlong` type in predicates [#45783](https://github.com/pingcap/tidb/issues/45783) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue of inconsistent data indexes caused by concurrent DML operations when adding a unique index [#52914](https://github.com/pingcap/tidb/issues/52914) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that TiDB might panic when parsing index data [#47115](https://github.com/pingcap/tidb/issues/47115) @[zyguan](https://github.com/zyguan) + - Fix the issue that column pruning without using shallow copies of slices might cause TiDB to panic [#52768](https://github.com/pingcap/tidb/issues/52768) @[winoros](https://github.com/winoros) + - Fix the issue that using a view does not work in recursive CTE [#49721](https://github.com/pingcap/tidb/issues/49721) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `LEADING` hint does not support querying block aliases [#44645](https://github.com/pingcap/tidb/issues/44645) @[qw4990](https://github.com/qw4990) + - Fix the incorrect result of the TopN operator in correlated subqueries [#52777](https://github.com/pingcap/tidb/issues/52777) @[yibin87](https://github.com/yibin87) + - Fix the issue that unstable unique IDs of columns might cause the `UPDATE` statement to return errors [#53236](https://github.com/pingcap/tidb/issues/53236) @[winoros](https://github.com/winoros) + - Fix the issue that TiDB keeps sending probe requests to a TiFlash node that has been offline [#46602](https://github.com/pingcap/tidb/issues/46602) @[zyguan](https://github.com/zyguan) + - Fix the issue that comparing a column of `YEAR` type with an unsigned integer that is out of range causes incorrect results [#50235](https://github.com/pingcap/tidb/issues/50235) @[qw4990](https://github.com/qw4990) + - Fix the issue that AutoID Leader change might cause the value of the auto-increment column to decrease in the case of `AUTO_ID_CACHE=1` [#52600](https://github.com/pingcap/tidb/issues/52600) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that non-BIGINT unsigned integers might produce incorrect results when compared with strings/decimals [#41736](https://github.com/pingcap/tidb/issues/41736) @[LittleFall](https://github.com/LittleFall) + - Fix the issue that data conversion from the `FLOAT` type to the `UNSIGNED` type returns incorrect results [#41736](https://github.com/pingcap/tidb/issues/41736) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that `VAR_SAMP()` cannot be used as a window function [#52933](https://github.com/pingcap/tidb/issues/52933) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that a wrong TableDual plan causes empty query results [#50051](https://github.com/pingcap/tidb/issues/50051) @[onlyacat](https://github.com/onlyacat) + - Fix the issue that the TiDB synchronously loading statistics mechanism retries to load empty statistics indefinitely and prints the `fail to get stats version for this histogram` log [#52657](https://github.com/pingcap/tidb/issues/52657) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that an empty projection causes TiDB to panic [#49109](https://github.com/pingcap/tidb/issues/49109) @[winoros](https://github.com/winoros) + - Fix the issue that the TopN operator might be pushed down incorrectly [#37986](https://github.com/pingcap/tidb/issues/37986) @[qw4990](https://github.com/qw4990) + - Fix the issue that TiDB panics when executing the `SHOW ERRORS` statement with a predicate that is always `true` [#46962](https://github.com/pingcap/tidb/issues/46962) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that the metadata lock fails to prevent DDL operations from executing in the plan cache scenario [#51407](https://github.com/pingcap/tidb/issues/51407) @[wjhuang2016](https://github.com/wjhuang2016) + ++ TiKV + + - Fix the issue that slow `check-leader` operations on one TiKV node cause `resolved-ts` on other TiKV nodes to fail to advance normally [#15999](https://github.com/tikv/tikv/issues/15999) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that the `CONV()` function in queries might overflow during numeric system conversion, leading to TiKV panic [#16969](https://github.com/tikv/tikv/issues/16969) @[gengliqi](https://github.com/gengliqi) + - Fix the issue of unstable test cases, ensuring that each test uses an independent temporary directory to avoid online configuration changes affecting other test cases [#16871](https://github.com/tikv/tikv/issues/16871) @[glorv](https://github.com/glorv) + - Fix the issue that the decimal part of the `DECIMAL` type is incorrect in some cases [#16913](https://github.com/tikv/tikv/issues/16913) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that resolve-ts is blocked when a stale Region peer ignores the GC message [#16504](https://github.com/tikv/tikv/issues/16504) @[crazycs520](https://github.com/crazycs520) + ++ PD + + - Fix the issue that the Leader fails to transfer when you switch it between two deployed data centers [#7992](https://github.com/tikv/pd/issues/7992) @[TonsnakeLin](https://github.com/TonsnakeLin) + - Fix the issue that down peers might not recover when using Placement Rules [#7808](https://github.com/tikv/pd/issues/7808) @[rleungx](https://github.com/rleungx) + - Fix the issue that the `Filter target` monitoring metric for PD does not provide scatter range information [#8125](https://github.com/tikv/pd/issues/8125) @[HuSharp](https://github.com/HuSharp) + ++ TiFlash + + - Fix the issue that TiFlash might fail to synchronize schemas after executing `ALTER TABLE ... EXCHANGE PARTITION` across databases [#7296](https://github.com/pingcap/tiflash/issues/7296) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that a query with an empty key range fails to correctly generate read tasks on TiFlash, which might block TiFlash queries [#9108](https://github.com/pingcap/tiflash/issues/9108) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that the `SUBSTRING_INDEX()` function might cause TiFlash to crash in some corner cases [#9116](https://github.com/pingcap/tiflash/issues/9116) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that TiFlash metadata might become corrupted and cause the process to panic when upgrading a cluster from a version earlier than v6.5.0 to v6.5.0 or later [#9039](https://github.com/pingcap/tiflash/issues/9039) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash might return transiently incorrect results in high-concurrency read scenarios [#8845](https://github.com/pingcap/tiflash/issues/8845) @[JinheLin](https://github.com/JinheLin) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that the test case `TestGetTSWithRetry` takes too long to execute [#52547](https://github.com/pingcap/tidb/issues/52547) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the Region fetched from PD does not have a Leader when restoring data using BR or importing data using TiDB Lightning in physical import mode [#51124](https://github.com/pingcap/tidb/issues/51124) [#50501](https://github.com/pingcap/tidb/issues/50501) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that a PD connection failure could cause the TiDB instance where the log backup advancer owner is located to panic [#52597](https://github.com/pingcap/tidb/issues/52597) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that after pausing, stopping, and rebuilding the log backup task, the task status is normal, but the checkpoint does not advance [#53047](https://github.com/pingcap/tidb/issues/53047) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that data restore is slowed down due to the absence of a leader on a TiKV node [#50566](https://github.com/pingcap/tidb/issues/50566) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the global checkpoint of log backup is advanced ahead of the actual backup file write point due to TiKV restart, which might cause a small amount of backup data loss [#16809](https://github.com/tikv/tikv/issues/16809) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that the transfer of PD leaders might cause BR to panic when restoring data [#53724](https://github.com/pingcap/tidb/issues/53724) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that TiKV might panic when resuming a paused log backup task with unstable network connections to PD [#17020](https://github.com/tikv/tikv/issues/17020) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that log backup might be paused after the advancer owner migration [#53561](https://github.com/pingcap/tidb/issues/53561) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that BR fails to correctly identify errors due to multiple nested retries during the restore process [#54053](https://github.com/pingcap/tidb/issues/54053) @[RidRisR](https://github.com/RidRisR) + + + TiCDC + + - Fix the issue that TiCDC fails to create a changefeed with Syncpoint enabled when the downstream database password is Base64 encoded [#10516](https://github.com/pingcap/tiflow/issues/10516) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that `DROP PRIMARY KEY` and `DROP UNIQUE KEY` statements are not replicated correctly [#10890](https://github.com/pingcap/tiflow/issues/10890) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that the default value of `TIMEZONE` type is not set according to the correct time zone [#10931](https://github.com/pingcap/tiflow/issues/10931) @[3AceShowHand](https://github.com/3AceShowHand) + + + TiDB Lightning + + - Fix the issue that killing the PD Leader causes TiDB Lightning to report the `invalid store ID 0` error during data import [#50501](https://github.com/pingcap/tidb/issues/50501) @[Leavrth](https://github.com/Leavrth) + - Fix the issue of missing data in the TiDB Lightning Grafana dashboard [#43357](https://github.com/pingcap/tidb/issues/43357) @[lichunzhu](https://github.com/lichunzhu) + - Fix the issue that TiDB Lightning might print sensitive information to logs in server mode [#36374](https://github.com/pingcap/tidb/issues/36374) @[kennytm](https://github.com/kennytm) + - Fix the issue that TiDB fails to generate auto-increment IDs and reports an error `Failed to read auto-increment value from storage engine` after importing a table with both `SHARD_ROW_ID_BITS` and `AUTO_ID_CACHE=1` set using TiDB Lightning [#52654](https://github.com/pingcap/tidb/issues/52654) @[D3Hunter](https://github.com/D3Hunter) + + + Dumpling + + - Fix the issue that Dumpling reports an error when exporting tables and views at the same time [#53682](https://github.com/pingcap/tidb/issues/53682) @[tangenta](https://github.com/tangenta) + + + TiDB Binlog + + - Fix the issue that deleting rows during the execution of `ADD COLUMN` might report an error `data and columnID count not match` when TiDB Binlog is enabled [#53133](https://github.com/pingcap/tidb/issues/53133) @[tangenta](https://github.com/tangenta) diff --git a/releases/release-6.5.11.md b/releases/release-6.5.11.md new file mode 100644 index 0000000000000..a47ad21581d14 --- /dev/null +++ b/releases/release-6.5.11.md @@ -0,0 +1,125 @@ +--- +title: TiDB 6.5.11 Release Notes +summary: Learn about the improvements and bug fixes in TiDB 6.5.11. +--- + +# TiDB 6.5.11 Release Notes + +Release date: September 20, 2024 + +TiDB version: 6.5.11 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) + +## Improvements + ++ TiDB + + - By batch deleting TiFlash placement rules, improve the processing speed of data GC after performing the `TRUNCATE` or `DROP` operation on partitioned tables [#54068](https://github.com/pingcap/tidb/issues/54068) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + ++ TiKV + + - Optimize the compaction trigger mechanism of RocksDB to accelerate disk space reclamation when handling a large number of DELETE versions [#17269](https://github.com/tikv/tikv/issues/17269) @[AndreMouche](https://github.com/AndreMouche) + ++ TiFlash + + - Reduce lock conflicts under highly concurrent data read operations and optimize short query performance [#9125](https://github.com/pingcap/tiflash/issues/9125) @[JinheLin](https://github.com/JinheLin) + - Optimize the execution efficiency of `LENGTH()` and `ASCII()` functions [#9344](https://github.com/pingcap/tiflash/issues/9344) @[xzhangxian1008](https://github.com/xzhangxian1008) + ++ Tools + + + TiCDC + + - When the downstream is TiDB with the `SUPER` permission granted, TiCDC supports querying the execution status of `ADD INDEX DDL` from the downstream database to avoid data replication failure due to timeout in retrying executing the DDL statement in some cases [#10682](https://github.com/pingcap/tiflow/issues/10682) @[CharlesCheung96](https://github.com/CharlesCheung96) + +## Bug fixes + ++ TiDB + + - Fix the issue that the recursive CTE operator incorrectly tracks memory usage [#54181](https://github.com/pingcap/tidb/issues/54181) @[guo-shaoge](https://github.com/guo-shaoge) + - Reset the parameters in the `Open` method of `PipelinedWindow` to fix the unexpected error that occurs when the `PipelinedWindow` is used as a child node of `Apply` due to the reuse of previous parameter values caused by repeated opening and closing operations [#53600](https://github.com/pingcap/tidb/issues/53600) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that the memory used by transactions might be tracked multiple times [#53984](https://github.com/pingcap/tidb/issues/53984) @[ekexium](https://github.com/ekexium) + - Fix the issue of abnormally high memory usage caused by `memTracker` not being detached when the `HashJoin` or `IndexLookUp` operator is the driven side sub-node of the `Apply` operator [#54005](https://github.com/pingcap/tidb/issues/54005) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that `INDEX_HASH_JOIN` cannot exit properly when SQL is abnormally interrupted [#54688](https://github.com/pingcap/tidb/issues/54688) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that table replication fails when the index length of the table replicated from DM exceeds the maximum length specified by `max-index-length` [#55138](https://github.com/pingcap/tidb/issues/55138) @[lance6716](https://github.com/lance6716) + - Fix the issue that indirect placeholder `?` references in a `GROUP BY` statement cannot find columns [#53872](https://github.com/pingcap/tidb/issues/53872) @[qw4990](https://github.com/qw4990) + - Fix the issue that the illegal column type `DECIMAL(0,0)` can be created in some cases [#53779](https://github.com/pingcap/tidb/issues/53779) @[tangenta](https://github.com/tangenta) + - Fix the issue that predicates cannot be pushed down properly when the filter condition of a SQL query contains virtual columns and the execution condition contains `UnionScan` [#54870](https://github.com/pingcap/tidb/issues/54870) @[qw4990](https://github.com/qw4990) + - Fix the issue that executing the `SELECT DISTINCT CAST(col AS DECIMAL), CAST(col AS SIGNED) FROM ...` query might return incorrect results [#53726](https://github.com/pingcap/tidb/issues/53726) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue of reusing wrong point get plans for `SELECT ... FOR UPDATE` [#54652](https://github.com/pingcap/tidb/issues/54652) @[qw4990](https://github.com/qw4990) + - Fix the issue that RANGE partitioned tables that are not strictly self-incrementing can be created [#54829](https://github.com/pingcap/tidb/issues/54829) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that the `TIMESTAMPADD()` function goes into an infinite loop when the first argument is `month` and the second argument is negative [#54908](https://github.com/pingcap/tidb/issues/54908) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that TiDB fails to reject unauthenticated user connections in some cases when using the `auth_socket` authentication plugin [#54031](https://github.com/pingcap/tidb/issues/54031) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that the network partition during adding indexes using the Distributed eXecution Framework (DXF) might cause inconsistent data indexes [#54897](https://github.com/pingcap/tidb/issues/54897) @[tangenta](https://github.com/tangenta) + - Fix the issue that the query might get stuck when terminated because the memory usage exceeds the limit set by `tidb_mem_quota_query` [#55042](https://github.com/pingcap/tidb/issues/55042) @[yibin87](https://github.com/yibin87) + - Fix the issue that improper use of metadata locks might lead to writing anomalous data when using the plan cache under certain circumstances [#53634](https://github.com/pingcap/tidb/issues/53634) @[zimulala](https://github.com/zimulala) + - Fix the issue that recursive CTE queries might result in invalid pointers [#54449](https://github.com/pingcap/tidb/issues/54449) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `tot_col_size` column in the `mysql.stats_histograms` table might be a negative number [#55126](https://github.com/pingcap/tidb/issues/55126) @[qw4990](https://github.com/qw4990) + - Fix the issue that obtaining the column information using `information_schema.columns` returns warning 1356 when a subquery is used as a column definition in a view definition [#54343](https://github.com/pingcap/tidb/issues/54343) @[lance6716](https://github.com/lance6716) + - Fix the issue that TiDB reports an error in the log when closing the connection in some cases [#53689](https://github.com/pingcap/tidb/issues/53689) @[jackysp](https://github.com/jackysp) + - Fix the issue that the performance of the `SELECT ... WHERE ... ORDER BY ...` statement execution is poor in some cases [#54969](https://github.com/pingcap/tidb/issues/54969) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the `SUB_PART` value in the `INFORMATION_SCHEMA.STATISTICS` table is `NULL` [#55812](https://github.com/pingcap/tidb/issues/55812) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that the query might return incorrect results instead of an error after being killed [#50089](https://github.com/pingcap/tidb/issues/50089) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that querying the `INFORMATION_SCHEMA.CLUSTER_SLOW_QUERY` table might cause TiDB to panic [#54324](https://github.com/pingcap/tidb/issues/54324) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that empty `groupOffset` in `StreamAggExec` might cause TiDB to panic [#53867](https://github.com/pingcap/tidb/issues/53867) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that disk files might not be deleted after the `Sort` operator spills and a query error occurs [#55061](https://github.com/pingcap/tidb/issues/55061) @[wshwsh12](https://github.com/wshwsh12) + - Fix the data race issue in `IndexNestedLoopHashJoin` [#49692](https://github.com/pingcap/tidb/issues/49692) @[solotzg](https://github.com/solotzg) + - Fix the issue that an error occurs when using `SHOW COLUMNS` to view columns in a view [#54964](https://github.com/pingcap/tidb/issues/54964) @[lance6716](https://github.com/lance6716) + - Fix the issue that an error occurs when a DML statement contains nested generated columns [#53967](https://github.com/pingcap/tidb/issues/53967) @[wjhuang2016](https://github.com/wjhuang2016) + ++ TiKV + + - Fix the issue that prevents master key rotation when the master key is stored in a Key Management Service (KMS) [#17410](https://github.com/tikv/tikv/issues/17410) @[hhwyt](https://github.com/hhwyt) + - Fix a traffic control issue that might occur after deleting large tables or partitions [#17304](https://github.com/tikv/tikv/issues/17304) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that TiKV might panic due to ingesting deleted `sst_importer` SST files [#15053](https://github.com/tikv/tikv/issues/15053) @[lance6716](https://github.com/lance6716) + - Fix the issue that TiKV might panic when a stale replica processes Raft snapshots, triggered by a slow split operation and immediate removal of the new replica [#17469](https://github.com/tikv/tikv/issues/17469) @[hbisheng](https://github.com/hbisheng) + - Fix the issue that TiKV might repeatedly panic when applying a corrupted Raft data snapshot [#15292](https://github.com/tikv/tikv/issues/15292) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that setting the gRPC message compression method via `grpc-compression-type` does not take effect on messages sent from TiKV to TiDB [#17176](https://github.com/tikv/tikv/issues/17176) @[ekexium](https://github.com/ekexium) + - Fix the issue that CDC and log-backup do not limit the timeout of `check_leader` using the `advance-ts-interval` configuration, causing the `resolved_ts` lag to be too large when TiKV restarts normally in some cases [#17107](https://github.com/tikv/tikv/issues/17107) @[MyonKeminta](https://github.com/MyonKeminta) + ++ PD + + - Fix the issue that some logs are not redacted [#8419](https://github.com/tikv/pd/issues/8419) @[rleungx](https://github.com/rleungx) + - Fix the issue that setting the TiKV configuration item [`coprocessor.region-split-size`](/tikv-configuration-file.md#region-split-size) to a value less than 1 MiB causes PD panic [#8323](https://github.com/tikv/pd/issues/8323) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that setting `replication.strictly-match-label` to `true` causes TiFlash to fail to start [#8480](https://github.com/tikv/pd/issues/8480) @[rleungx](https://github.com/rleungx) + - Fix the data race issue that PD encounters during operator checks [#8263](https://github.com/tikv/pd/issues/8263) @[lhy1024](https://github.com/lhy1024) + ++ TiFlash + + - Fix the issue that when using the `CAST()` function to convert a string to a datetime with a time zone or invalid characters, the result is incorrect [#8754](https://github.com/pingcap/tiflash/issues/8754) @[solotzg](https://github.com/solotzg) + - Fix the issue that TiFlash might panic when a database is deleted shortly after creation [#9266](https://github.com/pingcap/tiflash/issues/9266) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that setting the SSL certificate configuration to an empty string in TiFlash incorrectly enables TLS and causes TiFlash to fail to start [#9235](https://github.com/pingcap/tiflash/issues/9235) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that a network partition (network disconnection) between TiFlash and any PD might cause read request timeout errors [#9243](https://github.com/pingcap/tiflash/issues/9243) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that TiFlash might crash if an error occurs during the execution of a query containing outer join [#9190](https://github.com/pingcap/tiflash/issues/9190) @[windtalker](https://github.com/windtalker) + - Fix the issue that converting data types to `DECIMAL` might cause incorrect query results in some corner cases [#53892](https://github.com/pingcap/tidb/issues/53892) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that frequent `EXCHANGE PARTITION` and `DROP TABLE` operations over a long period in a cluster might slow down the replication of TiFlash table metadata and degrade the query performance [#9227](https://github.com/pingcap/tiflash/issues/9227) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that the checkpoint path of backup and restore is incompatible with some external storage [#55265](https://github.com/pingcap/tidb/issues/55265) @[Leavrth](https://github.com/Leavrth) + - Fix the inefficiency issue in scanning DDL jobs during incremental backups [#54139](https://github.com/pingcap/tidb/issues/54139) @[3pointer](https://github.com/3pointer) + - Fix the issue that the backup performance during checkpoint backups is affected due to interruptions in seeking Region leaders [#17168](https://github.com/tikv/tikv/issues/17168) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that DDLs requiring backfilling, such as `ADD INDEX` and `MODIFY COLUMN`, might not be correctly recovered during incremental restore [#54426](https://github.com/pingcap/tidb/issues/54426) @[3pointer](https://github.com/3pointer) + - Fix the issue that after a log backup PITR task fails and you stop it, the safepoints related to that task are not properly cleared in PD [#17316](https://github.com/tikv/tikv/issues/17316) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that backup tasks might get stuck if TiKV becomes unresponsive during the backup process [#53480](https://github.com/pingcap/tidb/issues/53480) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that BR logs might print sensitive credential information when log backup is enabled [#55273](https://github.com/pingcap/tidb/issues/55273) @[RidRisR](https://github.com/RidRisR) + + + TiCDC + + - Fix the issue that TiCDC might panic when the Sorter module reads disk data [#10853](https://github.com/pingcap/tiflow/issues/10853) @[hicqu](https://github.com/hicqu) + - Fix the issue that the Processor module might get stuck when the downstream Kafka is inaccessible [#11340](https://github.com/pingcap/tiflow/issues/11340) @[asddongmen](https://github.com/asddongmen) + + + TiDB Data Migration (DM) + + - Fix the issue that data replication is interrupted when the index length exceeds the default value of `max-index-length` [#11459](https://github.com/pingcap/tiflow/issues/11459) @[michaelmdeng](https://github.com/michaelmdeng) + - Fix the issue that schema tracker incorrectly handles LIST partition tables, causing DM errors [#11408](https://github.com/pingcap/tiflow/issues/11408) @[lance6716](https://github.com/lance6716) + - Fix the issue that DM returns an error when replicating the `ALTER TABLE ... DROP PARTITION` statement for LIST partitioned tables [#54760](https://github.com/pingcap/tidb/issues/54760) @[lance6716](https://github.com/lance6716) + - Fix the issue that DM does not set the default database when processing the `ALTER DATABASE` statement, which causes a replication error [#11503](https://github.com/pingcap/tiflow/issues/11503) @[lance6716](https://github.com/lance6716) + + + TiDB Lightning + + - Fix the issue that transaction conflicts occur during data import using TiDB Lightning [#49826](https://github.com/pingcap/tidb/issues/49826) @[lance6716](https://github.com/lance6716) + - Fix the issue that TiKV data might be corrupted when importing data after disabling the import mode of TiDB Lightning [#15003](https://github.com/tikv/tikv/issues/15003) [#47694](https://github.com/pingcap/tidb/issues/47694) @[lance6716](https://github.com/lance6716) + - Fix the issue that during importing data using TiDB Lightning, an error occurs when restarting TiKV [#15912](https://github.com/tikv/tikv/issues/15912) @[lance6716](https://github.com/lance6716) diff --git a/releases/release-6.5.12.md b/releases/release-6.5.12.md new file mode 100644 index 0000000000000..5b99c9c4d9147 --- /dev/null +++ b/releases/release-6.5.12.md @@ -0,0 +1,185 @@ +--- +title: TiDB 6.5.12 Release Notes +summary: Learn about the improvements and bug fixes in TiDB 6.5.12. +--- + +# TiDB 6.5.12 Release Notes + +Release date: February 27, 2025 + +TiDB version: 6.5.12 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v6.5/production-deployment-using-tiup) + +## Compatibility changes + +- Support the openEuler 22.03 LTS SP3/SP4 operating system. For more information, see [OS and platform requirements](https://docs.pingcap.com/tidb/v6.5/hardware-and-software-requirements#os-and-platform-requirements). +- Set a default limit of 2048 for DDL historical tasks retrieved through the [TiDB HTTP API](https://github.com/pingcap/tidb/blob/release-6.5/docs/tidb_http_api.md) to prevent OOM issues caused by excessive historical tasks [#55711](https://github.com/pingcap/tidb/issues/55711) @[joccau](https://github.com/joccau) +- Add a new system variable [`tidb_ddl_reorg_max_write_speed`](https://docs.pingcap.com/tidb/v6.5/system-variables#tidb_ddl_reorg_max_write_speed-new-in-v6512) to limit the maximum speed of the ingest phase when adding indexes [#57156](https://github.com/pingcap/tidb/issues/57156) @[CbcWestwolf](https://github.com/CbcWestwolf) + +## Improvements + ++ TiDB + + - Enhance the validity check for read timestamps [#57786](https://github.com/pingcap/tidb/issues/57786) @[MyonKeminta](https://github.com/MyonKeminta) + ++ TiKV + + - Add the detection mechanism for invalid `max_ts` updates [#17916](https://github.com/tikv/tikv/issues/17916) @[ekexium](https://github.com/ekexium) + ++ TiFlash + + - Improve the garbage collection speed of outdated data in the background for tables with clustered indexes [#9529](https://github.com/pingcap/tiflash/issues/9529) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Add a check to verify whether the target cluster is an empty cluster for full restore [#35744](https://github.com/pingcap/tidb/issues/35744) @[3pointer](https://github.com/3pointer) + - Add a check to verify whether the target cluster contains a table with the same name for non-full restore [#55087](https://github.com/pingcap/tidb/issues/55087) @[RidRisR](https://github.com/RidRisR) + - Except for the `br log restore` subcommand, all other `br log` subcommands support skipping the loading of the TiDB `domain` data structure to reduce memory consumption [#52088](https://github.com/pingcap/tidb/issues/52088) @[Leavrth](https://github.com/Leavrth) + - Disable the table-level checksum calculation during full backups by default (`--checksum=false`) to improve backup performance [#56373](https://github.com/pingcap/tidb/issues/56373) @[Tristan1900](https://github.com/Tristan1900) + + + TiDB Lightning + + - Add a row width check when parsing CSV files to prevent OOM issues [#58590](https://github.com/pingcap/tidb/issues/58590) @[D3Hunter](https://github.com/D3Hunter) + +## Bug fixes + ++ TiDB + + - Fix the issue that using subqueries after the `NATURAL JOIN` or `USING` clause might result in errors [#53766](https://github.com/pingcap/tidb/issues/53766) @[dash12653](https://github.com/dash12653) + - Fix the issue that the `CAST` function does not support explicitly setting the character set [#55677](https://github.com/pingcap/tidb/issues/55677) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that the `LOAD DATA ... REPLACE INTO` operation causes data inconsistency [#56408](https://github.com/pingcap/tidb/issues/56408) @[fzzf678](https://github.com/fzzf678) + - Fix the issue that TiDB does not check the index length limitation when executing `ADD INDEX` [#56930](https://github.com/pingcap/tidb/issues/56930) @[fzzf678](https://github.com/fzzf678) + - Fix the issue of illegal memory access that might occur when a Common Table Expression (CTE) has multiple data consumers and one consumer exits without reading any data [#55881](https://github.com/pingcap/tidb/issues/55881) @[windtalker](https://github.com/windtalker) + - Fix the issue that some predicates might be lost when constructing `IndexMerge` [#58476](https://github.com/pingcap/tidb/issues/58476) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that converting data from the `BIT` type to the `CHAR` type might cause TiKV panics [#56494](https://github.com/pingcap/tidb/issues/56494) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that using variables or parameters in the `CREATE VIEW` statement does not report errors [#53176](https://github.com/pingcap/tidb/issues/53176) @[mjonss](https://github.com/mjonss) + - Fix the issue that unreleased session resources might lead to memory leaks [#56271](https://github.com/pingcap/tidb/issues/56271) @[lance6716](https://github.com/lance6716) + - Fix the issue that executing `ADD INDEX` might fail after modifying the PD member in the distributed execution framework [#48680](https://github.com/pingcap/tidb/issues/48680) @[lance6716](https://github.com/lance6716) + - Fix the issue that using `ORDER BY` when querying `cluster_slow_query table` might generate unordered results [#51723](https://github.com/pingcap/tidb/issues/51723) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that stale read does not strictly verify the timestamp of the read operation, resulting in a small probability of affecting the consistency of the transaction when an offset exists between the TSO and the real physical time [#56809](https://github.com/pingcap/tidb/issues/56809) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that the performance of querying `INFORMATION_SCHEMA.columns` degrades [#58184](https://github.com/pingcap/tidb/issues/58184) @[lance6716](https://github.com/lance6716) + - Fix the issue that the `INSERT ... ON DUPLICATE KEY` statement is not compatible with `mysql_insert_id` [#55965](https://github.com/pingcap/tidb/issues/55965) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the optimizer incorrectly estimates the number of rows as 1 when accessing a unique index with the query condition `column IS NULL` [#56116](https://github.com/pingcap/tidb/issues/56116) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that part of the memory of the `IndexLookUp` operator is not tracked [#56440](https://github.com/pingcap/tidb/issues/56440) @[wshwsh12](https://github.com/wshwsh12) + - Fix the potential data race issue that might occur in TiDB's internal coroutine [#57798](https://github.com/pingcap/tidb/issues/57798) [#56053](https://github.com/pingcap/tidb/issues/56053) @[fishiu](https://github.com/fishiu) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the `read_from_storage` hint might not take effect when the query has an available Index Merge execution plan [#56217](https://github.com/pingcap/tidb/issues/56217) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that execution plan bindings cannot be created for the multi-table `DELETE` statement with aliases [#56726](https://github.com/pingcap/tidb/issues/56726) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `INDEX_HASH_JOIN` might hang during an abnormal exit [#54055](https://github.com/pingcap/tidb/issues/54055) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that two DDL Owners might exist at the same time [#54689](https://github.com/pingcap/tidb/issues/54689) @[joccau](https://github.com/joccau) + - Fix the issue that when querying the `information_schema.cluster_slow_query` table, if the time filter is not added, only the latest slow log file is queried [#56100](https://github.com/pingcap/tidb/issues/56100) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that `duplicate entry` might occur when adding unique indexes [#56161](https://github.com/pingcap/tidb/issues/56161) @[tangenta](https://github.com/tangenta) + - Fix the issue that the error message is incorrect in certain type conversion errors [#41730](https://github.com/pingcap/tidb/issues/41730) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the CTE defined in `VIEW` is incorrectly inlined [#56582](https://github.com/pingcap/tidb/issues/56582) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that the `UPDATE` statement incorrectly updates values of the `ENUM` type [#56832](https://github.com/pingcap/tidb/issues/56832) @[xhebox](https://github.com/xhebox) + - Fix the issue that executing the `UPDATE` statement after adding a `DATE` column results in the error `Incorrect date value: '0000-00-00'` in some cases [#59047](https://github.com/pingcap/tidb/issues/59047) @[mjonss](https://github.com/mjonss) + - Fix the issue that in the Prepare protocol, an error occurs when the client uses a non-UTF8 character set [#58870](https://github.com/pingcap/tidb/issues/58870) @[xhebox](https://github.com/xhebox) + - Fix the issue that querying temporary tables might trigger unexpected TiKV requests in some cases [#58875](https://github.com/pingcap/tidb/issues/58875) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the `ONLY_FULL_GROUP_BY` setting does not take effect on statements in views [#53175](https://github.com/pingcap/tidb/issues/53175) @[mjonss](https://github.com/mjonss) + - Fix the issue that querying partitioned tables using an `IN` condition containing a mismatched value type and a type conversion error leads to incorrect query results [#54746](https://github.com/pingcap/tidb/issues/54746) @[mjonss](https://github.com/mjonss) + - Fix the issue that querying slow logs might fail when certain fields contain empty values [#58147](https://github.com/pingcap/tidb/issues/58147) @[yibin87](https://github.com/yibin87) + - Fix the issue that the `RADIANS()` function computes values in an incorrect order [#57671](https://github.com/pingcap/tidb/issues/57671) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that the default value of the `BIT` column is incorrect [#57301](https://github.com/pingcap/tidb/issues/57301) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that an inline error might occur if a CTE contains `ORDER BY`, `LIMIT`, or `SELECT DISTINCT` clauses and is referenced by the recursive part of another CTE [#56603](https://github.com/pingcap/tidb/issues/56603) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that the timeout that occurs when loading statistics synchronically might not be handled correctly [#57710](https://github.com/pingcap/tidb/issues/57710) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that an incorrect database name might be returned when parsing the database name in a CTE [#54582](https://github.com/pingcap/tidb/issues/54582) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that TiDB might panic during startup due to invalid data binding [#58016](https://github.com/pingcap/tidb/issues/58016) @[qw4990](https://github.com/qw4990) + - Fix the issue that cost estimation might generate invalid INF/NaN values in certain extreme cases, which could lead to incorrect Join Reorder results [#56704](https://github.com/pingcap/tidb/issues/56704) @[winoros](https://github.com/winoros) + - Fix the issue that loading statistics manually might fail when the statistics file contains null values [#53966](https://github.com/pingcap/tidb/issues/53966) @[King-Dylan](https://github.com/King-Dylan) + - Fix the issue that creating two views with the same name does not report an error [#58769](https://github.com/pingcap/tidb/issues/58769) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that when a virtual generated column's dependencies contain a column with the `ON UPDATE` attribute, the data of the updated row and its index data might be inconsistent [#56829](https://github.com/pingcap/tidb/issues/56829) @[joechenrh](https://github.com/joechenrh) + - Fix the issue that the `INFORMATION_SCHEMA.TABLES` system table returns incorrect results [#57345](https://github.com/pingcap/tidb/issues/57345) @[tangenta](https://github.com/tangenta) + ++ TiKV + + - Fix the issue that Follower Read might read stale data [#17018](https://github.com/tikv/tikv/issues/17018) @[glorv](https://github.com/glorv) + - Fix the issue that TiKV might panic when destroying a peer [#18005](https://github.com/tikv/tikv/issues/18005) @[glorv](https://github.com/glorv) + - Fix the issue that time rollback might cause abnormal RocksDB flow control, leading to performance jitter [#17995](https://github.com/tikv/tikv/issues/17995) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that disk stalls might prevent leader migration, leading to performance jitter [#17363](https://github.com/tikv/tikv/issues/17363) @[hhwyt](https://github.com/hhwyt) + - Fix the issue that the latest written data might not be readable when only one-phase commit (1PC) is enabled and Async Commit is not enabled [#18117](https://github.com/tikv/tikv/issues/18117) @[zyguan](https://github.com/zyguan) + - Fix the issue that a deadlock might occur when GC Worker is under heavy load [#18214](https://github.com/tikv/tikv/issues/18214) @[zyguan](https://github.com/zyguan) + - Fix the issue that the **Storage async write duration** monitoring metric on the TiKV panel in Grafana is inaccurate [#17579](https://github.com/tikv/tikv/issues/17579) @[overvenus](https://github.com/overvenus) + - Fix the issue that TiKV might panic when executing queries containing `RADIANS()` or `DEGREES()` functions [#17852](https://github.com/tikv/tikv/issues/17852) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that merging Regions might cause TiKV to panic in rare cases [#17840](https://github.com/tikv/tikv/issues/17840) @[glorv](https://github.com/glorv) + - Fix the issue that the leader could not be quickly elected after Region split [#17602](https://github.com/tikv/tikv/issues/17602) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that encoding might fail when processing GBK/GB18030 encoded data [#17618](https://github.com/tikv/tikv/issues/17618) @[CbcWestwolf](https://github.com/CbcWestwolf) + ++ PD + + - Fix the issue that memory leaks might occur when allocating TSOs [#9004](https://github.com/tikv/pd/issues/9004) @[rleungx](https://github.com/rleungx) + - Fix the issue that the `tidb_enable_tso_follower_proxy` system variable might not take effect [#8947](https://github.com/tikv/pd/issues/8947) @[JmPotato](https://github.com/JmPotato) + - Fix a potential issue that might cause PD to panic [#8915](https://github.com/tikv/pd/issues/8915) @[bufferflies](https://github.com/bufferflies) + - Fix the issue that memory leaks might occur in long-running clusters [#9047](https://github.com/tikv/pd/issues/9047) @[bufferflies](https://github.com/bufferflies) + - Fix the issue that a PD node might still generate TSOs even when it is not the Leader [#9051](https://github.com/tikv/pd/issues/9051) @[rleungx](https://github.com/rleungx) + - Fix the issue that Region syncer might not exit in time during the PD Leader switch [#9017](https://github.com/tikv/pd/issues/9017) @[rleungx](https://github.com/rleungx) + - Fix the issue that when creating `evict-leader-scheduler` or `grant-leader-scheduler` encounters an error, the error message is not returned to pd-ctl [#8759](https://github.com/tikv/pd/issues/8759) @[okJiang](https://github.com/okJiang) + - Fix the memory leak issue in hotspot cache [#8698](https://github.com/tikv/pd/issues/8698) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that PD's Region API cannot be requested when a large number of Regions exist [#55872](https://github.com/pingcap/tidb/issues/55872) @[rleungx](https://github.com/rleungx) + - Fix the issue that `evict-leader-scheduler` fails to work properly when it is repeatedly created with the same Store ID [#8756](https://github.com/tikv/pd/issues/8756) @[okJiang](https://github.com/okJiang) + - Upgrade the version of Gin Web Framework from v1.9.1 to v1.10.0 to fix potential security vulnerabilities [#8643](https://github.com/tikv/pd/issues/8643) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that when using a wrong parameter in `evict-leader-scheduler`, PD does not report errors correctly and some schedulers are unavailable [#8619](https://github.com/tikv/pd/issues/8619) @[rleungx](https://github.com/rleungx) + - Fix the memory leak issue in label statistics [#8700](https://github.com/tikv/pd/issues/8700) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that TiDB Dashboard cannot read PD `trace` data correctly [#7253](https://github.com/tikv/pd/issues/7253) @[nolouch](https://github.com/nolouch) + - Fix the memory leak issue in Region statistics [#8710](https://github.com/tikv/pd/issues/8710) @[rleungx](https://github.com/rleungx) + - Fix the issue that PD cannot quickly re-elect a leader during etcd leader transition [#8823](https://github.com/tikv/pd/issues/8823) @[rleungx](https://github.com/rleungx) + ++ TiFlash + + - Fix the issue that the `SUBSTRING()` function does not support the `pos` and `len` arguments for certain integer types, causing query errors [#9473](https://github.com/pingcap/tiflash/issues/9473) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that some JSON functions unsupported by TiFlash are pushed down to TiFlash [#9444](https://github.com/pingcap/tiflash/issues/9444) @[windtalker](https://github.com/windtalker) + - Fix the issue that the `SUBSTRING()` function returns incorrect results when the second parameter is negative [#9604](https://github.com/pingcap/tiflash/issues/9604) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that `LPAD()` and `RPAD()` functions return incorrect results in some cases [#9465](https://github.com/pingcap/tiflash/issues/9465) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that executing `DROP TABLE` on large tables might cause TiFlash OOM [#9437](https://github.com/pingcap/tiflash/issues/9437) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash fails to start due to a division by zero error when retrieving the number of CPU cores [#9212](https://github.com/pingcap/tiflash/issues/9212) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that TiFlash might maintain high memory usage after importing large amounts of data [#9812](https://github.com/pingcap/tiflash/issues/9812) @[CalvinNeo](https://github.com/CalvinNeo) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that BR fails to restore due to getting the `rpcClient is idle` error when sending requests to TiKV [#58845](https://github.com/pingcap/tidb/issues/58845) @[Tristan1900](https://github.com/Tristan1900) + - Fix the issue that the `status` field is missing in the result when querying log backup tasks using `br log status --json` [#57959](https://github.com/pingcap/tidb/issues/57959) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that PD Leader I/O latency during log backup might increase checkpoint latency [#58574](https://github.com/pingcap/tidb/issues/58574) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that the `tiup br restore` command omits checking whether the target cluster table already exists during database or table restoration, which might overwrite existing tables [#58168](https://github.com/pingcap/tidb/issues/58168) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that log backup might unexpectedly enter a paused state when the advancer owner switches [#58031](https://github.com/pingcap/tidb/issues/58031) @[3pointer](https://github.com/3pointer) + - Fix the issue that log backups cannot resolve residual locks promptly, causing the checkpoint to fail to advance [#57134](https://github.com/pingcap/tidb/issues/57134) @[3pointer](https://github.com/3pointer) + - Fix the issue that BR integration test cases are unstable, and add a new test case to simulate snapshot or log backup file corruption [#53835](https://github.com/pingcap/tidb/issues/53835) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that logs might print out encrypted information [#57585](https://github.com/pingcap/tidb/issues/57585) @[kennytm](https://github.com/kennytm) + - Fix the issue that PITR tasks might return the `Information schema is out of date` error when there are a large number of tables in the cluster but the actual data size is small [#57743](https://github.com/pingcap/tidb/issues/57743) @[Tristan1900](https://github.com/Tristan1900) + + + TiCDC + + - Fix the issue that TiCDC uses incorrect table names for filtering during `RENAME TABLE` operations [#11946](https://github.com/pingcap/tiflow/issues/11946) @[wk989898](https://github.com/wk989898) + - Fix the issue that TiCDC reports errors when replicating `default NULL` SQL statements via the Avro protocol [#11994](https://github.com/pingcap/tiflow/issues/11994) @[wk989898](https://github.com/wk989898) + - Fix the issue that TiCDC fails to properly connect to PD after PD scale-in [#12004](https://github.com/pingcap/tiflow/issues/12004) @[lidezhu](https://github.com/lidezhu) + - Fix the issue that Initial Scan is not canceled after the changefeed is stopped or deleted [#11638](https://github.com/pingcap/tiflow/issues/11638) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that after the default value of a newly added column in the upstream is changed from `NOT NULL` to `NULL`, the default values of that column in the downstream are incorrect [#12037](https://github.com/pingcap/tiflow/issues/12037) @[wk989898](https://github.com/wk989898) + - Fix the issue that using the `--overwrite-checkpoint-ts` parameter in the `changefeed pause` command might cause the changefeed to be stuck [#12055](https://github.com/pingcap/tiflow/issues/12055) @[hongyunyan](https://github.com/hongyunyan) + - Fix the issue that TiCDC might panic when replicating `CREATE TABLE IF NOT EXISTS` or `CREATE DATABASE IF NOT EXISTS` statements [#11839](https://github.com/pingcap/tiflow/issues/11839) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that TiCDC might report an error when replicating a `TRUNCATE TABLE` DDL on a table without valid index [#11765](https://github.com/pingcap/tiflow/issues/11765) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that TiCDC mistakenly discards DDL tasks when the schema versions of DDL tasks become non-incremental during TiDB DDL owner changes [#11714](https://github.com/pingcap/tiflow/issues/11714) @[wlwilliamx](https://github.com/wlwilliamx) + - Fix the issue that the changefeed might get stuck after new TiKV nodes are added to the cluster [#11766](https://github.com/pingcap/tiflow/issues/11766) @[lidezhu](https://github.com/lidezhu) + - Fix the issue that out-of-order messages resent by the Sarama client cause Kafka message order to be incorrect [#11935](https://github.com/pingcap/tiflow/issues/11935) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that the Resolved TS latency monitoring in the Puller module displays incorrect values [#11561](https://github.com/pingcap/tiflow/issues/11561) @[wlwilliamx](https://github.com/wlwilliamx) + - Fix the issue that the redo module fails to properly report errors [#11744](https://github.com/pingcap/tiflow/issues/11744) @[CharlesCheung96](https://github.com/CharlesCheung96) + + + TiDB Data Migration (DM) + + - Fix the issue that multiple DM-master nodes might simultaneously become leaders, leading to data inconsistency [#11602](https://github.com/pingcap/tiflow/issues/11602) @[GMHDBJD](https://github.com/GMHDBJD) + - Fix the issue that connecting to MySQL 8.0 fails when the password length exceeds 19 characters [#11603](https://github.com/pingcap/tiflow/issues/11603) @[fishiu](https://github.com/fishiu) + - Fix the issue that the pre-check of `start-task` fails when both TLS and `shard-mode` are configured [#11842](https://github.com/pingcap/tiflow/issues/11842) @[sunxiaoguang](https://github.com/sunxiaoguang) + + + TiDB Lightning + + - Fix the issue that logs are not properly desensitized [#59086](https://github.com/pingcap/tidb/issues/59086) @[GMHDBJD](https://github.com/GMHDBJD) + - Fix the issue that the lack of caching in the encoding phase causes performance regression [#56705](https://github.com/pingcap/tidb/issues/56705) @[OliverS929](https://github.com/OliverS929) + - Fix the issue that the performance degrades when importing data from a cloud storage in high-concurrency scenarios [#57413](https://github.com/pingcap/tidb/issues/57413) @[xuanyu66](https://github.com/xuanyu66) + - Fix the issue that TiDB Lightning does not automatically retry when encountering `Lock wait timeout` errors during metadata updates [#53042](https://github.com/pingcap/tidb/issues/53042) @[guoshouyan](https://github.com/guoshouyan) + - Fix the issue that TiDB Lightning fails to receive oversized messages sent from TiKV [#56114](https://github.com/pingcap/tidb/issues/56114) @[fishiu](https://github.com/fishiu) + - Fix the issue that the error report output is truncated when importing data using TiDB Lightning [#58085](https://github.com/pingcap/tidb/issues/58085) @[lance6716](https://github.com/lance6716) + + + Dumpling + + - Fix the issue that Dumpling fails to retry properly when receiving a 503 error from Google Cloud Storage (GCS) [#56127](https://github.com/pingcap/tidb/issues/56127) @[OliverS929](https://github.com/OliverS929) diff --git a/releases/release-6.5.2.md b/releases/release-6.5.2.md index 86435e538c5c0..3825d7a6be554 100644 --- a/releases/release-6.5.2.md +++ b/releases/release-6.5.2.md @@ -94,7 +94,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- + TiCDC - - Fix the issue that the partition separator does not work when TiCDC replicates data to object storage [#8581](https://github.com/pingcap/tiflow/issues/8581) @[CharlesCheung96](https://github.com/CharlesCheung96) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that the partition separator does not work when TiCDC replicates data to object storage [#8581](https://github.com/pingcap/tiflow/issues/8581) @[CharlesCheung96](https://github.com/CharlesCheung96) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that table scheduling might cause data loss when TiCDC replicates data to object storage [#8256](https://github.com/pingcap/tiflow/issues/8256) @[zhaoxinyu](https://github.com/zhaoxinyu) - Fix the issue that the replication gets stuck due to non-reentrant DDL statements [#8662](https://github.com/pingcap/tiflow/issues/8662) @[hicqu](https://github.com/hicqu) - Fix the issue that TiCDC scaling might cause data loss when TiCDC replicates data to object storage [#8666](https://github.com/pingcap/tiflow/issues/8666) @[CharlesCheung96](https://github.com/CharlesCheung96) diff --git a/releases/release-6.5.3.md b/releases/release-6.5.3.md index 38e48b99e0b31..f1ca8f2605be4 100644 --- a/releases/release-6.5.3.md +++ b/releases/release-6.5.3.md @@ -15,7 +15,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- ### Behavior changes -- When processing update event, TiCDC splits an event into delete and insert events if the primary key or non-null unique index value is modified in the event. For more information, see [documentation](/ticdc/ticdc-behavior-change.md#transactions-containing-a-single-update-change). +- When processing update event, TiCDC splits an event into delete and insert events if the primary key or non-null unique index value is modified in the event. For more information, see [documentation](/ticdc/ticdc-split-update-behavior.md#transactions-containing-a-single-update-change). ## Improvements @@ -41,11 +41,11 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Optimize the way TiCDC handles DDLs so that DDLs do not block the use of other unrelated DML Events, and reduce memory usage [#8106](https://github.com/pingcap/tiflow/issues/8106) @[asddongmen](https://github.com/asddongmen) - Optimize the Decoder interface and add a new method `AddKeyValue` [#8861](https://github.com/pingcap/tiflow/issues/8861) @[3AceShowHand](https://github.com/3AceShowHand) - Optimize the directory structure when DDL events occur in the scenario of replicating data to object storage [#8890](https://github.com/pingcap/tiflow/issues/8890) @[CharlesCheung96](https://github.com/CharlesCheung96) - - Support replicating data to the Kafka-on-Pulsar downstream [#8892](https://github.com/pingcap/tiflow/issues/8892) @[hi-rustin](https://github.com/hi-rustin) - - Support using the OAuth protocol for validation when replicating data to Kafka [#8865](https://github.com/pingcap/tiflow/issues/8865) @[hi-rustin](https://github.com/hi-rustin) + - Support replicating data to the Kafka-on-Pulsar downstream [#8892](https://github.com/pingcap/tiflow/issues/8892) @[hi-rustin](https://github.com/Rustin170506) + - Support using the OAuth protocol for validation when replicating data to Kafka [#8865](https://github.com/pingcap/tiflow/issues/8865) @[hi-rustin](https://github.com/Rustin170506) - Optimize the way TiCDC handles the `UPDATE` statement during data replication using the Avro or CSV protocol, by splitting `UPDATE` into `DELETE` and `INSERT` statements, so that you can get the old value from the `DELETE` statement [#9086](https://github.com/pingcap/tiflow/issues/9086) @[3AceShowHand](https://github.com/3AceShowHand) - - Add a configuration item `insecure-skip-verify` to control whether to set the authentication algorithm in the scenario of enabling TLS [#8867](https://github.com/pingcap/tiflow/issues/8867) @[hi-rustin](https://github.com/hi-rustin) - - Optimize DDL replication operations to mitigate the impact of DDL operations on downstream latency [#8686](https://github.com/pingcap/tiflow/issues/8686) @[hi-rustin](https://github.com/hi-rustin) + - Add a configuration item `insecure-skip-verify` to control whether to set the authentication algorithm in the scenario of enabling TLS [#8867](https://github.com/pingcap/tiflow/issues/8867) @[hi-rustin](https://github.com/Rustin170506) + - Optimize DDL replication operations to mitigate the impact of DDL operations on downstream latency [#8686](https://github.com/pingcap/tiflow/issues/8686) @[hi-rustin](https://github.com/Rustin170506) - Optimize the method of setting GC TLS for the upstream when the TiCDC replication task fails [#8403](https://github.com/pingcap/tiflow/issues/8403) @[charleszheng44](https://github.com/charleszheng44) + TiDB Binlog @@ -117,12 +117,12 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix an OOM issue that might occur when there are as many as 50,000 tables [#7872](https://github.com/pingcap/tiflow/issues/7872) @[sdojjy](https://github.com/sdojjy) - Fix the issue that TiCDC gets stuck when an OOM occurs in upstream TiDB [#8561](https://github.com/pingcap/tiflow/issues/8561) @[overvenus](https://github.com/overvenus) - Fix the issue that TiCDC gets stuck when PD fails such as network isolation or PD Owner node reboot [#8808](https://github.com/pingcap/tiflow/issues/8808) [#8812](https://github.com/pingcap/tiflow/issues/8812) [#8877](https://github.com/pingcap/tiflow/issues/8877) @[asddongmen](https://github.com/asddongmen) - - Fix the issue of TiCDC time zone setting [#8798](https://github.com/pingcap/tiflow/issues/8798) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue of TiCDC time zone setting [#8798](https://github.com/pingcap/tiflow/issues/8798) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that checkpoint lag increases when one of the upstream TiKV nodes crashes [#8858](https://github.com/pingcap/tiflow/issues/8858) @[hicqu](https://github.com/hicqu) - Fix the issue that when replicating data to downstream MySQL, a replication error occurs after the `FLASHBACK CLUSTER TO TIMESTAMP` statement is executed in the upstream TiDB [#8040](https://github.com/pingcap/tiflow/issues/8040) @[asddongmen](https://github.com/asddongmen) - Fix the issue that when replicating data to object storage, the `EXCHANGE PARTITION` operation in the upstream cannot be properly replicated to the downstream [#8914](https://github.com/pingcap/tiflow/issues/8914) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the OOM issue caused by excessive memory usage of the sorter component in some special scenarios [#8974](https://github.com/pingcap/tiflow/issues/8974) @[hicqu](https://github.com/hicqu) - - Fix the issue that when the downstream is Kafka, TiCDC queries the downstream metadata too frequently and causes excessive workload in the downstream [#8957](https://github.com/pingcap/tiflow/issues/8957) [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that when the downstream is Kafka, TiCDC queries the downstream metadata too frequently and causes excessive workload in the downstream [#8957](https://github.com/pingcap/tiflow/issues/8957) [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that when a replication error occurs due to an oversized Kafka message, the message body is recorded in the log [#9031](https://github.com/pingcap/tiflow/issues/9031) @[darraes](https://github.com/darraes) - Fix the TiCDC node panic that occurs when the downstream Kafka sinks are rolling restarted [#9023](https://github.com/pingcap/tiflow/issues/9023) @[asddongmen](https://github.com/asddongmen) - Fix the issue that when replicating data to storage services, the JSON file corresponding to downstream DDL statements does not record the default values of table fields [#9066](https://github.com/pingcap/tiflow/issues/9066) @[CharlesCheung96](https://github.com/CharlesCheung96) diff --git a/releases/release-6.5.4.md b/releases/release-6.5.4.md index 0c2667d5eb67f..772a97dd52117 100644 --- a/releases/release-6.5.4.md +++ b/releases/release-6.5.4.md @@ -18,7 +18,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- ### Behavior changes -- For transactions containing multiple changes, if the primary key or non-null unique index value is modified in the update event, TiCDC splits an event into delete and insert events and ensures that all events follow the sequence of delete events preceding insert events. For more information, see [documentation](/ticdc/ticdc-behavior-change.md#transactions-containing-multiple-update-changes). +- For transactions containing multiple changes, if the primary key or non-null unique index value is modified in the update event, TiCDC splits an event into delete and insert events and ensures that all events follow the sequence of delete events preceding insert events. For more information, see [documentation](/ticdc/ticdc-split-update-behavior.md#transactions-containing-multiple-update-changes). ## Improvements @@ -166,7 +166,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix the issue that the replication task might get stuck when the downstream encounters a short-term failure [#9542](https://github.com/pingcap/tiflow/issues/9542) [#9272](https://github.com/pingcap/tiflow/issues/9272)[#9582](https://github.com/pingcap/tiflow/issues/9582) [#9592](https://github.com/pingcap/tiflow/issues/9592) @[hicqu](https://github.com/hicqu) - Fix the panic issue that might occur when the TiCDC node status changes [#9354](https://github.com/pingcap/tiflow/issues/9354) @[sdojjy](https://github.com/sdojjy) - Fix the issue that when Kafka Sink encounters errors it might indefinitely block changefeed progress [#9309](https://github.com/pingcap/tiflow/issues/9309) @[hicqu](https://github.com/hicqu) - - Fix the issue that when the downstream is Kafka, TiCDC queries the downstream metadata too frequently and causes excessive workload in the downstream [#8957](https://github.com/pingcap/tiflow/issues/8957) [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that when the downstream is Kafka, TiCDC queries the downstream metadata too frequently and causes excessive workload in the downstream [#8957](https://github.com/pingcap/tiflow/issues/8957) [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/Rustin170506) - Fix the data inconsistency issue that might occur when some TiCDC nodes are isolated from the network [#9344](https://github.com/pingcap/tiflow/issues/9344) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that the replication task might get stuck when the redo log is enabled and there is an exception downstream [#9172](https://github.com/pingcap/tiflow/issues/9172) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that changefeeds would fail due to the temporary unavailability of PD [#9294](https://github.com/pingcap/tiflow/issues/9294) @[asddongmen](https://github.com/asddongmen) diff --git a/releases/release-6.5.6.md b/releases/release-6.5.6.md index c280cf9d649db..44a828e1bfc4d 100644 --- a/releases/release-6.5.6.md +++ b/releases/release-6.5.6.md @@ -17,9 +17,10 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Introduce the [`tidb_opt_enable_hash_join`](https://docs.pingcap.com/tidb/v6.5/system-variables#tidb_opt_enable_hash_join-new-in-v656) system variable to control whether the optimizer selects hash joins for tables [#46695](https://github.com/pingcap/tidb/issues/46695) @[coderplay](https://github.com/coderplay) - After further testing, the default value of the TiCDC Changefeed configuration item [`case-sensitive`](/ticdc/ticdc-changefeed-config.md) is changed from `true` to `false`. This means that by default, table and database names in the TiCDC configuration file are case-insensitive [#10047](https://github.com/pingcap/tiflow/issues/10047) @[sdojjy](https://github.com/sdojjy) - TiCDC Changefeed introduces the following new configuration items: - - [`sql-mode`](/ticdc/ticdc-changefeed-config.md): enables you to set the [SQL mode](/ticdc/ticdc-ddl.md#sql-mode) used by TiCDC to parse DDL statements when TiCDC replicates data [#9876](https://github.com/pingcap/tiflow/issues/9876) @[asddongmen](https://github.com/asddongmen) + - [`sql-mode`](/ticdc/ticdc-changefeed-config.md): enables you to set the [SQL mode](https://docs.pingcap.com/tidb/v6.5/ticdc-ddl#sql-mode) used by TiCDC to parse DDL statements when TiCDC replicates data [#9876](https://github.com/pingcap/tiflow/issues/9876) @[asddongmen](https://github.com/asddongmen) - [`encoding-worker-num`](/ticdc/ticdc-changefeed-config.md) and [`flush-worker-num`](/ticdc/ticdc-changefeed-config.md): enables you to set different concurrency parameters for the redo module based on specifications of different machines [#10048](https://github.com/pingcap/tiflow/issues/10048) @[CharlesCheung96](https://github.com/CharlesCheung96) - [`compression`](/ticdc/ticdc-changefeed-config.md): enables you to configure the compression behavior of redo log files [#10176](https://github.com/pingcap/tiflow/issues/10176) @[sdojjy](https://github.com/sdojjy) + - [`changefeed-error-stuck-duration`](/ticdc/ticdc-changefeed-config.md): enables you to set the duration for which the changefeed is allowed to automatically retry when internal errors or exceptions occur [#9875](https://github.com/pingcap/tiflow/issues/9875) @[asddongmen](https://github.com/asddongmen) - [`sink.cloud-storage-config`](/ticdc/ticdc-changefeed-config.md): enables you to set the automatic cleanup of historical data when replicating data to object storage [#10109](https://github.com/pingcap/tiflow/issues/10109) @[CharlesCheung96](https://github.com/CharlesCheung96) ## Improvements @@ -76,13 +77,13 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix the issue of incorrect memory usage estimation in `INDEX_LOOKUP_HASH_JOIN` [#47788](https://github.com/pingcap/tidb/issues/47788) @[SeaRise](https://github.com/SeaRise) - Fix the issue that the zip file generated by `plan replayer` cannot be imported back into TiDB [#46474](https://github.com/pingcap/tidb/issues/46474) @[YangKeao](https://github.com/YangKeao) - Fix the incorrect cost estimation caused by an excessively large `N` in `LIMIT N` [#43285](https://github.com/pingcap/tidb/issues/43285) @[qw4990](https://github.com/qw4990) - - Fix the panic issue that might occur when constructing TopN structure for statistics [#35948](https://github.com/pingcap/tidb/issues/35948) @[hi-rustin](https://github.com/hi-rustin) + - Fix the panic issue that might occur when constructing TopN structure for statistics [#35948](https://github.com/pingcap/tidb/issues/35948) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that the result of `COUNT(INT)` calculated by MPP might be incorrect [#48643](https://github.com/pingcap/tidb/issues/48643) @[AilinKid](https://github.com/AilinKid) - Fix the issue that panic might occur when `tidb_enable_ordered_result_mode` is enabled [#45044](https://github.com/pingcap/tidb/issues/45044) @[qw4990](https://github.com/qw4990) - Fix the issue that the optimizer mistakenly selects IndexFullScan to reduce sort introduced by window functions [#46177](https://github.com/pingcap/tidb/issues/46177) @[qw4990](https://github.com/qw4990) - Fix the issue that the result might be incorrect when predicates are pushed down to common table expressions [#47881](https://github.com/pingcap/tidb/issues/47881) @[winoros](https://github.com/winoros) - Fix the issue that executing `UNION ALL` with the DUAL table as the first subnode might cause an error [#48755](https://github.com/pingcap/tidb/issues/48755) @[winoros](https://github.com/winoros) - - Fix the issue that column pruning can cause panic in specific situations [#47331](https://github.com/pingcap/tidb/issues/47331) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that column pruning can cause panic in specific situations [#47331](https://github.com/pingcap/tidb/issues/47331) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue of possible syntax error when a common table expression (CTE) containing aggregate or window functions is referenced by other recursive CTEs [#47603](https://github.com/pingcap/tidb/issues/47603) [#47711](https://github.com/pingcap/tidb/issues/47711) @[elsa0520](https://github.com/elsa0520) - Fix the issue that an exception might occur when using the `QB_NAME` hint in a prepared statement [#46817](https://github.com/pingcap/tidb/issues/46817) @[jackysp](https://github.com/jackysp) - Fix the issue of Goroutine leak when using `AUTO_ID_CACHE=1` [#46324](https://github.com/pingcap/tidb/issues/46324) @[tiancaiamao](https://github.com/tiancaiamao) diff --git a/releases/release-6.5.7.md b/releases/release-6.5.7.md index 5d63eb623d118..2bbb8f59e2bac 100644 --- a/releases/release-6.5.7.md +++ b/releases/release-6.5.7.md @@ -44,7 +44,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- + TiDB - - Fix the issue that `stats_meta` is not created following table creation [#38189](https://github.com/pingcap/tidb/issues/38189) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) + - Fix the issue that TiDB might not simultaneously create the new statistics metadata when a large number of `CREATE TABLE` statements are executed in a short period of time, causing subsequent query estimation to fail to get accurate row count information [#36004](https://github.com/pingcap/tidb/issues/36004) [#38189](https://github.com/pingcap/tidb/issues/38189) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) - Fix the issue that TiDB server might consume a significant amount of resources when the enterprise plugin for audit logging is used [#49273](https://github.com/pingcap/tidb/issues/49273) @[lcwangchao](https://github.com/lcwangchao) - Fix the incorrect error message for `ErrLoadDataInvalidURI` (invalid S3 URI error) [#48164](https://github.com/pingcap/tidb/issues/48164) @[lance6716](https://github.com/lance6716) - Fix the issue that high CPU usage of TiDB occurs due to long-term memory pressure caused by `tidb_server_memory_limit` [#48741](https://github.com/pingcap/tidb/issues/48741) @[XuHuaiyu](https://github.com/XuHuaiyu) diff --git a/releases/release-6.5.8.md b/releases/release-6.5.8.md index 742b7d4c8c46e..872f0d5b79372 100644 --- a/releases/release-6.5.8.md +++ b/releases/release-6.5.8.md @@ -48,7 +48,6 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- - Fix the issue that executing `SELECT INTO OUTFILE` using the `PREPARE` method incorrectly returns a success message instead of an error [#49166](https://github.com/pingcap/tidb/issues/49166) @[qw4990](https://github.com/qw4990) - Fix the issue that executing `UNIQUE` index lookup with an `ORDER BY` clause might cause an error [#49920](https://github.com/pingcap/tidb/issues/49920) @[jackysp](https://github.com/jackysp) - Fix the issue that the `DELETE` and `UPDATE` statements using index lookup might report an error when `tidb_multi_statement_mode` mode is enabled [#50012](https://github.com/pingcap/tidb/issues/50012) @[tangenta](https://github.com/tangenta) - - Fix the issue that TiDB might not simultaneously establish the new statistics metadata when executing a large number of `CREATE TABLE` statements in a short period of time [#36004](https://github.com/pingcap/tidb/issues/36004) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) - Fix the issue that the `LEADING` hint does not take effect in `UNION ALL` statements [#50067](https://github.com/pingcap/tidb/issues/50067) @[hawkingrei](https://github.com/hawkingrei) - Fix the issue that using old interfaces might cause inconsistent metadata for tables [#49751](https://github.com/pingcap/tidb/issues/49751) @[hawkingrei](https://github.com/hawkingrei) - Fix the issue that common hints do not take effect in `UNION ALL` statements [#50068](https://github.com/pingcap/tidb/issues/50068) @[hawkingrei](https://github.com/hawkingrei) diff --git a/releases/release-6.5.9.md b/releases/release-6.5.9.md index d4699e224d41b..6e94ff6e38c29 100644 --- a/releases/release-6.5.9.md +++ b/releases/release-6.5.9.md @@ -47,7 +47,6 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v6.5/quick-start-with- + TiDB - - Fix the issue that after a large number of tables are created, the newly created tables might lack the `stats_meta` information, causing subsequent query estimation to fail to get accurate row count information [#36004](https://github.com/pingcap/tidb/issues/36004) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) - Fix the issue that dropped tables are still counted by the Grafana `Stats Healthy Distribution` panel [#39349](https://github.com/pingcap/tidb/issues/39349) @[xuyifangreeneyes](https://github.com/xuyifangreeneyes) - Fix the issue that TiDB does not handle the `WHERE ` filtering condition in a SQL statement when the query of that statement involves the `MemTableScan` operator [#40937](https://github.com/pingcap/tidb/issues/40937) @[zhongzc](https://github.com/zhongzc) - Fix the issue that query results might be incorrect when the `HAVING` clause in a subquery contains correlated columns [#51107](https://github.com/pingcap/tidb/issues/51107) @[hawkingrei](https://github.com/hawkingrei) diff --git a/releases/release-6.6.0.md b/releases/release-6.6.0.md index 1a234a695f5dd..6c85367d59c3a 100644 --- a/releases/release-6.6.0.md +++ b/releases/release-6.6.0.md @@ -356,7 +356,8 @@ In v6.6.0-DMR, the key new features and improvements are as follows: | [`tidb_enable_foreign_key`](/system-variables.md#tidb_enable_foreign_key-new-in-v630) | Modified | This variable controls whether to enable the foreign key feature. The default value changes from `OFF` to `ON`, which means enabling foreign key by default. | | `tidb_enable_general_plan_cache` | Modified | This variable controls whether to enable General Plan Cache. Starting from v6.6.0, this variable is renamed to [`tidb_enable_non_prepared_plan_cache`](/system-variables.md#tidb_enable_non_prepared_plan_cache). | | [`tidb_enable_historical_stats`](/system-variables.md#tidb_enable_historical_stats) | Modified | This variable controls whether to enable historical statistics. The default value changes from `OFF` to `ON`, which means that historical statistics are enabled by default. | -| [`tidb_enable_telemetry`](/system-variables.md#tidb_enable_telemetry-new-in-v402) | Modified | The default value changes from `ON` to `OFF`, which means that telemetry is disabled by default in TiDB. | +| [`tidb_enable_plan_replayer_capture`](/system-variables.md#tidb_enable_plan_replayer_capture) | Modified | This variable takes effect starting from v6.6.0 and controls whether to enable the [`PLAN REPLAYER CAPTURE` feature](/sql-plan-replayer.md#use-plan-replayer-capture-to-capture-target-plans). The default value changes from `OFF` to `ON`, which means that the `PLAN REPLAYER CAPTURE` feature is enabled by default. | +| [`tidb_enable_telemetry`](/system-variables.md#tidb_enable_telemetry-new-in-v402-and-deprecated-in-v810) | Modified | The default value changes from `ON` to `OFF`, which means that telemetry is disabled by default in TiDB. | | `tidb_general_plan_cache_size` | Modified | This variable controls the maximum number of execution plans that can be cached by General Plan Cache. Starting from v6.6.0, this variable is renamed to [`tidb_non_prepared_plan_cache_size`](/system-variables.md#tidb_non_prepared_plan_cache_size). | | [`tidb_replica_read`](/system-variables.md#tidb_replica_read-new-in-v40) | Modified | A new value option `learner` is added for this variable to specify the learner replicas with which TiDB reads data from read-only nodes. | | [`tidb_replica_read`](/system-variables.md#tidb_replica_read-new-in-v40) | Modified | A new value option `prefer-leader` is added for this variable to improve the overall read availability of TiDB clusters. When this option is set, TiDB prefers to read from the leader replica. When the performance of the leader replica significantly decreases, TiDB automatically reads from follower replicas. | @@ -366,7 +367,6 @@ In v6.6.0-DMR, the key new features and improvements are as follows: | [`tidb_ddl_distribute_reorg`](https://docs.pingcap.com/tidb/v6.6/system-variables#tidb_ddl_distribute_reorg-new-in-v660) | Newly added | This variable controls whether to enable distributed execution of the DDL reorg phase to accelerate this phase. The default value `OFF` means not to enable distributed execution of the DDL reorg phase by default. Currently, this variable takes effect only for `ADD INDEX`. | | [`tidb_enable_historical_stats_for_capture`](/system-variables.md#tidb_enable_historical_stats_for_capture) | Newly added | This variable controls whether the information captured by `PLAN REPLAYER CAPTURE` includes historical statistics by default. The default value `OFF` means that historical statistics are not included by default. | | [`tidb_enable_plan_cache_for_param_limit`](/system-variables.md#tidb_enable_plan_cache_for_param_limit-new-in-v660) | Newly added | This variable controls whether Prepared Plan Cache caches execution plans that contain `COUNT` after `Limit`. The default value is `ON`, which means Prepared Plan Cache supports caching such execution plans. Note that Prepared Plan Cache does not support caching execution plans with a `COUNT` condition that counts a number greater than 10000. | -| [`tidb_enable_plan_replayer_capture`](/system-variables.md#tidb_enable_plan_replayer_capture) | Newly added | This variable controls whether to enable the [`PLAN REPLAYER CAPTURE` feature](/sql-plan-replayer.md#use-plan-replayer-capture-to-capture-target-plans). The default value `OFF` means to disable the `PLAN REPLAYER CAPTURE` feature. | | [`tidb_enable_resource_control`](/system-variables.md#tidb_enable_resource_control-new-in-v660) | Newly added | This variable controls whether to enable the resource control feature. The default value is `OFF`. When this variable is set to `ON`, the TiDB cluster supports resource isolation of applications based on resource groups. | | [`tidb_historical_stats_duration`](/system-variables.md#tidb_historical_stats_duration-new-in-v660) | Newly added | This variable controls how long the historical statistics are retained in storage. The default value is 7 days. | | [`tidb_index_join_double_read_penalty_cost_rate`](/system-variables.md#tidb_index_join_double_read_penalty_cost_rate-new-in-v660) | Newly added | This variable controls whether to add some penalty cost to the selection of index join. The default value `0` means that this feature is disabled by default. | @@ -385,7 +385,7 @@ In v6.6.0-DMR, the key new features and improvements are as follows: | TiKV | `raftdb.enable-statistics` | Deleted | This configuration item specifies whether to enable Raft RocksDB statistics. Starting from v6.6.0, this item is deleted. Raft RocksDB statistics are enabled for all clusters by default to help diagnostics. For details, see [#13942](https://github.com/tikv/tikv/pull/13942). | | TiKV | `storage.block-cache.shared` | Deleted | Starting from v6.6.0, this configuration item is deleted, and the block cache is enabled by default and cannot be disabled. For details, see [#12936](https://github.com/tikv/tikv/issues/12936). | | DM | `on-duplicate` | Deleted | This configuration item controls the methods to resolve conflicts during the full import phase. In v6.6.0, new configuration items `on-duplicate-logical` and `on-duplicate-physical` are introduced to replace `on-duplicate`. | -| TiDB | [`enable-telemetry`](/tidb-configuration-file.md#enable-telemetry-new-in-v402) | Modified | Starting from v6.6.0, the default value changes from `true` to `false`, which means that telemetry is disabled by default in TiDB. | +| TiDB | [`enable-telemetry`](/tidb-configuration-file.md#enable-telemetry-new-in-v402-and-deprecated-in-v810) | Modified | Starting from v6.6.0, the default value changes from `true` to `false`, which means that telemetry is disabled by default in TiDB. | | TiKV | [`rocksdb.defaultcf.block-size`](/tikv-configuration-file.md#block-size) and [`rocksdb.writecf.block-size`](/tikv-configuration-file.md#block-size) | Modified | The default values change from `64K` to `32K`. | | TiKV | [`rocksdb.defaultcf.block-cache-size`](/tikv-configuration-file.md#block-cache-size), [`rocksdb.writecf.block-cache-size`](/tikv-configuration-file.md#block-cache-size), [`rocksdb.lockcf.block-cache-size`](/tikv-configuration-file.md#block-cache-size) | Deprecated | Starting from v6.6.0, these configuration items are deprecated. For details, see [#12936](https://github.com/tikv/tikv/issues/12936). | | PD | [`enable-telemetry`](/pd-configuration-file.md#enable-telemetry) | Modified | Starting from v6.6.0, the default value changes from `true` to `false`, which means that telemetry is disabled by default in TiDB Dashboard. | @@ -471,7 +471,7 @@ In v6.6.0-DMR, the key new features and improvements are as follows: + TiCDC - Support batch `UPDATE` DML statements to improve TiCDC replication performance [#8084](https://github.com/pingcap/tiflow/issues/8084) @[amyangfei](https://github.com/amyangfei) - - Implement MQ sink and MySQL sink in the asynchronous mode to improve the sink throughput [#5928](https://github.com/pingcap/tiflow/issues/5928) @[hicqu](https://github.com/hicqu) @[hi-rustin](https://github.com/hi-rustin) + - Implement MQ sink and MySQL sink in the asynchronous mode to improve the sink throughput [#5928](https://github.com/pingcap/tiflow/issues/5928) @[hicqu](https://github.com/hicqu) @[hi-rustin](https://github.com/Rustin170506) + TiDB Data Migration (DM) @@ -588,7 +588,7 @@ In v6.6.0-DMR, the key new features and improvements are as follows: - Fix the issue of insufficient duration that redo log can tolerate for S3 storage failure [#8089](https://github.com/pingcap/tiflow/issues/8089) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that changefeed might get stuck in special scenarios such as when scaling in or scaling out TiKV or TiCDC nodes [#8174](https://github.com/pingcap/tiflow/issues/8174) @[hicqu](https://github.com/hicqu) - Fix the issue of too high traffic among TiKV nodes [#14092](https://github.com/tikv/tikv/issues/14092) @[overvenus](https://github.com/overvenus) - - Fix the performance issues of TiCDC in terms of CPU usage, memory control, and throughput when the pull-based sink is enabled [#8142](https://github.com/pingcap/tiflow/issues/8142) [#8157](https://github.com/pingcap/tiflow/issues/8157) [#8001](https://github.com/pingcap/tiflow/issues/8001) [#5928](https://github.com/pingcap/tiflow/issues/5928) @[hicqu](https://github.com/hicqu) @[hi-rustin](https://github.com/hi-rustin) + - Fix the performance issues of TiCDC in terms of CPU usage, memory control, and throughput when the pull-based sink is enabled [#8142](https://github.com/pingcap/tiflow/issues/8142) [#8157](https://github.com/pingcap/tiflow/issues/8157) [#8001](https://github.com/pingcap/tiflow/issues/8001) [#5928](https://github.com/pingcap/tiflow/issues/5928) @[hicqu](https://github.com/hicqu) @[hi-rustin](https://github.com/Rustin170506) + TiDB Data Migration (DM) diff --git a/releases/release-7.0.0.md b/releases/release-7.0.0.md index 34e2adb119703..b9ca5b16eff61 100644 --- a/releases/release-7.0.0.md +++ b/releases/release-7.0.0.md @@ -248,7 +248,7 @@ In v7.0.0-DMR, the key new features and improvements are as follows: * [DBeaver](https://dbeaver.io/) v23.0.1 supports TiDB by default [#17396](https://github.com/dbeaver/dbeaver/issues/17396) @[Icemap](https://github.com/Icemap) - Provides an independent TiDB module, icon, and logo. - - The default configuration supports [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless), making it easier to connect to TiDB Serverless. + - The default configuration supports [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless), making it easier to connect to TiDB Cloud Serverless. - Supports identifying TiDB versions to display or hide foreign key tabs. - Supports visualizing SQL execution plans in `EXPLAIN` results. - Supports highlighting TiDB keywords such as `PESSIMISTIC`, `OPTIMISTIC`, `AUTO_RANDOM`, `PLACEMENT`, `POLICY`, `REORGANIZE`, `EXCHANGE`, `CACHE`, `NONCLUSTERED`, and `CLUSTERED`. @@ -268,7 +268,7 @@ In v7.0.0-DMR, the key new features and improvements are as follows: For more information, see [documentation](/sql-statements/sql-statement-load-data.md). -* TiDB Lightning supports enabling compressed transfers when sending key-value pairs to TiKV (GA) [#41163](https://github.com/pingcap/tidb/issues/41163) @[gozssky](https://github.com/gozssky) +* TiDB Lightning supports enabling compressed transfers when sending key-value pairs to TiKV (GA) [#41163](https://github.com/pingcap/tidb/issues/41163) @[sleepymole](https://github.com/sleepymole) Starting from v6.6.0, TiDB Lightning supports compressing locally encoded and sorted key-value pairs for network transfer when sending them to TiKV, thus reducing the amount of data transferred over the network and lowering the network bandwidth overhead. In the earlier TiDB versions before this feature is supported, TiDB Lightning requires relatively high network bandwidth and incurs high traffic charges in case of large data volumes. @@ -345,6 +345,7 @@ In v7.0.0-DMR, the key new features and improvements are as follows: | [`tidb_opt_enable_late_materialization`](/system-variables.md#tidb_opt_enable_late_materialization-new-in-v700) | Newly added | This variable controls whether to enable the [TiFlash Late Materialization](/tiflash/tiflash-late-materialization.md) feature. The default value is `OFF`, which means the feature is not enabled. | | [`tidb_opt_ordering_index_selectivity_threshold`](/system-variables.md#tidb_opt_ordering_index_selectivity_threshold-new-in-v700) | Newly added | This variable controls how the optimizer selects indexes when the SQL statement contains `ORDER BY` and `LIMIT` clauses and has filtering conditions. | | [`tidb_pessimistic_txn_fair_locking`](/system-variables.md#tidb_pessimistic_txn_fair_locking-new-in-v700) | Newly added | Controls whether to enable the enhanced pessimistic lock-waking model to reduce the tail latency of transactions under single-row conflict scenarios. The default value is `ON`. When the cluster is upgraded from an earlier version to v7.0.0 or later, the value of this variable is set to `OFF`. | +| [`tidb_slow_txn_log_threshold`](/system-variables.md#tidb_slow_txn_log_threshold-new-in-v700) | Newly added | Sets the threshold for slow transaction logging. When the execution time of a transaction exceeds this threshold, TiDB logs detailed information about the transaction. The default value `0` means that this feature is disabled. | | [`tidb_ttl_running_tasks`](/system-variables.md#tidb_ttl_running_tasks-new-in-v700) | Newly added | This variable is used to limit the concurrency of TTL tasks across the entire cluster. The default value `-1` means that the TTL tasks are the same as the number of TiKV nodes. | ### Configuration file parameters @@ -411,15 +412,15 @@ In v7.0.0-DMR, the key new features and improvements are as follows: - Support splitting transactions in the redo applier to improve its throughput and reduce RTO in disaster recovery scenarios [#8318](https://github.com/pingcap/tiflow/issues/8318) @[CharlesCheung96](https://github.com/CharlesCheung96) - Improve the table scheduling to split a single table more evenly across various TiCDC nodes [#8247](https://github.com/pingcap/tiflow/issues/8247) @[overvenus](https://github.com/overvenus) - - Add the Large Row monitoring metrics in MQ sink [#8286](https://github.com/pingcap/tiflow/issues/8286) @[hi-rustin](https://github.com/hi-rustin) + - Add the Large Row monitoring metrics in MQ sink [#8286](https://github.com/pingcap/tiflow/issues/8286) @[hi-rustin](https://github.com/Rustin170506) - Reduce network traffic between TiKV and TiCDC nodes in scenarios where a Region contains data of multiple tables [#6346](https://github.com/pingcap/tiflow/issues/6346) @[overvenus](https://github.com/overvenus) - - Move the P99 metrics panel of Checkpoint TS and Resolved TS to the Lag analyze panel [#8524](https://github.com/pingcap/tiflow/issues/8524) @[hi-rustin](https://github.com/hi-rustin) + - Move the P99 metrics panel of Checkpoint TS and Resolved TS to the Lag analyze panel [#8524](https://github.com/pingcap/tiflow/issues/8524) @[hi-rustin](https://github.com/Rustin170506) - Support applying DDL events in redo logs [#8361](https://github.com/pingcap/tiflow/issues/8361) @[CharlesCheung96](https://github.com/CharlesCheung96) - Support splitting and scheduling tables to TiCDC nodes based on upstream write throughput [#7720](https://github.com/pingcap/tiflow/issues/7720) @[overvenus](https://github.com/overvenus) + TiDB Lightning - - TiDB Lightning Physical Import Mode supports separating data import and index import to improve import speed and stability [#42132](https://github.com/pingcap/tidb/issues/42132) @[gozssky](https://github.com/gozssky) + - TiDB Lightning Physical Import Mode supports separating data import and index import to improve import speed and stability [#42132](https://github.com/pingcap/tidb/issues/42132) @[sleepymole](https://github.com/sleepymole) Add the `add-index-by-sql` parameter. The default value is `false`, which means that TiDB Lightning encodes both row data and index data into KV pairs and import them into TiKV together. If you set it to `true`, it means that TiDB Lightning adds indexes via the `ADD INDEX` SQL statement after importing the row data to improve import speed and stability. @@ -504,7 +505,7 @@ We would like to thank the following contributors from the TiDB community: - [BornChanger](https://github.com/BornChanger) - [Dousir9](https://github.com/Dousir9) - [erwadba](https://github.com/erwadba) -- [HappyUncle](https://github.com/HappyUncle) +- [happy-v587](https://github.com/happy-v587) - [jiyfhust](https://github.com/jiyfhust) - [L-maple](https://github.com/L-maple) - [liumengya94](https://github.com/liumengya94) diff --git a/releases/release-7.1.0.md b/releases/release-7.1.0.md index 57c38a1c77750..d228392800fa8 100644 --- a/releases/release-7.1.0.md +++ b/releases/release-7.1.0.md @@ -182,7 +182,7 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr For more information, see [documentation](/ticdc/ticdc-integrity-check.md). -* TiCDC optimizes DDL replication operations [#8686](https://github.com/pingcap/tiflow/issues/8686) @[hi-rustin](https://github.com/hi-rustin) +* TiCDC optimizes DDL replication operations [#8686](https://github.com/pingcap/tiflow/issues/8686) @[hi-rustin](https://github.com/Rustin170506) Before v7.1.0, when you perform a DDL operation that affects all rows on a large table (such as adding or deleting a column), the replication latency of TiCDC would significantly increase. Starting from v7.1.0, TiCDC optimizes this replication operation and mitigates the impact of DDL operations on downstream latency. @@ -386,7 +386,7 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr + PD - - Add a controller that automatically adjusts the size of the store limit based on the execution details of the snapshot. To enable this controller, set `store-limit-version` to `v2`. Once enabled, you do not need to manually adjust the `store limit` configuration to control the speed of scaling in or scaling out [#6147](https://github.com/tikv/pd/issues/6147) @[bufferflies](https://github.com/bufferflies) + - Add a controller that automatically adjusts the size of the store limit based on the execution details of the snapshot. To enable this controller, set `store-limit-version` to `v2` (experimental). Once enabled, you do not need to manually adjust the `store limit` configuration to control the speed of scaling in or scaling out [#6147](https://github.com/tikv/pd/issues/6147) @[bufferflies](https://github.com/bufferflies) - Add historical load information to avoid frequent scheduling of Regions with unstable loads by the hotspot scheduler when the storage engine is raft-kv2 [#6297](https://github.com/tikv/pd/issues/6297) @[bufferflies](https://github.com/bufferflies) - Add a leader health check mechanism. When the PD server where the etcd leader is located cannot be elected as the leader, PD actively switches the etcd leader to ensure that the PD leader is available [#6403](https://github.com/tikv/pd/issues/6403) @[nolouch](https://github.com/nolouch) @@ -406,10 +406,10 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr - Optimize the directory structure when DDL events occur in the scenario of replicating data to object storage [#8890](https://github.com/pingcap/tiflow/issues/8890) @[CharlesCheung96](https://github.com/CharlesCheung96) - Optimize the method of setting GC TLS for the upstream when the TiCDC replication task fails [#8403](https://github.com/pingcap/tiflow/issues/8403) @[charleszheng44](https://github.com/charleszheng44) - - Support replicating data to the Kafka-on-Pulsar downstream [#8892](https://github.com/pingcap/tiflow/issues/8892) @[hi-rustin](https://github.com/hi-rustin) + - Support replicating data to the Kafka-on-Pulsar downstream [#8892](https://github.com/pingcap/tiflow/issues/8892) @[hi-rustin](https://github.com/Rustin170506) - Support using the open-protocol protocol to only replicate the changed columns after an update occurs when replicating data to Kafka [#8706](https://github.com/pingcap/tiflow/issues/8706) @[sdojjy](https://github.com/sdojjy) - Optimize the error handling of TiCDC in the downstream failures or other scenarios [#8657](https://github.com/pingcap/tiflow/issues/8657) @[hicqu](https://github.com/hicqu) - - Add a configuration item `insecure-skip-verify` to control whether to set the authentication algorithm in the scenario of enabling TLS [#8867](https://github.com/pingcap/tiflow/issues/8867) @[hi-rustin](https://github.com/hi-rustin) + - Add a configuration item `insecure-skip-verify` to control whether to set the authentication algorithm in the scenario of enabling TLS [#8867](https://github.com/pingcap/tiflow/issues/8867) @[hi-rustin](https://github.com/Rustin170506) + TiDB Lightning @@ -425,7 +425,6 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr - Fix the issue of missing table names in the `ADMIN SHOW DDL JOBS` result when a `DROP TABLE` operation is being executed [#42268](https://github.com/pingcap/tidb/issues/42268) @[tiancaiamao](https://github.com/tiancaiamao) - Fix the issue that `Ignore Event Per Minute` and `Stats Cache LRU Cost` charts might not be displayed normally in the Grafana monitoring panel [#42562](https://github.com/pingcap/tidb/issues/42562) @[pingandb](https://github.com/pingandb) - Fix the issue that the `ORDINAL_POSITION` column returns incorrect results when querying the `INFORMATION_SCHEMA.COLUMNS` table [#43379](https://github.com/pingcap/tidb/issues/43379) @[bb7133](https://github.com/bb7133) - - Fix the case sensitivity issue in some columns of the permission table [#41048](https://github.com/pingcap/tidb/issues/41048) @[bb7133](https://github.com/bb7133) - Fix the issue that after a new column is added in the cache table, the value is `NULL` instead of the default value of the column [#42928](https://github.com/pingcap/tidb/issues/42928) @[lqs](https://github.com/lqs) - Fix the issue that CTE results are incorrect when pushing down predicates [#43645](https://github.com/pingcap/tidb/issues/43645) @[winoros](https://github.com/winoros) - Fix the issue of DDL retry caused by write conflict when executing `TRUNCATE TABLE` for partitioned tables with many partitions and TiFlash replicas [#42940](https://github.com/pingcap/tidb/issues/42940) @[mjonss](https://github.com/mjonss) @@ -501,7 +500,7 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr + TiCDC - - Fix the issue of TiCDC time zone setting [#8798](https://github.com/pingcap/tiflow/issues/8798) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue of TiCDC time zone setting [#8798](https://github.com/pingcap/tiflow/issues/8798) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that TiCDC cannot automatically recover when PD address or leader fails [#8812](https://github.com/pingcap/tiflow/issues/8812) [#8877](https://github.com/pingcap/tiflow/issues/8877) @[asddongmen](https://github.com/asddongmen) - Fix the issue that checkpoint lag increases when one of the upstream TiKV nodes crashes [#8858](https://github.com/pingcap/tiflow/issues/8858) @[hicqu](https://github.com/hicqu) - Fix the issue that when replicating data to object storage, the `EXCHANGE PARTITION` operation in the upstream cannot be properly replicated to the downstream [#8914](https://github.com/pingcap/tiflow/issues/8914) @[CharlesCheung96](https://github.com/CharlesCheung96) @@ -534,7 +533,7 @@ Compared with the previous LTS 6.5.0, 7.1.0 not only includes new features, impr ## Performance test -To learn about the performance of TiDB v7.1.0, you can refer to the [TPC-C performance test report](https://docs.pingcap.com/tidbcloud/v7.1.0-performance-benchmarking-with-tpcc) and [Sysbench performance test report](https://docs.pingcap.com/tidbcloud/v7.1.0-performance-benchmarking-with-sysbench) of the TiDB Dedicated cluster. +To learn about the performance of TiDB v7.1.0, you can refer to the [TPC-C performance test report](https://docs.pingcap.com/tidbcloud/v7.1.0-performance-benchmarking-with-tpcc) and [Sysbench performance test report](https://docs.pingcap.com/tidbcloud/v7.1.0-performance-benchmarking-with-sysbench) of the TiDB Cloud Dedicated cluster. ## Contributors diff --git a/releases/release-7.1.1.md b/releases/release-7.1.1.md index 361f5e48a5879..cd905ea59ef0c 100644 --- a/releases/release-7.1.1.md +++ b/releases/release-7.1.1.md @@ -17,7 +17,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- ### Behavior changes -- When processing update event, TiCDC splits an event into delete and insert events if the primary key or non-null unique index value is modified in the event. For more information, see [documentation](/ticdc/ticdc-behavior-change.md#transactions-containing-a-single-update-change). +- When processing update event, TiCDC splits an event into delete and insert events if the primary key or non-null unique index value is modified in the event. For more information, see [documentation](/ticdc/ticdc-split-update-behavior.md#transactions-containing-a-single-update-change). ## Improvements @@ -37,7 +37,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- + TiCDC - Optimize the encoding format of binary fields when TiCDC replicates data to object storage services [#9373](https://github.com/pingcap/tiflow/issues/9373) @[CharlesCheung96](https://github.com/CharlesCheung96) - - Support the OAUTHBEARER authentication in the scenario of replication to Kafka [#8865](https://github.com/pingcap/tiflow/issues/8865) @[hi-rustin](https://github.com/hi-rustin) + - Support the OAUTHBEARER authentication in the scenario of replication to Kafka [#8865](https://github.com/pingcap/tiflow/issues/8865) @[hi-rustin](https://github.com/Rustin170506) + TiDB Lightning @@ -123,8 +123,8 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- - Fix the issue of excessive memory consumption when replicating to an object storage service [#8894](https://github.com/pingcap/tiflow/issues/8894) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that the replication task might get stuck when the redo log is enabled and there is an exception downstream [#9172](https://github.com/pingcap/tiflow/issues/9172) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that TiCDC keeps retrying when there is a downstream failure, which causes the retry time to be too long [#9272](https://github.com/pingcap/tiflow/issues/9272) @[asddongmen](https://github.com/asddongmen) - - Fix the issue of excessive downstream pressure caused by reading downstream metadata too frequently when replicating data to Kafka [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/hi-rustin) - - Fix the issue that when the downstream is Kafka, TiCDC queries the downstream metadata too frequently and causes excessive workload in the downstream [#8957](https://github.com/pingcap/tiflow/issues/8957) [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue of excessive downstream pressure caused by reading downstream metadata too frequently when replicating data to Kafka [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that when the downstream is Kafka, TiCDC queries the downstream metadata too frequently and causes excessive workload in the downstream [#8957](https://github.com/pingcap/tiflow/issues/8957) [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/Rustin170506) - Fix the OOM issue caused by excessive memory usage of the sorter component in some special scenarios [#8974](https://github.com/pingcap/tiflow/issues/8974) @[hicqu](https://github.com/hicqu) - Fix the issue that the `UPDATE` operation cannot output old values when the Avro or CSV protocol is used [#9086](https://github.com/pingcap/tiflow/issues/9086) @[3AceShowHand](https://github.com/3AceShowHand) - Fix the issue that when replicating data to storage services, the JSON file corresponding to downstream DDL statements does not record the default values of table fields [#9066](https://github.com/pingcap/tiflow/issues/9066) @[CharlesCheung96](https://github.com/CharlesCheung96) diff --git a/releases/release-7.1.2.md b/releases/release-7.1.2.md index 7963f74bb2f5f..f5f5c500e429f 100644 --- a/releases/release-7.1.2.md +++ b/releases/release-7.1.2.md @@ -22,7 +22,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- ### Behavior changes -- For transactions containing multiple changes, if the primary key or non-null unique index value is modified in the update event, TiCDC splits an event into delete and insert events and ensures that all events follow the sequence of delete events preceding insert events. For more information, see [documentation](/ticdc/ticdc-behavior-change.md#transactions-containing-multiple-update-changes). +- For transactions containing multiple changes, if the primary key or non-null unique index value is modified in the update event, TiCDC splits an event into delete and insert events and ensures that all events follow the sequence of delete events preceding insert events. For more information, see [documentation](/ticdc/ticdc-split-update-behavior.md#transactions-containing-multiple-update-changes). ## Improvements diff --git a/releases/release-7.1.3.md b/releases/release-7.1.3.md index 0f8e4ddcfc3b5..cabdb83df6bc1 100644 --- a/releases/release-7.1.3.md +++ b/releases/release-7.1.3.md @@ -15,7 +15,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- - After further testing, the default value of the TiCDC Changefeed configuration item [`case-sensitive`](/ticdc/ticdc-changefeed-config.md) is changed from `true` to `false`. This means that by default, table and database names in the TiCDC configuration file are case-insensitive [#10047](https://github.com/pingcap/tiflow/issues/10047) @[sdojjy](https://github.com/sdojjy) - TiCDC Changefeed introduces the following new configuration items: - - [`sql-mode`](/ticdc/ticdc-changefeed-config.md): enables you to set the [SQL mode](/ticdc/ticdc-ddl.md#sql-mode) used by TiCDC to parse DDL statements when TiCDC replicates data [#9876](https://github.com/pingcap/tiflow/issues/9876) @[asddongmen](https://github.com/asddongmen) + - [`sql-mode`](/ticdc/ticdc-changefeed-config.md): enables you to set the [SQL mode](https://docs.pingcap.com/tidb/v7.1/ticdc-ddl#sql-mode) used by TiCDC to parse DDL statements when TiCDC replicates data [#9876](https://github.com/pingcap/tiflow/issues/9876) @[asddongmen](https://github.com/asddongmen) - [`encoding-worker-num`](/ticdc/ticdc-changefeed-config.md) and [`flush-worker-num`](/ticdc/ticdc-changefeed-config.md): enables you to set different concurrency parameters for the redo module based on specifications of different machines [#10048](https://github.com/pingcap/tiflow/issues/10048) @[CharlesCheung96](https://github.com/CharlesCheung96) - [`compression`](/ticdc/ticdc-changefeed-config.md): enables you to configure the compression behavior of redo log files [#10176](https://github.com/pingcap/tiflow/issues/10176) @[sdojjy](https://github.com/sdojjy) - [`sink.cloud-storage-config`](/ticdc/ticdc-changefeed-config.md): enables you to set the automatic cleanup of historical data when replicating data to object storage [#10109](https://github.com/pingcap/tiflow/issues/10109) @[CharlesCheung96](https://github.com/CharlesCheung96) @@ -60,7 +60,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- - Fix the issue that queries containing CTEs report `runtime error: index out of range [32] with length 32` when `tidb_max_chunk_size` is set to a small value [#48808](https://github.com/pingcap/tidb/issues/48808) @[guo-shaoge](https://github.com/guo-shaoge) - Fix the issue that the query result is incorrect when an `ENUM` type column is used as the join key [#48991](https://github.com/pingcap/tidb/issues/48991) @[winoros](https://github.com/winoros) - Fix the parsing error caused by aggregate or window functions in recursive CTEs [#47711](https://github.com/pingcap/tidb/issues/47711) @[elsa0520](https://github.com/elsa0520) - - Fix the issue that `UPDATE` statements might be incorrectly converted to PointGet [#47445](https://github.com/pingcap/tidb/issues/47445) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that `UPDATE` statements might be incorrectly converted to PointGet [#47445](https://github.com/pingcap/tidb/issues/47445) @[hi-rustin](https://github.com/Rustin170506) - Fix the OOM issue that might occur when TiDB performs garbage collection on the `stats_history` table [#48431](https://github.com/pingcap/tidb/issues/48431) @[hawkingrei](https://github.com/hawkingrei) - Fix the issue that the same query plan has different `PLAN_DIGEST` values in some cases [#47634](https://github.com/pingcap/tidb/issues/47634) @[King-Dylan](https://github.com/King-Dylan) - Fix the issue that `GenJSONTableFromStats` cannot be killed when it consumes a large amount of memory [#47779](https://github.com/pingcap/tidb/issues/47779) @[hawkingrei](https://github.com/hawkingrei) @@ -82,10 +82,10 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- - Fix the issue of incorrect memory usage estimation in `INDEX_LOOKUP_HASH_JOIN` [#47788](https://github.com/pingcap/tidb/issues/47788) @[SeaRise](https://github.com/SeaRise) - Fix the issue of `IMPORT INTO` task failure caused by PD leader malfunction for 1 minute [#48307](https://github.com/pingcap/tidb/issues/48307) @[D3Hunter](https://github.com/D3Hunter) - Fix the panic issue of `batch-client` in `client-go` [#47691](https://github.com/pingcap/tidb/issues/47691) @[crazycs520](https://github.com/crazycs520) - - Fix the issue that column pruning can cause panic in specific situations [#47331](https://github.com/pingcap/tidb/issues/47331) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that column pruning can cause panic in specific situations [#47331](https://github.com/pingcap/tidb/issues/47331) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that TiDB does not read `cgroup` resource limits when it is started with `systemd` [#47442](https://github.com/pingcap/tidb/issues/47442) @[hawkingrei](https://github.com/hawkingrei) - Fix the issue of possible syntax error when a common table expression (CTE) containing aggregate or window functions is referenced by other recursive CTEs [#47603](https://github.com/pingcap/tidb/issues/47603) [#47711](https://github.com/pingcap/tidb/issues/47711) @[elsa0520](https://github.com/elsa0520) - - Fix the panic issue that might occur when constructing TopN structure for statistics [#35948](https://github.com/pingcap/tidb/issues/35948) @[hi-rustin](https://github.com/hi-rustin) + - Fix the panic issue that might occur when constructing TopN structure for statistics [#35948](https://github.com/pingcap/tidb/issues/35948) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that the result of `COUNT(INT)` calculated by MPP might be incorrect [#48643](https://github.com/pingcap/tidb/issues/48643) @[AilinKid](https://github.com/AilinKid) - Fix the issue that the chunk cannot be reused when the HashJoin operator performs probe [#48082](https://github.com/pingcap/tidb/issues/48082) @[wshwsh12](https://github.com/wshwsh12) diff --git a/releases/release-7.1.4.md b/releases/release-7.1.4.md index 04a5c88fe10cc..9f7629016e269 100644 --- a/releases/release-7.1.4.md +++ b/releases/release-7.1.4.md @@ -66,7 +66,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- - Fix the issue that CTE queries might report an error `type assertion for CTEStorageMap failed` during the retry process [#46522](https://github.com/pingcap/tidb/issues/46522) @[tiancaiamao](https://github.com/tiancaiamao) - Fix the issue of excessive statistical error in constructing statistics caused by Golang's implicit conversion algorithm [#49801](https://github.com/pingcap/tidb/issues/49801) @[qw4990](https://github.com/qw4990) - Fix the issue that errors might be returned during the concurrent merging of global statistics for partitioned tables [#48713](https://github.com/pingcap/tidb/issues/48713) @[hawkingrei](https://github.com/hawkingrei) - - Fix the issue of wrong query results due to TiDB incorrectly eliminating constant values in `group by` [#38756](https://github.com/pingcap/tidb/issues/38756) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue of wrong query results due to TiDB incorrectly eliminating constant values in `group by` [#38756](https://github.com/pingcap/tidb/issues/38756) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that `BIT` type columns might cause query errors due to decode failures when they are involved in calculations of some functions [#49566](https://github.com/pingcap/tidb/issues/49566) [#50850](https://github.com/pingcap/tidb/issues/50850) [#50855](https://github.com/pingcap/tidb/issues/50855) @[jiyfhust](https://github.com/jiyfhust) - Fix the issue that `LIMIT` in multi-level nested `UNION` queries might become ineffective [#49874](https://github.com/pingcap/tidb/issues/49874) @[Defined2014](https://github.com/Defined2014) - Fix the issue that the auto-increment ID allocation reports an error due to concurrent conflicts when using an auto-increment column with `AUTO_ID_CACHE=1` [#50519](https://github.com/pingcap/tidb/issues/50519) @[tiancaiamao](https://github.com/tiancaiamao) diff --git a/releases/release-7.1.5.md b/releases/release-7.1.5.md index 385347f612574..525360875600b 100644 --- a/releases/release-7.1.5.md +++ b/releases/release-7.1.5.md @@ -64,7 +64,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with- - Fix the issue that AutoID Leader change might cause the value of the auto-increment column to decrease in the case of `AUTO_ID_CACHE=1` [#52600](https://github.com/pingcap/tidb/issues/52600) @[tiancaiamao](https://github.com/tiancaiamao) - Fix the issue that executing `INSERT IGNORE` might result in inconsistency between the unique index and the data [#51784](https://github.com/pingcap/tidb/issues/51784) @[wjhuang2016](https://github.com/wjhuang2016) - Fix the issue that adding a unique index might cause TiDB to panic [#52312](https://github.com/pingcap/tidb/issues/52312) @[wjhuang2016](https://github.com/wjhuang2016) - - Fix the issue that the Window function might panic when there is a related subquery in it [#42734](https://github.com/pingcap/tidb/issues/42734) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that the Window function might panic when there is a related subquery in it [#42734](https://github.com/pingcap/tidb/issues/42734) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that the `init-stats` process might cause TiDB to panic and the `load stats` process to quit [#51581](https://github.com/pingcap/tidb/issues/51581) @[hawkingrei](https://github.com/hawkingrei) - Fix the performance regression issue caused by disabling predicate pushdown in TableDual [#50614](https://github.com/pingcap/tidb/issues/50614) @[time-and-fate](https://github.com/time-and-fate) - Fix the issue that query results might be incorrect when the `HAVING` clause in a subquery contains correlated columns [#51107](https://github.com/pingcap/tidb/issues/51107) @[hawkingrei](https://github.com/hawkingrei) diff --git a/releases/release-7.1.6.md b/releases/release-7.1.6.md new file mode 100644 index 0000000000000..30ffa0becccdb --- /dev/null +++ b/releases/release-7.1.6.md @@ -0,0 +1,300 @@ +--- +title: TiDB 7.1.6 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.1.6. +--- + +# TiDB 7.1.6 Release Notes + +Release date: November 21, 2024 + +TiDB version: 7.1.6 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.1/production-deployment-using-tiup) + +## Compatibility changes + +- Set a default limit of 2048 for DDL historical tasks retrieved through the [TiDB HTTP API](https://github.com/pingcap/tidb/blob/release-7.1/docs/tidb_http_api.md) to prevent OOM issues caused by excessive historical tasks [#55711](https://github.com/pingcap/tidb/issues/55711) @[joccau](https://github.com/joccau) +- In earlier versions, when processing a transaction containing `UPDATE` changes, if the primary key or non-null unique index value is modified in an `UPDATE` event, TiCDC splits this event into `DELETE` and `INSERT` events. Starting from v7.1.6, when using the MySQL sink, TiCDC splits an `UPDATE` event into `DELETE` and `INSERT` events if the transaction `commitTS` for the `UPDATE` change is less than TiCDC `thresholdTS` (which is the current timestamp fetched from PD when TiCDC starts replicating the corresponding table to the downstream). This behavior change addresses the issue of downstream data inconsistencies caused by the potentially incorrect order of `UPDATE` events received by TiCDC, which can lead to an incorrect order of split `DELETE` and `INSERT` events. For more information, see [documentation](https://docs.pingcap.com/tidb/v7.1/ticdc-split-update-behavior#split-update-events-for-mysql-sinks). [#10918](https://github.com/pingcap/tiflow/issues/10918) @[lidezhu](https://github.com/lidezhu) +- Must set the line terminator when using TiDB Lightning `strict-format` to import CSV files [#37338](https://github.com/pingcap/tidb/issues/37338) @[lance6716](https://github.com/lance6716) + +## Improvements + ++ TiDB + + - Adjust estimation results from 0 to 1 for equality conditions that do not hit TopN when statistics are entirely composed of TopN and the modified row count in the corresponding table statistics is non-zero [#47400](https://github.com/pingcap/tidb/issues/47400) @[terry1purcell](https://github.com/terry1purcell) + - Remove stores without Regions during MPP load balancing [#52313](https://github.com/pingcap/tidb/issues/52313) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Improve the MySQL compatibility of expression default values displayed in the output of `SHOW CREATE TABLE` [#52939](https://github.com/pingcap/tidb/issues/52939) @[CbcWestwolf](https://github.com/CbcWestwolf) + - By batch deleting TiFlash placement rules, improve the processing speed of data GC after performing the `TRUNCATE` or `DROP` operation on partitioned tables [#54068](https://github.com/pingcap/tidb/issues/54068) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Improve sync load performance to reduce latency in loading statistics [#52294](https://github.com/pingcap/tidb/issues/52294) [hawkingrei](https://github.com/hawkingrei) + ++ TiKV + + - Add slow logs for peer and store messages [#16600](https://github.com/tikv/tikv/issues/16600) @[Connor1996](https://github.com/Connor1996) + - Optimize the compaction trigger mechanism of RocksDB to accelerate disk space reclamation when handling a large number of DELETE versions [#17269](https://github.com/tikv/tikv/issues/17269) @[AndreMouche](https://github.com/AndreMouche) + - Optimize the jittery access delay when restarting TiKV due to waiting for the log to be applied, improving the stability of TiKV [#15874](https://github.com/tikv/tikv/issues/15874) @[LykxSassinator](https://github.com/LykxSassinator) + - Remove unnecessary async blocks to reduce memory usage [#16540](https://github.com/tikv/tikv/issues/16540) @[overvenus](https://github.com/overvenus) + ++ TiFlash + + - Optimize the execution efficiency of `LENGTH()` and `ASCII()` functions [#9344](https://github.com/pingcap/tiflash/issues/9344) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Mitigate the issue that TiFlash might panic due to updating certificates after TLS is enabled [#8535](https://github.com/pingcap/tiflash/issues/8535) @[windtalker](https://github.com/windtalker) + - Improve the cancel mechanism of the JOIN operator, so that the JOIN operator can respond to cancel requests in a timely manner [#9430](https://github.com/pingcap/tiflash/issues/9430) @[windtalker](https://github.com/windtalker) + - Reduce lock conflicts under highly concurrent data read operations and optimize short query performance [#9125](https://github.com/pingcap/tiflash/issues/9125) @[JinheLin](https://github.com/JinheLin) + - Improve the garbage collection speed of outdated data in the background for tables with clustered indexes [#9529](https://github.com/pingcap/tiflash/issues/9529) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Enhance the tolerance of log backup to merge operations. When encountering a reasonably long merge operation, log backup tasks are less likely to enter the error state [#16554](https://github.com/tikv/tikv/issues/16554) @[YuJuncen](https://github.com/YuJuncen) + - BR cleans up empty SST files during data recovery [#16005](https://github.com/tikv/tikv/issues/16005) @[Leavrth](https://github.com/Leavrth) + - Increase the number of retries for failures caused by DNS errors [#53029](https://github.com/pingcap/tidb/issues/53029) @[YuJuncen](https://github.com/YuJuncen) + - Increase the number of retries for failures caused by the absence of a leader in a Region [#54017](https://github.com/pingcap/tidb/issues/54017) @[Leavrth](https://github.com/Leavrth) + - Except for the `br log restore` subcommand, all other `br log` subcommands support skipping the loading of the TiDB `domain` data structure to reduce memory consumption [#52088](https://github.com/pingcap/tidb/issues/52088) @[Leavrth](https://github.com/Leavrth) + - Support checking whether the disk space in TiKV is sufficient before TiKV downloads each SST file. If the space is insufficient, BR terminates the restore and returns an error [#17224](https://github.com/tikv/tikv/issues/17224) @[RidRisR](https://github.com/RidRisR) + - Support setting Alibaba Cloud access credentials through environment variables [#45551](https://github.com/pingcap/tidb/issues/45551) @[RidRisR](https://github.com/RidRisR) + - Reduce unnecessary log printing during backup [#55902](https://github.com/pingcap/tidb/issues/55902) @[Leavrth](https://github.com/Leavrth) + + + TiCDC + + - Support directly outputting raw events when the downstream is a Message Queue (MQ) or cloud storage [#11211](https://github.com/pingcap/tiflow/issues/11211) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Improve memory stability during data recovery using redo logs to reduce the probability of OOM [#10900](https://github.com/pingcap/tiflow/issues/10900) @[CharlesCheung96](https://github.com/CharlesCheung96) + - When the downstream is TiDB with the `SUPER` permission granted, TiCDC supports querying the execution status of `ADD INDEX DDL` from the downstream database to avoid data replication failure due to timeout in retrying executing the DDL statement in some cases [#10682](https://github.com/pingcap/tiflow/issues/10682) @[CharlesCheung96](https://github.com/CharlesCheung96) + + + TiDB Data Migration (DM) + + - Upgrade `go-mysql` to 1.9.1 to support connecting to MySQL server 8.0 using passwords longer than 19 characters [#11603](https://github.com/pingcap/tiflow/pull/11603) @[fishiu](https://github.com/fishiu) + +## Bug fixes + ++ TiDB + + - Fix the issue of inconsistent data indexes caused by concurrent DML operations when adding a unique index [#52914](https://github.com/pingcap/tidb/issues/52914) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that comparing a column of `YEAR` type with an unsigned integer that is out of range causes incorrect results [#50235](https://github.com/pingcap/tidb/issues/50235) @[qw4990](https://github.com/qw4990) + - Fix the issue that `INDEX_HASH_JOIN` cannot exit properly when SQL is abnormally interrupted [#54688](https://github.com/pingcap/tidb/issues/54688) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that the network partition during adding indexes using the Distributed eXecution Framework (DXF) might cause inconsistent data indexes [#54897](https://github.com/pingcap/tidb/issues/54897) @[tangenta](https://github.com/tangenta) + - Fix the issue that using `SHOW WARNINGS;` to obtain warnings might cause a panic [#48756](https://github.com/pingcap/tidb/issues/48756) @[xhebox](https://github.com/xhebox) + - Fix the issue that querying the `INFORMATION_SCHEMA.CLUSTER_SLOW_QUERY` table might cause TiDB to panic [#54324](https://github.com/pingcap/tidb/issues/54324) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue of abnormally high memory usage caused by `memTracker` not being detached when the `HashJoin` or `IndexLookUp` operator is the driven side sub-node of the `Apply` operator [#54005](https://github.com/pingcap/tidb/issues/54005) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that recursive CTE queries might result in invalid pointers [#54449](https://github.com/pingcap/tidb/issues/54449) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that an empty projection causes TiDB to panic [#49109](https://github.com/pingcap/tidb/issues/49109) @[winoros](https://github.com/winoros) + - Fix the issue that TiDB might return incorrect query results when you query tables with virtual columns in transactions that involve data modification operations [#53951](https://github.com/pingcap/tidb/issues/53951) @[qw4990](https://github.com/qw4990) + - Fix the issue that for tables containing auto-increment columns with `AUTO_ID_CACHE=1`, setting `auto_increment_increment` and `auto_increment_offset` system variables to non-default values might cause incorrect auto-increment ID allocation [#52622](https://github.com/pingcap/tidb/issues/52622) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that subqueries included in the `ALL` function might cause incorrect results [#52755](https://github.com/pingcap/tidb/issues/52755) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that predicates cannot be pushed down properly when the filter condition of a SQL query contains virtual columns and the execution condition contains `UnionScan` [#54870](https://github.com/pingcap/tidb/issues/54870) @[qw4990](https://github.com/qw4990) + - Fix the issue that subqueries in an `UPDATE` list might cause TiDB to panic [#52687](https://github.com/pingcap/tidb/issues/52687) @[winoros](https://github.com/winoros) + - Fix the issue that indirect placeholder `?` references in a `GROUP BY` statement cannot find columns [#53872](https://github.com/pingcap/tidb/issues/53872) @[qw4990](https://github.com/qw4990) + - Fix the issue that disk files might not be deleted after the `Sort` operator spills and a query error occurs [#55061](https://github.com/pingcap/tidb/issues/55061) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue of reusing wrong point get plans for `SELECT ... FOR UPDATE` [#54652](https://github.com/pingcap/tidb/issues/54652) @[qw4990](https://github.com/qw4990) + - Fix the issue that `max_execute_time` settings at multiple levels interfere with each other [#50914](https://github.com/pingcap/tidb/issues/50914) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue that the histogram and TopN in the primary key column statistics are not loaded after restarting TiDB [#37548](https://github.com/pingcap/tidb/issues/37548) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the TopN operator might be pushed down incorrectly [#37986](https://github.com/pingcap/tidb/issues/37986) @[qw4990](https://github.com/qw4990) + - Fix the issue that the performance of the `SELECT ... WHERE ... ORDER BY ...` statement execution is poor in some cases [#54969](https://github.com/pingcap/tidb/issues/54969) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that TiDB reports an error in the log when closing the connection in some cases [#53689](https://github.com/pingcap/tidb/issues/53689) @[jackysp](https://github.com/jackysp) + - Fix the issue that the illegal column type `DECIMAL(0,0)` can be created in some cases [#53779](https://github.com/pingcap/tidb/issues/53779) @[tangenta](https://github.com/tangenta) + - Fix the issue that obtaining the column information using `information_schema.columns` returns warning 1356 when a subquery is used as a column definition in a view definition [#54343](https://github.com/pingcap/tidb/issues/54343) @[lance6716](https://github.com/lance6716) + - Fix the issue that the optimizer incorrectly estimates the number of rows as 1 when accessing a unique index with the query condition `column IS NULL` [#56116](https://github.com/pingcap/tidb/issues/56116) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `SELECT INTO OUTFILE` does not work when clustered indexes are used as predicates [#42093](https://github.com/pingcap/tidb/issues/42093) @[qw4990](https://github.com/qw4990) + - Fix the issue of incorrect WARNINGS information when using Optimizer Hints [#53767](https://github.com/pingcap/tidb/issues/53767) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the Sync Load QPS monitoring metric is incorrect [#53558](https://github.com/pingcap/tidb/issues/53558) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that executing `CREATE OR REPLACE VIEW` concurrently might result in the `table doesn't exist` error [#53673](https://github.com/pingcap/tidb/issues/53673) @[tangenta](https://github.com/tangenta) + - Fix the issue that restoring a table with `AUTO_ID_CACHE=1` using the `RESTORE` statement might cause a `Duplicate entry` error [#52680](https://github.com/pingcap/tidb/issues/52680) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the `SUB_PART` value in the `INFORMATION_SCHEMA.STATISTICS` table is `NULL` [#55812](https://github.com/pingcap/tidb/issues/55812) @[Defined2014](https://github.com/Defined2014) + - Fix the overflow issue of the `Longlong` type in predicates [#45783](https://github.com/pingcap/tidb/issues/45783) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that incorrect results are returned when the cached execution plans contain the comparison between date types and `unix_timestamp` [#48165](https://github.com/pingcap/tidb/issues/48165) @[qw4990](https://github.com/qw4990) + - Fix the issue that the `LENGTH()` condition is unexpectedly removed when the collation is `utf8_bin` or `utf8mb4_bin` [#53730](https://github.com/pingcap/tidb/issues/53730) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that when an `UPDATE` or `DELETE` statement contains a recursive CTE, the statement might report an error or not take effect [#55666](https://github.com/pingcap/tidb/issues/55666) @[time-and-fate](https://github.com/time-and-fate) + - Fix the issue that TiDB might hang or return incorrect results when executing a query containing a correlated subquery and CTE [#55551](https://github.com/pingcap/tidb/issues/55551) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that statistics for string columns with non-binary collations might fail to load when initializing statistics [#55684](https://github.com/pingcap/tidb/issues/55684) @[winoros](https://github.com/winoros) + - Fix the issue that IndexJoin produces duplicate rows when calculating hash values in the Left Outer Anti Semi type [#52902](https://github.com/pingcap/tidb/issues/52902) @[yibin87](https://github.com/yibin87) + - Fix the issue that a query statement that contains `UNION` might return incorrect results [#52985](https://github.com/pingcap/tidb/issues/52985) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that empty `groupOffset` in `StreamAggExec` might cause TiDB to panic [#53867](https://github.com/pingcap/tidb/issues/53867) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that RANGE partitioned tables that are not strictly self-incrementing can be created [#54829](https://github.com/pingcap/tidb/issues/54829) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that the query might get stuck when terminated because the memory usage exceeds the limit set by `tidb_mem_quota_query` [#55042](https://github.com/pingcap/tidb/issues/55042) @[yibin87](https://github.com/yibin87) + - Fix the issue that the `STATE` field in the `INFORMATION_SCHEMA.TIDB_TRX` table is empty due to the `size` of the `STATE` field not being defined [#53026](https://github.com/pingcap/tidb/issues/53026) @[cfzjywxk](https://github.com/cfzjywxk) + - Fix the data race issue in `IndexNestedLoopHashJoin` [#49692](https://github.com/pingcap/tidb/issues/49692) @[solotzg](https://github.com/solotzg) + - Fix the issue that a wrong TableDual plan causes empty query results [#50051](https://github.com/pingcap/tidb/issues/50051) @[onlyacat](https://github.com/onlyacat) + - Fix the issue that the `tot_col_size` column in the `mysql.stats_histograms` table might be a negative number [#55126](https://github.com/pingcap/tidb/issues/55126) @[qw4990](https://github.com/qw4990) + - Fix the issue that data conversion from the `FLOAT` type to the `UNSIGNED` type returns incorrect results [#41736](https://github.com/pingcap/tidb/issues/41736) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that TiDB fails to reject unauthenticated user connections in some cases when using the `auth_socket` authentication plugin [#54031](https://github.com/pingcap/tidb/issues/54031) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that the `memory_quota` hint might not work in subqueries [#53834](https://github.com/pingcap/tidb/issues/53834) @[qw4990](https://github.com/qw4990) + - Fix the issue that the metadata lock fails to prevent DDL operations from executing in the plan cache scenario [#51407](https://github.com/pingcap/tidb/issues/51407) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that using `CURRENT_DATE()` as the default value for a column results in incorrect query results [#53746](https://github.com/pingcap/tidb/issues/53746) @[tangenta](https://github.com/tangenta) + - Fix the issue that the `COALESCE()` function returns incorrect result type for `DATE` type parameters [#46475](https://github.com/pingcap/tidb/issues/46475) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Reset the parameters in the `Open` method of `PipelinedWindow` to fix the unexpected error that occurs when the `PipelinedWindow` is used as a child node of `Apply` due to the reuse of previous parameter values caused by repeated opening and closing operations [#53600](https://github.com/pingcap/tidb/issues/53600) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the incorrect result of the TopN operator in correlated subqueries [#52777](https://github.com/pingcap/tidb/issues/52777) @[yibin87](https://github.com/yibin87) + - Fix the issue that the recursive CTE operator incorrectly tracks memory usage [#54181](https://github.com/pingcap/tidb/issues/54181) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that an error occurs when using `SHOW COLUMNS` to view columns in a view [#54964](https://github.com/pingcap/tidb/issues/54964) @[lance6716](https://github.com/lance6716) + - Fix the issue that reducing the value of `tidb_ttl_delete_worker_count` during TTL job execution makes the job fail to complete [#55561](https://github.com/pingcap/tidb/issues/55561) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that using a view does not work in recursive CTE [#49721](https://github.com/pingcap/tidb/issues/49721) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that TiDB does not create corresponding statistics metadata (`stats_meta`) when creating a table with foreign keys [#53652](https://github.com/pingcap/tidb/issues/53652) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the query might return incorrect results instead of an error after being killed [#50089](https://github.com/pingcap/tidb/issues/50089) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that the statistics synchronous loading mechanism might fail unexpectedly under high query concurrency [#52294](https://github.com/pingcap/tidb/issues/52294) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that certain filter conditions in queries might cause the planner module to report an `invalid memory address or nil pointer dereference` error [#53582](https://github.com/pingcap/tidb/issues/53582) [#53580](https://github.com/pingcap/tidb/issues/53580) [#53594](https://github.com/pingcap/tidb/issues/53594) [#53603](https://github.com/pingcap/tidb/issues/53603) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the TiDB synchronously loading statistics mechanism retries to load empty statistics indefinitely and prints the `fail to get stats version for this histogram` log [#52657](https://github.com/pingcap/tidb/issues/52657) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `TIMESTAMPADD()` function goes into an infinite loop when the first argument is `month` and the second argument is negative [#54908](https://github.com/pingcap/tidb/issues/54908) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that TiDB might crash when `tidb_mem_quota_analyze` is enabled and the memory used by updating statistics exceeds the limit [#52601](https://github.com/pingcap/tidb/issues/52601) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `duplicate entry` might occur when adding unique indexes [#56161](https://github.com/pingcap/tidb/issues/56161) @[tangenta](https://github.com/tangenta) + - Fix the issue that the query latency of stale reads increases, caused by information schema cache misses [#53428](https://github.com/pingcap/tidb/issues/53428) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that the `Distinct_count` information in GlobalStats might be incorrect [#53752](https://github.com/pingcap/tidb/issues/53752) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that executing the `SELECT DISTINCT CAST(col AS DECIMAL), CAST(col AS SIGNED) FROM ...` query might return incorrect results [#53726](https://github.com/pingcap/tidb/issues/53726) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `read_from_storage` hint might not take effect when the query has an available Index Merge execution plan [#56217](https://github.com/pingcap/tidb/issues/56217) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that the `TIMESTAMPADD()` function returns incorrect results [#41052](https://github.com/pingcap/tidb/issues/41052) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that `PREPARE`/`EXECUTE` statements with the `CONV` expression containing a `?` argument might result in incorrect query results when executed multiple times [#53505](https://github.com/pingcap/tidb/issues/53505) @[qw4990](https://github.com/qw4990) + - Fix the issue that the memory used by transactions might be tracked multiple times [#53984](https://github.com/pingcap/tidb/issues/53984) @[ekexium](https://github.com/ekexium) + - Fix the issue that column pruning without using shallow copies of slices might cause TiDB to panic [#52768](https://github.com/pingcap/tidb/issues/52768) @[winoros](https://github.com/winoros) + - Fix the issue that a SQL binding containing window functions might not take effect in some cases [#55981](https://github.com/pingcap/tidb/issues/55981) @[winoros](https://github.com/winoros) + - Fix the issue that TiDB might panic when parsing index data [#47115](https://github.com/pingcap/tidb/issues/47115) @[zyguan](https://github.com/zyguan) + - Fix the issue that TiDB might report an error due to GC when loading statistics at startup [#53592](https://github.com/pingcap/tidb/issues/53592) @[you06](https://github.com/you06) + - Fix the issue that an error occurs when a DML statement contains nested generated columns [#53967](https://github.com/pingcap/tidb/issues/53967) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that TiDB panics when executing the `SHOW ERRORS` statement with a predicate that is always `true` [#46962](https://github.com/pingcap/tidb/issues/46962) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that improper use of metadata locks might lead to writing anomalous data when using the plan cache under certain circumstances [#53634](https://github.com/pingcap/tidb/issues/53634) @[zimulala](https://github.com/zimulala) + - Fix the issue of data index inconsistency caused by retries during index addition [#55808](https://github.com/pingcap/tidb/issues/55808) @[lance6716](https://github.com/lance6716) + - Fix the issue that unstable unique IDs of columns might cause the `UPDATE` statement to return errors [#53236](https://github.com/pingcap/tidb/issues/53236) @[winoros](https://github.com/winoros) + - Fix the issue that after a statement within a transaction is killed by OOM, if TiDB continues to execute the next statement within the same transaction, you might get an error `Trying to start aggressive locking while it's already started` and a panic occurs [#53540](https://github.com/pingcap/tidb/issues/53540) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that executing `RECOVER TABLE BY JOB JOB_ID;` might cause TiDB to panic [#55113](https://github.com/pingcap/tidb/issues/55113) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that executing `ADD INDEX` might fail after modifying the PD member in the distributed execution framework [#48680](https://github.com/pingcap/tidb/issues/48680) @[lance6716](https://github.com/lance6716) + - Fix the issue that two DDL Owners might exist at the same time [#54689](https://github.com/pingcap/tidb/issues/54689) @[joccau](https://github.com/joccau) + - Fix the issue that TiDB rolling restart during the execution of `ADD INDEX` might cause the adding index operation to fail [#52805](https://github.com/pingcap/tidb/issues/52805) @[tangenta](https://github.com/tangenta) + - Fix the issue that the `LOAD DATA ... REPLACE INTO` operation causes data inconsistency [#56408](https://github.com/pingcap/tidb/issues/56408) @[fzzf678](https://github.com/fzzf678) + - Fix the issue that the `AUTO_INCREMENT` field is not correctly set after importing data using the `IMPORT INTO` statement [#56476](https://github.com/pingcap/tidb/issues/56476) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that TiDB does not check for the existence of local files before restoring from a checkpoint [#53009](https://github.com/pingcap/tidb/issues/53009) @[lance6716](https://github.com/lance6716) + - Fix the issue that the DM schema tracker cannot create indexes longer than the default length [#55138](https://github.com/pingcap/tidb/issues/55138) @[lance6716](https://github.com/lance6716) + - Fix the issue that `ALTER TABLE` does not handle the `AUTO_INCREMENT` field correctly [#47899](https://github.com/pingcap/tidb/issues/47899) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that unreleased session resources might lead to memory leaks [#56271](https://github.com/pingcap/tidb/issues/56271) @[lance6716](https://github.com/lance6716) + - Fix the issue that float or integer overflow affects the plan cache [#46538](https://github.com/pingcap/tidb/issues/46538) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that part of the memory of the `IndexLookUp` operator is not tracked [#56440](https://github.com/pingcap/tidb/issues/56440) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that stale read does not strictly verify the timestamp of the read operation, resulting in a small probability of affecting the consistency of the transaction when an offset exists between the TSO and the real physical time [#56809](https://github.com/pingcap/tidb/issues/56809) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that TTL might fail if TiKV is not selected as the storage engine [#56402](https://github.com/pingcap/tidb/issues/56402) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that TTL tasks cannot be canceled when there is a write conflict [#56422](https://github.com/pingcap/tidb/issues/56422) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that inserting oversized numbers in scientific notation causes an error `ERROR 1264 (22003)`, to make the behavior consistent with MySQL [#47787](https://github.com/pingcap/tidb/issues/47787) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that when canceling a TTL task, the corresponding SQL is not killed forcibly [#56511](https://github.com/pingcap/tidb/issues/56511) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that the `INSERT ... ON DUPLICATE KEY` statement is not compatible with `mysql_insert_id` [#55965](https://github.com/pingcap/tidb/issues/55965) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that audit log filtering does not take effect when SQL cannot build an execution plan [#50988](https://github.com/pingcap/tidb/issues/50988) @[CbcWestwolf](https://github.com/CbcWestwolf) + - Fix the issue that existing TTL tasks are executed unexpectedly frequently in a cluster that is upgraded from v6.5 to v7.5 or later [#56539](https://github.com/pingcap/tidb/issues/56539) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that the `CAST` function does not support explicitly setting the character set [#55677](https://github.com/pingcap/tidb/issues/55677) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that TiDB does not check the index length limitation when executing `ADD INDEX` [#56930](https://github.com/pingcap/tidb/issues/56930) @[fzzf678](https://github.com/fzzf678) + ++ TiKV + + - Add the `RawKvMaxTimestampNotSynced` error, log detailed error information in `errorpb.Error.max_ts_not_synced`, and add a retry mechanism for the `must_raw_put` operation when this error occurs [#16789](https://github.com/tikv/tikv/issues/16789) @[pingyu](https://github.com/pingyu) + - Fix a traffic control issue that might occur after deleting large tables or partitions [#17304](https://github.com/tikv/tikv/issues/17304) @[SpadeA-Tang](https://github.com/SpadeA-Tang) + - Fix the panic issue that occurs when read threads access outdated indexes in the MemTable of the Raft Engine [#17383](https://github.com/tikv/tikv/issues/17383) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that CDC and log-backup do not limit the timeout of `check_leader` using the `advance-ts-interval` configuration, causing the `resolved_ts` lag to be too large when TiKV restarts normally in some cases [#17107](https://github.com/tikv/tikv/issues/17107) @[SpadeA-Tang](https://github.com/SpadeA-Tang) + - Fix the issue that SST files imported by TiDB Lightning are lost after TiKV is restarted [#15912](https://github.com/tikv/tikv/issues/15912) @[lance6716](https://github.com/lance6716) + - Fix the issue that TiKV might panic due to ingesting deleted `sst_importer` SST files [#15053](https://github.com/tikv/tikv/issues/15053) @[lance6716](https://github.com/lance6716) + - Fix the issue that when there are a large number of Regions in a TiKV instance, TiKV might be OOM during data import [#16229](https://github.com/tikv/tikv/issues/16229) @[SpadeA-Tang](https://github.com/SpadeA-Tang) + - Fix the issue that bloom filters are incompatible between earlier versions (earlier than v7.1) and later versions [#17272](https://github.com/tikv/tikv/issues/17272) @[v01dstar](https://github.com/v01dstar) + - Fix the issue that setting the gRPC message compression method via `grpc-compression-type` does not take effect on messages sent from TiKV to TiDB [#17176](https://github.com/tikv/tikv/issues/17176) @[ekexium](https://github.com/ekexium) + - Fix the issue of unstable test cases, ensuring that each test uses an independent temporary directory to avoid online configuration changes affecting other test cases [#16871](https://github.com/tikv/tikv/issues/16871) @[glorv](https://github.com/glorv) + - Fix the issue that when a large number of transactions are queuing for lock release on the same key and the key is frequently updated, excessive pressure on deadlock detection might cause TiKV OOM issues [#17394](https://github.com/tikv/tikv/issues/17394) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that the decimal part of the `DECIMAL` type is incorrect in some cases [#16913](https://github.com/tikv/tikv/issues/16913) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that the `CONV()` function in queries might overflow during numeric system conversion, leading to TiKV panic [#16969](https://github.com/tikv/tikv/issues/16969) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that TiKV might panic when a stale replica processes Raft snapshots, triggered by a slow split operation and immediate removal of the new replica [#17469](https://github.com/tikv/tikv/issues/17469) @[hbisheng](https://github.com/hbisheng) + - Fix the issue that highly concurrent Coprocessor requests might cause TiKV OOM [#16653](https://github.com/tikv/tikv/issues/16653) @[overvenus](https://github.com/overvenus) + - Fix the issue that prevents master key rotation when the master key is stored in a Key Management Service (KMS) [#17410](https://github.com/tikv/tikv/issues/17410) @[hhwyt](https://github.com/hhwyt) + - Fix the issue that the output of the `raft region` command in tikv-ctl does not include the Region status information [#17037](https://github.com/tikv/tikv/issues/17037) @[glorv](https://github.com/glorv) + - Fix the issue that the **Storage async write duration** monitoring metric on the TiKV panel in Grafana is inaccurate [#17579](https://github.com/tikv/tikv/issues/17579) @[overvenus](https://github.com/overvenus) + - Fix the issue that TiKV converts the time zone incorrectly for Brazil and Egypt [#16220](https://github.com/tikv/tikv/issues/16220) @[overvenus](https://github.com/overvenus) + ++ PD + + - Fix the memory leak issue in label statistics [#8700](https://github.com/tikv/pd/issues/8700) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that resource groups print excessive logs [#8159](https://github.com/tikv/pd/issues/8159) @[nolouch](https://github.com/nolouch) + - Fix the performance jitter issue caused by frequent creation of random number generator [#8674](https://github.com/tikv/pd/issues/8674) @[rleungx](https://github.com/rleungx) + - Fix the memory leak issue in Region statistics [#8710](https://github.com/tikv/pd/issues/8710) @[rleungx](https://github.com/rleungx) + - Fix the memory leak issue in hotspot cache [#8698](https://github.com/tikv/pd/issues/8698) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that `evict-leader-scheduler` fails to work properly when it is repeatedly created with the same Store ID [#8756](https://github.com/tikv/pd/issues/8756) @[okJiang](https://github.com/okJiang) + - Fix the issue that setting `replication.strictly-match-label` to `true` causes TiFlash to fail to start [#8480](https://github.com/tikv/pd/issues/8480) @[rleungx](https://github.com/rleungx) + - Fix the issue that changing the log level via the configuration file does not take effect [#8117](https://github.com/tikv/pd/issues/8117) @[rleungx](https://github.com/rleungx) + - Fix the issue that resource groups could not effectively limit resource usage under high concurrency [#8435](https://github.com/tikv/pd/issues/8435) @[nolouch](https://github.com/nolouch) + - Fix the data race issue that PD encounters during operator checks [#8263](https://github.com/tikv/pd/issues/8263) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that a resource group encounters quota limits when requesting tokens for more than 500 ms [#8349](https://github.com/tikv/pd/issues/8349) @[nolouch](https://github.com/nolouch) + - Fix the issue that some logs are not redacted [#8419](https://github.com/tikv/pd/issues/8419) @[rleungx](https://github.com/rleungx) + - Fix the issue that no error is reported when binding a role to a resource group [#54417](https://github.com/pingcap/tidb/issues/54417) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that PD's Region API cannot be requested when a large number of Regions exist [#55872](https://github.com/pingcap/tidb/issues/55872) @[rleungx](https://github.com/rleungx) + - Fix the issue that a large number of retries occur when canceling resource groups queries [#8217](https://github.com/tikv/pd/issues/8217) @[nolouch](https://github.com/nolouch) + - Fix the issue that the encryption manager is not initialized before use [#8384](https://github.com/tikv/pd/issues/8384) @[rleungx](https://github.com/rleungx) + - Fix the issue that the `Filter target` monitoring metric for PD does not provide scatter range information [#8125](https://github.com/tikv/pd/issues/8125) @[HuSharp](https://github.com/HuSharp) + - Fix the data race issue of resource groups [#8267](https://github.com/tikv/pd/issues/8267) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that setting the TiKV configuration item [`coprocessor.region-split-size`](/tikv-configuration-file.md#region-split-size) to a value less than 1 MiB causes PD panic [#8323](https://github.com/tikv/pd/issues/8323) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that when using a wrong parameter in `evict-leader-scheduler`, PD does not report errors correctly and some schedulers are unavailable [#8619](https://github.com/tikv/pd/issues/8619) @[rleungx](https://github.com/rleungx) + - Fix the issue that slots are not fully deleted in a resource group client, which causes the number of the allocated tokens to be less than the specified value [#7346](https://github.com/tikv/pd/issues/7346) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that down peers might not recover when using Placement Rules [#7808](https://github.com/tikv/pd/issues/7808) @[rleungx](https://github.com/rleungx) + ++ TiFlash + + - Fix the issue that TiFlash metadata might become corrupted and cause the process to panic when upgrading a cluster from a version earlier than v6.5.0 to v6.5.0 or later [#9039](https://github.com/pingcap/tiflash/issues/9039) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that some queries might report a column type mismatch error after late materialization is enabled [#9175](https://github.com/pingcap/tiflash/issues/9175) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that some queries might report errors when late materialization is enabled [#9472](https://github.com/pingcap/tiflash/issues/9472) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that some JSON functions unsupported by TiFlash are pushed down to TiFlash [#9444](https://github.com/pingcap/tiflash/issues/9444) @[windtalker](https://github.com/windtalker) + - Fix the issue that setting the SSL certificate configuration to an empty string in TiFlash incorrectly enables TLS and causes TiFlash to fail to start [#9235](https://github.com/pingcap/tiflash/issues/9235) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that a network partition (network disconnection) between TiFlash and any PD might cause read request timeout errors [#9243](https://github.com/pingcap/tiflash/issues/9243) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that a large number of duplicate rows might be read in FastScan mode after importing data via BR or TiDB Lightning [#9118](https://github.com/pingcap/tiflash/issues/9118) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that TiFlash fails to parse the table schema when the table contains Bit-type columns with a default value that contains invalid characters [#9461](https://github.com/pingcap/tiflash/issues/9461) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that queries with virtual generated columns might return incorrect results after late materialization is enabled [#9188](https://github.com/pingcap/tiflash/issues/9188) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that TiFlash might fail to synchronize schemas after executing `ALTER TABLE ... EXCHANGE PARTITION` across databases [#7296](https://github.com/pingcap/tiflash/issues/7296) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash might panic when a database is deleted shortly after creation [#9266](https://github.com/pingcap/tiflash/issues/9266) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that when using the `CAST()` function to convert a string to a datetime with a time zone or invalid characters, the result is incorrect [#8754](https://github.com/pingcap/tiflash/issues/8754) @[solotzg](https://github.com/solotzg) + - Fix the issue that TiFlash might return transiently incorrect results in high-concurrency read scenarios [#8845](https://github.com/pingcap/tiflash/issues/8845) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that the `SUBSTRING_INDEX()` function might cause TiFlash to crash in some corner cases [#9116](https://github.com/pingcap/tiflash/issues/9116) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that frequent `EXCHANGE PARTITION` and `DROP TABLE` operations over a long period in a cluster might slow down the replication of TiFlash table metadata and degrade the query performance [#9227](https://github.com/pingcap/tiflash/issues/9227) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that a query with an empty key range fails to correctly generate read tasks on TiFlash, which might block TiFlash queries [#9108](https://github.com/pingcap/tiflash/issues/9108) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that the sign in the result of the `CAST AS DECIMAL` function is incorrect in certain cases [#9301](https://github.com/pingcap/tiflash/issues/9301) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that the `SUBSTRING()` function does not support the `pos` and `len` arguments for certain integer types, causing query errors [#9473](https://github.com/pingcap/tiflash/issues/9473) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that executing `DROP TABLE` on large tables might cause TiFlash OOM [#9437](https://github.com/pingcap/tiflash/issues/9437) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that BR integration test cases are unstable, and add a new test case to simulate snapshot or log backup file corruption [#53835](https://github.com/pingcap/tidb/issues/53835) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that DDLs requiring backfilling, such as `ADD INDEX` and `MODIFY COLUMN`, might not be correctly recovered during incremental restore [#54426](https://github.com/pingcap/tidb/issues/54426) @[3pointer](https://github.com/3pointer) + - Fix the issue that after a log backup PITR task fails and you stop it, the safepoints related to that task are not properly cleared in PD [#17316](https://github.com/tikv/tikv/issues/17316) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that log backup might be paused after the advancer owner migration [#53561](https://github.com/pingcap/tidb/issues/53561) @[RidRisR](https://github.com/RidRisR) + - Fix the inefficiency issue in scanning DDL jobs during incremental backups [#54139](https://github.com/pingcap/tidb/issues/54139) @[3pointer](https://github.com/3pointer) + - Fix the issue that the backup performance during checkpoint backups is affected due to interruptions in seeking Region leaders [#17168](https://github.com/tikv/tikv/issues/17168) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that BR logs might print sensitive credential information when log backup is enabled [#55273](https://github.com/pingcap/tidb/issues/55273) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that BR fails to correctly identify errors due to multiple nested retries during the restore process [#54053](https://github.com/pingcap/tidb/issues/54053) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that TiKV might panic when resuming a paused log backup task with unstable network connections to PD [#17020](https://github.com/tikv/tikv/issues/17020) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that backup tasks might get stuck if TiKV becomes unresponsive during the backup process [#53480](https://github.com/pingcap/tidb/issues/53480) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the checkpoint path of backup and restore is incompatible with some external storage [#55265](https://github.com/pingcap/tidb/issues/55265) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the Region fetched from PD does not have a Leader when restoring data using BR or importing data using TiDB Lightning in physical import mode [#51124](https://github.com/pingcap/tidb/issues/51124) [#50501](https://github.com/pingcap/tidb/issues/50501) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the transfer of PD leaders might cause BR to panic when restoring data [#53724](https://github.com/pingcap/tidb/issues/53724) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that after pausing, stopping, and rebuilding the log backup task, the task status is normal, but the checkpoint does not advance [#53047](https://github.com/pingcap/tidb/issues/53047) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that log backups cannot resolve residual locks promptly, causing the checkpoint to fail to advance [#57134](https://github.com/pingcap/tidb/issues/57134) @[3pointer](https://github.com/3pointer) + + + TiCDC + + - Fix the issue that the default value of `TIMEZONE` type is not set according to the correct time zone [#10931](https://github.com/pingcap/tiflow/issues/10931) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that TiCDC might panic when the Sorter module reads disk data [#10853](https://github.com/pingcap/tiflow/issues/10853) @[hicqu](https://github.com/hicqu) + - Fix the issue that data inconsistency might occur when restarting Changefeed repeatedly when performing a large number of `UPDATE` operations in a multi-node environment [#11219](https://github.com/pingcap/tiflow/issues/11219) @[lidezhu](https://github.com/lidezhu) + - Fix the issue that after filtering out `add table partition` events is configured in `ignore-event`, TiCDC does not replicate other types of DML changes for related partitions to the downstream [#10524](https://github.com/pingcap/tiflow/issues/10524) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that TiCDC might get stuck when replicating data to Kafka [#9855](https://github.com/pingcap/tiflow/issues/9855) @[hicqu](https://github.com/hicqu) + - Fix the issue that `DROP PRIMARY KEY` and `DROP UNIQUE KEY` statements are not replicated correctly [#10890](https://github.com/pingcap/tiflow/issues/10890) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that the Processor module might get stuck when the downstream Kafka is inaccessible [#11340](https://github.com/pingcap/tiflow/issues/11340) @[asddongmen](https://github.com/asddongmen) + + + TiDB Data Migration (DM) + + - Fix the issue that DM does not set the default database when processing the `ALTER DATABASE` statement, which causes a replication error [#11503](https://github.com/pingcap/tiflow/issues/11503) @[lance6716](https://github.com/lance6716) + - Fix the issue that multiple DM-master nodes might simultaneously become leaders, leading to data inconsistency [#11602](https://github.com/pingcap/tiflow/issues/11602) @[GMHDBJD](https://github.com/GMHDBJD) + - Fix the connection blocking issue by upgrading `go-mysql` [#11041](https://github.com/pingcap/tiflow/issues/11041) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that data replication is interrupted when the index length exceeds the default value of `max-index-length` [#11459](https://github.com/pingcap/tiflow/issues/11459) @[michaelmdeng](https://github.com/michaelmdeng) + - Fix the issue that DM returns an error when replicating the `ALTER TABLE ... DROP PARTITION` statement for LIST partitioned tables [#54760](https://github.com/pingcap/tidb/issues/54760) @[lance6716](https://github.com/lance6716) + + + TiDB Lightning + + - Fix the issue that TiDB Lightning fails to receive oversized messages sent from TiKV [#56114](https://github.com/pingcap/tidb/issues/56114) @[fishiu](https://github.com/fishiu) + - Fix the issue that TiKV data might be corrupted when importing data after disabling the import mode of TiDB Lightning [#15003](https://github.com/tikv/tikv/issues/15003) [#47694](https://github.com/pingcap/tidb/issues/47694) @[lance6716](https://github.com/lance6716) + - Fix the issue that transaction conflicts occur during data import using TiDB Lightning [#49826](https://github.com/pingcap/tidb/issues/49826) @[lance6716](https://github.com/lance6716) + - Fix the issue that TiDB Lightning might fail to import data when EBS BR is running [#49517](https://github.com/pingcap/tidb/issues/49517) @[mittalrishabh](https://github.com/mittalrishabh) + - Fix the issue that TiDB Lightning reports a `verify allocator base failed` error when two instances simultaneously start parallel import tasks and are assigned the same task ID [#55384](https://github.com/pingcap/tidb/issues/55384) @[ei-sugimoto](https://github.com/ei-sugimoto) + - Fix the issue that killing the PD Leader causes TiDB Lightning to report the `invalid store ID 0` error during data import [#50501](https://github.com/pingcap/tidb/issues/50501) @[Leavrth](https://github.com/Leavrth) + + + Dumpling + + - Fix the issue that Dumpling reports an error when exporting tables and views at the same time [#53682](https://github.com/pingcap/tidb/issues/53682) @[tangenta](https://github.com/tangenta) + + + TiDB Binlog + + - Fix the issue that deleting rows during the execution of `ADD COLUMN` might report an error `data and columnID count not match` when TiDB Binlog is enabled [#53133](https://github.com/pingcap/tidb/issues/53133) @[tangenta](https://github.com/tangenta) diff --git a/releases/release-7.2.0.md b/releases/release-7.2.0.md index 168dc1bebdde0..af7949497d7eb 100644 --- a/releases/release-7.2.0.md +++ b/releases/release-7.2.0.md @@ -172,7 +172,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with- ### Behavior changes -- When processing update event, TiCDC splits an event into delete and insert events if the primary key or non-null unique index value is modified in the event. For more information, see [documentation](/ticdc/ticdc-behavior-change.md#transactions-containing-a-single-update-change). +- When processing update event, TiCDC splits an event into delete and insert events if the primary key or non-null unique index value is modified in the event. For more information, see [documentation](/ticdc/ticdc-split-update-behavior.md#transactions-containing-a-single-update-change). ### System variables @@ -240,7 +240,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with- + TiCDC - Optimize the structure of the directory where data files are stored when a DDL operation occurs in the scenario of replication to an object storage service [#8891](https://github.com/pingcap/tiflow/issues/8891) @[CharlesCheung96](https://github.com/CharlesCheung96) - - Support the OAUTHBEARER authentication in the scenario of replication to Kafka [#8865](https://github.com/pingcap/tiflow/issues/8865) @[hi-rustin](https://github.com/hi-rustin) + - Support the OAUTHBEARER authentication in the scenario of replication to Kafka [#8865](https://github.com/pingcap/tiflow/issues/8865) @[hi-rustin](https://github.com/Rustin170506) - Add the option of outputting only the handle keys for the `DELETE` operation in the scenario of replication to Kafka [#9143](https://github.com/pingcap/tiflow/issues/9143) @[3AceShowHand](https://github.com/3AceShowHand) + TiDB Data Migration (DM) @@ -263,6 +263,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with- - Fix the issue that the stale read global optimization does not take effect due to the lack of `TxnScope` in Coprocessor tasks [#43365](https://github.com/pingcap/tidb/issues/43365) @[you06](https://github.com/you06) - Fix the issue that follower read does not handle flashback errors before retrying, which causes query errors [#43673](https://github.com/pingcap/tidb/issues/43673) @[you06](https://github.com/you06) - Fix the issue that data and indexes are inconsistent when the `ON UPDATE` statement does not correctly update the primary key [#44565](https://github.com/pingcap/tidb/issues/44565) @[zyguan](https://github.com/zyguan) + - Fix the case sensitivity issue in some columns of the permission table [#41048](https://github.com/pingcap/tidb/issues/41048) @[bb7133](https://github.com/bb7133) - Modify the upper limit of the `UNIX_TIMESTAMP()` function to `3001-01-19 03:14:07.999999 UTC` to be consistent with that of MySQL 8.0.28 or later versions [#43987](https://github.com/pingcap/tidb/issues/43987) @[YangKeao](https://github.com/YangKeao) - Fix the issue that adding an index fails in the ingest mode [#44137](https://github.com/pingcap/tidb/issues/44137) @[tangenta](https://github.com/tangenta) - Fix the issue that canceling a DDL task in the rollback state causes errors in related metadata [#44143](https://github.com/pingcap/tidb/issues/44143) @[wjhuang2016](https://github.com/wjhuang2016) @@ -304,10 +305,10 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.2/quick-start-with- - Fix the issue that Resolved TS does not advance properly in some cases [#8963](https://github.com/pingcap/tiflow/issues/8963) @[CharlesCheung96](https://github.com/CharlesCheung96) - Fix the issue that the `UPDATE` operation cannot output old values when the Avro or CSV protocol is used [#9086](https://github.com/pingcap/tiflow/issues/9086) @[3AceShowHand](https://github.com/3AceShowHand) - - Fix the issue of excessive downstream pressure caused by reading downstream metadata too frequently when replicating data to Kafka [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue of excessive downstream pressure caused by reading downstream metadata too frequently when replicating data to Kafka [#8959](https://github.com/pingcap/tiflow/issues/8959) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue of too many downstream logs caused by frequently setting the downstream bidirectional replication-related variables when replicating data to TiDB or MySQL [#9180](https://github.com/pingcap/tiflow/issues/9180) @[asddongmen](https://github.com/asddongmen) - Fix the issue that the PD node crashing causes the TiCDC node to restart [#8868](https://github.com/pingcap/tiflow/issues/8868) @[asddongmen](https://github.com/asddongmen) - - Fix the issue that TiCDC cannot create a changefeed with a downstream Kafka-on-Pulsar [#8892](https://github.com/pingcap/tiflow/issues/8892) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that TiCDC cannot create a changefeed with a downstream Kafka-on-Pulsar [#8892](https://github.com/pingcap/tiflow/issues/8892) @[hi-rustin](https://github.com/Rustin170506) + TiDB Lightning @@ -324,7 +325,7 @@ We would like to thank the following contributors from the TiDB community: - [darraes](https://github.com/darraes) - [demoManito](https://github.com/demoManito) - [dhysum](https://github.com/dhysum) -- [HappyUncle](https://github.com/HappyUncle) +- [happy-v587](https://github.com/happy-v587) - [jiyfhust](https://github.com/jiyfhust) - [L-maple](https://github.com/L-maple) - [nyurik](https://github.com/nyurik) diff --git a/releases/release-7.4.0.md b/releases/release-7.4.0.md index 64faee8d149eb..af131dc40c5e3 100644 --- a/releases/release-7.4.0.md +++ b/releases/release-7.4.0.md @@ -185,7 +185,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with- For more information, see [documentation](/tidb-resource-control.md#manage-background-tasks). -* Lock statistics becomes generally available (GA) [#46351](https://github.com/pingcap/tidb/issues/46351) @[hi-rustin](https://github.com/hi-rustin) +* Lock statistics becomes generally available (GA) [#46351](https://github.com/pingcap/tidb/issues/46351) @[hi-rustin](https://github.com/Rustin170506) In v7.4.0, [lock statistics](/statistics.md#lock-statistics) becomes generally available. Now, to ensure operational security, locking and unlocking statistics require the same privileges as collecting statistics. In addition, TiDB supports locking and unlocking statistics for specific partitions, providing greater flexibility. If you are confident in queries and execution plans in the database and want to prevent any changes from occurring, you can lock statistics to enhance stability. @@ -199,6 +199,14 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with- For more information, see [documentation](/system-variables.md#tidb_opt_enable_hash_join-new-in-v656-v712-and-v740). +* Memory control for the statistics cache is generally available (GA) [#45367](https://github.com/pingcap/tidb/issues/45367) @[hawkingrei](https://github.com/hawkingrei) + + TiDB instances can cache table statistics to accelerate execution plan generation and improve SQL performance. Starting from v6.1.0, TiDB introduces the system variable [`tidb_stats_cache_mem_quota`](https://github.com/system-variables.md#tidb_stats_cache_mem_quota-new-in-v610). By configuring this system variable, you can set a memory usage limit for the statistics cache. When the cache reaches its limit, TiDB automatically evicts inactive cache entries, helping control instance memory usage and improve stability. + + Starting from v7.4.0, this feature becomes generally available (GA). + + For more information, see [documentation](/system-variables.md#tidb_stats_cache_mem_quota-new-in-v610). + ### SQL * TiDB supports partition type management [#42728](https://github.com/pingcap/tidb/issues/42728) @[mjonss](https://github.com/mjonss) @@ -286,7 +294,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with- - The [`information_schema.CHECK_CONSTRAINTS`](/information-schema/information-schema-check-constraints.md) table is added for improved compatibility with MySQL 8.0. -- For transactions containing multiple changes, if the primary key or non-null unique index value is modified in the update event, TiCDC splits an event into delete and insert events and ensures that all events follow the sequence of delete events preceding insert events. For more information, see [documentation](/ticdc/ticdc-behavior-change.md#transactions-containing-multiple-update-changes). +- For transactions containing multiple changes, if the primary key or non-null unique index value is modified in the update event, TiCDC splits an event into delete and insert events and ensures that all events follow the sequence of delete events preceding insert events. For more information, see [documentation](/ticdc/ticdc-split-update-behavior.md#transactions-containing-multiple-update-changes). ### System variables @@ -321,10 +329,11 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.4/quick-start-with- | TiCDC | [`large-message-handle-compression`](/ticdc/ticdc-sink-to-kafka.md#ticdc-data-compression) | Newly added | Controls whether to enable compression during encoding. The default value is empty, which means not enabled. | | TiCDC | [`large-message-handle-option`](/ticdc/ticdc-sink-to-kafka.md#send-large-messages-to-external-storage) | Modified | This configuration item adds a new value `claim-check`. When it is set to `claim-check`, TiCDC Kafka sink supports sending the message to external storage when the message size exceeds the limit and sends a message to Kafka containing the address of this large message in external storage. | -## Deprecated features +## Deprecated and removed features + [Mydumper](https://docs.pingcap.com/tidb/v4.0/mydumper-overview) will be deprecated in v7.5.0 and most of its features have been replaced by [Dumpling](/dumpling-overview.md). It is strongly recommended that you use Dumpling instead of mydumper. + TiKV-importer will be deprecated in v7.5.0. It is strongly recommended that you use the [Physical Import Mode of TiDB Lightning](/tidb-lightning/tidb-lightning-physical-import-mode.md) as an alternative. ++ The `enable-old-value` parameter of TiCDC is removed. [#9667](https://github.com/pingcap/tiflow/issues/9667) @[3AceShowHand](https://github.com/3AceShowHand) ## Improvements @@ -496,4 +505,4 @@ We would like to thank the following contributors from the TiDB community: - [shawn0915](https://github.com/shawn0915) - [tedyu](https://github.com/tedyu) - [yumchina](https://github.com/yumchina) -- [ZhuohaoHe](https://github.com/ZhuohaoHe) +- [ZzzhHe](https://github.com/ZzzhHe) diff --git a/releases/release-7.5.0.md b/releases/release-7.5.0.md index 4f46e13c688d7..124a9836322bb 100644 --- a/releases/release-7.5.0.md +++ b/releases/release-7.5.0.md @@ -13,7 +13,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with- TiDB 7.5.0 is a Long-Term Support Release (LTS). -Compared with the previous LTS 7.1.0, 7.5.0 includes new features, improvements, and bug fixes released in [7.2.0-DMR](/releases/release-7.2.0.md), [7.3.0-DMR](/releases/release-7.3.0.md), and [7.4.0-DMR](/releases/release-7.4.0.md). When you upgrade from 7.1.x to 7.5.0, you can download the [TiDB Release Notes PDF](https://download.pingcap.org/tidb-v7.2-to-v7.5-en-release-notes.pdf) to view all release notes between the two LTS versions. The following table lists some highlights from 7.2.0 to 7.5.0: +Compared with the previous LTS 7.1.0, 7.5.0 includes new features, improvements, and bug fixes released in [7.2.0-DMR](/releases/release-7.2.0.md), [7.3.0-DMR](/releases/release-7.3.0.md), and [7.4.0-DMR](/releases/release-7.4.0.md). When you upgrade from 7.1.x to 7.5.0, you can download the [TiDB Release Notes PDF](https://docs-download.pingcap.com/pdf/tidb-v7.2-to-v7.5-en-release-notes.pdf) to view all release notes between the two LTS versions. The following table lists some highlights from 7.2.0 to 7.5.0: @@ -172,6 +172,7 @@ Compared with the previous LTS 7.1.0, 7.5.0 includes new features, improvements, | Configuration file | Configuration parameter | Change type | Description | | -------- | -------- | -------- | -------- | | TiDB | [`tikv-client.copr-req-timeout`](/tidb-configuration-file.md#copr-req-timeout-new-in-v750) | Newly added | Sets the timeout of a single Coprocessor request. | +| TiKV | [`raftstore.inspect-interval`](/tikv-configuration-file.md#inspect-interval) | Modified | Changes the default value from `500ms` to `100ms` after optimizing the algorithm to improve the sensitivity of slow node detection. | | TiKV | [`raftstore.region-compact-min-redundant-rows`](/tikv-configuration-file.md#region-compact-min-redundant-rows-new-in-v710) | Modified | Sets the number of redundant MVCC rows required to trigger RocksDB compaction. Starting from v7.5.0, this configuration item takes effect for the `"raft-kv"` storage engine. | | TiKV | [`raftstore.region-compact-redundant-rows-percent`](/tikv-configuration-file.md#region-compact-redundant-rows-percent-new-in-v710) | Modified | Sets the percentage of redundant MVCC rows required to trigger RocksDB compaction. Starting from v7.5.0, this configuration item takes effect for the `"raft-kv"` storage engine. | | TiKV | [`raftstore.evict-cache-on-memory-ratio`](/tikv-configuration-file.md#evict-cache-on-memory-ratio-new-in-v750) | Newly added | When the memory usage of TiKV exceeds 90% of the system available memory, and the memory occupied by Raft entry cache exceeds the `evict-cache-on-memory-ratio` of used memory, TiKV evicts the Raft entry cache. | @@ -245,7 +246,7 @@ Starting from v7.5.0, the following contents are removed from the `TiDB-communit - Fix the issue of uneven workload caused by the rejoining of a TiFlash node that has been offline for a long time [#35418](https://github.com/pingcap/tidb/issues/35418) @[windtalker](https://github.com/windtalker) - Fix the issue that the chunk cannot be reused when the HashJoin operator performs probe [#48082](https://github.com/pingcap/tidb/issues/48082) @[wshwsh12](https://github.com/wshwsh12) - Fix the issue that the `COALESCE()` function returns incorrect result type for `DATE` type parameters [#46475](https://github.com/pingcap/tidb/issues/46475) @[xzhangxian1008](https://github.com/xzhangxian1008) - - Fix the issue that `UPDATE` statements with subqueries are incorrectly converted to PointGet [#48171](https://github.com/pingcap/tidb/issues/48171) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that `UPDATE` statements with subqueries are incorrectly converted to PointGet [#48171](https://github.com/pingcap/tidb/issues/48171) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that incorrect results are returned when the cached execution plans contain the comparison between date types and `unix_timestamp` [#48165](https://github.com/pingcap/tidb/issues/48165) @[qw4990](https://github.com/qw4990) - Fix the issue that an error is reported when default inline common table expressions (CTEs) with aggregate functions or window functions are referenced by recursive CTEs [#47881](https://github.com/pingcap/tidb/issues/47881) @[elsa0520](https://github.com/elsa0520) - Fix the issue that the optimizer mistakenly selects IndexFullScan to reduce sort introduced by window functions [#46177](https://github.com/pingcap/tidb/issues/46177) @[qw4990](https://github.com/qw4990) @@ -294,7 +295,7 @@ Starting from v7.5.0, the following contents are removed from the `TiDB-communit ## Performance test -To learn about the performance of TiDB v7.5.0, you can refer to the [TPC-C performance test report](https://docs.pingcap.com/tidbcloud/v7.5.0-performance-benchmarking-with-tpcc) and [Sysbench performance test report](https://docs.pingcap.com/tidbcloud/v7.5.0-performance-benchmarking-with-sysbench) of the TiDB Dedicated cluster. +To learn about the performance of TiDB v7.5.0, you can refer to the [TPC-C performance test report](https://docs.pingcap.com/tidbcloud/v7.5.0-performance-benchmarking-with-tpcc) and [Sysbench performance test report](https://docs.pingcap.com/tidbcloud/v7.5.0-performance-benchmarking-with-sysbench) of the TiDB Cloud Dedicated cluster. ## Contributors diff --git a/releases/release-7.5.1.md b/releases/release-7.5.1.md index de7c5c368cadf..e519214530e67 100644 --- a/releases/release-7.5.1.md +++ b/releases/release-7.5.1.md @@ -43,6 +43,10 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with- - Support multiple accelerated `ADD INDEX` DDL tasks to be queued for execution, instead of falling back to normal `ADD INDEX` tasks [#47758](https://github.com/pingcap/tidb/issues/47758) @[tangenta](https://github.com/tangenta) - Improve the speed of adding indexes to empty tables [#49682](https://github.com/pingcap/tidb/issues/49682) @[zimulala](https://github.com/zimulala) ++ TiKV + + - Enhance the slow store detection algorithm by improving its sensitivity and reducing the false-positive rate, especially in intensive read and write load scenarios [#15909](https://github.com/tikv/tikv/issues/15909) @[LykxSassinator](https://github.com/LykxSassinator) + + TiFlash - Improve the calculation method for [Request Unit (RU)](/tidb-resource-control.md#what-is-request-unit-ru) to make RU values more stable [#8391](https://github.com/pingcap/tiflash/issues/8391) @[guo-shaoge](https://github.com/guo-shaoge) @@ -85,11 +89,11 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with- - Fix the issue that wrong results might be returned when TiFlash late materialization processes associated columns [#49241](https://github.com/pingcap/tidb/issues/49241) [#51204](https://github.com/pingcap/tidb/issues/51204) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) - Fix the issue that the background job thread of TiDB might panic when TiDB records historical statistics [#49076](https://github.com/pingcap/tidb/issues/49076) @[hawkingrei](https://github.com/hawkingrei) - Fix the error that might occur when TiDB merges histograms of global statistics for partitioned tables [#49023](https://github.com/pingcap/tidb/issues/49023) @[hawkingrei](https://github.com/hawkingrei) - - Fix the issue that the historical statistics of the `stats_meta` table are not updated after a partition is dropped [#49334](https://github.com/pingcap/tidb/issues/49334) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that the historical statistics of the `stats_meta` table are not updated after a partition is dropped [#49334](https://github.com/pingcap/tidb/issues/49334) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue of incorrect query results caused by multi-valued indexes mistakenly selected as the `Index Join` probe side [#50382](https://github.com/pingcap/tidb/issues/50382) @[AilinKid](https://github.com/AilinKid) - Fix the issue that the `USE_INDEX_MERGE` hint does not take effect on multi-valued indexes [#50553](https://github.com/pingcap/tidb/issues/50553) @[AilinKid](https://github.com/AilinKid) - - Fix the issue that users might get errors when querying the `INFORMATION_SCHEMA.ANALYZE_STATUS` system table [#48835](https://github.com/pingcap/tidb/issues/48835) @[hi-rustin](https://github.com/hi-rustin) - - Fix the issue of wrong query results due to TiDB incorrectly eliminating constant values in `group by` [#38756](https://github.com/pingcap/tidb/issues/38756) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that users might get errors when querying the `INFORMATION_SCHEMA.ANALYZE_STATUS` system table [#48835](https://github.com/pingcap/tidb/issues/48835) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue of wrong query results due to TiDB incorrectly eliminating constant values in `group by` [#38756](https://github.com/pingcap/tidb/issues/38756) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that the `processed_rows` of the `ANALYZE` task on a table might exceed the total number of rows in that table [#50632](https://github.com/pingcap/tidb/issues/50632) @[hawkingrei](https://github.com/hawkingrei) - Fix the issue that TiDB might panic when using the `EXECUTE` statement to execute `PREPARE STMT` after the `tidb_enable_prepared_plan_cache` system variable is enabled and then disabled [#49344](https://github.com/pingcap/tidb/issues/49344) @[qw4990](https://github.com/qw4990) - Fix the `Column ... in from clause is ambiguous` error that might occur when a query uses `NATURAL JOIN` [#32044](https://github.com/pingcap/tidb/issues/32044) @[AilinKid](https://github.com/AilinKid) @@ -159,6 +163,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with- - Fix the issue that the joint state of DR Auto-Sync might time out when scaling out [#15817](https://github.com/tikv/tikv/issues/15817) @[Connor1996](https://github.com/Connor1996) - Fix the issue that Resolved TS might be blocked for two hours [#11847](https://github.com/tikv/tikv/issues/11847) [#15520](https://github.com/tikv/tikv/issues/15520) [#39130](https://github.com/pingcap/tidb/issues/39130) @[overvenus](https://github.com/overvenus) - Fix the issue that `cast_duration_as_time` might return incorrect results [#16211](https://github.com/tikv/tikv/issues/16211) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that TiKV hangs in corner cases (for example, when disk I/O operations are blocked), which affects availability [#16368](https://github.com/tikv/tikv/issues/16368) @[LykxSassinator](https://github.com/LykxSassinator) + PD diff --git a/releases/release-7.5.2.md b/releases/release-7.5.2.md new file mode 100644 index 0000000000000..36baba41161ed --- /dev/null +++ b/releases/release-7.5.2.md @@ -0,0 +1,253 @@ +--- +title: TiDB 7.5.2 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.5.2. +--- + +# TiDB 7.5.2 Release Notes + +Release date: June 13, 2024 + +TiDB version: 7.5.2 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.5/production-deployment-using-tiup) + +## Compatibility changes + +- Add a TiKV configuration item [`track-and-verify-wals-in-manifest`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file#track-and-verify-wals-in-manifest-new-in-v659-v715-and-v752) for RocksDB, which helps you investigate possible corruption of Write Ahead Log (WAL) [#16549](https://github.com/tikv/tikv/issues/16549) @[v01dstar](https://github.com/v01dstar) +- Must set the line terminator when using TiDB Lightning `strict-format` or `SPLIT_FILE` to import CSV files [#37338](https://github.com/pingcap/tidb/issues/37338) @[lance6716](https://github.com/lance6716) +- Add the `sink.open.output-old-value` configuration item for TiCDC Open Protocol to control whether to output the value before the update to the downstream [#10916](https://github.com/pingcap/tiflow/issues/10916) @[sdojjy](https://github.com/sdojjy) +- In earlier versions, when processing a transaction containing `UPDATE` changes, if the primary key or non-null unique index value is modified in an `UPDATE` event, TiCDC splits this event into `DELETE` and `INSERT` events. Starting from v7.5.2, when using the MySQL sink, TiCDC splits an `UPDATE` event into `DELETE` and `INSERT` events if the transaction `commitTS` for the `UPDATE` change is less than TiCDC `thresholdTS` (which is the current timestamp fetched from PD when TiCDC starts replicating the corresponding table to the downstream). This behavior change addresses the issue of downstream data inconsistencies caused by the potentially incorrect order of `UPDATE` events received by TiCDC, which can lead to an incorrect order of split `DELETE` and `INSERT` events. For more information, see [documentation](https://docs.pingcap.com/tidb/v7.5/ticdc-split-update-behavior#split-update-events-for-mysql-sinks). [#10918](https://github.com/pingcap/tiflow/issues/10918) @[lidezhu](https://github.com/lidezhu) + +## Improvements + ++ TiDB + + - Optimize the issue that the `ANALYZE` statement blocks the metadata lock [#47475](https://github.com/pingcap/tidb/issues/47475) @[wjhuang2016](https://github.com/wjhuang2016) + - Improve the MySQL compatibility of expression default values displayed in the output of `SHOW CREATE TABLE` [#52939](https://github.com/pingcap/tidb/issues/52939) @[CbcWestwolf](https://github.com/CbcWestwolf) + - Enhance the handling of DNF items that are always `false` by directly ignoring such filter conditions, thus avoiding unnecessary full table scans [#40997](https://github.com/pingcap/tidb/issues/40997) @[hi-rustin](https://github.com/Rustin170506) + - Optimize the statistics for the execution process of the TiFlash `TableScan` operator in `EXPLAIN ANALYZE` [#51727](https://github.com/pingcap/tidb/issues/51727) @[JinheLin](https://github.com/JinheLin) + - Remove stores without Regions during MPP load balancing [#52313](https://github.com/pingcap/tidb/issues/52313) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Support loading Regions in batch from PD to speed up the conversion process from the KV range to Regions when querying large tables [#51326](https://github.com/pingcap/tidb/issues/51326) @[SeaRise](https://github.com/SeaRise) + - On the `Resource Control` monitoring page, add a new panel `RU(Max)` to show the maximum RU consumption rate for each resource group [#49318](https://github.com/pingcap/tidb/issues/49318) @[nolouch](https://github.com/nolouch) + - Improve sync load performance to reduce latency in loading statistics [#52994](https://github.com/pingcap/tidb/issues/52294) [hawkingrei](https://github.com/hawkingrei) + - Increase concurrency of statistics initialization to speed up startup [#52466](https://github.com/pingcap/tidb/issues/52466) [#52102](https://github.com/pingcap/tidb/issues/52102) [#52553](https://github.com/pingcap/tidb/issues/52553) [hawkingrei](https://github.com/hawkingrei) + ++ TiKV + + - Adjust the log level of coprocessor errors from `warn` to `debug` to reduce unnecessary logs of the cluster [#15881](https://github.com/tikv/tikv/issues/15881) @[cfzjywxk](https://github.com/cfzjywxk) + - Add monitoring metrics for the queue time for processing CDC events to facilitate troubleshooting downstream CDC event latency issues [#16282](https://github.com/tikv/tikv/issues/16282) @[hicqu](https://github.com/hicqu) + - Avoid performing IO operations on snapshot files in raftstore threads to improve TiKV stability [#16564](https://github.com/tikv/tikv/issues/16564) @[Connor1996](https://github.com/Connor1996) + - Add slow logs for peer and store messages [#16600](https://github.com/tikv/tikv/issues/16600) @[Connor1996](https://github.com/Connor1996) + - When TiKV detects the existence of corrupted SST files, it logs the specific reasons for the corruption [#16308](https://github.com/tikv/tikv/issues/16308) @[overvenus](https://github.com/overvenus) + - Remove unnecessary async blocks to reduce memory usage [#16540](https://github.com/tikv/tikv/issues/16540) @[overvenus](https://github.com/overvenus) + - Accelerate the shutdown speed of TiKV [#16680](https://github.com/tikv/tikv/issues/16680) @[LykxSassinator](https://github.com/LykxSassinator) + ++ PD + + - Upgrade the etcd version to v3.4.30 [#7904](https://github.com/tikv/pd/issues/7904) @[JmPotato](https://github.com/JmPotato) + - Add the monitoring metric for the maximum Request Unit (RU) per second [#7908](https://github.com/tikv/pd/issues/7908) @[nolouch](https://github.com/nolouch) + ++ TiFlash + + - Mitigate the issue that TiFlash might panic due to updating certificates after TLS is enabled [#8535](https://github.com/pingcap/tiflash/issues/8535) @[windtalker](https://github.com/windtalker) + ++ Tools + + + Backup & Restore (BR) + + - BR cleans up empty SST files during data recovery [#16005](https://github.com/tikv/tikv/issues/16005) @[Leavrth](https://github.com/Leavrth) + - Add PITR integration test cases to cover compatibility testing for log backup and adding index acceleration [#51987](https://github.com/pingcap/tidb/issues/51987) @[Leavrth](https://github.com/Leavrth) + - Enhance the tolerance of log backup to merge operations. When encountering a reasonably long merge operation, log backup tasks are less likely to enter the error state [#16554](https://github.com/tikv/tikv/issues/16554) @[YuJuncen](https://github.com/YuJuncen) + - Improve the table creation performance of the `RESTORE` statement in scenarios with large datasets [#48301](https://github.com/pingcap/tidb/issues/48301) @[Leavrth](https://github.com/Leavrth) + - Support pre-allocating Table ID during the restore process to maximize the reuse of Table ID and improve restore performance [#51736](https://github.com/pingcap/tidb/issues/51736) @[Leavrth](https://github.com/Leavrth) + - Remove the invalid verification for active DDL jobs when log backup starts [#52733](https://github.com/pingcap/tidb/issues/52733) @[Leavrth](https://github.com/Leavrth) + - Remove an outdated compatibility check when using Google Cloud Storage (GCS) as the external storage [#50533](https://github.com/pingcap/tidb/issues/50533) @[lance6716](https://github.com/lance6716) + - Increase the number of retries for failures caused by DNS errors [#53029](https://github.com/pingcap/tidb/issues/53029) @[YuJuncen](https://github.com/YuJuncen) + + + TiCDC + + - Improve memory stability during data recovery using redo logs to reduce the probability of OOM [#10900](https://github.com/pingcap/tiflow/issues/10900) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Significantly improve the stability of data replication in transaction conflict scenarios, with up to 10 times performance improvement [#10896](https://github.com/pingcap/tiflow/issues/10896) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Enable the PD client forwarding function to make TiCDC more stable during network isolation between TiCDC and the PD leader [#10849](https://github.com/pingcap/tiflow/issues/10849) @[asddongmen](https://github.com/asddongmen) + - Improve the initialization speed of replication tasks [#11124](https://github.com/pingcap/tiflow/issues/11124) @[asddongmen](https://github.com/asddongmen) + - Initialize replication tasks asynchronously to reduce initialization time for the processor and owner [#10845](https://github.com/pingcap/tiflow/issues/10845) @[sdojjy](https://github.com/sdojjy) + - Detect the Kafka cluster version automatically to improve compatibility with Kafka [#10852](https://github.com/pingcap/tiflow/issues/10852) @[wk989898](https://github.com/wk989898) + +## Bug fixes + ++ TiDB + + - Fix the issue of inconsistent data indexes caused by concurrent DML operations when adding a unique index [#52914](https://github.com/pingcap/tidb/issues/52914) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue of inconsistent data indexes caused by adding indexes with multi-schema changes on partitioned tables [#52080](https://github.com/pingcap/tidb/issues/52080) @[tangenta](https://github.com/tangenta) + - Fix the issue of inconsistent data indexes caused by adding multi-valued indexes [#51162](https://github.com/pingcap/tidb/issues/51162) @[ywqzzy](https://github.com/ywqzzy) + - Fix the issue that DDL operations get stuck due to network problems [#47060](https://github.com/pingcap/tidb/issues/47060) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that TiDB might report an error due to GC when loading statistics at startup [#53592](https://github.com/pingcap/tidb/issues/53592) @[you06](https://github.com/you06) + - Fix the issue that TiDB might send requests to unready TiKV nodes [#50758](https://github.com/pingcap/tidb/issues/50758) @[zyguan](https://github.com/zyguan) + - Fix the issue that Stale Read might miss after a TiKV rolling restart [#52193](https://github.com/pingcap/tidb/issues/52193) @[zyguan](https://github.com/zyguan) + - Fix the issue that data race might occur during KV request retries, leading to TiDB panics [#51921](https://github.com/pingcap/tidb/issues/51921) @[zyguan](https://github.com/zyguan) + - Fix the issue that TiDB might panic when parsing index data [#47115](https://github.com/pingcap/tidb/issues/47115) @[zyguan](https://github.com/zyguan) + - Fix the issue that TiDB might panic when the JOIN condition contains an implicit type conversion [#46556](https://github.com/pingcap/tidb/issues/46556) @[qw4990](https://github.com/qw4990) + - Fix the issue that comparing a column of `YEAR` type with an unsigned integer that is out of range causes incorrect results [#50235](https://github.com/pingcap/tidb/issues/50235) @[qw4990](https://github.com/qw4990) + - Fix the issue that subqueries in an `UPDATE` list might cause TiDB to panic [#52687](https://github.com/pingcap/tidb/issues/52687) @[winoros](https://github.com/winoros) + - Fix the overflow issue of the `Longlong` type in predicates [#45783](https://github.com/pingcap/tidb/issues/45783) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `SELECT INTO OUTFILE` does not work when clustered indexes are used as predicates [#42093](https://github.com/pingcap/tidb/issues/42093) @[qw4990](https://github.com/qw4990) + - Fix the issue that the TopN operator might be pushed down incorrectly [#37986](https://github.com/pingcap/tidb/issues/37986) @[qw4990](https://github.com/qw4990) + - Fix the issue that an empty projection causes TiDB to panic [#49109](https://github.com/pingcap/tidb/issues/49109) @[winoros](https://github.com/winoros) + - Fix the issue that Index Merge incorrectly pushes partial limit down when index plans are kept ordered [#52947](https://github.com/pingcap/tidb/issues/52947) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that using a view does not work in recursive CTE [#49721](https://github.com/pingcap/tidb/issues/49721) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that unstable unique IDs of columns might cause the `UPDATE` statement to return errors [#53236](https://github.com/pingcap/tidb/issues/53236) @[winoros](https://github.com/winoros) + - Fix the issue that TiDB panics when executing the `SHOW ERRORS` statement with a predicate that is always `true` [#46962](https://github.com/pingcap/tidb/issues/46962) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that the `final` AggMode and the `non-final` AggMode cannot coexist in Massively Parallel Processing (MPP) [#51362](https://github.com/pingcap/tidb/issues/51362) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that a wrong TableDual plan causes empty query results [#50051](https://github.com/pingcap/tidb/issues/50051) @[onlyacat](https://github.com/onlyacat) + - Fix the issue that TiDB might panic when initializing statistics after enabling both `lite-init-stats` and `concurrently-init-stats` [#52223](https://github.com/pingcap/tidb/issues/52223) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `NO_JOIN` hints do not work with `CREATE BINDING` [#52813](https://github.com/pingcap/tidb/issues/52813) @[qw4990](https://github.com/qw4990) + - Fix the issue that subqueries included in the `ALL` function might cause incorrect results [#52755](https://github.com/pingcap/tidb/issues/52755) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `VAR_SAMP()` cannot be used as a window function [#52933](https://github.com/pingcap/tidb/issues/52933) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that column pruning without using shallow copies of slices might cause TiDB to panic [#52768](https://github.com/pingcap/tidb/issues/52768) @[winoros](https://github.com/winoros) + - Fix the issue that adding a unique index might cause TiDB to panic [#52312](https://github.com/pingcap/tidb/issues/52312) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that the TiDB server is marked as health before the initialization is complete [#51596](https://github.com/pingcap/tidb/issues/51596) @[shenqidebaozi](https://github.com/shenqidebaozi) + - Fix the issue that the type returned by the `IFNULL` function is inconsistent with MySQL [#51765](https://github.com/pingcap/tidb/issues/51765) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that parallel `Apply` might generate incorrect results when the table has a clustered index [#51372](https://github.com/pingcap/tidb/issues/51372) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that query results might be incorrect when the `HAVING` clause in a subquery contains correlated columns [#51107](https://github.com/pingcap/tidb/issues/51107) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that querying the `TIDB_HOT_REGIONS` table might incorrectly return `INFORMATION_SCHEMA` tables [#50810](https://github.com/pingcap/tidb/issues/50810) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that automatic statistics collection is triggered before the initialization of statistics finishes [#52346](https://github.com/pingcap/tidb/issues/52346) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that AutoID Leader change might cause the value of the auto-increment column to decrease in the case of `AUTO_ID_CACHE=1` [#52600](https://github.com/pingcap/tidb/issues/52600) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that query results might be incorrect when you use Common Table Expressions (CTE) to access partitioned tables with missing statistics [#51873](https://github.com/pingcap/tidb/issues/51873) @[qw4990](https://github.com/qw4990) + - Fix the incorrect calculation and display of the number of connections (Connection Count) on the TiDB Dashboard Monitoring page [#51889](https://github.com/pingcap/tidb/issues/51889) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that DDL operations get stuck when restoring a table with the foreign key [#51838](https://github.com/pingcap/tidb/issues/51838) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that getting the default value of a column returns an error if the column default value is dropped [#50043](https://github.com/pingcap/tidb/issues/50043) [#51324](https://github.com/pingcap/tidb/issues/51324) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that TiDB does not listen to the corresponding port when `force-init-stats` is configured [#51473](https://github.com/pingcap/tidb/issues/51473) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the query result is incorrect when the `IN()` predicate contains `NULL` [#51560](https://github.com/pingcap/tidb/issues/51560) @[winoros](https://github.com/winoros) + - Fix the issue that the TiDB synchronously loading statistics mechanism retries to load empty statistics indefinitely and prints the `fail to get stats version for this histogram` log [#52657](https://github.com/pingcap/tidb/issues/52657) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `EXCHANGE PARTITION` incorrectly processes foreign keys [#51807](https://github.com/pingcap/tidb/issues/51807) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that `LIMIT` might not be pushed down to the `OR` type `Index Merge` [#48588](https://github.com/pingcap/tidb/issues/48588) @[AilinKid](https://github.com/AilinKid) + - Fix the incorrect result of the TopN operator in correlated subqueries [#52777](https://github.com/pingcap/tidb/issues/52777) @[yibin87](https://github.com/yibin87) + - Fix the issue that the `CPS by type` metric displays incorrect values [#52605](https://github.com/pingcap/tidb/issues/52605) @[nolouch](https://github.com/nolouch) + - Fix the issue that the `EXPLAIN` statement might display incorrect column IDs in the result when statistics for certain columns are not fully loaded [#52207](https://github.com/pingcap/tidb/issues/52207) @[time-and-fate](https://github.com/time-and-fate) + - Fix the issue that expressions containing different collations might cause the query to panic when the new framework for collations is disabled [#52772](https://github.com/pingcap/tidb/issues/52772) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that executing SQL statements containing tables with multi-valued indexes might return the `Can't find a proper physical plan for this query` error [#49438](https://github.com/pingcap/tidb/issues/49438) @[qw4990](https://github.com/qw4990) + - Fix the issue that TiDB cannot correctly convert the type of a system variable in an expression [#43527](https://github.com/pingcap/tidb/issues/43527) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that executing `INSERT IGNORE` might result in inconsistency between the unique index and the data [#51784](https://github.com/pingcap/tidb/issues/51784) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that automatic statistics collection gets stuck after an OOM error occurs [#51993](https://github.com/pingcap/tidb/issues/51993) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that TiDB might crash when `tidb_mem_quota_analyze` is enabled and the memory used by updating statistics exceeds the limit [#52601](https://github.com/pingcap/tidb/issues/52601) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that `max_execute_time` settings at multiple levels interfere with each other [#50914](https://github.com/pingcap/tidb/issues/50914) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue of index inconsistency caused by adding multiple indexes using a single SQL statement [#51746](https://github.com/pingcap/tidb/issues/51746) @[tangenta](https://github.com/tangenta) + - Fix the issue that the Window function might panic when there is a related subquery in it [#42734](https://github.com/pingcap/tidb/issues/42734) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that TiDB crashes when `shuffleExec` quits unexpectedly [#48230](https://github.com/pingcap/tidb/issues/48230) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that the status gets stuck when rolling back the partition DDL tasks [#51090](https://github.com/pingcap/tidb/issues/51090) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue that querying JSON of `BINARY` type might cause an error in some cases [#51547](https://github.com/pingcap/tidb/issues/51547) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that adding indexes to large tables fails after enabling the Distributed eXecution Framework (DXF) [#52640](https://github.com/pingcap/tidb/issues/52640) @[tangenta](https://github.com/tangenta) + - Fix the issue that the TTL feature causes data hotspots due to incorrect data range splitting in some cases [#51527](https://github.com/pingcap/tidb/issues/51527) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that `ALTER TABLE ... COMPACT TIFLASH REPLICA` might incorrectly end when the primary key type is `VARCHAR` [#51810](https://github.com/pingcap/tidb/issues/51810) @[breezewish](https://github.com/breezewish) + - Fix the issue of inconsistent data indexes caused by cluster upgrade during adding indexes [#52411](https://github.com/pingcap/tidb/issues/52411) @[tangenta](https://github.com/tangenta) + - Fix the performance regression issue caused by disabling predicate pushdown in TableDual [#50614](https://github.com/pingcap/tidb/issues/50614) @[time-and-fate](https://github.com/time-and-fate) + - Fix the issue that the TiDB server adds a label via the HTTP interface and returns success, but does not take effect [#51427](https://github.com/pingcap/tidb/issues/51427) @[you06](https://github.com/you06) + - Fix the issue that adding indexes in the ingest mode might cause inconsistent data index in some corner cases [#51954](https://github.com/pingcap/tidb/issues/51954) @[lance6716](https://github.com/lance6716) + - Fix the issue that the `init-stats` process might cause TiDB to panic and the `load stats` process to quit [#51581](https://github.com/pingcap/tidb/issues/51581) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the configuration file does not take effect when it contains an invalid configuration item [#51399](https://github.com/pingcap/tidb/issues/51399) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that query execution using MPP might lead to incorrect query results when a SQL statement contains `JOIN` and the `SELECT` list in the statement contains only constants [#50358](https://github.com/pingcap/tidb/issues/50358) @[yibin87](https://github.com/yibin87) + - Fix the issue that in `determinate` mode (`tidb_opt_objective='determinate'`), if a query does not contain predicates, statistics might not be loaded [#48257](https://github.com/pingcap/tidb/issues/48257) @[time-and-fate](https://github.com/time-and-fate) + - Fix the issue that the `SURVIVAL_PREFERENCES` attribute might not appear in the output of the `SHOW CREATE PLACEMENT POLICY` statement under certain conditions [#51699](https://github.com/pingcap/tidb/issues/51699) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that IndexJoin produces duplicate rows when calculating hash values in the Left Outer Anti Semi type [#52902](https://github.com/pingcap/tidb/issues/52902) @[yibin87](https://github.com/yibin87) + - Fix the issue that the `TIMESTAMPADD()` function returns incorrect results [#41052](https://github.com/pingcap/tidb/issues/41052) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that data conversion from the `FLOAT` type to the `UNSIGNED` type returns incorrect results [#41736](https://github.com/pingcap/tidb/issues/41736) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that the `TRUNCATE()` function returns incorrect results when its second argument is a large negative number [#52978](https://github.com/pingcap/tidb/issues/52978) @[yibin87](https://github.com/yibin87) + - Fix the issue that duplicated panel IDs in Grafana might cause abnormal display [#51556](https://github.com/pingcap/tidb/issues/51556) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that TiDB unexpectedly restarts when logging gRPC errors [#51301](https://github.com/pingcap/tidb/issues/51301) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that TiDB loading statistics during startup might cause OOM [#52219](https://github.com/pingcap/tidb/issues/52219) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the TTL job for a table does not stop after the table is deleted [#51540](https://github.com/pingcap/tidb/issues/51540) @[YangKeao](https://github.com/YangKeao) + ++ TiKV + + - Fix the issue that `thread_id` values are incorrectly displayed as `0x5` in TiKV logs [#16398](https://github.com/tikv/tikv/issues/16398) @[overvenus](https://github.com/overvenus) + - Fix the issue of unstable test cases, ensuring that each test uses an independent temporary directory to avoid online configuration changes affecting other test cases [#16871](https://github.com/tikv/tikv/issues/16871) @[glorv](https://github.com/glorv) + - Fix the issue that TiKV might panic during the conversion from Binary to JSON [#16616](https://github.com/tikv/tikv/issues/16616) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the output of the `raft region` command in tikv-ctl does not include the Region status information [#17037](https://github.com/tikv/tikv/issues/17037) @[glorv](https://github.com/glorv) + - Fix the issue that slow `check-leader` operations on one TiKV node cause `resolved-ts` on other TiKV nodes to fail to advance normally [#15999](https://github.com/tikv/tikv/issues/15999) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that after the process of destroying Peer is interrupted by applying snapshots, it does not resume even after applying snapshots is complete [#16561](https://github.com/tikv/tikv/issues/16561) @[tonyxuqqi](https://github.com/tonyxuqqi) + - Fix the issue that the decimal part of the `DECIMAL` type is incorrect in some cases [#16913](https://github.com/tikv/tikv/issues/16913) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that the `CONV()` function in queries might overflow during numeric system conversion, leading to TiKV panic [#16969](https://github.com/tikv/tikv/issues/16969) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that TiKV converts the time zone incorrectly for Brazil and Egypt [#16220](https://github.com/tikv/tikv/issues/16220) @[overvenus](https://github.com/overvenus) + - Fix the issue that the monitoring metric `tikv_unified_read_pool_thread_count` has no data in some cases [#16629](https://github.com/tikv/tikv/issues/16629) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that inactive Write Ahead Logs (WALs) in RocksDB might corrupt data [#16705](https://github.com/tikv/tikv/issues/16705) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that resolve-ts is blocked when a stale Region peer ignores the GC message [#16504](https://github.com/tikv/tikv/issues/16504) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that during the execution of an optimistic transaction, if other transactions initiate the resolving lock operation on it, there is a small chance that the atomicity of the transaction might be broken if the transaction's primary key has data that was previously committed in Async Commit or 1PC mode [#16620](https://github.com/tikv/tikv/issues/16620) @[MyonKeminta](https://github.com/MyonKeminta) + ++ PD + + - Fix the issue of connection panic after fault recovery in the TiDB network partition [#7926](https://github.com/tikv/pd/issues/7926) @[CabinfeverB](https://github.com/CabinfeverB) + - Fix the issue that scheduling might be incorrectly paused after online data recovery [#8095](https://github.com/tikv/pd/issues/8095) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that the CPS By Type monitoring types display incorrectly after enabling resource groups [#52605](https://github.com/pingcap/tidb/issues/52605) @[nolouch](https://github.com/nolouch) + - Fix the issue that changing the log level via the configuration file does not take effect [#8117](https://github.com/tikv/pd/issues/8117) @[rleungx](https://github.com/rleungx) + - Fix the issue that a large number of retries occur when canceling resource groups queries [#8217](https://github.com/tikv/pd/issues/8217) @[nolouch](https://github.com/nolouch) + - Fix the issue that `ALTER PLACEMENT POLICY` cannot modify the placement policy [#52257](https://github.com/pingcap/tidb/issues/52257) [#51712](https://github.com/pingcap/tidb/issues/51712) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue that down peers might not recover when using Placement Rules [#7808](https://github.com/tikv/pd/issues/7808) @[rleungx](https://github.com/rleungx) + - Fix the issue that manually transferring the PD leader might fail [#8225](https://github.com/tikv/pd/issues/8225) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that the scheduling of write hotspots might break placement policy constraints [#7848](https://github.com/tikv/pd/issues/7848) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that slots are not fully deleted in a resource group client, which causes the number of the allocated tokens to be less than the specified value [#7346](https://github.com/tikv/pd/issues/7346) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that the scaling progress is not correctly displayed [#7726](https://github.com/tikv/pd/issues/7726) @[CabinfeverB](https://github.com/CabinfeverB) + - Fix the issue that the Leader fails to transfer when you switch it between two deployed data centers [#7992](https://github.com/tikv/pd/issues/7992) @[TonsnakeLin](https://github.com/TonsnakeLin) + - Fix the issue that the `Filter target` monitoring metric for PD does not provide scatter range information [#8125](https://github.com/tikv/pd/issues/8125) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that the query result of `SHOW CONFIG` includes the deprecated configuration item `trace-region-flow` [#7917](https://github.com/tikv/pd/issues/7917) @[rleungx](https://github.com/rleungx) + ++ TiFlash + + - Fix the issue that in the disaggregated storage and compute architecture, null values might be incorrectly returned in queries after adding non-null columns in DDL operations [#9084](https://github.com/pingcap/tiflash/issues/9084) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue of query timeout when executing queries on partitioned tables that contains the empty partition [#9024](https://github.com/pingcap/tiflash/issues/9024) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that in the disaggregated storage and compute architecture, TiFlash might panic when the compute node process is stopped [#8860](https://github.com/pingcap/tiflash/issues/8860) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that querying generated columns returns an error [#8787](https://github.com/pingcap/tiflash/issues/8787) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that TiFlash metadata might become corrupted and cause the process to panic when upgrading a cluster from a version earlier than v6.5.0 to v6.5.0 or later [#9039](https://github.com/pingcap/tiflash/issues/9039) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that the `ENUM` column might cause TiFlash to crash during chunk encoding [#8674](https://github.com/pingcap/tiflash/issues/8674) @[yibin87](https://github.com/yibin87) + - Fix incorrect `local_region_num` values in logs [#8895](https://github.com/pingcap/tiflash/issues/8895) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that in the disaggregated storage and compute architecture, TiFlash might panic during shutdown [#8837](https://github.com/pingcap/tiflash/issues/8837) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash might return transiently incorrect results in high-concurrency read scenarios [#8845](https://github.com/pingcap/tiflash/issues/8845) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that in the disaggregated storage and compute architecture, the disk `used_size` metric displayed in Grafana is incorrect after you modify the value of the `storage.remote.cache.capacity` configuration item for TiFlash compute nodes [#8920](https://github.com/pingcap/tiflash/issues/8920) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that in the disaggregated storage and compute architecture, queries might be permanently blocked after network isolation [#8806](https://github.com/pingcap/tiflash/issues/8806) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that TiFlash might panic when you insert data to columns with invalid default values in non-strict `sql_mode` [#8803](https://github.com/pingcap/tiflash/issues/8803) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + ++ Tools + + + Backup & Restore (BR) + + - Fix a rare issue that special event timing might cause the data loss in log backup [#16739](https://github.com/tikv/tikv/issues/16739) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that too many logs are printed when a full backup fails [#51572](https://github.com/pingcap/tidb/issues/51572) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that a PD connection failure could cause the TiDB instance where the log backup advancer owner is located to panic [#52597](https://github.com/pingcap/tidb/issues/52597) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that the global checkpoint of log backup is advanced ahead of the actual backup file write point due to TiKV restart, which might cause a small amount of backup data loss [#16809](https://github.com/tikv/tikv/issues/16809) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that TiKV might panic when resuming a paused log backup task with unstable network connections to PD [#17020](https://github.com/tikv/tikv/issues/17020) @[YuJuncen](https://github.com/YuJuncen) + - Fix an unstable test case [#52547](https://github.com/pingcap/tidb/issues/52547) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the Region fetched from PD does not have a Leader when restoring data using BR or importing data using TiDB Lightning in physical import mode [#51124](https://github.com/pingcap/tidb/issues/51124) [#50501](https://github.com/pingcap/tidb/issues/50501) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that TiKV panics when a full backup fails to find a peer in some extreme cases [#16394](https://github.com/tikv/tikv/issues/16394) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that removing a log backup task after it is paused does not immediately restore the GC safepoint [#52082](https://github.com/pingcap/tidb/issues/52082) @[3pointer](https://github.com/3pointer) + - Fix the unstable test case `TestClearCache` [#50743](https://github.com/pingcap/tidb/issues/50743) @[3pointer](https://github.com/3pointer) + - Fix the issue that BR fails to restore a transactional KV cluster due to an empty `EndKey` [#52574](https://github.com/pingcap/tidb/issues/52574) @[3pointer](https://github.com/3pointer) + - Fix the issue that the transfer of PD leaders might cause BR to panic when restoring data [#53724](https://github.com/pingcap/tidb/issues/53724) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that BR could not back up the `AUTO_RANDOM` ID allocation progress in a union clustered index that contains an `AUTO_RANDOM` column [#52255](https://github.com/pingcap/tidb/issues/52255) @[Leavrth](https://github.com/Leavrth) + + + TiCDC + + - Fix the issue that high latency in the PD disk I/O causes severe latency in data replication [#9054](https://github.com/pingcap/tiflow/issues/9054) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that calling the API (`/api/v2/owner/resign`) that evicts the TiCDC owner node causes the TiCDC task to restart unexpectedly [#10781](https://github.com/pingcap/tiflow/issues/10781) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that data is written to a wrong CSV file due to wrong BarrierTS in scenarios where DDL statements are executed frequently [#10668](https://github.com/pingcap/tiflow/issues/10668) @[lidezhu](https://github.com/lidezhu) + - Fix the issue that TiCDC fails to validate `TIMESTAMP` type checksum due to time zone mismatch after data integrity validation for single-row data is enabled [#10573](https://github.com/pingcap/tiflow/issues/10573) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that a changefeed with eventual consistency enabled might fail when the object storage sink encounters a temporary failure [#10710](https://github.com/pingcap/tiflow/issues/10710) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that `DROP PRIMARY KEY` and `DROP UNIQUE KEY` statements are not replicated correctly [#10890](https://github.com/pingcap/tiflow/issues/10890) @[asddongmen](https://github.com/asddongmen) + - Fix the issue TiCDC panics when scheduling table replication tasks [#10613](https://github.com/pingcap/tiflow/issues/10613) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that when the downstream Pulsar is stopped, removing the changefeed causes the normal TiCDC process to get stuck, which causes other changefeed processes to get stuck [#10629](https://github.com/pingcap/tiflow/issues/10629) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that restarting PD might cause the TiCDC node to restart with an error [#10799](https://github.com/pingcap/tiflow/issues/10799) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that the old value part of `open-protocol` incorrectly outputs the default value according to the `STRING` type instead of its actual type [#10803](https://github.com/pingcap/tiflow/issues/10803) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that the default value of `TIMEZONE` type is not set according to the correct time zone [#10931](https://github.com/pingcap/tiflow/issues/10931) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that TiCDC fails to execute the `Exchange Partition ... With Validation` DDL downstream after it is written upstream, causing the changefeed to get stuck [#10859](https://github.com/pingcap/tiflow/issues/10859) @[hongyunyan](https://github.com/hongyunyan) + - Fix the issue that data race in the KV client causes TiCDC to panic [#10718](https://github.com/pingcap/tiflow/issues/10718) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that updating the primary key or the unique key in upstream might cause data inconsistency between upstream and downstream [#10918](https://github.com/pingcap/tiflow/issues/10918) @[lidezhu](https://github.com/lidezhu) + + + TiDB Data Migration (DM) + + - Fix the connection blocking issue by upgrading `go-mysql` [#11041](https://github.com/pingcap/tiflow/issues/11041) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that data is lost when the upstream primary key is of binary type [#10672](https://github.com/pingcap/tiflow/issues/10672) @[GMHDBJD](https://github.com/GMHDBJD) + + + TiDB Lightning + + - Fix the issue that TiDB Lightning might fail to import data when EBS BR is running [#49517](https://github.com/pingcap/tidb/issues/49517) @[mittalrishabh](https://github.com/mittalrishabh) + - Fix the issue that TiDB Lightning reports `no database selected` during data import due to incompatible SQL statements in the source files [#51800](https://github.com/pingcap/tidb/issues/51800) @[lance6716](https://github.com/lance6716) + - Fix the issue that killing the PD Leader causes TiDB Lightning to report the `invalid store ID 0` error during data import [#50501](https://github.com/pingcap/tidb/issues/50501) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that TiDB Lightning panics when importing an empty table of Parquet format [#52518](https://github.com/pingcap/tidb/issues/52518) @[kennytm](https://github.com/kennytm) diff --git a/releases/release-7.5.3.md b/releases/release-7.5.3.md new file mode 100644 index 0000000000000..f635f9d5e0983 --- /dev/null +++ b/releases/release-7.5.3.md @@ -0,0 +1,130 @@ +--- +title: TiDB 7.5.3 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.5.3. +--- + +# TiDB 7.5.3 Release Notes + +Release date: August 5, 2024 + +TiDB version: 7.5.3 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.5/production-deployment-using-tiup) + +## Compatibility changes + +- Add a new system table [`INFORMATION_SCHEMA.KEYWORDS`](/information-schema/information-schema-keywords.md) to display the information of all keywords supported by TiDB [#48801](https://github.com/pingcap/tidb/issues/48801) @[dveeden](https://github.com/dveeden) + +## Improvements + ++ TiDB + + - By batch deleting TiFlash placement rules, improve the processing speed of data GC after performing the `TRUNCATE` or `DROP` operation on partitioned tables [#54068](https://github.com/pingcap/tidb/issues/54068) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + ++ TiFlash + + - Mitigate the issue that TiFlash might panic due to updating certificates after TLS is enabled [#8535](https://github.com/pingcap/tiflash/issues/8535) @[windtalker](https://github.com/windtalker) + - Reduce lock conflicts under highly concurrent data read operations and optimize short query performance [#9125](https://github.com/pingcap/tiflash/issues/9125) @[JinheLin](https://github.com/JinheLin) + ++ Tools + + + Backup & Restore (BR) + + - Except for the `br log restore` subcommand, all other `br log` subcommands support skipping the loading of the TiDB `domain` data structure to reduce memory consumption [#52088](https://github.com/pingcap/tidb/issues/52088) @[Leavrth](https://github.com/Leavrth) + - Support automatically abandoning log backup tasks when encountering a large checkpoint lag, to avoid prolonged blocking GC and potential cluster issues [#50803](https://github.com/pingcap/tidb/issues/50803) @[RidRisR](https://github.com/RidRisR) + - Increase the number of retries for failures caused by DNS errors [#53029](https://github.com/pingcap/tidb/issues/53029) @[YuJuncen](https://github.com/YuJuncen) + - Add PITR integration test cases to cover compatibility testing for log backup and adding index acceleration [#51987](https://github.com/pingcap/tidb/issues/51987) @[Leavrth](https://github.com/Leavrth) + - Increase the number of retries for failures caused by the absence of a leader in a Region [#54017](https://github.com/pingcap/tidb/issues/54017) @[Leavrth](https://github.com/Leavrth) + - Support setting Alibaba Cloud access credentials through environment variables [#45551](https://github.com/pingcap/tidb/issues/45551) @[RidRisR](https://github.com/RidRisR) + + + TiCDC + + - Support directly outputting raw events when the downstream is a Message Queue (MQ) or cloud storage [#11211](https://github.com/pingcap/tiflow/issues/11211) @[CharlesCheung96](https://github.com/CharlesCheung96) + +## Bug fixes + ++ TiDB + + - Fix the issue that loading index statistics might cause memory leaks [#54022](https://github.com/pingcap/tidb/issues/54022) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that the `UPDATE` operation can cause TiDB OOM in multi-table scenarios [#53742](https://github.com/pingcap/tidb/issues/53742) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that indirect placeholder `?` references in a `GROUP BY` statement cannot find columns [#53872](https://github.com/pingcap/tidb/issues/53872) @[qw4990](https://github.com/qw4990) + - Fix the issue that the `LENGTH()` condition is unexpectedly removed when the collation is `utf8_bin` or `utf8mb4_bin` [#53730](https://github.com/pingcap/tidb/issues/53730) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that inserting an overlarge number in scientific notation returns a warning instead of an error, making it consistent with MySQL [#47787](https://github.com/pingcap/tidb/issues/47787) @[qw4990](https://github.com/qw4990) + - Fix the issue that recursive CTE queries might result in invalid pointers [#54449](https://github.com/pingcap/tidb/issues/54449) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that statistics collection does not update the `stats_history` table when encountering duplicate primary keys [#47539](https://github.com/pingcap/tidb/issues/47539) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that when queries contain non-correlated subqueries and `LIMIT` clauses, column pruning might be incomplete, resulting in a less optimal plan [#54213](https://github.com/pingcap/tidb/issues/54213) @[qw4990](https://github.com/qw4990) + - Fix the issue of abnormally high memory usage caused by `memTracker` not being detached when the `HashJoin` or `IndexLookUp` operator is the driven side sub-node of the `Apply` operator [#54005](https://github.com/pingcap/tidb/issues/54005) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that the recursive CTE operator incorrectly tracks memory usage [#54181](https://github.com/pingcap/tidb/issues/54181) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that the memory used by transactions might be tracked multiple times [#53984](https://github.com/pingcap/tidb/issues/53984) @[ekexium](https://github.com/ekexium) + - Fix the issue that using `SHOW WARNINGS;` to obtain warnings might cause a panic [#48756](https://github.com/pingcap/tidb/issues/48756) @[xhebox](https://github.com/xhebox) + - Fix the issue that updating an `UNSIGNED` type of field to `-1` returns `null` instead of `0` when `sql_mode=''` [#47816](https://github.com/pingcap/tidb/issues/47816) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that the `TIMESTAMPADD()` function goes into an infinite loop when the first argument is `month` and the second argument is negative [#54908](https://github.com/pingcap/tidb/issues/54908) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that the Connection Count monitoring metric in Grafana is incorrect when some connections exit before the handshake is complete [#54428](https://github.com/pingcap/tidb/issues/54428) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the Connection Count of each resource group is incorrect when using TiProxy and resource groups [#54545](https://github.com/pingcap/tidb/issues/54545) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that executing `CREATE OR REPLACE VIEW` concurrently might result in the `table doesn't exist` error [#53673](https://github.com/pingcap/tidb/issues/53673) @[tangenta](https://github.com/tangenta) + - Fix the issue that TiDB might return incorrect query results when you query tables with virtual columns in transactions that involve data modification operations [#53951](https://github.com/pingcap/tidb/issues/53951) @[qw4990](https://github.com/qw4990) + - Fix the issue that executing the `SELECT DISTINCT CAST(col AS DECIMAL), CAST(col AS SIGNED) FROM ...` query might return incorrect results [#53726](https://github.com/pingcap/tidb/issues/53726) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue of incorrect WARNINGS information when using Optimizer Hints [#53767](https://github.com/pingcap/tidb/issues/53767) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the illegal column type `DECIMAL(0,0)` can be created in some cases [#53779](https://github.com/pingcap/tidb/issues/53779) @[tangenta](https://github.com/tangenta) + - Fix the issue that the `memory_quota` hint might not work in subqueries [#53834](https://github.com/pingcap/tidb/issues/53834) @[qw4990](https://github.com/qw4990) + - Fix the issue that JSON-related functions return errors inconsistent with MySQL in some cases [#53799](https://github.com/pingcap/tidb/issues/53799) @[dveeden](https://github.com/dveeden) + - Fix the issue that improper use of metadata locks might lead to writing anomalous data when using the plan cache under certain circumstances [#53634](https://github.com/pingcap/tidb/issues/53634) @[zimulala](https://github.com/zimulala) + - Fix the issue that certain filter conditions in queries might cause the planner module to report an `invalid memory address or nil pointer dereference` error [#53582](https://github.com/pingcap/tidb/issues/53582) [#53580](https://github.com/pingcap/tidb/issues/53580) [#53594](https://github.com/pingcap/tidb/issues/53594) [#53603](https://github.com/pingcap/tidb/issues/53603) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that after a statement within a transaction is killed by OOM, if TiDB continues to execute the next statement within the same transaction, you might get an error `Trying to start aggressive locking while it's already started` and a panic occurs [#53540](https://github.com/pingcap/tidb/issues/53540) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that executing `ALTER TABLE ... REMOVE PARTITIONING` might cause data loss [#53385](https://github.com/pingcap/tidb/issues/53385) @[mjonss](https://github.com/mjonss) + - Fix the issue that `PREPARE`/`EXECUTE` statements with the `CONV` expression containing a `?` argument might result in incorrect query results when executed multiple times [#53505](https://github.com/pingcap/tidb/issues/53505) @[qw4990](https://github.com/qw4990) + - Fix the issue that TiDB fails to reject unauthenticated user connections in some cases when using the `auth_socket` authentication plugin [#54031](https://github.com/pingcap/tidb/issues/54031) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that the query latency of stale reads increases, caused by information schema cache misses [#53428](https://github.com/pingcap/tidb/issues/53428) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that the `STATE` field in the `INFORMATION_SCHEMA.TIDB_TRX` table is empty due to the `size` of the `STATE` field not being defined [#53026](https://github.com/pingcap/tidb/issues/53026) @[cfzjywxk](https://github.com/cfzjywxk) + - Fix the issue that the `tidb_enable_async_merge_global_stats` and `tidb_analyze_partition_concurrency` system variables do not take effect during automatic statistics collection [#53972](https://github.com/pingcap/tidb/issues/53972) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that using `CURRENT_DATE()` as the default value for a column results in incorrect query results [#53746](https://github.com/pingcap/tidb/issues/53746) @[tangenta](https://github.com/tangenta) + - Fix the issue of reusing wrong point get plans for `SELECT ... FOR UPDATE` [#54652](https://github.com/pingcap/tidb/issues/54652) @[qw4990](https://github.com/qw4990) + ++ TiKV + + - Fix the issue that setting the gRPC message compression method via `grpc-compression-type` does not take effect on messages sent from TiKV to TiDB [#17176](https://github.com/tikv/tikv/issues/17176) @[ekexium](https://github.com/ekexium) + - Fix the issue that highly concurrent Coprocessor requests might cause TiKV OOM [#16653](https://github.com/tikv/tikv/issues/16653) @[overvenus](https://github.com/overvenus) + - Fix the issue that CDC and log-backup do not limit the timeout of `check_leader` using the `advance-ts-interval` configuration, causing the `resolved_ts` lag to be too large when TiKV restarts normally in some cases [#17107](https://github.com/tikv/tikv/issues/17107) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that TiKV might repeatedly panic when applying a corrupted Raft data snapshot [#15292](https://github.com/tikv/tikv/issues/15292) @[LykxSassinator](https://github.com/LykxSassinator) + ++ PD + + - Fix the issue that slots are not fully deleted in a resource group client, which causes the number of the allocated tokens to be less than the specified value [#7346](https://github.com/tikv/pd/issues/7346) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that a resource group encounters quota limits when requesting tokens for more than 500 ms [#8349](https://github.com/tikv/pd/issues/8349) @[nolouch](https://github.com/nolouch) + - Fix the data race issue of resource groups [#8267](https://github.com/tikv/pd/issues/8267) @[HuSharp](https://github.com/HuSharp) + - Fix the data race issue that PD encounters during operator checks [#8263](https://github.com/tikv/pd/issues/8263) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that deleted nodes still appear in the candidate connection list in etcd client [#8286](https://github.com/tikv/pd/issues/8286) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that setting the TiKV configuration item [`coprocessor.region-split-size`](/tikv-configuration-file.md#region-split-size) to a value less than 1 MiB causes PD panic [#8323](https://github.com/tikv/pd/issues/8323) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that the encryption manager is not initialized before use [#8384](https://github.com/tikv/pd/issues/8384) @[releungx](https://github.com/releungx) + - Fix the issue that PD logs are not fully redacted when the PD configuration item [`security.redact-info-log`](/pd-configuration-file.md#redact-info-log-new-in-v50) is enabled [#8419](https://github.com/tikv/pd/issues/8419) @[releungx](https://github.com/releungx) + - Fix the issue that no error is reported when binding a role to a resource group [#54417](https://github.com/pingcap/tidb/issues/54417) @[JmPotato](https://github.com/JmPotato) + ++ TiFlash + + - Fix the issue that a large number of duplicate rows might be read in FastScan mode after importing data via BR or TiDB Lightning [#9118](https://github.com/pingcap/tiflash/issues/9118) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that the `SUBSTRING_INDEX()` function might cause TiFlash to crash in some corner cases [#9116](https://github.com/pingcap/tiflash/issues/9116) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that setting the SSL certificate configuration to an empty string in TiFlash incorrectly enables TLS and causes TiFlash to fail to start [#9235](https://github.com/pingcap/tiflash/issues/9235) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that some queries might report a column type mismatch error after late materialization is enabled [#9175](https://github.com/pingcap/tiflash/issues/9175) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that queries with virtual generated columns might return incorrect results after late materialization is enabled [#9188](https://github.com/pingcap/tiflash/issues/9188) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that TiFlash might panic after executing `RENAME TABLE ... TO ...` on a partitioned table with empty partitions across databases [#9132](https://github.com/pingcap/tiflash/issues/9132) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash might panic when a database is deleted shortly after creation [#9266](https://github.com/pingcap/tiflash/issues/9266) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that log backup might be paused after the advancer owner migration [#53561](https://github.com/pingcap/tidb/issues/53561) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that BR fails to correctly identify errors due to multiple nested retries during the restore process [#54053](https://github.com/pingcap/tidb/issues/54053) @[RidRisR](https://github.com/RidRisR) + - Fix the inefficiency issue in scanning DDL jobs during incremental backups [#54139](https://github.com/pingcap/tidb/issues/54139) @[3pointer](https://github.com/3pointer) + - Fix the issue that the backup performance during checkpoint backups is affected due to interruptions in seeking Region leaders [#17168](https://github.com/tikv/tikv/issues/17168) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that after pausing, stopping, and rebuilding the log backup task, the task status is normal, but the checkpoint does not advance [#53047](https://github.com/pingcap/tidb/issues/53047) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that DDLs requiring backfilling, such as `ADD INDEX` and `MODIFY COLUMN`, might not be correctly recovered during incremental restore [#54426](https://github.com/pingcap/tidb/issues/54426) @[3pointer](https://github.com/3pointer) + + + TiCDC + + - Fix the issue that the checksum is not correctly set to `0` after splitting `UPDATE` events [#11402](https://github.com/pingcap/tiflow/issues/11402) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that the Processor module might get stuck when the downstream Kafka is inaccessible [#11340](https://github.com/pingcap/tiflow/issues/11340) @[asddongmen](https://github.com/asddongmen) + + + Dumpling + + - Fix the issue that Dumpling reports an error when exporting tables and views at the same time [#53682](https://github.com/pingcap/tidb/issues/53682) @[tangenta](https://github.com/tangenta) diff --git a/releases/release-7.5.4.md b/releases/release-7.5.4.md new file mode 100644 index 0000000000000..5abb7291674d6 --- /dev/null +++ b/releases/release-7.5.4.md @@ -0,0 +1,125 @@ +--- +title: TiDB 7.5.4 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.5.4. +--- + +# TiDB 7.5.4 Release Notes + +Release date: October 15, 2024 + +TiDB version: 7.5.4 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.5/production-deployment-using-tiup) + +## Compatibility changes + +- Set a default limit of 2048 for DDL historical tasks retrieved through the [TiDB HTTP API](https://github.com/pingcap/tidb/blob/release-7.5/docs/tidb_http_api.md) to prevent OOM issues caused by excessive historical tasks [#55711](https://github.com/pingcap/tidb/issues/55711) @[joccau](https://github.com/joccau) + +## Improvements + ++ TiDB + + - Support applying the `tidb_redact_log` setting to the output of `EXPLAIN` statements and further optimize the logic in processing logs [#54565](https://github.com/pingcap/tidb/issues/54565) @[hawkingrei](https://github.com/hawkingrei) + - Optimize the query speed of TiDB slow queries [#54630](https://github.com/pingcap/tidb/pull/54630) @[yibin87](https://github.com/yibin87) + ++ TiKV + + - Optimize the trigger mechanism of RocksDB compaction to accelerate disk space reclamation when handling a large number of DELETE versions [#17269](https://github.com/tikv/tikv/issues/17269) @[AndreMouche](https://github.com/AndreMouche) + - Reduce the memory usage of the peer message channel [#16229](https://github.com/tikv/tikv/issues/16229) @[Connor1996](https://github.com/Connor1996) + - Optimize the jittery access delay when restarting TiKV due to waiting for the log to be applied, improving the stability of TiKV [#15874](https://github.com/tikv/tikv/issues/15874) @[LykxSassinator](https://github.com/LykxSassinator) + - Optimize TiKV's `DiskFull` detection to make it compatible with RaftEngine's `spill-dir` configuration, ensuring that this feature works consistently [#17356](https://github.com/tikv/tikv/issues/17356) @[LykxSassinator](https://github.com/LykxSassinator) + ++ TiFlash + + - Optimize the execution efficiency of `LENGTH()` and `ASCII()` functions [#9344](https://github.com/pingcap/tiflash/issues/9344) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Mitigate the issue that TiFlash might panic due to updating certificates after TLS is enabled [#8535](https://github.com/pingcap/tiflash/issues/8535) @[windtalker](https://github.com/windtalker) + - Improve the cancel mechanism of the JOIN operator, so that the JOIN operator can respond to cancel requests in a timely manner [#9430](https://github.com/pingcap/tiflash/issues/9430) @[windtalker](https://github.com/windtalker) + ++ Tools + + + Backup & Restore (BR) + + - Support checking whether the disk space in TiKV is sufficient before TiKV downloads each SST file. If the space is insufficient, BR terminates the restore and returns an error [#17224](https://github.com/tikv/tikv/issues/17224) @[RidRisR](https://github.com/RidRisR) + + + TiCDC + + - When the downstream is TiDB with the `SUPER` permission granted, TiCDC supports querying the execution status of `ADD INDEX DDL` from the downstream database to avoid data replication failure due to timeout in retrying executing the DDL statement in some cases [#10682](https://github.com/pingcap/tiflow/issues/10682) @[CharlesCheung96](https://github.com/CharlesCheung96) + +## Bug fixes + ++ TiDB + + - Fix the issue that `FLASHBACK DATABASE` fails when many tables exist in the database [#54415](https://github.com/pingcap/tidb/issues/54415) @[lance6716](https://github.com/lance6716) + - Fix the issue that RANGE partitioned tables that are not strictly self-incrementing can be created [#54829](https://github.com/pingcap/tidb/issues/54829) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that a query statement that contains `UNION` might return incorrect results [#52985](https://github.com/pingcap/tidb/issues/52985) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that `INDEX_HASH_JOIN` cannot exit properly when SQL is abnormally interrupted [#54688](https://github.com/pingcap/tidb/issues/54688) @[wshwsh12](https://github.com/wshwsh12) + - Reset the parameters in the `Open` method of `PipelinedWindow` to fix the unexpected error that occurs when the `PipelinedWindow` is used as a child node of `Apply` due to the reuse of previous parameter values caused by repeated opening and closing operations [#53600](https://github.com/pingcap/tidb/issues/53600) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that the query latency of stale reads increases, caused by information schema cache misses [#53428](https://github.com/pingcap/tidb/issues/53428) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that disk files might not be deleted after the `Sort` operator spills and a query error occurs [#55061](https://github.com/pingcap/tidb/issues/55061) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that the query might return incorrect results instead of an error after being killed [#50089](https://github.com/pingcap/tidb/issues/50089) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that table replication fails when the index length of the table replicated from DM exceeds the maximum length specified by `max-index-length` [#55138](https://github.com/pingcap/tidb/issues/55138) @[lance6716](https://github.com/lance6716) + - Fix the issue that the `SUB_PART` value in the `INFORMATION_SCHEMA.STATISTICS` table is `NULL` [#55812](https://github.com/pingcap/tidb/issues/55812) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that an error occurs when a DML statement contains nested generated columns [#53967](https://github.com/pingcap/tidb/issues/53967) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that the `tot_col_size` column in the `mysql.stats_histograms` table might be a negative number [#55126](https://github.com/pingcap/tidb/issues/55126) @[qw4990](https://github.com/qw4990) + - Fix the data race issue in `IndexNestedLoopHashJoin` [#49692](https://github.com/pingcap/tidb/issues/49692) @[solotzg](https://github.com/solotzg) + - Fix the issue that the query might get stuck when terminated because the memory usage exceeds the limit set by `tidb_mem_quota_query` [#55042](https://github.com/pingcap/tidb/issues/55042) @[yibin87](https://github.com/yibin87) + - Fix the issue that `columnEvaluator` cannot identify the column references in the input chunk, which leads to `runtime error: index out of range` when executing SQL statements [#53713](https://github.com/pingcap/tidb/issues/53713) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that the performance of the `SELECT ... WHERE ... ORDER BY ...` statement execution is poor in some cases [#54969](https://github.com/pingcap/tidb/issues/54969) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that empty `groupOffset` in `StreamAggExec` might cause TiDB to panic [#53867](https://github.com/pingcap/tidb/issues/53867) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that TiDB queries cannot be canceled during cop task construction [#55957](https://github.com/pingcap/tidb/issues/55957) @[yibin87](https://github.com/yibin87) + - Fix the issue that an `out of range` error might occur when a small display width is specified for a column of the integer type [#55837](https://github.com/pingcap/tidb/issues/55837) @[windtalker](https://github.com/windtalker) + - Fix the issue that `duplicate entry` might occur when adding unique indexes [#56161](https://github.com/pingcap/tidb/issues/56161) @[tangenta](https://github.com/tangenta) + - Fix the issue that TiDB panics when importing temporary tables using the `IMPORT INTO` statement [#55970](https://github.com/pingcap/tidb/issues/55970) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue of data index inconsistency caused by retries during index addition [#55808](https://github.com/pingcap/tidb/issues/55808) @[lance6716](https://github.com/lance6716) + ++ TiKV + + - Fix the issue that TiKV might panic when a stale replica processes Raft snapshots, triggered by a slow split operation and immediate removal of the new replica [#17469](https://github.com/tikv/tikv/issues/17469) @[hbisheng](https://github.com/hbisheng) + - Fix the flow control issue that might occur after deleting large tables or partitions [#17304](https://github.com/tikv/tikv/issues/17304) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that bloom filters are incompatible between earlier versions (earlier than v7.1) and later versions [#17272](https://github.com/tikv/tikv/issues/17272) @[v01dstar](https://github.com/v01dstar) + - Fix the issue that prevents master key rotation when the master key is stored in a Key Management Service (KMS) [#17410](https://github.com/tikv/tikv/issues/17410) @[hhwyt](https://github.com/hhwyt) + - Fix the issue that the **Storage async write duration** monitoring metric on the TiKV panel in Grafana is inaccurate [#17579](https://github.com/tikv/tikv/issues/17579) @[overvenus](https://github.com/overvenus) + - Fix the issue that when a large number of transactions are queuing for lock release on the same key and the key is frequently updated, excessive pressure on deadlock detection might cause TiKV OOM issues [#17394](https://github.com/tikv/tikv/issues/17394) @[MyonKeminta](https://github.com/MyonKeminta) + ++ PD + + - Fix the issue that PD's Region API cannot be requested when a large number of Regions exist [#55872](https://github.com/pingcap/tidb/issues/55872) @[rleungx](https://github.com/rleungx) + - Fix the issue that when using a wrong parameter in `evict-leader-scheduler`, PD does not report errors correctly and some schedulers are unavailable [#8619](https://github.com/tikv/pd/issues/8619) @[rleungx](https://github.com/rleungx) + - Fix the issue that data race might occur in the scheduling server when a PD leader is switched in the microservice mode [#8538](https://github.com/tikv/pd/issues/8538) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that slots are not fully deleted in a resource group client, which causes the number of the allocated tokens to be less than the specified value [#7346](https://github.com/tikv/pd/issues/7346) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that the time data type in the `INFORMATION_SCHEMA.RUNAWAY_WATCHES` table is incorrect [#54770](https://github.com/pingcap/tidb/issues/54770) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that setting `replication.strictly-match-label` to `true` causes TiFlash to fail to start [#8480](https://github.com/tikv/pd/issues/8480) @[rleungx](https://github.com/rleungx) + ++ TiFlash + + - Fix the issue that TiFlash write nodes might fail to restart in the disaggregated storage and compute architecture [#9282](https://github.com/pingcap/tiflash/issues/9282) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that a network partition (network disconnection) between TiFlash and any PD might cause read request timeout errors [#9243](https://github.com/pingcap/tiflash/issues/9243) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that when using the `CAST()` function to convert a string to a datetime with a time zone or invalid characters, the result is incorrect [#8754](https://github.com/pingcap/tiflash/issues/8754) @[solotzg](https://github.com/solotzg) + - Fix the issue that read snapshots of TiFlash write nodes are not released in a timely manner in the disaggregated storage and compute architecture [#9298](https://github.com/pingcap/tiflash/issues/9298) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that TiFlash fails to parse the table schema when the table contains Bit-type columns with a default value that contains invalid characters [#9461](https://github.com/pingcap/tiflash/issues/9461) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that some queries might report errors when late materialization is enabled [#9472](https://github.com/pingcap/tiflash/issues/9472) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that converting a data type to the `DECIMAL` type might lead to wrong query results in some extreme cases [#53892](https://github.com/pingcap/tidb/issues/53892) @[guo-shaoge](https://github.com/guo-shaoge) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that backup tasks might get stuck if TiKV becomes unresponsive during the backup process [#53480](https://github.com/pingcap/tidb/issues/53480) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the checkpoint path of backup and restore is incompatible with some external storage [#55265](https://github.com/pingcap/tidb/issues/55265) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that after a log backup PITR task fails and you stop it, the safepoints related to that task are not properly cleared in PD [#17316](https://github.com/tikv/tikv/issues/17316) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that BR logs might print sensitive credential information when log backup is enabled [#55273](https://github.com/pingcap/tidb/issues/55273) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that BR integration test cases are unstable, and add a new test case to simulate snapshot or log backup file corruption [#53835](https://github.com/pingcap/tidb/issues/53835) @[Leavrth](https://github.com/Leavrth) + + + TiCDC + + - Fix the issue that the **barrier-ts** monitoring metric for changefeed checkpoints might be inaccurate [#11553](https://github.com/pingcap/tiflow/issues/11553) @[3AceShowHand](https://github.com/3AceShowHand) + + + TiDB Data Migration (DM) + + - Fix the issue that data replication is interrupted when the index length exceeds the default value of `max-index-length` [#11459](https://github.com/pingcap/tiflow/issues/11459) @[michaelmdeng](https://github.com/michaelmdeng) + - Fix the issue that DM does not set the default database when processing the `ALTER DATABASE` statement, which causes a replication error [#11503](https://github.com/pingcap/tiflow/issues/11503) @[lance6716](https://github.com/lance6716) + - Fix the issue that multiple DM-master nodes might simultaneously become leaders, leading to data inconsistency [#11602](https://github.com/pingcap/tiflow/issues/11602) @[GMHDBJD](https://github.com/GMHDBJD) + + + TiDB Lightning + + - Fix the issue that transaction conflicts occur during data import using TiDB Lightning [#49826](https://github.com/pingcap/tidb/issues/49826) @[lance6716](https://github.com/lance6716) diff --git a/releases/release-7.5.5.md b/releases/release-7.5.5.md new file mode 100644 index 0000000000000..d372ac42bd61c --- /dev/null +++ b/releases/release-7.5.5.md @@ -0,0 +1,167 @@ +--- +title: TiDB 7.5.5 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.5.5. +--- + +# TiDB 7.5.5 Release Notes + +Release date: December 31, 2024 + +TiDB version: 7.5.5 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.5/production-deployment-using-tiup) + +## Compatibility changes + +- Add a new system variable [`tidb_ddl_reorg_max_write_speed`](https://docs.pingcap.com/tidb/v7.5/system-variables#tidb_ddl_reorg_max_write_speed-new-in-v6512-and-v755) to limit the maximum speed of the ingest phase when adding indexes [#57156](https://github.com/pingcap/tidb/issues/57156) @[CbcWestwolf](https://github.com/CbcWestwolf) +- Change the default value of the TiKV configuration item [`raft-client-queue-size`](/tikv-configuration-file.md#raft-client-queue-size) from `8192` to `16384` [#17101](https://github.com/tikv/tikv/issues/17101) @[Connor1996](https://github.com/Connor1996) + +## Improvements + ++ TiDB + + - Adjust estimation results from 0 to 1 for equality conditions that do not hit TopN when statistics are entirely composed of TopN and the modified row count in the corresponding table statistics is non-zero [#47400](https://github.com/pingcap/tidb/issues/47400) @[terry1purcell](https://github.com/terry1purcell) + ++ TiKV + + - Fix the issue that when Raft and RocksDB are deployed on different disks, the slow disk detection does not work for the disk where RocksDB is located [#17884](https://github.com/tikv/tikv/issues/17884) @[LykxSassinator](https://github.com/LykxSassinator) + - Add slow logs for peer and store messages [#16600](https://github.com/tikv/tikv/issues/16600) @[Connor1996](https://github.com/Connor1996) + - Optimize the jittery access delay when restarting TiKV due to waiting for the log to be applied, improving the stability of TiKV [#15874](https://github.com/tikv/tikv/issues/15874) @[LykxSassinator](https://github.com/LykxSassinator) + ++ TiFlash + + - Mitigate the issue that TiFlash might panic due to updating certificates after TLS is enabled [#8535](https://github.com/pingcap/tiflash/issues/8535) @[windtalker](https://github.com/windtalker) + - Improve the garbage collection speed of outdated data in the background for tables with clustered indexes [#9529](https://github.com/pingcap/tiflash/issues/9529) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Optimize the retry strategy for TiFlash compute nodes in the disaggregated storage and compute architecture to handle exceptions when downloading files from Amazon S3 [#9695](https://github.com/pingcap/tiflash/issues/9695) @[JinheLin](https://github.com/JinheLin) + ++ Tools + + + Backup & Restore (BR) + + - Reduce unnecessary log printing during backup [#55902](https://github.com/pingcap/tidb/issues/55902) @[Leavrth](https://github.com/Leavrth) + + + TiDB Data Migration (DM) + + - Lower the log level for unrecognized MariaDB binlog events [#10204](https://github.com/pingcap/tiflow/issues/10204) @[dveeden](https://github.com/dveeden) + +## Bug fixes + ++ TiDB + + - Fix the issue that TiDB cannot resume Reorg DDL tasks from the previous progress after the DDL owner node is switched [#56506](https://github.com/pingcap/tidb/issues/56506) @[tangenta](https://github.com/tangenta) + - Fix the issue that invalid `NULL` values can be inserted in non-strict mode (`sql_mode = ''`) [#56381](https://github.com/pingcap/tidb/issues/56381) @[joechenrh](https://github.com/joechenrh) + - Fix the issue that the data in the **Stats Healthy Distribution** panel of Grafana might be incorrect [#57176](https://github.com/pingcap/tidb/issues/57176) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue of illegal memory access that might occur when a Common Table Expression (CTE) has multiple data consumers and one consumer exits without reading any data [#55881](https://github.com/pingcap/tidb/issues/55881) @[windtalker](https://github.com/windtalker) + - Fix the issue that existing TTL tasks are executed unexpectedly frequently in a cluster that is upgraded from v6.5 to v7.5 or later [#56539](https://github.com/pingcap/tidb/issues/56539) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that TTL tasks are not canceled after the `tidb_ttl_job_enable` variable is disabled [#57404](https://github.com/pingcap/tidb/issues/57404) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the query latency of stale reads increases, caused by information schema cache misses [#53428](https://github.com/pingcap/tidb/issues/53428) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that stale read does not strictly verify the timestamp of the read operation, resulting in a small probability of affecting the consistency of the transaction when an offset exists between the TSO and the real physical time [#56809](https://github.com/pingcap/tidb/issues/56809) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that the `AUTO_INCREMENT` field is not correctly set after importing data using the `IMPORT INTO` statement [#56476](https://github.com/pingcap/tidb/issues/56476) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that two DDL Owners might exist at the same time [#54689](https://github.com/pingcap/tidb/issues/54689) @[joccau](https://github.com/joccau) + - Fix the issue that TTL might fail if TiKV is not selected as the storage engine [#56402](https://github.com/pingcap/tidb/issues/56402) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that TiDB does not check the index length limitation when executing `ADD INDEX` [#56930](https://github.com/pingcap/tidb/issues/56930) @[fzzf678](https://github.com/fzzf678) + - Fix the issue that when canceling a TTL task, the corresponding SQL is not killed forcibly [#56511](https://github.com/pingcap/tidb/issues/56511) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that execution plan bindings cannot be created for the multi-table `DELETE` statement with aliases [#56726](https://github.com/pingcap/tidb/issues/56726) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that when using `ANALYZE` to collect statistics for a table, if the table contains expression indexes of virtually generated columns, the execution reports an error [#57079](https://github.com/pingcap/tidb/issues/57079) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that Plan Replayer might report an error when importing a table structure containing Placement Rules [#54961](https://github.com/pingcap/tidb/issues/54961) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that when parsing a database name in CTE, it returns a wrong database name [#54582](https://github.com/pingcap/tidb/issues/54582) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `INSERT ... ON DUPLICATE KEY` statement is not compatible with `mysql_insert_id` [#55965](https://github.com/pingcap/tidb/issues/55965) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that improper use of metadata locks might lead to writing anomalous data when using the plan cache under certain circumstances [#53634](https://github.com/pingcap/tidb/issues/53634) @[zimulala](https://github.com/zimulala) + - Fix the issue that the performance is unstable when adding indexes using Global Sort [#54147](https://github.com/pingcap/tidb/issues/54147) @[tangenta](https://github.com/tangenta) + - Fix the issue that Plan Replayer might report an error when importing a table structure containing foreign keys [#56456](https://github.com/pingcap/tidb/issues/56456) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that using the `RANGE COLUMNS` partition function and the `utf8mb4_0900_ai_ci` collation at the same time could result in incorrect query results [#57261](https://github.com/pingcap/tidb/issues/57261) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that using subqueries after the `NATURAL JOIN` or `USING` clause might result in errors [#53766](https://github.com/pingcap/tidb/issues/53766) @[dash12653](https://github.com/dash12653) + - Fix the issue that TTL tasks cannot be canceled when there is a write conflict [#56422](https://github.com/pingcap/tidb/issues/56422) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that if a CTE contains the `ORDER BY`, `LIMIT`, or `SELECT DISTINCT` clause and is referenced by the recursive part of another CTE, it might be incorrectly inlined and result in an execution error [#56603](https://github.com/pingcap/tidb/issues/56603) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that the `UPDATE` statement incorrectly updates values of the `ENUM` type [#56832](https://github.com/pingcap/tidb/issues/56832) @[xhebox](https://github.com/xhebox) + - Fix the issue that executing `RECOVER TABLE BY JOB JOB_ID;` might cause TiDB to panic [#55113](https://github.com/pingcap/tidb/issues/55113) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that the `read_from_storage` hint might not take effect when the query has an available Index Merge execution plan [#56217](https://github.com/pingcap/tidb/issues/56217) @[AilinKid](https://github.com/AilinKid) + - Fix the issue that `INDEX_HASH_JOIN` might hang during an abnormal exit [#54055](https://github.com/pingcap/tidb/issues/54055) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that querying system tables related to the Distributed eXecution Framework (DXF) might lead to upgrade failures [#49263](https://github.com/pingcap/tidb/issues/49263) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that the DDL internal transaction error `GC life time is shorter than transaction duration` causes index addition to fail [#57043](https://github.com/pingcap/tidb/issues/57043) @[tangenta](https://github.com/tangenta) + - Fix the issue that when executing `EXCHANGE PARTITION` and encountering invalid rows, InfoSchema is fully loaded and the error `failed to load schema diff` is reported [#56685](https://github.com/pingcap/tidb/issues/56685) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that collation is not handled correctly when `tidb_ddl_enable_fast_reorg` and `new_collations_enabled_on_first_bootstrap` are enabled, resulting in inconsistent data indexes [#58036](https://github.com/pingcap/tidb/issues/58036) @[djshow832](https://github.com/djshow832) + - Fix the issue that data indexes are inconsistent because plan cache uses the wrong schema when adding indexes [#56733](https://github.com/pingcap/tidb/issues/56733) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that executing `ALTER TABLE TIFLASH REPLICA` during upgrade causes the TiDB node to crash [#57863](https://github.com/pingcap/tidb/issues/57863) @[tangenta](https://github.com/tangenta) + - Fix the issue that the performance of querying `INFORMATION_SCHEMA.columns` degrades [#58184](https://github.com/pingcap/tidb/issues/58184) @[lance6716](https://github.com/lance6716) + - Fix the issue that the default timeout for querying the TiFlash system table is too short [#57816](https://github.com/pingcap/tidb/issues/57816) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that the value of the `default_collation_for_utf8mb4` variable does not work for the `SET NAMES` statement [#56439](https://github.com/pingcap/tidb/issues/56439) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that TTL internal coroutine might panic when you manually delete a timer in the `mysql.tidb_timer` table [#57112](https://github.com/pingcap/tidb/issues/57112) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that when converting a normal table to a partitioned table using the `ALTER TABLE` statement, insufficient checks might result in incorrect data [#55721](https://github.com/pingcap/tidb/issues/55721) @[mjonss](https://github.com/mjonss) + - Fix the issue that when setting `tidb_gogc_tuner_max_value` and `tidb_gogc_tuner_min_value`, if the maximum value is null, an incorrect warning message occurs [#57889](https://github.com/pingcap/tidb/issues/57889) @[hawkingrei](https://github.com/hawkingrei) + - Fix the potential data race issue that might occur in TiDB's internal coroutine [#57798](https://github.com/pingcap/tidb/issues/57798) [#56053](https://github.com/pingcap/tidb/issues/56053) @[fishiu](https://github.com/fishiu) @[tiancaiamao](https://github.com/tiancaiamao) + - Update `golang-jwt` and `jwt` to prevent potential security risks [#57135](https://github.com/pingcap/tidb/issues/57135) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that concurrent writes might result in duplicate data when converting a table with clustered indexes to a partitioned table using the `ALTER TABLE` statement [#57510](https://github.com/pingcap/tidb/issues/57510) @[mjonss](https://github.com/mjonss) + ++ TiKV + + - Fix the issue that merging Regions might cause TiKV to panic in rare cases [#17840](https://github.com/tikv/tikv/issues/17840) @[glorv](https://github.com/glorv) + - Fix the issue that when Raft and RocksDB are deployed on different disks, the slow disk detection does not work for the disk where RocksDB is located [#17884](https://github.com/tikv/tikv/issues/17884) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that the output of jprof is not correctly captured and processed when the `log-file` parameter is not specified [#17607](https://github.com/tikv/tikv/issues/17607) @[Hexilee](https://github.com/Hexilee) + - Fix the issue that latency might increase when hibernated Regions are awake [#17101](https://github.com/tikv/tikv/issues/17101) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that TiKV might panic when executing queries containing `RADIANS()` or `DEGREES()` functions [#17852](https://github.com/tikv/tikv/issues/17852) @[gengliqi](https://github.com/gengliqi) + - Fix the panic issue that occurs when read threads access outdated indexes in the MemTable of the Raft Engine [#17383](https://github.com/tikv/tikv/issues/17383) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that when a large number of transactions are queuing for lock release on the same key and the key is frequently updated, excessive pressure on deadlock detection might cause TiKV OOM issues [#17394](https://github.com/tikv/tikv/issues/17394) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that write jitter might occur when all hibernated Regions are awakened [#17101](https://github.com/tikv/tikv/issues/17101) @[hhwyt](https://github.com/hhwyt) + - Fix the issue that TiKV converts the time zone incorrectly for Brazil and Egypt [#16220](https://github.com/tikv/tikv/issues/16220) @[overvenus](https://github.com/overvenus) + - Fix the issue that Online Unsafe Recovery cannot handle merge abort [#15580](https://github.com/tikv/tikv/issues/15580) @[v01dstar](https://github.com/v01dstar) + - Fix the issue that CPU profiling flag is not reset correctly when an error occurs [#17234](https://github.com/tikv/tikv/issues/17234) @[Connor1996](https://github.com/Connor1996) + - Fix the issue that large batch writes cause performance jitter when `raft-entry-max-size` is set too high [#17701](https://github.com/tikv/tikv/issues/17701) @[SpadeA-Tang](https://github.com/SpadeA-Tang) + - Fix the issue that improper error handling in the conflict detection interface of the import module might cause TiKV to panic [#17830](https://github.com/tikv/tikv/issues/17830) @[joccau](https://github.com/joccau) + ++ PD + + - Fix the issue that when creating `evict-leader-scheduler` or `grant-leader-scheduler` encounters an error, the error message is not returned to pd-ctl [#8759](https://github.com/tikv/pd/issues/8759) @[okJiang](https://github.com/okJiang) + - Fix the issue that PD cannot quickly re-elect a leader during etcd leader transition [#8823](https://github.com/tikv/pd/issues/8823) @[rleungx](https://github.com/rleungx) + - Fix the memory leak issue in label statistics [#8700](https://github.com/tikv/pd/issues/8700) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that `evict-leader-scheduler` fails to work properly when it is repeatedly created with the same Store ID [#8756](https://github.com/tikv/pd/issues/8756) @[okJiang](https://github.com/okJiang) + - Fix the issue that slots are not fully deleted in a resource group client, which causes the number of the allocated tokens to be less than the specified value [#7346](https://github.com/tikv/pd/issues/7346) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that when using a wrong parameter in `evict-leader-scheduler`, PD does not report errors correctly and some schedulers are unavailable [#8619](https://github.com/tikv/pd/issues/8619) @[rleungx](https://github.com/rleungx) + - Fix the memory leak issue in hotspot cache [#8698](https://github.com/tikv/pd/issues/8698) @[lhy1024](https://github.com/lhy1024) + - Fix the performance jitter issue caused by frequent creation of random number generator [#8674](https://github.com/tikv/pd/issues/8674) @[rleungx](https://github.com/rleungx) + ++ TiFlash + + - Fix the TiFlash panic issue when TiFlash encounters conflicts during concurrent DDL execution [#8578](https://github.com/pingcap/tiflash/issues/8578) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that `LPAD()` and `RPAD()` functions return incorrect results in some cases [#9465](https://github.com/pingcap/tiflash/issues/9465) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that the `SUBSTRING()` function returns incorrect results when the second parameter is negative [#9604](https://github.com/pingcap/tiflash/issues/9604) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that TiFlash fails to parse the table schema when the table contains Bit-type columns with a default value that contains invalid characters [#9461](https://github.com/pingcap/tiflash/issues/9461) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that querying new columns might return incorrect results under the disaggregated storage and compute architecture [#9665](https://github.com/pingcap/tiflash/issues/9665) @[zimulala](https://github.com/zimulala) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that when resuming from a checkpoint after data restore fails, an error `the target cluster is not fresh` occurs [#50232](https://github.com/pingcap/tidb/issues/50232) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that log backups cannot resolve residual locks promptly, causing the checkpoint to fail to advance [#57134](https://github.com/pingcap/tidb/issues/57134) @[3pointer](https://github.com/3pointer) + - Fix the issue that logs might print out encrypted information [#57585](https://github.com/pingcap/tidb/issues/57585) @[kennytm](https://github.com/kennytm) + - Fix the issue that the `TestStoreRemoved` test case is unstable [#52791](https://github.com/pingcap/tidb/issues/52791) @[YuJuncen](https://github.com/YuJuncen) + - Fix potential security vulnerabilities by upgrading the `k8s.io/api` library version [#57790](https://github.com/pingcap/tidb/issues/57790) @[BornChanger](https://github.com/BornChanger) + - Fix the issue that PITR tasks might return the `Information schema is out of date` error when there are a large number of tables in the cluster but the actual data size is small [#57743](https://github.com/pingcap/tidb/issues/57743) @[Tristan1900](https://github.com/Tristan1900) + + + TiCDC + + - Fix the issue that TiCDC might report an error when replicating a `TRUNCATE TABLE` DDL on a table without valid index [#11765](https://github.com/pingcap/tiflow/issues/11765) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that the `tableID` for partitioned tables is not correctly set in Simple Protocol messages [#11846](https://github.com/pingcap/tiflow/issues/11846) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that the redo module fails to properly report errors [#11744](https://github.com/pingcap/tiflow/issues/11744) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that after filtering out `add table partition` events is configured in `ignore-event`, TiCDC does not replicate other types of DML changes for related partitions to the downstream [#10524](https://github.com/pingcap/tiflow/issues/10524) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that TiCDC mistakenly discards DDL tasks when the schema versions of DDL tasks become non-incremental during TiDB DDL owner changes [#11714](https://github.com/pingcap/tiflow/issues/11714) @[wlwilliamx](https://github.com/wlwilliamx) + + + TiDB Data Migration (DM) + + - Fix the issue that automatically generated IDs in a table might have significant jumps after data import in physical import mode [#11768](https://github.com/pingcap/tiflow/issues/11768) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that the pre-check of `start-task` fails when both TLS and `shard-mode` are configured [#11842](https://github.com/pingcap/tiflow/issues/11842) @[sunxiaoguang](https://github.com/sunxiaoguang) + - Fix the issue that connecting to MySQL 8.0 fails when the password length exceeds 19 characters [#11603](https://github.com/pingcap/tiflow/issues/11603) @[fishiu](https://github.com/fishiu) + + + TiDB Lightning + + - Fix the issue that TiDB Lightning fails to receive oversized messages sent from TiKV [#56114](https://github.com/pingcap/tidb/issues/56114) @[fishiu](https://github.com/fishiu) + - Fix the issue that the `AUTO_INCREMENT` value is set too high after importing data using the physical import mode [#56814](https://github.com/pingcap/tidb/issues/56814) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that TiDB Lightning does not automatically retry when encountering `Lock wait timeout` errors during metadata updates [#53042](https://github.com/pingcap/tidb/issues/53042) @[guoshouyan](https://github.com/guoshouyan) + - Fix the issue that the performance degrades when importing data from a cloud storage in high-concurrency scenarios [#57413](https://github.com/pingcap/tidb/issues/57413) @[xuanyu66](https://github.com/xuanyu66) + - Fix the issue that TiDB Lightning might get stuck for a long time during the preparation phase when importing a large number of Parquet files [#56104](https://github.com/pingcap/tidb/issues/56104) @[zeminzhou](https://github.com/zeminzhou) + - Fix the issue that the error report output is truncated when importing data using TiDB Lightning [#58085](https://github.com/pingcap/tidb/issues/58085) @[lance6716](https://github.com/lance6716) + + + Dumpling + + - Fix the issue that Dumpling fails to retry properly when receiving a 503 error from Google Cloud Storage (GCS) [#56127](https://github.com/pingcap/tidb/issues/56127) @[OliverS929](https://github.com/OliverS929) diff --git a/releases/release-7.5.6.md b/releases/release-7.5.6.md new file mode 100644 index 0000000000000..dc43357dff058 --- /dev/null +++ b/releases/release-7.5.6.md @@ -0,0 +1,137 @@ +--- +title: TiDB 7.5.6 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 7.5.6. +--- + +# TiDB 7.5.6 Release Notes + +Release date: March 14, 2025 + +TiDB version: 7.5.6 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.5/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v7.5/production-deployment-using-tiup) + +## Compatibility changes + +- Support the openEuler 22.03 LTS SP3/SP4 operating system. For more information, see [OS and platform requirements](https://docs.pingcap.com/tidb/v7.5/hardware-and-software-requirements#os-and-platform-requirements). + +## Improvements + ++ TiDB + + - Adjust estimation results from 0 to 1 for equality conditions that do not hit TopN when statistics are entirely composed of TopN and the modified row count in the corresponding table statistics is non-zero [#47400](https://github.com/pingcap/tidb/issues/47400) @[terry1purcell](https://github.com/terry1purcell) + - Enhance the timestamp validity check [#57786](https://github.com/pingcap/tidb/issues/57786) @[MyonKeminta](https://github.com/MyonKeminta) + - Limit the execution of GC for TTL tables and related statistics collection tasks to the owner node, thereby reducing overhead [#59357](https://github.com/pingcap/tidb/issues/59357) @[lcwangchao](https://github.com/lcwangchao) + ++ TiKV + + - Add the detection mechanism for invalid `max_ts` updates [#17916](https://github.com/tikv/tikv/issues/17916) @[ekexium](https://github.com/ekexium) + - Add slow logs for peer and store messages [#16600](https://github.com/tikv/tikv/issues/16600) @[Connor1996](https://github.com/Connor1996) + - Optimize the jittery access delay when restarting TiKV due to waiting for the log to be applied, improving the stability of TiKV [#15874](https://github.com/tikv/tikv/issues/15874) @[LykxSassinator](https://github.com/LykxSassinator) + ++ TiFlash + + - Mitigate the issue that TiFlash might panic due to updating certificates after TLS is enabled [#8535](https://github.com/pingcap/tiflash/issues/8535) @[windtalker](https://github.com/windtalker) + ++ Tools + + + Backup & Restore (BR) + + - Disable the table-level checksum calculation during full backups by default (`--checksum=false`) to improve backup performance [#56373](https://github.com/pingcap/tidb/issues/56373) @[Tristan1900](https://github.com/Tristan1900) + - Add a check to verify whether the target cluster contains a table with the same name for non-full restore [#55087](https://github.com/pingcap/tidb/issues/55087) @[RidRisR](https://github.com/RidRisR) + + + TiDB Lightning + + - Add a row width check when parsing CSV files to prevent OOM issues [#58590](https://github.com/pingcap/tidb/issues/58590) @[D3Hunter](https://github.com/D3Hunter) + +## Bug fixes + ++ TiDB + + - Fix the issue that some predicates might be lost when constructing `IndexMerge` [#58476](https://github.com/pingcap/tidb/issues/58476) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `ONLY_FULL_GROUP_BY` setting does not take effect on statements in views [#53175](https://github.com/pingcap/tidb/issues/53175) @[mjonss](https://github.com/mjonss) + - Fix the issue that creating two views with the same name does not report an error [#58769](https://github.com/pingcap/tidb/issues/58769) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that converting data from the `BIT` type to the `CHAR` type might cause TiKV panics [#56494](https://github.com/pingcap/tidb/issues/56494) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that a TTL job that loses the heartbeat blocks other jobs from getting heartbeats [#57915](https://github.com/pingcap/tidb/issues/57915) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that querying partitioned tables using an `IN` condition containing a mismatched value type and a type conversion error leads to incorrect query results [#54746](https://github.com/pingcap/tidb/issues/54746) @[mjonss](https://github.com/mjonss) + - Fix the issue that the default value of the `BIT` column is incorrect [#57301](https://github.com/pingcap/tidb/issues/57301) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that in the Prepare protocol, an error occurs when the client uses a non-UTF8 character set [#58870](https://github.com/pingcap/tidb/issues/58870) @[xhebox](https://github.com/xhebox) + - Fix the issue that using variables or parameters in the `CREATE VIEW` statement does not report errors [#53176](https://github.com/pingcap/tidb/issues/53176) @[mjonss](https://github.com/mjonss) + - Fix the issue that loading statistics manually might fail when the statistics file contains null values [#53966](https://github.com/pingcap/tidb/issues/53966) @[King-Dylan](https://github.com/King-Dylan) + - Fix the issue that when querying the `information_schema.cluster_slow_query` table, if the time filter is not added, only the latest slow log file is queried [#56100](https://github.com/pingcap/tidb/issues/56100) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that querying temporary tables might trigger unexpected TiKV requests in some cases [#58875](https://github.com/pingcap/tidb/issues/58875) @[tiancaiamao](https://github.com/tiancaiamao) + - Fix the issue that the data in the **Stats Healthy Distribution** panel of Grafana might be incorrect [#57176](https://github.com/pingcap/tidb/issues/57176) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that some TTL jobs might hang when modifying `tidb_ttl_delete_rate_limit` [#58484](https://github.com/pingcap/tidb/issues/58484) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that improper exception handling for statistics causes in-memory statistics to be mistakenly deleted when background tasks time out [#57901](https://github.com/pingcap/tidb/issues/57901) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that using `ORDER BY` when querying `cluster_slow_query table` might generate unordered results [#51723](https://github.com/pingcap/tidb/issues/51723) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that when a virtual generated column's dependencies contain a column with the `ON UPDATE` attribute, the data of the updated row and its index data might be inconsistent [#56829](https://github.com/pingcap/tidb/issues/56829) @[joechenrh](https://github.com/joechenrh) + - Fix the issue that TTL jobs cannot be canceled if the TiDB heartbeat is lost [#57784](https://github.com/pingcap/tidb/issues/57784) @[YangKeao](https://github.com/YangKeao) + - When the parameter is of `Enum`, `Bit`, or `Set` type, the `Conv()` function is no longer pushed down to TiKV [#51877](https://github.com/pingcap/tidb/issues/51877) @[yibin87](https://github.com/yibin87) + - Fix the issue that after executing `ALTER TABLE ... PLACEMENT POLICY ...` in a cluster with TiFlash nodes in the disaggregated storage and compute architecture, Region peers might be accidentally added to TiFlash Compute nodes [#58633](https://github.com/pingcap/tidb/issues/58633) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that job status is overwritten when the DDL owner changes [#52747](https://github.com/pingcap/tidb/issues/52747) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that queries with `is null` conditions on Hash partitioned tables cause a panic [#58374](https://github.com/pingcap/tidb/issues/58374) @[Defined2014](https://github.com/Defined2014/) + - Fix the issue that an error occurs when querying partitioned tables that contain generated columns [#58475](https://github.com/pingcap/tidb/issues/58475) @[joechenrh](https://github.com/joechenrh) + - Fix the issue that TTL jobs might be ignored or processed multiple times [#59347](https://github.com/pingcap/tidb/issues/59347) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that incorrect judgment in exchange partition causes execution failure [#59534](https://github.com/pingcap/tidb/issues/59534) @[mjonss](https://github.com/mjonss) + - Fix the issue that setting the `tidb_audit_log` variable with multi-level relative paths causes errors in the log directory [#58971](https://github.com/pingcap/tidb/issues/58971) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that different data types on both sides of the equality condition in Join might cause incorrect results in TiFlash [#59877](https://github.com/pingcap/tidb/issues/59877) @[yibin87](https://github.com/yibin87) + ++ TiKV + + - Fix the issue that the leader could not be quickly elected after Region split [#17602](https://github.com/tikv/tikv/issues/17602) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that time rollback might cause abnormal RocksDB flow control, leading to performance jitter [#17995](https://github.com/tikv/tikv/issues/17995) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that the latest written data might not be readable when only one-phase commit (1PC) is enabled and Async Commit is not enabled [#18117](https://github.com/tikv/tikv/issues/18117) @[zyguan](https://github.com/zyguan) + - Fix the issue that TiKV converts the time zone incorrectly for Brazil and Egypt [#16220](https://github.com/tikv/tikv/issues/16220) @[overvenus](https://github.com/overvenus) + - Fix the issue that a deadlock might occur when GC Worker is under heavy load [#18214](https://github.com/tikv/tikv/issues/18214) @[zyguan](https://github.com/zyguan) + - Fix the issue that disk stalls might prevent leader migration, leading to performance jitter [#17363](https://github.com/tikv/tikv/issues/17363) @[hhwyt](https://github.com/hhwyt) + - Fix the issue that encoding might fail when processing GBK/GB18030 encoded data [#17618](https://github.com/tikv/tikv/issues/17618) @[CbcWestwolf](https://github.com/CbcWestwolf) + - Fix the issue that CDC connections might cause resource leakage when encountering exceptions [#18245](https://github.com/tikv/tikv/issues/18245) @[wlwilliamx](https://github.com/wlwilliamx) + - Fix the issue that Region merge might lead to TiKV abnormal exit due to Raft index mismatch [#18129](https://github.com/tikv/tikv/issues/18129) @[glorv](https://github.com/glorv) + - Fix the issue that Resolved-TS monitoring and logs might be abnormal [#17989](https://github.com/tikv/tikv/issues/17989) @[ekexium](https://github.com/ekexium) + - Fix the issue that an incompatibility with the Titan component causes upgrade failure [#18263](https://github.com/tikv/tikv/issues/18263) @[v01dstar](https://github.com/v01dstar) @[LykxSassinator](https://github.com/LykxSassinator) + ++ PD + + - Fix the issue that the default value of `max-size` for a single log file is not correctly set [#9037](https://github.com/tikv/pd/issues/9037) @[rleungx](https://github.com/rleungx) + - Fix the issue that the value of the `flow-round-by-digit` configuration item might be overwritten after a restart [#8980](https://github.com/tikv/pd/issues/8980) @[nolouch](https://github.com/nolouch) + - Fix the issue that operations in data import or adding index scenarios might fail due to unstable PD network [#8962](https://github.com/tikv/pd/issues/8962) @[okJiang](https://github.com/okJiang) + - Fix the issue that PD might panic when the `tidb_enable_tso_follower_proxy` system variable is enabled [#8950](https://github.com/tikv/pd/issues/8950) @[okJiang](https://github.com/okJiang) + - Fix the issue that the `tidb_enable_tso_follower_proxy` system variable might not take effect [#8947](https://github.com/tikv/pd/issues/8947) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that memory leaks might occur when allocating TSOs [#9004](https://github.com/tikv/pd/issues/9004) @[rleungx](https://github.com/rleungx) + - Fix the issue that Region syncer might not exit in time during the PD Leader switch [#9017](https://github.com/tikv/pd/issues/9017) @[rleungx](https://github.com/rleungx) + - Fix the issue that a PD node might still generate TSOs even when it is not the Leader [#9051](https://github.com/tikv/pd/issues/9051) @[rleungx](https://github.com/rleungx) + ++ TiFlash + + - Fix the issue that TiFlash might unexpectedly reject processing Raft messages when memory usage is low [#9745](https://github.com/pingcap/tiflash/issues/9745) @[CalvinNeo](https://github.com/CalvinNeo) + - Fix the issue that TiFlash might maintain high memory usage after importing large amounts of data [#9812](https://github.com/pingcap/tiflash/issues/9812) @[CalvinNeo](https://github.com/CalvinNeo) + - Fix the issue that queries on a partitioned table might return errors after executing `ALTER TABLE ... RENAME COLUMN` on that partitioned table [#9787](https://github.com/pingcap/tiflash/issues/9787) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that TiFlash might fail to print error stack traces when it unexpectedly exits in certain situations [#9902](https://github.com/pingcap/tiflash/issues/9902) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that TiFlash startup might be blocked when `profiles.default.init_thread_count_scale` is set to `0` [#9906](https://github.com/pingcap/tiflash/issues/9906) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that a `Not found column` error might occur when a query involves virtual columns and triggers remote reads [#9561](https://github.com/pingcap/tiflash/issues/9561) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that in the disaggregated storage and compute architecture, TiFlash compute nodes might be incorrectly selected as target nodes for adding Region peers [#9750](https://github.com/pingcap/tiflash/issues/9750) @[JaySon-Huang](https://github.com/JaySon-Huang) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that log backup fails to exit properly when encountering a fatal error due to not being able to access PD [#18087](https://github.com/tikv/tikv/issues/18087) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that BR fails to restore due to getting the `rpcClient is idle` error when sending requests to TiKV [#58845](https://github.com/pingcap/tidb/issues/58845) @[Tristan1900](https://github.com/Tristan1900) + - Fix the issue that PITR fails to restore indexes larger than 3072 bytes [#58430](https://github.com/pingcap/tidb/issues/58430) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that the `status` field is missing in the result when querying log backup tasks using `br log status --json` [#57959](https://github.com/pingcap/tidb/issues/57959) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that log backup might unexpectedly enter a paused state when the advancer owner switches [#58031](https://github.com/pingcap/tidb/issues/58031) @[3pointer](https://github.com/3pointer) + + + TiCDC + + - Fix the issue that enabling TiCDC in scenarios with many small tables might cause TiKV to restart [#18142](https://github.com/tikv/tikv/issues/18142) @[hicqu](https://github.com/hicqu) + - Fix the issue that after the default value of a newly added column in the upstream is changed from `NOT NULL` to `NULL`, the default values of that column in the downstream are incorrect [#12037](https://github.com/pingcap/tiflow/issues/12037) @[wk989898](https://github.com/wk989898) + - Fix the issue that out-of-order messages resent by the Sarama client cause Kafka message order to be incorrect [#11935](https://github.com/pingcap/tiflow/issues/11935) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that TiCDC uses incorrect table names for filtering during `RENAME TABLE` operations [#11946](https://github.com/pingcap/tiflow/issues/11946) @[wk989898](https://github.com/wk989898) + - Fix the issue that goroutines leak occurs after a changefeed is deleted [#11954](https://github.com/pingcap/tiflow/issues/11954) @[hicqu](https://github.com/hicqu) + - Fix the issue that using the `--overwrite-checkpoint-ts` parameter in the `changefeed pause` command might cause the changefeed to be stuck [#12055](https://github.com/pingcap/tiflow/issues/12055) @[hongyunyan](https://github.com/hongyunyan) + - Fix the issue that TiCDC reports errors when replicating `default NULL` SQL statements via the Avro protocol [#11994](https://github.com/pingcap/tiflow/issues/11994) @[wk989898](https://github.com/wk989898) + - Fix the issue that TiCDC fails to properly connect to PD after PD scale-in [#12004](https://github.com/pingcap/tiflow/issues/12004) @[lidezhu](https://github.com/lidezhu) + + + TiDB Lightning + + - Fix the issue that logs are not properly desensitized [#59086](https://github.com/pingcap/tidb/issues/59086) @[GMHDBJD](https://github.com/GMHDBJD) diff --git a/releases/release-7.6.0.md b/releases/release-7.6.0.md index edbb3ff6089cc..a1b14bd97b533 100644 --- a/releases/release-7.6.0.md +++ b/releases/release-7.6.0.md @@ -119,7 +119,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.6/quick-start-with- Starting from v7.6.0, TiDB supports periodic full compaction for TiKV. This feature serves as an enhancement to Garbage Collection (GC) to eliminate redundant data versions. In scenarios where application activity shows obvious peaks and valleys, you can use this feature to perform data compaction during idle periods to improve the performance during peak periods. - You can set the specific times that TiKV initiates periodic full compaction by configuring the TiKV configuration item [`periodic-full-compact-start-times`](/tikv-configuration-file.md#periodic-full-compact-start-times-new-in-v760) and limit the maximum CPU usage rate for TiKV periodic full compaction by configuring [`periodic-full-compact-start-max-cpu`](/tikv-configuration-file.md#periodic-full-compact-start-max-cpu-new-in-v760). The default value of `periodic-full-compact-start-max-cpu` is 10%, which means that periodic full compaction is triggered only when the CPU utilization of TiKV is lower than 10%, thereby reducing the impact on application traffic. + You can set the specific times that TiKV initiates periodic full compaction by configuring the TiKV configuration item [`periodic-full-compact-start-times`](/tikv-configuration-file.md#periodic-full-compact-start-times-new-in-v760) and limit the maximum CPU usage rate for TiKV periodic full compaction by configuring [`periodic-full-compact-start-max-cpu`](/tikv-configuration-file.md#periodic-full-compact-start-max-cpu-new-in-v760). The default value of `periodic-full-compact-start-max-cpu` is `0.1`, which means that periodic full compaction is triggered only when the CPU utilization of TiKV is lower than 10%, thereby reducing the impact on application traffic. For more information, see [documentation](/tikv-configuration-file.md#periodic-full-compact-start-times-new-in-v760). @@ -200,7 +200,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.6/quick-start-with- * Support dynamically modifying the size limit of a single row record in TiDB [#49237](https://github.com/pingcap/tidb/pull/49237) @[zyguan](https://github.com/zyguan) - Before v7.6.0, the size of a single row record in a transaction is limited by the TiDB configuration item [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50). If the size limit is exceeded, TiDB returns the `entry too large` error. In this case, you need to manually modify the TiDB configuration file and restart TiDB to make the modification take effect. To reduce your management overhead, TiDB v7.6.0 introduces the system variable [`tidb_txn_entry_size_limit`](/system-variables.md#tidb_txn_entry_size_limit-new-in-v760), which supports dynamically modifying the value of the `txn-entry-size-limit` configuration item. The default value of this variable is `0`, which means that TiDB uses the value of the configuration item `txn-entry-size-limit` by default. When this variable is set to a non-zero value, TiDB limits the size of a row record in transactions to the value of this variable. This improvement enhances the flexibility for you to adjust system configurations without restarting TiDB. + Before v7.6.0, the size of a single row record in a transaction is limited by the TiDB configuration item [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500). If the size limit is exceeded, TiDB returns the `entry too large` error. In this case, you need to manually modify the TiDB configuration file and restart TiDB to make the modification take effect. To reduce your management overhead, TiDB v7.6.0 introduces the system variable [`tidb_txn_entry_size_limit`](/system-variables.md#tidb_txn_entry_size_limit-new-in-v760), which supports dynamically modifying the value of the `txn-entry-size-limit` configuration item. The default value of this variable is `0`, which means that TiDB uses the value of the configuration item `txn-entry-size-limit` by default. When this variable is set to a non-zero value, TiDB limits the size of a row record in transactions to the value of this variable. This improvement enhances the flexibility for you to adjust system configurations without restarting TiDB. For more information, see [documentation](/system-variables.md#tidb_txn_entry_size_limit-new-in-v760). @@ -208,7 +208,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.6/quick-start-with- Starting from v5.1.0, when you back up snapshots, BR automatically backs up system tables in the `mysql` schema, but does not restore these system tables by default. In v6.2.0, BR adds the parameter `--with-sys-table` to support restoring data in some system tables, providing more flexibility in operations. - To further reduce your management overhead and provide more intuitive default behavior, starting from v7.6.0, BR enables the parameter `--with-sys-table` by default and supports restoring user data for the `cloud_admin` user. This means that BR restores some system tables by default during restoration, especially user account and table statistics data. This improvement makes backup and restore operations more intuitive, thereby reducing the burden of manual configuration and improving the overall operation experience. + To further reduce your management overhead and provide more intuitive default behavior, starting from v7.6.0, BR enables the parameter `--with-sys-table` by default. This means that BR restores some system tables by default during restoration, especially user account and table statistics data. This improvement makes backup and restore operations more intuitive, thereby reducing the burden of manual configuration and improving the overall operation experience. For more information, see [documentation](/br/br-snapshot-guide.md). @@ -222,7 +222,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.6/quick-start-with- * [Statement Summary Tables](/statement-summary-tables.md): add the resource group name, RU consumption, and time for waiting for resources. * In the system variable [`tidb_last_query_info`](/system-variables.md#tidb_last_query_info-new-in-v4014), add a new entry `ru_consumption` to indicate the consumed [RU](/tidb-resource-control.md#what-is-request-unit-ru) by SQL statements. You can use this variable to get the resource consumption of the last statement in the session. * Add database metrics based on resource groups: QPS/TPS, execution time (P999/P99/P95), number of failures, and number of connections. - * Add the system table [`request_unit_by_group`](/mysql-schema.md#system-tables-related-to-resource-control) to record the history records of daily consumed RUs of all resource groups. + * Add the system table [`request_unit_by_group`](/mysql-schema/mysql-schema.md#system-tables-related-to-resource-control) to record the history records of daily consumed RUs of all resource groups. For more information, see [Identify Slow Queries](/identify-slow-queries.md), [Statement Summary Tables](/statement-summary-tables.md), and [Key Monitoring Metrics of Resource Control](/grafana-resource-control-dashboard.md). @@ -272,8 +272,9 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.6/quick-start-with- | [`tidb_ddl_version`](https://docs.pingcap.com/tidb/v7.6/system-variables#tidb_ddl_version-new-in-v760) | Newly added | Controls whether to enable [TiDB DDL V2](https://docs.pingcap.com/tidb/v7.6/ddl-v2). Set the value to `2` to enable it and `1` to disable it. The default value is `1`. When TiDB DDL V2 is enabled, DDL statements will be executed using TiDB DDL V2. The execution speed of DDL statements for creating tables is increased by 10 times compared with TiDB DDL V1. | | [`tidb_enable_global_index`](/system-variables.md#tidb_enable_global_index-new-in-v760) | Newly added | Controls whether to support creating `Global indexes` for partitioned tables. The default value is `OFF`. `Global index` is currently in the development stage. **It is not recommended to modify the value of this system variable**. | | [`tidb_idle_transaction_timeout`](/system-variables.md#tidb_idle_transaction_timeout-new-in-v760) | Newly added | Controls the idle timeout for transactions in a user session. When a user session is in a transactional state and remains idle for a duration exceeding the value of this variable, TiDB will terminate the session. The default value `0` means unlimited. | +| [`tidb_ignore_inlist_plan_digest`](/system-variables.md#tidb_ignore_inlist_plan_digest-new-in-v760) | Newly added | Controls whether TiDB ignores the element differences in the `IN` list across different queries when generating Plan Digests. The default value `OFF` means differences are not ignored. | | [`tidb_opt_enable_fuzzy_binding`](/system-variables.md#tidb_opt_enable_fuzzy_binding-new-in-v760) | Newly added | Controls whether to enable the cross-database binding feature. The default value `OFF` means cross-database binding is disabled. | -| [`tidb_txn_entry_size_limit`](/system-variables.md#tidb_txn_entry_size_limit-new-in-v760) | Newly added | Dynamically modifies the TiDB configuration item [`performance.txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50). It limits the size of a single row of data in TiDB. The default value of this variable is `0`, which means that TiDB uses the value of the configuration item `txn-entry-size-limit` by default. When this variable is set to a non-zero value, `txn-entry-size-limit` is also set to the same value. | +| [`tidb_txn_entry_size_limit`](/system-variables.md#tidb_txn_entry_size_limit-new-in-v760) | Newly added | Dynamically modifies the TiDB configuration item [`performance.txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500). It limits the size of a single row of data in TiDB. The default value of this variable is `0`, which means that TiDB uses the value of the configuration item `txn-entry-size-limit` by default. When this variable is set to a non-zero value, `txn-entry-size-limit` is also set to the same value. | | [`pd_enable_follower_handle_region`](/system-variables.md#pd_enable_follower_handle_region-new-in-v760) | Newly added | Controls whether to enable the [Active PD Follower](/tune-region-performance.md#use-the-active-pd-follower-feature-to-enhance-the-scalability-of-pds-region-information-query-service) feature (experimental). When the value is `OFF`, TiDB only obtains Region information from the PD leader. When the value is `ON`, TiDB evenly distributes requests for Region information to all PD servers, and PD followers can also handle Region requests, thereby reducing the CPU pressure on the PD leader. | ### Configuration file parameters @@ -281,12 +282,15 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v7.6/quick-start-with- | Configuration file | Configuration parameter | Change type | Description | | -------- | -------- | -------- | -------- | | TiDB | [`tls-version`](/tidb-configuration-file.md#tls-version) | Modified | The default value is "". The default supported TLS versions of TiDB are changed from `TLS1.1` or higher to `TLS1.2` or higher. | +| TiKV | [`raftstore.report-min-resolved-ts-interval`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file/#report-min-resolved-ts-interval-new-in-v600) | Renamed | To make the name more accurate, this configuration item is renamed to [`raftstore.pd-report-min-resolved-ts-interval`](/tikv-configuration-file.md#pd-report-min-resolved-ts-interval-new-in-v760). `raftstore.report-min-resolved-ts-interval` is no longer effective. | | TiKV | [`blob-file-compression`](/tikv-configuration-file.md#blob-file-compression) | Modified | The algorithm used for compressing values in Titan, which takes value as the unit. Starting from TiDB v7.6.0, the default compression algorithm is `zstd`. | | TiKV | [`rocksdb.defaultcf.titan.min-blob-size`](/tikv-configuration-file.md#min-blob-size) | Modified | Starting from TiDB v7.6.0, the default value for new clusters is `32KB`. For existing clusters upgrading to v7.6.0, the default value `1KB` remains unchanged. | | TiKV | [`rocksdb.titan.enabled`](/tikv-configuration-file.md#enabled) | Modified | Enables or disables Titan. For v7.5.0 and earlier versions, the default value is `false`. Starting from v7.6.0, the default value is `true` for only new clusters. Existing clusters upgraded to v7.6.0 or later versions will retain the original configuration. | +| TiKV | [`cdc.incremental-scan-concurrency-limit`](/tikv-configuration-file.md#incremental-scan-concurrency-limit-new-in-v760) | Newly added | Sets the maximum queue length for the tasks of incrementally scanning historical data waiting to be executed. The default value is `10000`, which means that at most 10000 tasks can be queued for execution. | | TiKV | [`gc.num-threads`](/tikv-configuration-file.md#num-threads-new-in-v658-v714-v751-and-v760) | Newly added | When `enable-compaction-filter` is set to `false`, this parameter controls the number of GC threads. The default value is `1`. | | TiKV | [`raftstore.periodic-full-compact-start-times`](/tikv-configuration-file.md#periodic-full-compact-start-times-new-in-v760) | Newly added | Sets the specific times that TiKV initiates periodic full compaction. The default value `[]` means periodic full compaction is disabled. | | TiKV | [`raftstore.periodic-full-compact-start-max-cpu`](/tikv-configuration-file.md#periodic-full-compact-start-max-cpu-new-in-v760) | Newly added | Limits the maximum CPU usage rate for TiKV periodic full compaction. The default value is `0.1`. | +| TiKV | [`raftstore.pd-report-min-resolved-ts-interval`](/tikv-configuration-file.md#pd-report-min-resolved-ts-interval-new-in-v760) | Newly added | Renamed from [`raftstore.report-min-resolved-ts-interval`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file#report-min-resolved-ts-interval-new-in-v600). It specifies the minimum interval for TiKV to report Resolved TS to the PD leader. The default value is `"1s"`. | | TiKV | [`zstd-dict-size`](/tikv-configuration-file.md#zstd-dict-size) | Newly added | Specifies the `zstd` dictionary compression size. The default value is `"0KB"`, which means to disable the `zstd` dictionary compression. | | TiFlash | [`logger.level`](/tiflash/tiflash-configuration.md#configure-the-tiflashtoml-file) | Modified | Changes the default value from `"debug"` to `"INFO"` to reduce the cost of logging. | | TiDB Lightning| [`tidb.pd-addr`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) | Modified | Configures the addresses of the PD Servers. Starting from v7.6.0, TiDB supports setting multiple PD addresses. | @@ -326,7 +330,7 @@ Starting from v7.6.0, the `TiDB-community-server` [binary-package](/binary-packa - Modify the `CANCEL IMPORT JOB` statement to a synchronous statement [#48736](https://github.com/pingcap/tidb/issues/48736) @[D3Hunter](https://github.com/D3Hunter) - Improve the speed of adding indexes to empty tables [#49682](https://github.com/pingcap/tidb/issues/49682) @[zimulala](https://github.com/zimulala) - When the columns of a correlated subquery are not referenced by the upper-level operator, the correlated subquery can be eliminated directly [#45822](https://github.com/pingcap/tidb/issues/45822) @[King-Dylan](https://github.com/King-Dylan) - - `EXCHANGE PARTITION` operations now trigger maintenance updates of statistics [#47354](https://github.com/pingcap/tidb/issues/47354) @[hi-rustin](https://github.com/hi-rustin) + - `EXCHANGE PARTITION` operations now trigger maintenance updates of statistics [#47354](https://github.com/pingcap/tidb/issues/47354) @[hi-rustin](https://github.com/Rustin170506) - TiDB supports building binary files that meet the requirements of Federal Information Processing Standards (FIPS) [#47948](https://github.com/pingcap/tidb/issues/47948) @[tiancaiamao](https://github.com/tiancaiamao) - Optimize the TiDB implementation when handling some type conversions and fix related issues [#47945](https://github.com/pingcap/tidb/issues/47945) [#47864](https://github.com/pingcap/tidb/issues/47864) [#47829](https://github.com/pingcap/tidb/issues/47829) [#47816](https://github.com/pingcap/tidb/issues/47816) @[YangKeao](https://github.com/YangKeao) @[lcwangchao](https://github.com/lcwangchao) - When obtaining the schema version, TiDB uses the KV timeout feature to read by default, reducing the impact of slow meta Region leader reads on schema version updates [#48125](https://github.com/pingcap/tidb/pull/48125) @[cfzjywxk](https://github.com/cfzjywxk) @@ -428,16 +432,16 @@ Starting from v7.6.0, the `TiDB-community-server` [binary-package](/binary-packa - Fix the issue that MPP plans might not be selected when indexes on generated columns are set as visible [#47766](https://github.com/pingcap/tidb/issues/47766) @[AilinKid](https://github.com/AilinKid) - Fix the issue that `LIMIT` might not be pushed down to the `OR` type `Index Merge` [#48588](https://github.com/pingcap/tidb/issues/48588) @[AilinKid](https://github.com/AilinKid) - Fix the issue that duplicate built-in rows might exist in the `mysql.bind_info` table after BR import [#46527](https://github.com/pingcap/tidb/issues/46527) @[qw4990](https://github.com/qw4990) - - Fix the issue that statistics for partitioned tables are not updated as expected after partitions are dropped [#48182](https://github.com/pingcap/tidb/issues/48182) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that statistics for partitioned tables are not updated as expected after partitions are dropped [#48182](https://github.com/pingcap/tidb/issues/48182) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that errors might be returned during the concurrent merging of global statistics for partitioned tables [#48713](https://github.com/pingcap/tidb/issues/48713) @[hawkingrei](https://github.com/hawkingrei) - Fix the issue that query results might be incorrect when the `LIKE` operator is used for index range scans on a column with PADDING SPACE [#48821](https://github.com/pingcap/tidb/issues/48821) @[time-and-fate](https://github.com/time-and-fate) - Fix the issue that generated columns might trigger concurrent read and write on memory and result in data race [#44919](https://github.com/pingcap/tidb/issues/44919) @[tangenta](https://github.com/tangenta) - Fix the issue that `ANALYZE TABLE` might still collect Top1 statistics even when `WITH 0 TOPN` (indicating not collecting topN statistics) is specified [#49080](https://github.com/pingcap/tidb/issues/49080) @[hawkingrei](https://github.com/hawkingrei) - Fix the issue that illegal optimizer hints might cause valid hints to be ineffective [#49308](https://github.com/pingcap/tidb/issues/49308) @[hawkingrei](https://github.com/hawkingrei) - - Fix the issue that statistics for Hash partitioned tables are not correspondingly updated when you add, drop, reorganize, or `TRUNCATE` partitions [#48235](https://github.com/pingcap/tidb/issues/48235) [#48233](https://github.com/pingcap/tidb/issues/48233) [#48226](https://github.com/pingcap/tidb/issues/48226) [#48231](https://github.com/pingcap/tidb/issues/48231) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that statistics for Hash partitioned tables are not correspondingly updated when you add, drop, reorganize, or `TRUNCATE` partitions [#48235](https://github.com/pingcap/tidb/issues/48235) [#48233](https://github.com/pingcap/tidb/issues/48233) [#48226](https://github.com/pingcap/tidb/issues/48226) [#48231](https://github.com/pingcap/tidb/issues/48231) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that after the time window for automatic statistics updates is configured, statistics might still be updated outside that time window [#49552](https://github.com/pingcap/tidb/issues/49552) @[hawkingrei](https://github.com/hawkingrei) - - Fix the issue that old statistics are not automatically deleted when a partitioned table is converted to a non-partitioned table [#49547](https://github.com/pingcap/tidb/issues/49547) @[hi-rustin](https://github.com/hi-rustin) - - Fix the issue that old statistics are not automatically deleted when you clear data from a non-partitioned table using `TRUNCATE TABLE` [#49663](https://github.com/pingcap/tidb/issues/49663) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that old statistics are not automatically deleted when a partitioned table is converted to a non-partitioned table [#49547](https://github.com/pingcap/tidb/issues/49547) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that old statistics are not automatically deleted when you clear data from a non-partitioned table using `TRUNCATE TABLE` [#49663](https://github.com/pingcap/tidb/issues/49663) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that enforced sorting might become ineffective when a query uses optimizer hints (such as `STREAM_AGG()`) that enforce sorting and its execution plan contains `IndexMerge` [#49605](https://github.com/pingcap/tidb/issues/49605) @[AilinKid](https://github.com/AilinKid) - Fix the issue that histogram statistics might not be parsed into readable strings when the histogram boundary contains `NULL` [#49823](https://github.com/pingcap/tidb/issues/49823) @[AilinKid](https://github.com/AilinKid) - Fix the issue that executing queries containing the `GROUP_CONCAT(ORDER BY)` syntax might return errors [#49986](https://github.com/pingcap/tidb/issues/49986) @[AilinKid](https://github.com/AilinKid) diff --git a/releases/release-8.0.0.md b/releases/release-8.0.0.md index 5c161a906af04..1ac5b5a30f57b 100644 --- a/releases/release-8.0.0.md +++ b/releases/release-8.0.0.md @@ -37,12 +37,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v8.0/quick-start-with- - + @@ -102,15 +97,15 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v8.0/quick-start-with- For more information, see [documentation](/tiflash/tiflash-supported-pushdown-calculations.md). -* The concurrent HashAgg algorithm of TiDB supports disk spill (experimental) [#35637](https://github.com/pingcap/tidb/issues/35637) @[xzhangxian1008](https://github.com/xzhangxian1008) +* The parallel HashAgg algorithm of TiDB supports disk spill (experimental) [#35637](https://github.com/pingcap/tidb/issues/35637) @[xzhangxian1008](https://github.com/xzhangxian1008) - In earlier versions of TiDB, the concurrency algorithm of the HashAgg operator does not support disk spill. If the execution plan of a SQL statement contains the concurrent HashAgg operator, all the data for that SQL statement can only be processed in memory. Consequently, TiDB has to process a large amount of data in memory. When the data size exceeds the memory limit, TiDB can only choose the non-concurrent algorithm, which does not leverage concurrency for performance improvement. + In earlier versions of TiDB, the concurrency algorithm of the HashAgg operator does not support disk spill. If the execution plan of a SQL statement contains the parallel HashAgg operator, all the data for that SQL statement can only be processed in memory. Consequently, TiDB has to process a large amount of data in memory. When the data size exceeds the memory limit, TiDB can only choose the non-parallel algorithm, which does not leverage concurrency for performance improvement. - In v8.0.0, the concurrent HashAgg algorithm of TiDB supports disk spill. Under any concurrent conditions, the HashAgg operator can automatically trigger data spill based on memory usage, thus balancing performance and data throughput. Currently, as an experimental feature, TiDB introduces the `tidb_enable_concurrent_hashagg_spill` variable to control whether to enable the concurrent HashAgg algorithm that supports disk spill. When this variable is `ON`, it means enabled. This variable will be deprecated after the feature is generally available in a future release. + In v8.0.0, the parallel HashAgg algorithm of TiDB supports disk spill. Under any parallel conditions, the HashAgg operator can automatically trigger data spill based on memory usage, thus balancing performance and data throughput. Currently, as an experimental feature, TiDB introduces the `tidb_enable_parallel_hashagg_spill` variable to control whether to enable the parallel HashAgg algorithm that supports disk spill. When this variable is `ON`, it means enabled. This variable will be deprecated after the feature is generally available in a future release. - For more information, see [documentation](/system-variables.md#tidb_enable_concurrent_hashagg_spill-new-in-v800). + For more information, see [documentation](/system-variables.md#tidb_enable_parallel_hashagg_spill-new-in-v800). -* Introduce the priority queue for automatic statistics collection [#50132](https://github.com/pingcap/tidb/issues/50132) @[hi-rustin](https://github.com/hi-rustin) +* Introduce the priority queue for automatic statistics collection [#50132](https://github.com/pingcap/tidb/issues/50132) @[hi-rustin](https://github.com/Rustin170506) Maintaining optimizer statistics up-to-date is the key to stabilizing database performance. Most users rely on the [automatic statistics collection](/statistics.md#automatic-update) provided by TiDB to collect the latest statistics. Automatic statistics collection checks the status of statistics for all objects, and adds unhealthy objects to a queue for sequential collections. In previous versions, the order is random, which could result in excessive waits for more worthy candidates to be updated, causing potential performance regressions. @@ -146,16 +141,6 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v8.0/quick-start-with- For more information, see [documentation](/system-variables.md#tidb_low_resolution_tso_update_interval-new-in-v800). -### Reliability - -* Support caching required schema information according to the LRU algorithm to reduce memory consumption on the TiDB server (experimental) [#50959](https://github.com/pingcap/tidb/issues/50959) @[gmhdbjd](https://github.com/gmhdbjd) - - Before v8.0.0, each TiDB node caches the schema information of all tables. In scenarios with hundreds of thousands of tables, just caching these table schemas could consume a significant amount of memory. - - Starting from v8.0.0, TiDB introduces the system variable [`tidb_schema_cache_size`](/system-variables.md#tidb_schema_cache_size-new-in-v800), which enables you to set an upper limit for caching schema information, thereby preventing excessive memory usage. When you enable this feature, TiDB uses the Least Recently Used (LRU) algorithm to cache the required tables, effectively reducing the memory consumed by the schema information. - - For more information, see [documentation](/system-variables.md#tidb_schema_cache_size-new-in-v800). - ### Availability * The proxy component TiProxy becomes generally available (GA) [#413](https://github.com/pingcap/tiproxy/issues/413) @[djshow832](https://github.com/djshow832) @[xhebox](https://github.com/xhebox) @@ -181,7 +166,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v8.0/quick-start-with- Starting from v8.0.0, TiDB supports a DML type for handling a large amount of data. This DML type writes data to TiKV in a timely manner during execution, avoiding the continuous storage of all transaction data in memory, and thus supports handling a large amount of data that exceeds the memory limit. This DML type ensures transaction integrity and uses the same syntax as standard DML. `INSERT`, `UPDATE`, `REPLACE`, and `DELETE` statements can use this new DML type to execute large-scale DML operations. - This DML type is implemented by the [Pipelined DML](https://github.com/pingcap/tidb/blob/master/docs/design/2024-01-09-pipelined-DML.md) feature and only takes effect on statements with auto-commit enabled. You can control whether to enable this DML type by setting the system variable [`tidb_dml_type`](/system-variables.md#tidb_dml_type-new-in-v800). + This DML type is implemented by the [Pipelined DML](https://github.com/pingcap/tidb/blob/release-8.0/docs/design/2024-01-09-pipelined-DML.md) feature and only takes effect on statements with auto-commit enabled. You can control whether to enable this DML type by setting the system variable [`tidb_dml_type`](/system-variables.md#tidb_dml_type-new-in-v800). For more information, see [documentation](/system-variables.md#tidb_dml_type-new-in-v800). @@ -306,6 +291,8 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v8.0/quick-start-with- > > This section provides compatibility changes you need to know when you upgrade from v7.6.0 to the current version (v8.0.0). If you are upgrading from v7.5.0 or earlier versions to the current version, you might also need to check the compatibility changes introduced in intermediate versions. +- Upgrade the default Prometheus version deployed by TiUP from 2.27.1 to 2.49.1. +- Upgrade the default Grafana version deployed by TiUP from 7.5.11 to 7.5.17. - Remove witness-related schedulers that are not GA but are enabled by default [#7765](https://github.com/tikv/pd/pull/7765) @[rleungx](https://github.com/rleungx) ### Behavior changes @@ -329,7 +316,7 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v8.0/quick-start-with- | [`div_precision_increment`](/system-variables.md#div_precision_increment-new-in-v800) | Newly added | Controls the number of digits by which to increase the scale of the result of a division operation performed using the `/` operator. This variable is the same as MySQL. | | [`tidb_dml_type`](/system-variables.md#tidb_dml_type-new-in-v800) | Newly added | Controls the execution mode of DML statements. The value options are `"standard"` and `"bulk"`. | | [`tidb_enable_auto_analyze_priority_queue`](/system-variables.md#tidb_enable_auto_analyze_priority_queue-new-in-v800) | Newly added | Controls whether to enable the priority queue to schedule the tasks of automatically collecting statistics. When this variable is enabled, TiDB prioritizes collecting statistics for the tables that most need statistics. | -| [`tidb_enable_concurrent_hashagg_spill`](/system-variables.md#tidb_enable_concurrent_hashagg_spill-new-in-v800) | Newly added | Controls whether TiDB supports disk spill for the concurrent HashAgg algorithm. When it is `ON`, disk spill can be triggered for the concurrent HashAgg algorithm. This variable will be deprecated when this feature is generally available in a future release. | +| [`tidb_enable_parallel_hashagg_spill`](/system-variables.md#tidb_enable_parallel_hashagg_spill-new-in-v800) | Newly added | Controls whether TiDB supports disk spill for the parallel HashAgg algorithm. When it is `ON`, disk spill can be triggered for the parallel HashAgg algorithm. This variable will be deprecated when this feature is generally available in a future release. | | [`tidb_enable_fast_create_table`](/system-variables.md#tidb_enable_fast_create_table-new-in-v800) | Newly added | Controls whether to enable [TiDB Accerates Table Creation](/accelerated-table-creation.md). Set the value to `ON` to enable it and `OFF` to disable it. The default value is `ON`. When this variable is enabled, TiDB accelerates table creation by using [`CREATE TABLE`](/sql-statements/sql-statement-create-table.md). | | [`tidb_load_binding_timeout`](/system-variables.md#tidb_load_binding_timeout-new-in-v800) | Newly added | Controls the timeout of loading bindings. If the execution time of loading bindings exceeds this value, the loading will stop. | | [`tidb_low_resolution_tso_update_interval`](/system-variables.md#tidb_low_resolution_tso_update_interval-new-in-v800) | Newly added | Controls the interval for updating TiDB [cache timestamp](/system-variables.md#tidb_low_resolution_tso). | @@ -349,14 +336,19 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v8.0/quick-start-with- | TiKV | [`log-backup.initial-scan-rate-limit`](/tikv-configuration-file.md#initial-scan-rate-limit-new-in-v620) | Modified | Adds a limit of `1MiB` as the minimum value. | | TiKV | [`raftstore.store-io-pool-size`](/tikv-configuration-file.md#store-io-pool-size-new-in-v530) | Modified | Changes the default value from `0` to `1` to improve TiKV performance, meaning that the size of the StoreWriter thread pool now defaults to `1`. | | TiKV | [`rocksdb.defaultcf.titan.blob-cache-size`](/tikv-configuration-file.md#blob-cache-size) | Modified | Starting from v8.0.0, TiKV introduces the `shared-blob-cache` configuration item and enables it by default, so there is no need to set `blob-cache-size` separately. The configuration of `blob-cache-size` only takes effect when `shared-blob-cache` is set to `false`. | +| TiKV | [`rocksdb.titan.max-background-gc`](/tikv-configuration-file.md#max-background-gc) | Modified | Changes the default value from `4` to `1` to reduce the occupation of thread resources by the Titan GC process. | | TiKV | [`security.encryption.master-key.vendor`](/encryption-at-rest.md#specify-a-master-key-via-kms) | Modified | Adds `gcp` as an available type for the service provider. | +| TiKV | [`storage.block-cache.low-pri-pool-ratio`](/tikv-configuration-file.md#low-pri-pool-ratio-new-in-v800) | Newly added | Specifies the proportion of the entire block cache that the Titan component can use. The default value is `0.2`. | | TiKV | [`rocksdb.defaultcf.titan.shared-blob-cache`](/tikv-configuration-file.md#shared-blob-cache-new-in-v800) | Newly added | Controls whether to enable the shared cache for Titan blob files and RocksDB block files. The default value is `true`. | | TiKV | [`security.encryption.master-key.gcp.credential-file-path`](/encryption-at-rest.md#specify-a-master-key-via-kms) | Newly added | Specifies the path to the Google Cloud authentication credentials file when `security.encryption.master-key.vendor` is `gcp`. | +| PD | [`schedule.enable-heartbeat-breakdown-metrics`](/pd-configuration-file.md#enable-heartbeat-breakdown-metrics-new-in-v800) | Newly added | Controls whether to enable breakdown metrics for Region heartbeats. These metrics measure the time consumed in each stage of Region heartbeat processing, facilitating analysis through monitoring. The default value is `true`. | +| PD | [`schedule.enable-heartbeat-concurrent-runner`](/pd-configuration-file.md#enable-heartbeat-concurrent-runner-new-in-v800) | Newly added | Controls whether to enable asynchronous concurrent processing for Region heartbeats. When enabled, an independent executor handles Region heartbeat requests asynchronously and concurrently, which can improve heartbeat processing throughput and reduce latency. The default value is `true`. | | TiDB Lightning | [`tikv-importer.duplicate-resolution`](/tidb-lightning/tidb-lightning-physical-import-mode-usage.md#the-old-version-of-conflict-detection-deprecated-in-v800) | Deprecated | Controls whether to detect and resolve unique key conflicts in physical import mode. Starting from v8.0.0, it is replaced by [`conflict.strategy`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task). | | TiDB Lightning | [`conflict.precheck-conflict-before-import`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) | Newly added | Controls whether to enable preprocess conflict detection, which checks conflicts in data before importing it to TiDB. The default value of this parameter is `false`, which means that TiDB Lightning only checks conflicts after the data import. This parameter can be used only in the physical import mode (`tikv-importer.backend = "local"`). | | TiDB Lightning | [`logical-import-batch-rows`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) | Newly added | Controls the maximum number of rows inserted per transaction in the logical import mode. The default value is `65536` rows. | | TiDB Lightning | [`logical-import-batch-size`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) | Newly added | Controls the maximum size of each SQL query executed on the downstream TiDB server in the logical import mode. The default value is `"96KiB"`. The unit can be KB, KiB, MB, or MiB. | | Data Migration | [`secret-key-path`](/dm/dm-master-configuration-file.md) | Newly added | Specifies the file path of the secret key, which is used to encrypt and decrypt upstream and downstream passwords. The file must contain a 64-character hexadecimal AES-256 secret key. | +| TiCDC | [`debezium-disable-schema`](/ticdc/ticdc-changefeed-config.md) | Newly added | Controls whether to disable the output of schema information. This parameter only takes effect when the sink type is MQ and the output protocol is Debezium. | | TiCDC | [`tls-certificate-file`](/ticdc/ticdc-sink-to-pulsar.md) | Newly added | Specifies the path to the encrypted certificate file on the client, which is required when Pulsar enables TLS encrypted transmission. | | TiCDC | [`tls-key-file-path`](/ticdc/ticdc-sink-to-pulsar.md) | Newly added | Specifies the path to the encrypted private key on the client, which is required when Pulsar enables TLS encrypted transmission. | @@ -459,11 +451,11 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v8.0/quick-start-with- + TiDB - - Fix the issue that `auto analyze` is triggered multiple times when there is no data change [#51775](https://github.com/pingcap/tidb/issues/51775) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that `auto analyze` is triggered multiple times when there is no data change [#51775](https://github.com/pingcap/tidb/issues/51775) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that the `auto analyze` concurrency is set incorrectly [#51749](https://github.com/pingcap/tidb/issues/51749) @[hawkingrei](https://github.com/hawkingrei) - Fix the issue of index inconsistency caused by adding multiple indexes using a single SQL statement [#51746](https://github.com/pingcap/tidb/issues/51746) @[tangenta](https://github.com/tangenta) - Fix the `Column ... in from clause is ambiguous` error that might occur when a query uses `NATURAL JOIN` [#32044](https://github.com/pingcap/tidb/issues/32044) @[AilinKid](https://github.com/AilinKid) - - Fix the issue of wrong query results due to TiDB incorrectly eliminating constant values in `group by` [#38756](https://github.com/pingcap/tidb/issues/38756) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue of wrong query results due to TiDB incorrectly eliminating constant values in `group by` [#38756](https://github.com/pingcap/tidb/issues/38756) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that the `LEADING` hint does not take effect in `UNION ALL` statements [#50067](https://github.com/pingcap/tidb/issues/50067) @[hawkingrei](https://github.com/hawkingrei) - Fix the issue that `BIT` type columns might cause query errors due to decode failures when they are involved in calculations of some functions [#49566](https://github.com/pingcap/tidb/issues/49566) [#50850](https://github.com/pingcap/tidb/issues/50850) [#50855](https://github.com/pingcap/tidb/issues/50855) @[jiyfhust](https://github.com/jiyfhust) - Fix the issue that TiDB might panic when performing a rolling upgrade using `tiup cluster upgrade/start` due to an interaction issue with PD [#50152](https://github.com/pingcap/tidb/issues/50152) @[zimulala](https://github.com/zimulala) @@ -475,13 +467,13 @@ Quick access: [Quick start](https://docs.pingcap.com/tidb/v8.0/quick-start-with- - Fix the issue that when executing the `CREATE GLOBAL BINDING` statement, if the schema name is in uppercase, the binding does not take effect [#50646](https://github.com/pingcap/tidb/issues/50646) @[qw4990](https://github.com/qw4990) - Fix the issue that `Index Path` selects duplicated indexes [#50496](https://github.com/pingcap/tidb/issues/50496) @[AilinKid](https://github.com/AilinKid) - Fix the issue that `PLAN REPLAYER` fails to load bindings when the `CREATE GLOBAL BINDING` statement contains `IN()` [#43192](https://github.com/pingcap/tidb/issues/43192) @[King-Dylan](https://github.com/King-Dylan) - - Fix the issue that when multiple `analyze` tasks fail, the failure reasons are not recorded correctly [#50481](https://github.com/pingcap/tidb/issues/50481) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that when multiple `analyze` tasks fail, the failure reasons are not recorded correctly [#50481](https://github.com/pingcap/tidb/issues/50481) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that `tidb_stats_load_sync_wait` does not take effect [#50872](https://github.com/pingcap/tidb/issues/50872) @[jiyfhust](https://github.com/jiyfhust) - Fix the issue that `max_execute_time` settings at multiple levels interfere with each other [#50914](https://github.com/pingcap/tidb/issues/50914) @[jiyfhust](https://github.com/jiyfhust) - - Fix the issue of thread safety caused by concurrent updating of statistics [#50835](https://github.com/pingcap/tidb/issues/50835) @[hi-rustin](https://github.com/hi-rustin) - - Fix the issue that executing `auto analyze` on a partition table might cause TiDB to panic [#51187](https://github.com/pingcap/tidb/issues/51187) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue of thread safety caused by concurrent updating of statistics [#50835](https://github.com/pingcap/tidb/issues/50835) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that executing `auto analyze` on a partition table might cause TiDB to panic [#51187](https://github.com/pingcap/tidb/issues/51187) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that SQL bindings might not work when `IN()` in a SQL statement contains a different number of values [#51222](https://github.com/pingcap/tidb/issues/51222) @[hawkingrei](https://github.com/hawkingrei) - - Fix the issue that TiDB cannot correctly convert the type of a system variable in an expression [#43527](https://github.com/pingcap/tidb/issues/43527) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that TiDB cannot correctly convert the type of a system variable in an expression [#43527](https://github.com/pingcap/tidb/issues/43527) @[hi-rustin](https://github.com/Rustin170506) - Fix the issue that TiDB does not listen to the corresponding port when `force-init-stats` is configured [#51473](https://github.com/pingcap/tidb/issues/51473) @[hawkingrei](https://github.com/hawkingrei) - Fix the issue that in `determinate` mode (`tidb_opt_objective='determinate'`), if a query does not contain predicates, statistics might not be loaded [#48257](https://github.com/pingcap/tidb/issues/48257) @[time-and-fate](https://github.com/time-and-fate) - Fix the issue that the `init-stats` process might cause TiDB to panic and the `load stats` process to quit [#51581](https://github.com/pingcap/tidb/issues/51581) @[hawkingrei](https://github.com/hawkingrei) diff --git a/releases/release-8.1.0.md b/releases/release-8.1.0.md new file mode 100644 index 0000000000000..146dc816ffaa6 --- /dev/null +++ b/releases/release-8.1.0.md @@ -0,0 +1,370 @@ +--- +title: TiDB 8.1.0 Release Notes +summary: Learn about the new features, compatibility changes, improvements, and bug fixes in TiDB 8.1.0. +--- + +# TiDB 8.1.0 Release Notes + + + +Release date: May 24, 2024 + +TiDB version: 8.1.0 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v8.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v8.1/production-deployment-using-tiup) + +TiDB 8.1.0 is a Long-Term Support Release (LTS). + +Compared with the previous LTS 7.5.0, 8.1.0 includes new features, improvements, and bug fixes released in [7.6.0-DMR](/releases/release-7.6.0.md) and [8.0.0-DMR](/releases/release-8.0.0.md). When you upgrade from 7.5.x to 8.1.0, you can download the [TiDB Release Notes PDF](https://docs-download.pingcap.com/pdf/tidb-v7.6-to-v8.1-en-release-notes.pdf) to view all release notes between the two LTS versions. The following table lists some highlights from 7.6.0 to 8.1.0: + +
Enhance the stability of caching the schema information when there is a massive number of tables (experimental)SaaS companies using TiDB as the system of record for their multi-tenant applications often need to store a substantial number of tables. In previous versions, handling table counts in the order of a million or more was feasible, but it had the potential to degrade the overall user experience. TiDB v8.0.0 improves the situation with the following enhancements: -
    -
  • Introduce a new schema information caching system, incorporating a lazy-loading Least Recently Used (LRU) cache for table metadata and more efficiently managing schema version changes.
    • -
  • Implement a priority queue for auto analyze, making the process less rigid and enhancing stability across a wider array of tables.
    • -
- 		SaaS companies using TiDB as the system of record for their multi-tenant applications often need to store a substantial number of tables. In previous versions, handling table counts in the order of a million or more was feasible, but it had the potential to degrade the overall user experience. TiDB v8.0.0 improves the situation by implementing a priority queue for auto analyze, making the process less rigid and enhancing stability across a wider array of tables.
DB Operations and Observability
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
CategoryFeature/EnhancementDescription
Scalability and PerformanceAcceleration of cluster snapshot restore speed (GA in v8.0.0)With this feature, BR can fully leverage the scale advantage of a cluster, enabling all TiKV nodes in the cluster to participate in the preparation step of data restores. This feature can significantly improve the restore speed of large datasets in large-scale clusters. Real-world tests show that this feature can saturate the download bandwidth, with the download speed improving by 8 to 10 times, and the end-to-end restore speed improving by approximately 1.5 to 3 times.
Achieve up to 10 times faster for creating tables in batch (experimental, introduced in v7.6.0)With the implementation of the new DDL architecture in v7.6.0, the performance of batch table creation has witnessed a remarkable improvement, up to 10 times faster. This substantial enhancement drastically reduces the time needed for creating numerous tables. This acceleration is particularly noteworthy in SaaS scenarios, where the prevalence of high volumes of tables, ranging from tens to hundreds of thousands, is a common challenge.
Use Active PD Followers to enhance PD's Region information query service (experimental, introduced in v7.6.0)TiDB v7.6.0 introduces an experimental feature "Active PD Follower", which allows PD followers to provide Region information query services. This feature improves the capability of the PD cluster to handle GetRegion and ScanRegions requests in clusters with a large number of TiDB nodes and Regions, thereby reducing the CPU pressure on PD leaders.
Bulk DML for much larger transactions (experimental, introduced in v8.0.0)Large batch DML jobs, such as extensive cleanup jobs, joins, or aggregations, can consume a significant amount of memory and have previously been limited at very large scales. Bulk DML (tidb_dml_type = "bulk") is a new DML type for handling large batch DML tasks more efficiently while providing transaction guarantees and mitigating OOM issues. This feature differs from import, load, and restore operations when used for data loading.
Enhance the stability of caching the schema information when there is a massive number of tables (experimental, introduced in v8.0.0)SaaS companies using TiDB as the system of record for their multi-tenant applications often need to store a substantial number of tables. In previous versions, handling table counts in the order of a million or more was feasible, but it had the potential to degrade the overall user experience. TiDB v8.0.0 improves the situation by implementing a priority queue for auto analyze, making the process less rigid and enhancing stability across a wider array of tables.
Reliability and availabilityGlobal Sort (GA in v8.0.0)The Global Sort feature aims to improve the stability and efficiency of IMPORT INTO and CREATE INDEX. By globally sorting the data to be processed, this feature improves the stability, controllability, and scalability of data writing to TiKV, consequently enhancing the user experience and service quality of data import and index creation. With global sorting enabled, each IMPORT INTO or CREATE INDEX statement now supports importing or adding indexes for up to 40 TiB of data.
Cross-database SQL binding (introduced in v7.6.0)When managing hundreds of databases with the same schema, it is often necessary to apply SQL bindings across these databases. For example, in SaaS or PaaS data platforms, each user typically operates separate databases with the same schema and runs similar SQL queries on them. In this case, it is impractical to bind SQL for each database one by one. TiDB v7.6.0 introduces cross-database SQL bindings that enable matching bindings across all schema-equivalent databases.
Support TiProxy (GA in v8.0.0)Full support for the TiProxy service, easily deployable via deployment tooling, to manage and maintain connections to TiDB so that they live through rolling restarts, upgrades, or scaling events.
Data Migration (DM) officially supports MySQL 8.0 (GA in v7.6.0)Previously, using DM to migrate data from MySQL 8.0 is an experimental feature and is not available for production environments. TiDB v7.6.0 enhances the stability and compatibility of this feature to help you smoothly and quickly migrate data from MySQL 8.0 to TiDB in production environments. In v7.6.0, this feature becomes generally available (GA).
TiDB resource control supports managing queries that consume more resources than expected (GA in v8.1.0)Through the rules of resource groups, TiDB can automatically identify queries that consume more resources than expected, and then limit or cancel these queries. Even if the queries are not identified by the rules, you can still manually add query characteristics and take corresponding measures to reduce the impact of the sudden query performance problem on the entire database.
DB Operations and ObservabilitySupport monitoring index usage statistics (introduced in v8.0.0)Proper index design is a crucial prerequisite to maintaining database performance. TiDB v8.0.0 introduces the INFORMATION_SCHEMA.TIDB_INDEX_USAGE table and the sys.schema_unused_indexes view to provide usage statistics of indexes. This feature helps you assess the efficiency of indexes in the database and optimize the index design.
Data MigrationTiCDC supports the Simple protocol (introduced in v8.0.0)TiCDC introduces a new protocol, the Simple protocol. This protocol provides in-band schema tracking capabilities by embedding table schema information in DDL and BOOTSTRAP events.
TiCDC supports the Debezium format protocol (introduced in v8.0.0)TiCDC introduces a new protocol, the Debezium protocol. TiCDC can now publish data change events to a Kafka sink using a protocol that generates Debezium style messages.
TiCDC supports client authentication (introduced in v8.1.0)TiCDC supports client authentication using mutual Transport Layer Security (mTLS) or TiDB username and password. This feature enables CLI or OpenAPI clients to authenticate their connections to TiCDC.
+ +## Feature details + +### Reliability + +* Support managing queries that consume more resources than expected (GA) [#43691](https://github.com/pingcap/tidb/issues/43691) @[nolouch](https://github.com/nolouch) + + Sudden SQL query performance problems can cause a decline in overall database performance, which is the most common challenge to database stability. The reasons for these problems are diverse, such as untested new SQL statements, drastic changes in data volume, and sudden changes in execution plans. These problems are difficult to avoid completely at the source. TiDB v7.2.0 has introduced the capability to manage queries that consume more resources than expected to quickly reduce the impact of sudden query performance problems. This feature becomes generally available in v8.1.0. + + You can set the maximum execution time for a query in a resource group. When the execution time of a query exceeds the set value, the priority of the query is automatically reduced or the query is canceled. You can also set immediately matching identified queries through text or execution plans within a period of time, to avoid excessive resource consumption during the identification phase when the concurrency of problematic queries is too high. + + TiDB also supports the manual marking of queries. By using the [`QUERY WATCH`](/sql-statements/sql-statement-query-watch.md) command, you can mark queries based on the SQL text, SQL Digest, or execution plan. The queries that match the mark can be downgraded or canceled, achieving the purpose of adding a SQL blocklist. + + The automatic management capability of queries that consume more resources than expected provides users with an effective means to quickly mitigate the impact of query problems on overall performance before the root cause is identified, thereby improving the stability of the database. + + For more information, see [documentation](/tidb-resource-control.md#manage-queries-that-consume-more-resources-than-expected-runaway-queries). + +### SQL + +* Support using more expressions to set default column values when creating a table (GA) [#50936](https://github.com/pingcap/tidb/issues/50936) @[zimulala](https://github.com/zimulala) + + Before v8.0.0, when you create a table, the default value of a column is limited to strings, numbers, dates, and certain expressions. Starting from v8.0.0, you can use more expressions as the default column values. For example, you can set the default value of a column to `DATE_FORMAT`. This feature helps you meet more diverse requirements. In v8.1.0, this feature becomes GA. + + Starting from v8.1.0, you can use expressions as default values when adding columns by `ADD COLUMN`. + + For more information, see [documentation](/data-type-default-values.md#specify-expressions-as-default-values). + +### DB operations + +* Enable the TiDB Distributed eXecution Framework (DXF) by default to enhance the performance and stability of `ADD INDEX` or `IMPORT INTO` tasks in parallel execution [#52441](https://github.com/pingcap/tidb/issues/52441) @[D3Hunter](https://github.com/D3Hunter) + + The DXF becomes generally available (GA) in v7.5.0, but it is disabled by default. This means that an `ADD INDEX` or `IMPORT INTO` task is executed only by one TiDB node by default. + + Starting from v8.1.0, TiDB enables this feature by default ([`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) defaults to `ON`). When enabled, the DXF can schedule multiple TiDB nodes to execute the same `ADD INDEX` or `IMPORT INTO` task in parallel, fully utilizing the resources of the TiDB cluster and greatly improving the performance of these tasks. In addition, you can linearly improve the performance of `ADD INDEX` and `IMPORT INTO` tasks by adding TiDB nodes and configuring [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) for the newly added nodes. + + For more information, see [documentation](/tidb-distributed-execution-framework.md). + +### Security + +* Enhance TiDB log desensitization (GA) [#52364](https://github.com/pingcap/tidb/issues/52364) @[xhebox](https://github.com/xhebox) + + The enhanced TiDB log desensitization supports removing sensitive data when users view logs, implemented by marking SQL text information in log files. You can control whether to mark log information to enable secure use of TiDB logs in different scenarios, enhancing the security and flexibility of using log desensitization. To use this feature, set the system variable `tidb_redact_log` to `MARKER`, and then the SQL text in TiDB's runtime logs is marked. In addition, you can use the `collect-log` subcommand on the TiDB server to remove marked sensitive data from the logs and display the logs in a secure manner. You can also remove all markers and get the normal logs. This feature became generally available in v8.1.0. + + For more information, see [documentation](/system-variables.md#tidb_redact_log). + +### Data migration + +* Support the `IMPORT INTO ... FROM SELECT` syntax (GA) [#49883](https://github.com/pingcap/tidb/issues/49883) @[D3Hunter](https://github.com/D3Hunter) + + Before v8.0.0, importing query results into a target table could only be done using the `INSERT INTO ... SELECT` statement, which is relatively inefficient in some large dataset scenarios. In v8.0.0, TiDB introduces `IMPORT INTO ... FROM SELECT` as an experimental feature, which enables you to import the results of a `SELECT` query into an empty TiDB target table. It achieves up to 8 times the performance of `INSERT INTO ... SELECT` and significantly reduces the import time. In addition, you can use `IMPORT INTO ... FROM SELECT` to import historical data queried with [`AS OF TIMESTAMP`](/as-of-timestamp.md). + + In v8.1.0, the `IMPORT INTO ... FROM SELECT` syntax becomes generally available (GA), enriching the functionality scenarios of the `IMPORT INTO` statement. + + For more information, see [documentation](/sql-statements/sql-statement-import-into.md). + +* TiDB Lightning simplifies conflict resolution strategies and supports handling conflicting data using the `replace` strategy (GA) [#51036](https://github.com/pingcap/tidb/issues/51036) @[lyzx2001](https://github.com/lyzx2001) + + Before v8.0.0, TiDB Lightning has [one data conflict resolution strategy](/tidb-lightning/tidb-lightning-logical-import-mode-usage.md#conflict-detection) for the logical import mode and [two data conflict resolution strategies](/tidb-lightning/tidb-lightning-physical-import-mode-usage.md#conflict-detection) for the physical import mode, which are not easy to understand and configure. + + In v8.0.0, TiDB Lightning deprecates the [old version of conflict detection](/tidb-lightning/tidb-lightning-physical-import-mode-usage.md#the-old-version-of-conflict-detection-deprecated-in-v800) strategy for the physical import mode, enables you to control the conflict detection strategy for both logical and physical import modes via the [`conflict.strategy`](/tidb-lightning/tidb-lightning-configuration.md) parameter (experimental), and simplifies the configuration of this parameter. In addition, in the physical import mode, the `replace` strategy supports retaining the latest data and overwriting the old data when the import detects data with primary key or unique key conflicts. In v8.1.0, the capability to handle conflicting data with the `replace` strategy becomes generally available (GA). + + For more information, see [documentation](/tidb-lightning/tidb-lightning-configuration.md). + +* TiCDC supports client authentication [#10636](https://github.com/pingcap/tiflow/issues/10636) @[CharlesCheung96](https://github.com/CharlesCheung96) + + In v8.1.0, TiCDC supports client authentication when you are using the TiCDC CLI or OpenAPI. This feature enables you to configure TiCDC to require client authentication using client certificates, thereby establishing mutual Transport Layer Security (mTLS). Additionally, you can configure authentication based on TiDB username and password. + + For more information, see [documentation](/ticdc/ticdc-client-authentication.md). + +## Compatibility changes + +> **Note:** +> +> This section provides compatibility changes you need to know when you upgrade from v8.0.0 to the current version (v8.1.0). If you are upgrading from v7.6.0 or earlier versions to the current version, you might also need to check the compatibility changes introduced in intermediate versions. + +### Behavior changes + +* In earlier versions, the `tidb.tls` configuration item in TiDB Lightning treats values `"false"` and `""` the same, as well as treating the values `"preferred"` and `"skip-verify"` the same. Starting from v8.1.0, TiDB Lightning distinguishes the behavior of `"false"`, `""`, `"skip-verify"`, and `"preferred"` for `tidb.tls`. For more information, see [TiDB Lightning configuration](/tidb-lightning/tidb-lightning-configuration.md). +* For tables with `AUTO_ID_CACHE=1`, TiDB supports a [centralized auto-increment ID allocating service](/auto-increment.md#mysql-compatibility-mode). In earlier versions, the primary TiDB node of this service automatically performs a `forceRebase` operation when the TiDB process exits (for example, during the TiDB node restart) to keep auto-assigned IDs as consecutive as possible. However, when there are too many tables with `AUTO_ID_CACHE=1`, executing `forceRebase` becomes very time-consuming, preventing TiDB from restarting promptly and even blocking data writes, thus affecting system availability. To resolve this issue, starting from v8.1.0, TiDB removes the `forceRebase` behavior, but this change will cause some auto-assigned IDs to be non-consecutive during the failover. +* In earlier versions, when processing a transaction containing `UPDATE` changes, if the primary key or non-null unique index value is modified in an `UPDATE` event, TiCDC splits this event into `DELETE` and `INSERT` events. In v8.1.0, when using the MySQL sink, TiCDC splits an `UPDATE` event into `DELETE` and `INSERT` events if the transaction `commitTS` for the `UPDATE` change is less than TiCDC `thresholdTS` (which is the current timestamp that TiCDC fetches from PD at TiCDC startup). This behavior change addresses the issue of downstream data inconsistencies caused by the potentially incorrect order of `UPDATE` events received by TiCDC, which can lead to an incorrect order of split `DELETE` and `INSERT` events. For more information, see [documentation](/ticdc/ticdc-split-update-behavior.md#split-update-events-for-mysql-sinks). + +### System variables + +| Variable name | Change type | Description | +|--------|------------------------------|------| +| [`tidb_enable_telemetry`](/system-variables.md#tidb_enable_telemetry-new-in-v402-and-deprecated-in-v810) | Deprecated | Starting from v8.1.0, the telemetry feature in TiDB is removed, and this variable is no longer functional. It is retained solely for compatibility with earlier versions. | +| [`tidb_auto_analyze_ratio`](/system-variables.md#tidb_auto_analyze_ratio) | Modified | Changes the value range from `[0, 18446744073709551615]` to `(0, 1]`. | +| [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) | Modified | Changes the default value from `OFF` to `ON`. This means that Distributed eXecution Framework (DXF) is enabled by default, which fully utilizes the resources of the TiDB cluster and greatly improves the performance of `ADD INDEX` and `IMPORT INTO` tasks. If you want to upgrade a cluster with the DXF enabled to v8.1.0 or later, disable the DXF (by setting `tidb_enable_dist_task` to `OFF`) before the upgrade, which avoids `ADD INDEX` operations during the upgrade causing data index inconsistency. After the upgrade, you can manually enable the DXF. | +| [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) | Modified | Changes the optional value from `""` or `background` to a string with a length of up to 64 characters, which enables you to control the service scope of each TiDB node more flexibly. Valid characters include digits `0-9`, letters `a-zA-Z`, underscores `_`, and hyphens `-`. The Distributed eXecution Framework (DXF) determines which TiDB nodes can be scheduled to execute distributed tasks based on the value of this variable. For specific rules, see [Task scheduling](/tidb-distributed-execution-framework.md#task-scheduling). | + +### Configuration file parameters + +| Configuration file | Configuration parameter | Change type | Description | +| -------- | -------- | -------- | -------- | +| TiDB| [`enable-telemetry`](/tidb-configuration-file.md#enable-telemetry-new-in-v402-and-deprecated-in-v810) | Deprecated | Starting from v8.1.0, the telemetry feature in TiDB is removed, and this configuration item is no longer functional. It is retained solely for compatibility with earlier versions. | +| TiDB| [`concurrently-init-stats`](/tidb-configuration-file.md#concurrently-init-stats-new-in-v810-and-v752) | Newly added | Controls whether to initialize statistics concurrently during TiDB startup. The default value is `false`. | +| PD | [`enable-telemetry`](/pd-configuration-file.md#enable-telemetry) | Deprecated | Starting from v8.1.0, the telemetry feature in TiDB Dashboard is removed, and this configuration item is no longer functional. It is retained solely for compatibility with earlier versions. | +| TiDB Lightning | [`conflict.max-record-rows`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-configuration) | Modified | Starting from v8.1.0, there is no need to configure `conflict.max-record-rows` manually, because TiDB Lightning automatically assigns the value of `conflict.max-record-rows` with the value of `conflict.threshold`, regardless of the user input. `conflict.max-record-rows` will be deprecated in a future release. | +| TiDB Lightning | [`conflict.threshold`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) | Modified | Changes the default value from `9223372036854775807` to `10000` to quickly interrupt abnormal tasks so that you can make corresponding adjustments as soon as possible. This saves time and computational resources by avoiding the scenario where a large amount of conflicting data is discovered after the import, caused by abnormal data sources or incorrect table schema definitions. | +| TiKV | [`memory.enable-thread-exclusive-arena`](/tikv-configuration-file.md#enable-thread-exclusive-arena-new-in-v810) | Newly added | Controls whether to display the memory allocation status at the TiKV thread level to track the memory usage of each TiKV thread. The default value is `true`. | +| TiCDC | [`security.client-allowed-user`](/ticdc/ticdc-server-config.md#cdc-server-configuration-file-parameters) | Newly added | Lists the usernames that are allowed for client authentication. Authentication requests with usernames not in this list will be rejected. The default value is null. | +| TiCDC | [`security.client-user-required`](/ticdc/ticdc-server-config.md#cdc-server-configuration-file-parameters) | Newly added | Controls whether to use username and password for client authentication. The default value is `false`. | +| TiCDC | [`security.mtls`](/ticdc/ticdc-server-config.md#cdc-server-configuration-file-parameters) | Newly added | Controls whether to enable the TLS client authentication. The default value is `false`. | +| TiCDC | [`sink.debezium.output-old-value`](/ticdc/ticdc-changefeed-config.md#changefeed-configuration-parameters) | Newly added | Controls whether to output the value before the row data changes. The default value is `true`. When it is disabled, the `UPDATE` event does not output the "before" field. | +| TiCDC | [`sink.open.output-old-value`](/ticdc/ticdc-changefeed-config.md#changefeed-configuration-parameters) | Newly added | Controls whether to output the value before the row data changes. The default value is `true`. When it is disabled, the `UPDATE` event does not output the "p" field. | + +## Deprecated features + +* Starting from v8.1.0, the telemetry feature in TiDB and TiDB Dashboard is removed: + + * The system variable [`tidb_enable_telemetry`](/system-variables.md#tidb_enable_telemetry-new-in-v402-and-deprecated-in-v810), the TiDB configuration item [`enable-telemetry`](/tidb-configuration-file.md#enable-telemetry-new-in-v402-and-deprecated-in-v810), and the PD configuration item [`enable-telemetry`](/pd-configuration-file.md#enable-telemetry) are deprecated and no longer functional. + * The `ADMIN SHOW TELEMETRY` syntax is removed. + * The `TELEMETRY` and `TELEMETRY_ID` keywords are removed. + +* It is planned to redesign [the auto-evolution of execution plan bindings](/sql-plan-management.md#baseline-evolution) in subsequent releases, and the related variables and behavior will change. +* The TiDB Lightning parameter [`conflict.max-record-rows`](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) is scheduled for deprecation in a future release and will be subsequently removed. This parameter will be replaced by `conflict.threshold`, which means that the maximum number of conflicting records is consistent with the maximum number of conflicting records that can be tolerated in a single import task. +* Starting from v8.0.0, TiDB Lightning deprecates the [old version of conflict detection](/tidb-lightning/tidb-lightning-physical-import-mode-usage.md#the-old-version-of-conflict-detection-deprecated-in-v800) strategy for the physical import mode, and enables you to control the conflict detection strategy for both logical and physical import modes via the [`conflict.strategy`](/tidb-lightning/tidb-lightning-configuration.md) parameter. The [`duplicate-resolution`](/tidb-lightning/tidb-lightning-configuration.md) parameter for the old version of conflict detection will be removed in a future release. + +## Improvements + ++ TiDB + + - Improve the MySQL compatibility of foreign keys displayed in the output of `SHOW CREATE TABLE` [#51837](https://github.com/pingcap/tidb/issues/51837) @[negachov](https://github.com/negachov) + - Improve the MySQL compatibility of expression default values displayed in the output of `SHOW CREATE TABLE` [#52939](https://github.com/pingcap/tidb/issues/52939) @[CbcWestwolf](https://github.com/CbcWestwolf) + - Support adding multiple indexes concurrently in the ingest mode [#52596](https://github.com/pingcap/tidb/issues/52596) @[lance6716](https://github.com/lance6716) + - Support configuring the system variable `tidb_service_scope` with various values, enhancing the utilization of the Distributed eXecution Framework (DXF) [#52441](https://github.com/pingcap/tidb/issues/52441) @[ywqzzy](https://github.com/ywqzzy) + - Enhance the handling of DNF items that are always `false` by directly ignoring such filter conditions, thus avoiding unnecessary full table scans [#40997](https://github.com/pingcap/tidb/issues/40997) @[hi-rustin](https://github.com/Rustin170506) + - Support using Optimizer Fix Controls to remove the limitation that the optimizer does not automatically choose Index Merge for a query when the optimizer can choose the single index scan method (other than full table scan) for the query [#52869](https://github.com/pingcap/tidb/issues/52869) @[time-and-fate](https://github.com/time-and-fate) + - Add the `total_kv_read_wall_time` metric to the column `execution info` of Coprocessor operators [#28937](https://github.com/pingcap/tidb/issues/28937) @[cfzjywxk](https://github.com/cfzjywxk) + - Add the `RU (max)` metric on the Resource Control dashboard [#49318](https://github.com/pingcap/tidb/issues/49318) @[nolouch](https://github.com/nolouch) + - Add a timeout mechanism for LDAP authentication to avoid the issue of resource lock (RLock) not being released in time [#51883](https://github.com/pingcap/tidb/issues/51883) @[YangKeao](https://github.com/YangKeao) + ++ TiKV + + - Avoid performing IO operations on snapshot files in Raftstore threads to improve TiKV stability [#16564](https://github.com/tikv/tikv/issues/16564) @[Connor1996](https://github.com/Connor1996) + - Accelerate the shutdown speed of TiKV [#16680](https://github.com/tikv/tikv/issues/16680) @[LykxSassinator](https://github.com/LykxSassinator) + - Add metrics for memory usage per thread [#15927](https://github.com/tikv/tikv/issues/15927) @[Connor1996](https://github.com/Connor1996) + ++ PD + + - Optimize the logic for `OperatorController` to reduce the overhead of competition locks [#7897](https://github.com/tikv/pd/issues/7897) @[nolouch](https://github.com/nolouch) + ++ TiFlash + + - Mitigate the issue that TiFlash might panic due to updating certificates after TLS is enabled [#8535](https://github.com/pingcap/tiflash/issues/8535) @[windtalker](https://github.com/windtalker) + ++ Tools + + + Backup & Restore (BR) + + - Add PITR integration test cases to cover compatibility testing for log backup and adding index acceleration [#51987](https://github.com/pingcap/tidb/issues/51987) @[Leavrth](https://github.com/Leavrth) + - Remove the invalid verification for active DDL jobs when log backup starts [#52733](https://github.com/pingcap/tidb/issues/52733) @[Leavrth](https://github.com/Leavrth) + - Add test cases to test compatibility between PITR and the acceleration of adding indexes feature [#51988](https://github.com/pingcap/tidb/issues/51988) @[Leavrth](https://github.com/Leavrth) + - BR cleans up empty SST files during data recovery [#16005](https://github.com/tikv/tikv/issues/16005) @[Leavrth](https://github.com/Leavrth) + + + TiCDC + + - Improve memory stability during data recovery using redo logs to reduce the probability of OOM [#10900](https://github.com/pingcap/tiflow/issues/10900) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Significantly improve the stability of data replication in transaction conflict scenarios, with up to 10 times performance improvement [#10896](https://github.com/pingcap/tiflow/issues/10896) @[CharlesCheung96](https://github.com/CharlesCheung96) + +## Bug fixes + ++ TiDB + + - Fix the issue that executing SQL statements containing tables with multi-valued indexes might return the `Can't find a proper physical plan for this query` error [#49438](https://github.com/pingcap/tidb/issues/49438) @[qw4990](https://github.com/qw4990) + - Fix the issue that automatic statistics collection gets stuck after an OOM error occurs [#51993](https://github.com/pingcap/tidb/issues/51993) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that after using BR to restore a table that has no statistics, the statistics health of that table is still 100% [#29769](https://github.com/pingcap/tidb/issues/29769) @[winoros](https://github.com/winoros) + - Fix the issue that TiDB creates statistics for system tables during upgrade [#52040](https://github.com/pingcap/tidb/issues/52040) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that automatic statistics collection is triggered before the initialization of statistics finishes [#52346](https://github.com/pingcap/tidb/issues/52346) @[hi-rustin](https://github.com/Rustin170506) + - Fix the issue that TiDB might crash when `tidb_mem_quota_analyze` is enabled and the memory used by updating statistics exceeds the limit [#52601](https://github.com/pingcap/tidb/issues/52601) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the TiDB synchronously loading statistics mechanism retries to load empty statistics indefinitely and prints the `fail to get stats version for this histogram` log [#52657](https://github.com/pingcap/tidb/issues/52657) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that expressions containing different collations might cause the query to panic when the new framework for collations is disabled [#52772](https://github.com/pingcap/tidb/issues/52772) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that the `CPS by type` metric displays incorrect values [#52605](https://github.com/pingcap/tidb/issues/52605) @[nolouch](https://github.com/nolouch) + - Fix the issue that the nil pointer error occurs when you query `INFORMATION_SCHEMA.TIKV_REGION_STATUS` [#52013](https://github.com/pingcap/tidb/issues/52013) @[JmPotato](https://github.com/JmPotato) + - Fix the incorrect error message displayed when an invalid default value is specified for a column [#51592](https://github.com/pingcap/tidb/issues/51592) @[danqixu](https://github.com/danqixu) + - Fix the issue that adding indexes in the ingest mode might cause inconsistent data index in some corner cases [#51954](https://github.com/pingcap/tidb/issues/51954) @[lance6716](https://github.com/lance6716) + - Fix the issue that DDL operations get stuck when restoring a table with the foreign key [#51838](https://github.com/pingcap/tidb/issues/51838) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that adding indexes fails when the TiDB network is isolated [#51846](https://github.com/pingcap/tidb/issues/51846) @[ywqzzy](https://github.com/ywqzzy) + - Fix the issue that adding an index with the same name after renaming an index causes an error [#51431](https://github.com/pingcap/tidb/issues/51431) @[lance6716](https://github.com/lance6716) + - Fix the issue of inconsistent data indexes caused by cluster upgrade during adding indexes [#52411](https://github.com/pingcap/tidb/issues/52411) @[tangenta](https://github.com/tangenta) + - Fix the issue that adding indexes to large tables fails after enabling the Distributed eXecution Framework (DXF) [#52640](https://github.com/pingcap/tidb/issues/52640) @[tangenta](https://github.com/tangenta) + - Fix the issue that adding indexes concurrently reports the error `no such file or directory` [#52475](https://github.com/pingcap/tidb/issues/52475) @[tangenta](https://github.com/tangenta) + - Fix the issue that the temporary data cannot be cleaned up after adding indexes fails [#52639](https://github.com/pingcap/tidb/issues/52639) @[lance6716](https://github.com/lance6716) + - Fix the issue that the metadata lock fails to prevent DDL operations from executing in the plan cache scenario [#51407](https://github.com/pingcap/tidb/issues/51407) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that the `IMPORT INTO` operation gets stuck when importing a large amount of data [#52884](https://github.com/pingcap/tidb/issues/52884) @[lance6716](https://github.com/lance6716) + - Fix the issue that TiDB unexpectedly restarts when logging gRPC errors [#51301](https://github.com/pingcap/tidb/issues/51301) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that IndexHashJoin outputs redundant data when calculating Anti Left Outer Semi Join [#52923](https://github.com/pingcap/tidb/issues/52923) @[yibin87](https://github.com/yibin87) + - Fix the incorrect result of the TopN operator in correlated subqueries [#52777](https://github.com/pingcap/tidb/issues/52777) @[yibin87](https://github.com/yibin87) + - Fix the inaccurate execution time statistics of HashJoin probe [#52222](https://github.com/pingcap/tidb/issues/52222) @[windtalker](https://github.com/windtalker) + - Fix the issue that using `TABLESAMPLE` returns incorrect results in static partition pruning mode (`tidb_partition_prune_mode='static'`) [#52282](https://github.com/pingcap/tidb/issues/52282) @[tangenta](https://github.com/tangenta) + - Fix the issue that TTL is deviated by 1 hour in daylight saving time [#51675](https://github.com/pingcap/tidb/issues/51675) @[lcwangchao](https://github.com/lcwangchao) + - Fix the incorrect calculation and display of the number of connections (Connection Count) on the TiDB Dashboard Monitoring page [#51889](https://github.com/pingcap/tidb/issues/51889) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the status gets stuck when rolling back the partition DDL tasks [#51090](https://github.com/pingcap/tidb/issues/51090) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue that the value of `max_remote_stream` is incorrect when executing `EXPLAIN ANALYZE` [#52646](https://github.com/pingcap/tidb/issues/52646) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that querying the `TIDB_HOT_REGIONS` table might incorrectly return `INFORMATION_SCHEMA` tables [#50810](https://github.com/pingcap/tidb/issues/50810) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that the `EXPLAIN` statement might display incorrect column IDs in the result when statistics for certain columns are not fully loaded [#52207](https://github.com/pingcap/tidb/issues/52207) @[time-and-fate](https://github.com/time-and-fate) + - Fix the issue that the type returned by the `IFNULL` function is inconsistent with MySQL [#51765](https://github.com/pingcap/tidb/issues/51765) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that adding a unique index might cause TiDB to panic [#52312](https://github.com/pingcap/tidb/issues/52312) @[wjhuang2016](https://github.com/wjhuang2016) + ++ TiKV + + - Fix the issue that resolve-ts is blocked when a stale Region peer ignores the GC message [#16504](https://github.com/tikv/tikv/issues/16504) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that inactive Write Ahead Logs (WALs) in RocksDB might corrupt data [#16705](https://github.com/tikv/tikv/issues/16705) @[Connor1996](https://github.com/Connor1996) + ++ PD + + - Fix the issue that TSO might get stuck when toggling the PD microservice mode on and off [#7849](https://github.com/tikv/pd/issues/7849) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that the `State` monitoring metric for DR Auto-Sync does not show any data [#7974](https://github.com/tikv/pd/issues/7974) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that checking binary versions might cause PD panic [#7978](https://github.com/tikv/pd/issues/7978) @[JmPotato](https://github.com/JmPotato) + - Fix the type conversion error that occurs when TTL parameters are parsed [#7980](https://github.com/tikv/pd/issues/7980) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that the Leader fails to transfer when you switch it between two deployed data centers [#7992](https://github.com/tikv/pd/issues/7992) @[TonsnakeLin](https://github.com/TonsnakeLin) + - Fix the issue that `PrintErrln` in pd-ctl fails to output error messages to `stderr` [#8022](https://github.com/tikv/pd/issues/8022) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that PD might panic when generating `Merge` schedules [#8049](https://github.com/tikv/pd/issues/8049) @[nolouch](https://github.com/nolouch) + - Fix the panic issue caused by `GetAdditionalInfo` [#8079](https://github.com/tikv/pd/issues/8079) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that the `Filter target` monitoring metric for PD does not provide scatter range information [#8125](https://github.com/tikv/pd/issues/8125) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that the query result of `SHOW CONFIG` includes the deprecated configuration item `trace-region-flow` [#7917](https://github.com/tikv/pd/issues/7917) @[rleungx](https://github.com/rleungx) + - Fix the issue that the scaling progress is not correctly displayed [#7726](https://github.com/tikv/pd/issues/7726) @[CabinfeverB](https://github.com/CabinfeverB) + ++ TiFlash + + - Fix the issue that TiFlash might panic when you insert data to columns with invalid default values in non-strict `sql_mode` [#8803](https://github.com/pingcap/tiflash/issues/8803) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that TiFlash might return transiently incorrect results in high-concurrency read scenarios [#8845](https://github.com/pingcap/tiflash/issues/8845) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that in the disaggregated storage and compute architecture, the disk `used_size` metric displayed in Grafana is incorrect after you modify the value of the `storage.remote.cache.capacity` configuration item for TiFlash compute nodes [#8920](https://github.com/pingcap/tiflash/issues/8920) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that TiFlash metadata might become corrupted and cause the process to panic when upgrading a cluster from a version earlier than v6.5.0 to v6.5.0 or later [#9039](https://github.com/pingcap/tiflash/issues/9039) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that in the disaggregated storage and compute architecture, TiFlash might panic when the compute node process is stopped [#8860](https://github.com/pingcap/tiflash/issues/8860) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that TiFlash might return an error when executing queries containing virtual generated columns [#8787](https://github.com/pingcap/tiflash/issues/8787) @[guo-shaoge](https://github.com/guo-shaoge) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that BR cannot back up the `AUTO_RANDOM` ID allocation progress in a union clustered index that contains an `AUTO_RANDOM` column [#52255](https://github.com/pingcap/tidb/issues/52255) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that removing a log backup task after it is paused does not immediately restore the GC safepoint [#52082](https://github.com/pingcap/tidb/issues/52082) @[3pointer](https://github.com/3pointer) + - Fix a rare issue that special event timing might cause the data loss in log backup [#16739](https://github.com/tikv/tikv/issues/16739) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that the global checkpoint of log backup is advanced ahead of the actual backup file write point due to TiKV restart, which might cause a small amount of backup data loss [#16809](https://github.com/tikv/tikv/issues/16809) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that the confusing information related to `--concurrency` appears in the log during a full backup [#50837](https://github.com/pingcap/tidb/issues/50837) @[BornChanger](https://github.com/BornChanger) + - Fix the issue that the Region fetched from PD does not have a Leader when restoring data using BR or importing data using TiDB Lightning in physical import mode [#51124](https://github.com/pingcap/tidb/issues/51124) [#50501](https://github.com/pingcap/tidb/issues/50501) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that after pausing, stopping, and rebuilding the log backup task, the task status is normal, but the checkpoint does not advance [#53047](https://github.com/pingcap/tidb/issues/53047) @[RidRisR](https://github.com/RidRisR) + - Fix the unstable test case `TestClearCache` [#51671](https://github.com/pingcap/tidb/issues/51671) @[zxc111](https://github.com/zxc111) + - Fix the unstable test case `TestGetMergeRegionSizeAndCount` [#52095](https://github.com/pingcap/tidb/issues/52095) @[3pointer](https://github.com/3pointer) + - Fix the unstable integration test `br_tikv_outage` [#52673](https://github.com/pingcap/tidb/issues/52673) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that the test case `TestGetTSWithRetry` takes too long to execute [#52547](https://github.com/pingcap/tidb/issues/52547) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that TiKV might panic when resuming a paused log backup task with unstable network connections to PD [#17020](https://github.com/tikv/tikv/issues/17020) @[YuJuncen](https://github.com/YuJuncen) + + + TiCDC + + - Fix the issue that calling the API (`/api/v2/owner/resign`) that evicts the TiCDC owner node causes the TiCDC task to restart unexpectedly [#10781](https://github.com/pingcap/tiflow/issues/10781) @[sdojjy](https://github.com/sdojjy) + - Fix the issue that when the downstream Pulsar is stopped, removing the changefeed causes the normal TiCDC process to get stuck, which causes other changefeed processes to get stuck [#10629](https://github.com/pingcap/tiflow/issues/10629) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that the **Ownership history** panel in Grafana is unstable [#10796](https://github.com/pingcap/tiflow/issues/10796) @[hongyunyan](https://github.com/hongyunyan) + - Fix the issue that restarting PD might cause the TiCDC node to restart with an error [#10799](https://github.com/pingcap/tiflow/issues/10799) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that high latency in the PD disk I/O causes severe latency in data replication [#9054](https://github.com/pingcap/tiflow/issues/9054) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that the default value of `TIMEZONE` type is not set according to the correct time zone [#10931](https://github.com/pingcap/tiflow/issues/10931) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that `DROP PRIMARY KEY` and `DROP UNIQUE KEY` statements are not replicated correctly [#10890](https://github.com/pingcap/tiflow/issues/10890) @[asddongmen](https://github.com/asddongmen) + - Fix the issue that TiCDC fails to execute the `Exchange Partition ... With Validation` DDL downstream after it is written upstream, causing the changefeed to get stuck [#10859](https://github.com/pingcap/tiflow/issues/10859) @[hongyunyan](https://github.com/hongyunyan) + + + TiDB Lightning + + - Fix the issue that TiDB Lightning reports `no database selected` during data import due to incompatible SQL statements in the source files [#51800](https://github.com/pingcap/tidb/issues/51800) @[lance6716](https://github.com/lance6716) + - Fix the issue that TiDB Lightning might print sensitive information to logs in server mode [#36374](https://github.com/pingcap/tidb/issues/36374) @[kennytm](https://github.com/kennytm) + - Fix the issue that killing the PD Leader causes TiDB Lightning to report the `invalid store ID 0` error during data import [#50501](https://github.com/pingcap/tidb/issues/50501) @[Leavrth](https://github.com/Leavrth) + - Fix the issue that TiDB Lightning reports the `Unknown column in where clause` error when processing conflicting data using the `replace` strategy [#52886](https://github.com/pingcap/tidb/issues/52886) @[lyzx2001](https://github.com/lyzx2001) + - Fix the issue that TiDB Lightning panics when importing an empty table of Parquet format [#52518](https://github.com/pingcap/tidb/issues/52518) @[kennytm](https://github.com/kennytm) + +## Performance test + +To learn about the performance of TiDB v8.1.0, you can refer to the [TPC-C performance test report](https://docs.pingcap.com/tidbcloud/v8.1-performance-benchmarking-with-tpcc) and [Sysbench performance test report](https://docs.pingcap.com/tidbcloud/v8.1-performance-benchmarking-with-sysbench) of the TiDB Cloud Dedicated cluster. + +## Contributors + +We would like to thank the following contributors from the TiDB community: + +- [arturmelanchyk](https://github.com/arturmelanchyk) (First-time contributor) +- [CabinfeverB](https://github.com/CabinfeverB) +- [danqixu](https://github.com/danqixu) (First-time contributor) +- [imalasong](https://github.com/imalasong) (First-time contributor) +- [jiyfhust](https://github.com/jiyfhust) +- [negachov](https://github.com/negachov) (First-time contributor) +- [testwill](https://github.com/testwill) +- [yzhan1](https://github.com/yzhan1) (First-time contributor) +- [zxc111](https://github.com/zxc111) (First-time contributor) diff --git a/releases/release-8.1.1.md b/releases/release-8.1.1.md new file mode 100644 index 0000000000000..76a2d88cea525 --- /dev/null +++ b/releases/release-8.1.1.md @@ -0,0 +1,206 @@ +--- +title: TiDB 8.1.1 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 8.1.1. +--- + +# TiDB 8.1.1 Release Notes + +Release date: August 27, 2024 + +TiDB version: 8.1.1 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v8.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v8.1/production-deployment-using-tiup) + +## Compatibility changes + +- When using TiDB Lightning to import a CSV file, if you set `strict-format = true` to split a large CSV file into multiple small CSV files to improve concurrency and import performance, you need to explicitly specify `terminator`. The values can be `\r`, `\n` or `\r\n`. Failure to specify a line terminator might result in an exception when parsing the CSV file data. [#37338](https://github.com/pingcap/tidb/issues/37338) @[lance6716](https://github.com/lance6716) +- When using [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) to import a CSV file, if you specify the `SPLIT_FILE` parameter to split a large CSV file into multiple small CSV files to improve concurrency and import performance, you need to explicitly specify the line terminator `LINES_TERMINATED_BY`. The values can be `\r`, `\n` or `\r\n`. Failure to specify a line terminator might result in an exception when parsing the CSV file data. [#37338](https://github.com/pingcap/tidb/issues/37338) @[lance6716](https://github.com/lance6716) +- Change the default value of the [`tidb_enable_parallel_hashagg_spill`](https://docs.pingcap.com/tidb/v8.1/system-variables#tidb_enable_parallel_hashagg_spill-new-in-v800) variable from `ON` to `OFF` to avoid incorrect query results caused by disk spill during parallel calculation. For clusters upgraded to v8.1.1 from v8.0.0 or v8.1.0, this variable will remain at its previous default value of `ON` after the upgrade, and it is recommended to manually change it to `OFF`. [#55290](https://github.com/pingcap/tidb/issues/55290) @[xzhangxian1008](https://github.com/xzhangxian1008) + +## Offline package changes + +In v8.1.1, `arbiter` is removed from the `TiDB-community-toolkit` [binary package](/binary-package.md). + +## Improvements + ++ TiDB + + - By batch deleting TiFlash placement rules, improve the processing speed of data GC after performing the `TRUNCATE` or `DROP` operation on partitioned tables [#54068](https://github.com/pingcap/tidb/issues/54068) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Remove stores without Regions during MPP load balancing [#52313](https://github.com/pingcap/tidb/issues/52313) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Temporarily adjust the priority of statistics synchronously loading tasks to high to avoid widespread timeouts during TiKV high loads, as these timeouts might result in statistics not being loaded [#50332](https://github.com/pingcap/tidb/issues/50332) @[winoros](https://github.com/winoros) + - The `EXPLAIN` statement supports applying the `tidb_redact_log` setting and further optimizes the log handling logic. + - Support applying the `tidb_redact_log` setting to the output of `EXPLAIN` statements and further optimize the logic in processing logs [#54565](https://github.com/pingcap/tidb/issues/54565) @[hawkingrei](https://github.com/hawkingrei) + ++ PD + + - Improve the retry logic of the HTTP client [#8142](https://github.com/tikv/pd/issues/8142) @[JmPotato](https://github.com/JmPotato) + ++ TiFlash + + - Mitigate the issue that TiFlash might panic due to updating certificates after TLS is enabled [#8535](https://github.com/pingcap/tiflash/issues/8535) @[windtalker](https://github.com/windtalker) + - Reduce lock conflicts under highly concurrent data read operations and optimize short query performance [#9125](https://github.com/pingcap/tiflash/issues/9125) @[JinheLin](https://github.com/JinheLin) + ++ Tools + + + Backup & Restore (BR) + + - Support encryption of temporary files generated during log backup [#15083](https://github.com/tikv/tikv/issues/15083) @[YuJuncen](https://github.com/YuJuncen) + - Except for the `br log restore` subcommand, all other `br log` subcommands support skipping the loading of the TiDB `domain` data structure to reduce memory consumption [#52088](https://github.com/pingcap/tidb/issues/52088) @[Leavrth](https://github.com/Leavrth) + - Support setting Alibaba Cloud access credentials through environment variables [#45551](https://github.com/pingcap/tidb/issues/45551) @[RidRisR](https://github.com/RidRisR) + - Support checking whether the disk space in TiKV is sufficient before TiKV downloads each SST file. If the space is insufficient, BR terminates the restore and returns an error [#17224](https://github.com/tikv/tikv/issues/17224) @[RidRisR](https://github.com/RidRisR) + + + TiCDC + + - Support sending BOOTSTRAP messages of all tables to the downstream in one go when a changefeed using Simple Protocol starts [#11315](https://github.com/pingcap/tiflow/issues/11315) @[asddongmen](https://github.com/asddongmen) + - Support directly outputting raw events when the downstream is a Message Queue (MQ) or cloud storage [#11211](https://github.com/pingcap/tiflow/issues/11211) @[CharlesCheung96](https://github.com/CharlesCheung96) + +## Bug fixes + ++ TiDB + + - Fix the issue that the disk spill for the HashAgg operator causes incorrect query results during parallel calculation [#55290](https://github.com/pingcap/tidb/issues/55290) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that `INDEX_HASH_JOIN` cannot exit properly when SQL is abnormally interrupted [#54688](https://github.com/pingcap/tidb/issues/54688) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that RANGE partitioned tables that are not strictly self-incrementing can be created [#54829](https://github.com/pingcap/tidb/issues/54829) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that `PointGet` execution plans for `_tidb_rowid` can be generated [#54583](https://github.com/pingcap/tidb/issues/54583) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that internal SQL statements in the slow log are redacted to null by default [#54190](https://github.com/pingcap/tidb/issues/54190) [#52743](https://github.com/pingcap/tidb/issues/52743) [#53264](https://github.com/pingcap/tidb/issues/53264) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that the `UPDATE` operation can cause TiDB OOM in multi-table scenarios [#53742](https://github.com/pingcap/tidb/issues/53742) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the Window function might panic when there is a related subquery in it [#42734](https://github.com/pingcap/tidb/issues/42734) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that the `LENGTH()` condition is unexpectedly removed when the collation is `utf8_bin` or `utf8mb4_bin` [#53730](https://github.com/pingcap/tidb/issues/53730) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that after a statement within a transaction is killed by OOM, if TiDB continues to execute the next statement within the same transaction, you might get an error `Trying to start aggressive locking while it's already started` and a panic occurs [#53540](https://github.com/pingcap/tidb/issues/53540) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that `PREPARE`/`EXECUTE` statements with the `CONV` expression containing a `?` argument might result in incorrect query results when executed multiple times [#53505](https://github.com/pingcap/tidb/issues/53505) @[qw4990](https://github.com/qw4990) + - Fix the issue that the recursive CTE operator incorrectly tracks memory usage [#54181](https://github.com/pingcap/tidb/issues/54181) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that using `SHOW WARNINGS;` to obtain warnings might cause a panic [#48756](https://github.com/pingcap/tidb/issues/48756) @[xhebox](https://github.com/xhebox) + - Fix the issue that the TopN operator might be pushed down incorrectly [#37986](https://github.com/pingcap/tidb/issues/37986) @[qw4990](https://github.com/qw4990) + - Fix the issue that TiDB panics when executing the `SHOW ERRORS` statement with a predicate that is always `true` [#46962](https://github.com/pingcap/tidb/issues/46962) @[elsa0520](https://github.com/elsa0520) + - Fix the issue that the `STATE` field in the `INFORMATION_SCHEMA.TIDB_TRX` table is empty due to the `size` of the `STATE` field not being defined [#53026](https://github.com/pingcap/tidb/issues/53026) @[cfzjywxk](https://github.com/cfzjywxk) + - Fix the issue that executing the `SELECT DISTINCT CAST(col AS DECIMAL), CAST(col AS SIGNED) FROM ...` query might return incorrect results [#53726](https://github.com/pingcap/tidb/issues/53726) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that DDL statements incorrectly use etcd and cause tasks to queue up [#52335](https://github.com/pingcap/tidb/issues/52335) @[wjhuang2016](https://github.com/wjhuang2016) + - Fix the issue that the `Distinct_count` information in GlobalStats might be incorrect [#53752](https://github.com/pingcap/tidb/issues/53752) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `tidb_enable_async_merge_global_stats` and `tidb_analyze_partition_concurrency` system variables do not take effect during automatic statistics collection [#53972](https://github.com/pingcap/tidb/issues/53972) @[hi-rustin](https://github.com/hi-rustin) + - Fix the issue that the `TIMESTAMPADD()` function goes into an infinite loop when the first argument is `month` and the second argument is negative [#54908](https://github.com/pingcap/tidb/issues/54908) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that the Connection Count monitoring metric in Grafana is incorrect when some connections exit before the handshake is complete [#54428](https://github.com/pingcap/tidb/issues/54428) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that the Connection Count of each resource group is incorrect when using TiProxy and resource groups [#54545](https://github.com/pingcap/tidb/issues/54545) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that using a view does not work in recursive CTE [#49721](https://github.com/pingcap/tidb/issues/49721) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `final` AggMode and the `non-final` AggMode cannot coexist in Massively Parallel Processing (MPP) [#51362](https://github.com/pingcap/tidb/issues/51362) @[AilinKid](https://github.com/AilinKid) + - Fix the issue of incorrect WARNINGS information when using Optimizer Hints [#53767](https://github.com/pingcap/tidb/issues/53767) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue of abnormally high memory usage caused by `memTracker` not being detached when the `HashJoin` or `IndexLookUp` operator is the driven side sub-node of the `Apply` operator [#54005](https://github.com/pingcap/tidb/issues/54005) @[XuHuaiyu](https://github.com/XuHuaiyu) + - Fix the issue that the illegal column type `DECIMAL(0,0)` can be created in some cases [#53779](https://github.com/pingcap/tidb/issues/53779) @[tangenta](https://github.com/tangenta) + - Fix the issue of potential data races during the execution of `(*PointGetPlan).StatsInfo()` [#49803](https://github.com/pingcap/tidb/issues/49803) [#43339](https://github.com/pingcap/tidb/issues/43339) @[qw4990](https://github.com/qw4990) + - Fix the issue that improper use of metadata locks might lead to writing anomalous data when using the plan cache under certain circumstances [#53634](https://github.com/pingcap/tidb/issues/53634) @[zimulala](https://github.com/zimulala) + - Fix the issue that JSON-related functions return errors inconsistent with MySQL in some cases [#53799](https://github.com/pingcap/tidb/issues/53799) @[dveeden](https://github.com/dveeden) + - Fix the issue that TiDB does not create corresponding statistics metadata (`stats_meta`) when creating a table with foreign keys [#53652](https://github.com/pingcap/tidb/issues/53652) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the `memory_quota` hint might not work in subqueries [#53834](https://github.com/pingcap/tidb/issues/53834) @[qw4990](https://github.com/qw4990) + - Fix the issue that TiDB might report an error due to GC when loading statistics at startup [#53592](https://github.com/pingcap/tidb/issues/53592) @[you06](https://github.com/you06) + - Fix the issue that executing `CREATE OR REPLACE VIEW` concurrently might result in the `table doesn't exist` error [#53673](https://github.com/pingcap/tidb/issues/53673) @[tangenta](https://github.com/tangenta) + - Fix the issue that the query latency of stale reads increases, caused by information schema cache misses [#53428](https://github.com/pingcap/tidb/issues/53428) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that `SELECT INTO OUTFILE` does not work when clustered indexes are used as predicates [#42093](https://github.com/pingcap/tidb/issues/42093) @[qw4990](https://github.com/qw4990) + - Fix the issue that comparing a column of `YEAR` type with an unsigned integer that is out of range causes incorrect results [#50235](https://github.com/pingcap/tidb/issues/50235) @[qw4990](https://github.com/qw4990) + - Fix the issue that TiDB might return incorrect query results when you query tables with virtual columns in transactions that involve data modification operations [#53951](https://github.com/pingcap/tidb/issues/53951) @[qw4990](https://github.com/qw4990) + - Fix the issue that TiDB fails to reject unauthenticated user connections in some cases when using the `auth_socket` authentication plugin [#54031](https://github.com/pingcap/tidb/issues/54031) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that when queries contain non-correlated subqueries and `LIMIT` clauses, column pruning might be incomplete, resulting in a less optimal plan [#54213](https://github.com/pingcap/tidb/issues/54213) @[qw4990](https://github.com/qw4990) + - Fix the issue that non-BIGINT unsigned integers might produce incorrect results when compared with strings/decimals [#41736](https://github.com/pingcap/tidb/issues/41736) @[LittleFall](https://github.com/LittleFall) + - Fix the issue that setting `max-index-length` causes TiDB to panic when adding indexes using the Distributed eXecution Framework (DXF) [#53281](https://github.com/pingcap/tidb/issues/53281) @[zimulala](https://github.com/zimulala) + - Fix the issue that certain filter conditions in queries might cause the planner module to report an `invalid memory address or nil pointer dereference` error [#53582](https://github.com/pingcap/tidb/issues/53582) [#53580](https://github.com/pingcap/tidb/issues/53580) [#53594](https://github.com/pingcap/tidb/issues/53594) [#53603](https://github.com/pingcap/tidb/issues/53603) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that recursive CTE queries might result in invalid pointers [#54449](https://github.com/pingcap/tidb/issues/54449) @[hawkingrei](https://github.com/hawkingrei) + - Fix the overflow issue of the `Longlong` type in predicates [#45783](https://github.com/pingcap/tidb/issues/45783) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that indirect placeholder `?` references in a `GROUP BY` statement cannot find columns [#53872](https://github.com/pingcap/tidb/issues/53872) @[qw4990](https://github.com/qw4990) + - Fix the issue that the memory used by transactions might be tracked multiple times [#53984](https://github.com/pingcap/tidb/issues/53984) @[ekexium](https://github.com/ekexium) + - Fix the issue that using `CURRENT_DATE()` as the default value for a column results in incorrect query results [#53746](https://github.com/pingcap/tidb/issues/53746) @[tangenta](https://github.com/tangenta) + - Fix the issue that the performance is unstable when adding indexes using Global Sort [#54147](https://github.com/pingcap/tidb/issues/54147) @[tangenta](https://github.com/tangenta) + - Fix the issue that `SHOW IMPORT JOBS` reports an error `Unknown column 'summary'` after upgrading from v7.1 [#54241](https://github.com/pingcap/tidb/issues/54241) @[tangenta](https://github.com/tangenta) + - Fix the issue that `root` user cannot query `tidb_mdl_view` [#53292](https://github.com/pingcap/tidb/issues/53292) @[tangenta](https://github.com/tangenta) + - Fix the issue that the network partition during adding indexes using the Distributed eXecution Framework (DXF) might cause inconsistent data indexes [#54897](https://github.com/pingcap/tidb/issues/54897) @[tangenta](https://github.com/tangenta) + - Fix the issue that the error occurred during initialization of the TiDB Lightning physical import mode might cause resource leaks [#53659](https://github.com/pingcap/tidb/issues/53659) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that obtaining the column information using `information_schema.columns` returns warning 1356 when a subquery is used as a column definition in a view definition [#54343](https://github.com/pingcap/tidb/issues/54343) @[lance6716](https://github.com/lance6716) + - Fix the issue that using index acceleration to add a unique index might cause a `Duplicate entry` error when the owner is switched [#49233](https://github.com/pingcap/tidb/issues/49233) @[lance6716](https://github.com/lance6716) + - Fix the unclear error message when setting `global.tidb_cloud_storage_uri` [#54096](https://github.com/pingcap/tidb/issues/54096) @[lance6716](https://github.com/lance6716) + - Fix the issue that the Sync Load QPS monitoring metric is incorrect [#53558](https://github.com/pingcap/tidb/issues/53558) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that some statistics information might be missed when loading initial statistics concurrently [#53607](https://github.com/pingcap/tidb/issues/53607) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue of reusing wrong point get plans for `SELECT ... FOR UPDATE` [#54652](https://github.com/pingcap/tidb/issues/54652) @[qw4990](https://github.com/qw4990) + ++ TiKV + + - Fix the issue that CDC and log-backup do not limit the timeout of `check_leader` using the `advance-ts-interval` configuration, causing the `resolved_ts` lag to be too large when TiKV restarts normally in some cases [#17107](https://github.com/tikv/tikv/issues/17107) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that setting the gRPC message compression method via `grpc-compression-type` does not take effect on messages sent from TiKV to TiDB [#17176](https://github.com/tikv/tikv/issues/17176) @[ekexium](https://github.com/ekexium) + - Fix the failure of `make docker` and `make docker_test` [#17075](https://github.com/tikv/tikv/issues/17075) @[shunki-fujita](https://github.com/shunki-fujita) + - Fix the issue that the **gRPC request sources duration** metric is displayed incorrectly in the monitoring dashboard [#17133](https://github.com/tikv/tikv/issues/17133) @[King-Dylan](https://github.com/King-Dylan) + - Fix the issue that the output of the `raft region` command in tikv-ctl does not include the Region status information [#17037](https://github.com/tikv/tikv/issues/17037) @[glorv](https://github.com/glorv) + - Fix the issue that changing the `raftstore.periodic-full-compact-start-times` configuration item online might cause TiKV to panic [#17066](https://github.com/tikv/tikv/issues/17066) @[SpadeA-Tang](https://github.com/SpadeA-Tang) + - Fix the issue that TiKV might repeatedly panic when applying a corrupted Raft data snapshot [#15292](https://github.com/tikv/tikv/issues/15292) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that releasing cache entries before they are persisted causes TiKV to panic [#17040](https://github.com/tikv/tikv/issues/17040) @[glorv](https://github.com/glorv) + ++ PD + + - Fix the issue that an incorrect PD API is called when you retrieve table attributes [#55188](https://github.com/pingcap/tidb/issues/55188) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that the time data type in the `INFORMATION_SCHEMA.RUNAWAY_WATCHES` table is incorrect [#54770](https://github.com/pingcap/tidb/issues/54770) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that some logs are not redacted [#8419](https://github.com/tikv/pd/issues/8419) @[rleungx](https://github.com/rleungx) + - Fix the issue of missing data in the `Filter` monitoring metric [#8098](https://github.com/tikv/pd/issues/8098) @[nolouch](https://github.com/nolouch) + - Fix the issue that the HTTP client might panic when TLS is enabled [#8237](https://github.com/tikv/pd/issues/8237) @[okJiang](https://github.com/okJiang) + - Fix the issue that the encryption manager is not initialized before use [#8384](https://github.com/tikv/pd/issues/8384) @[rleungx](https://github.com/rleungx) + - Fix the issue that resource groups could not effectively limit resource usage under high concurrency [#8435](https://github.com/tikv/pd/issues/8435) @[nolouch](https://github.com/nolouch) + - Fix the data race issue related to `store limit` [#8253](https://github.com/tikv/pd/issues/8253) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that the scaling progress is displayed incorrectly after the `scheduling` microservice is enabled [#8331](https://github.com/tikv/pd/issues/8331) @[rleungx](https://github.com/rleungx) + - Fix the issue that the TSO node is not dynamically updated after the `tso` microservice is enabled [#8154](https://github.com/tikv/pd/issues/8154) @[rleungx](https://github.com/rleungx) + - Fix the data race issue of resource groups [#8267](https://github.com/tikv/pd/issues/8267) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that a resource group encounters quota limits when requesting tokens for more than 500 ms [#8349](https://github.com/tikv/pd/issues/8349) @[nolouch](https://github.com/nolouch) + - Fix the issue that manually transferring the PD leader might fail [#8225](https://github.com/tikv/pd/issues/8225) @[HuSharp](https://github.com/HuSharp) + - Fix the issue that deleted nodes still appear in the candidate connection list in etcd client [#8286](https://github.com/tikv/pd/issues/8286) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that `ALTER PLACEMENT POLICY` cannot modify the placement policy [#52257](https://github.com/pingcap/tidb/issues/52257) [#51712](https://github.com/pingcap/tidb/issues/51712) @[jiyfhust](https://github.com/jiyfhust) + - Fix the issue that the scheduling of write hotspots might break placement policy constraints [#7848](https://github.com/tikv/pd/issues/7848) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that down peers might not recover when using Placement Rules [#7808](https://github.com/tikv/pd/issues/7808) @[rleungx](https://github.com/rleungx) + - Fix the issue that a large number of retries occur when canceling resource groups queries [#8217](https://github.com/tikv/pd/issues/8217) @[nolouch](https://github.com/nolouch) + - Fix the data race issue that PD encounters during operator checks [#8263](https://github.com/tikv/pd/issues/8263) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that no error is reported when binding a role to a resource group [#54417](https://github.com/pingcap/tidb/issues/54417) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that setting the TiKV configuration item [`coprocessor.region-split-size`](/tikv-configuration-file.md#region-split-size) to a value less than 1 MiB causes PD panic [#8323](https://github.com/tikv/pd/issues/8323) @[JmPotato](https://github.com/JmPotato) + ++ TiFlash + + - Fix the issue that a network partition (network disconnection) between TiFlash and any PD might cause read request timeout errors [#9243](https://github.com/pingcap/tiflash/issues/9243) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that the `SUBSTRING_INDEX()` function might cause TiFlash to crash in some corner cases [#9116](https://github.com/pingcap/tiflash/issues/9116) @[wshwsh12](https://github.com/wshwsh12) + - Fix the issue that a large number of duplicate rows might be read in FastScan mode after importing data via BR or TiDB Lightning [#9118](https://github.com/pingcap/tiflash/issues/9118) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that TiFlash might panic when a database is deleted shortly after creation [#9266](https://github.com/pingcap/tiflash/issues/9266) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that setting the SSL certificate configuration to an empty string in TiFlash incorrectly enables TLS and causes TiFlash to fail to start [#9235](https://github.com/pingcap/tiflash/issues/9235) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that in the disaggregated storage and compute architecture, null values might be incorrectly returned in queries after adding non-null columns in DDL operations [#9084](https://github.com/pingcap/tiflash/issues/9084) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that TiFlash might panic after executing `RENAME TABLE ... TO ...` on a partitioned table with empty partitions across databases [#9132](https://github.com/pingcap/tiflash/issues/9132) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue of query timeout when executing queries on partitioned tables that contain empty partitions [#9024](https://github.com/pingcap/tiflash/issues/9024) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that some queries might report a column type mismatch error after late materialization is enabled [#9175](https://github.com/pingcap/tiflash/issues/9175) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that queries with virtual generated columns might return incorrect results after late materialization is enabled [#9188](https://github.com/pingcap/tiflash/issues/9188) @[JinheLin](https://github.com/JinheLin) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that the backup performance during checkpoint backups is affected due to interruptions in seeking Region leaders [#17168](https://github.com/tikv/tikv/issues/17168) @[Leavrth](https://github.com/Leavrth) + - Fix the inefficiency issue in scanning DDL jobs during incremental backups [#54139](https://github.com/pingcap/tidb/issues/54139) @[3pointer](https://github.com/3pointer) + - Fix the issue that BR fails to correctly identify errors due to multiple nested retries during the restore process [#54053](https://github.com/pingcap/tidb/issues/54053) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that BR fails to restore a transactional KV cluster due to an empty `EndKey` [#52574](https://github.com/pingcap/tidb/issues/52574) @[3pointer](https://github.com/3pointer) + - Fix the issue that log backup might be paused after the advancer owner migration [#53561](https://github.com/pingcap/tidb/issues/53561) @[RidRisR](https://github.com/RidRisR) + - Fix the issue that DDLs requiring backfilling, such as `ADD INDEX` and `MODIFY COLUMN`, might not be correctly recovered during incremental restore [#54426](https://github.com/pingcap/tidb/issues/54426) @[3pointer](https://github.com/3pointer) + - Fix the issue that a PD connection failure could cause the TiDB instance where the log backup advancer owner is located to panic [#52597](https://github.com/pingcap/tidb/issues/52597) @[YuJuncen](https://github.com/YuJuncen) + + + TiCDC + + - Fix the issue that Region changes cause downstream panic [#17233](https://github.com/tikv/tikv/issues/17233) @[hicqu](https://github.com/hicqu) + - Fix the issue that TiCDC fails to decode primary keys in clustered index tables correctly when the new collation is disabled in the upstream [#11371](https://github.com/pingcap/tiflow/issues/11371) @[lidezhu](https://github.com/lidezhu) + - Fix the issue that the checksum is not correctly set to `0` after splitting `UPDATE` events [#11402](https://github.com/pingcap/tiflow/issues/11402) @[3AceShowHand](https://github.com/3AceShowHand) + - Fix the issue that data inconsistency might occur when restarting Changefeed repeatedly when performing a large number of `UPDATE` operations in a multi-node environment [#11219](https://github.com/pingcap/tiflow/issues/11219) @[lidezhu](https://github.com/lidezhu) + - Fix the issue that the Processor module might get stuck when the downstream Kafka is inaccessible [#11340](https://github.com/pingcap/tiflow/issues/11340) @[asddongmen](https://github.com/asddongmen) + + + TiDB Data Migration (DM) + + - Fix the issue that `SET` statements cause DM to panic during the migration of MariaDB data [#10206](https://github.com/pingcap/tiflow/issues/10206) @[dveeden](https://github.com/dveeden) + - Fix the connection blocking issue by upgrading `go-mysql` [#11041](https://github.com/pingcap/tiflow/issues/11041) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that data replication is interrupted when the index length exceeds the default value of `max-index-length` [#11459](https://github.com/pingcap/tiflow/issues/11459) @[michaelmdeng](https://github.com/michaelmdeng) + - Fix the issue that schema tracker incorrectly handles LIST partition tables, causing DM errors [#11408](https://github.com/pingcap/tiflow/issues/11408) @[lance6716](https://github.com/lance6716) + + + TiDB Lightning + + - Fix the issue that TiDB Lightning outputs a confusing `WARN` log when it fails to obtain the keyspace name [#54232](https://github.com/pingcap/tidb/issues/54232) @[kennytm](https://github.com/kennytm) + + + Dumpling + + - Fix the issue that Dumpling reports an error when exporting tables and views at the same time [#53682](https://github.com/pingcap/tidb/issues/53682) @[tangenta](https://github.com/tangenta) + + + TiDB Binlog + + - Fix the issue that deleting rows during the execution of `ADD COLUMN` might report an error `data and columnID count not match` when TiDB Binlog is enabled [#53133](https://github.com/pingcap/tidb/issues/53133) @[tangenta](https://github.com/tangenta) \ No newline at end of file diff --git a/releases/release-8.1.2.md b/releases/release-8.1.2.md new file mode 100644 index 0000000000000..e7f993e297844 --- /dev/null +++ b/releases/release-8.1.2.md @@ -0,0 +1,159 @@ +--- +title: TiDB 8.1.2 Release Notes +summary: Learn about the compatibility changes, improvements, and bug fixes in TiDB 8.1.2. +--- + +# TiDB 8.1.2 Release Notes + +Release date: December 26, 2024 + +TiDB version: 8.1.2 + +Quick access: [Quick start](https://docs.pingcap.com/tidb/v8.1/quick-start-with-tidb) | [Production deployment](https://docs.pingcap.com/tidb/v8.1/production-deployment-using-tiup) + +## Compatibility changes + +- Add the TiKV configuration item [`server.snap-min-ingest-size`](/tikv-configuration-file.md#snap-min-ingest-size-new-in-v812), which specifies the minimum threshold for whether TiKV adopts the ingest method when processing snapshots. The default value is `2MiB`. + + - When the snapshot size exceeds this threshold, TiKV adopts the ingest method, which imports SST files from the snapshot into RocksDB. This method is faster for large files. + - When the snapshot size does not exceed this threshold, TiKV adopts the direct write method, which writes each piece of data into RocksDB individually. This method is more efficient for small files. + +## Improvements + ++ TiDB + + - Add metrics about Request Unit (RU) settings [#8444](https://github.com/tikv/pd/issues/8444) @[nolouch](https://github.com/nolouch) + ++ TiKV + + - Improve the speed of Region Merge in scenarios with empty tables and small Regions [#17376](https://github.com/tikv/tikv/issues/17376) @[LykxSassinator](https://github.com/LykxSassinator) + - Optimize TiKV's `DiskFull` detection to make it compatible with RaftEngine's `spill-dir` configuration, ensuring that this feature works consistently [#17356](https://github.com/tikv/tikv/issues/17356) @[LykxSassinator](https://github.com/LykxSassinator) + - Optimize the trigger mechanism of RocksDB compaction to accelerate disk space reclamation when handling a large number of DELETE versions [#17269](https://github.com/tikv/tikv/issues/17269) @[AndreMouche](https://github.com/AndreMouche) + - Support modifying the `import.num-threads` configuration item dynamically [#17807](https://github.com/tikv/tikv/issues/17807) @[RidRisR](https://github.com/RidRisR) + - Replace the Rusoto library with AWS Rust SDK to access external storage (such as Amazon S3) for backup and restore, which enhances compatibility with AWS features such as IMDSv2 and EKS Pod Identity [#12371](https://github.com/tikv/tikv/issues/12371) @[akoshchiy](https://github.com/akoshchiy) + ++ TiFlash + + - Improve the garbage collection speed of outdated data in the background for tables with clustered indexes [#9529](https://github.com/pingcap/tiflash/issues/9529) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Mitigate the issue that TiFlash might panic due to updating certificates after TLS is enabled [#8535](https://github.com/pingcap/tiflash/issues/8535) @[windtalker](https://github.com/windtalker) + - Reduce the number of threads that TiFlash needs to create when processing disaggregated storage and compute requests, helping avoid crashes of TiFlash compute nodes when processing a large number of such requests [#9334](https://github.com/pingcap/tiflash/issues/9334) @[JinheLin](https://github.com/JinheLin) + - Improve the cancel mechanism of the JOIN operator, so that the JOIN operator can respond to cancel requests in a timely manner [#9430](https://github.com/pingcap/tiflash/issues/9430) @[windtalker](https://github.com/windtalker) + - Optimize the execution efficiency of `LENGTH()` and `ASCII()` functions [#9344](https://github.com/pingcap/tiflash/issues/9344) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Optimize the retry strategy for TiFlash compute nodes in the disaggregated storage and compute architecture to handle exceptions when downloading files from Amazon S3 [#9695](https://github.com/pingcap/tiflash/issues/9695) @[JinheLin](https://github.com/JinheLin) + ++ Tools + + + Backup & Restore (BR) + + - Reduce unnecessary log printing during backup [#55902](https://github.com/pingcap/tidb/issues/55902) @[Leavrth](https://github.com/Leavrth) + - Disable the table-level checksum calculation (`--checksum=false`) during full backups by default to improve backup performance [#56373](https://github.com/pingcap/tidb/issues/56373) @[Tristan1900](https://github.com/Tristan1900) + + + TiCDC + + - TiCDC supports querying the status of asynchronously executed DDL tasks after being granted the `SUPER` privilege, preventing execution errors caused by repeatedly executing DDL tasks on the same table [#11521](https://github.com/pingcap/tiflow/issues/11521) @[CharlesCheung96](https://github.com/CharlesCheung96) + - When the downstream is TiDB with the `SUPER` permission granted, TiCDC supports querying the execution status of `ADD INDEX DDL` from the downstream database to avoid data replication failure due to timeout in retrying executing the DDL statement in some cases [#10682](https://github.com/pingcap/tiflow/issues/10682) @[CharlesCheung96](https://github.com/CharlesCheung96) + +## Bug fixes + ++ TiDB + + - Fix the issue that existing TTL tasks are executed unexpectedly frequently in a cluster that is upgraded from v6.5 to v7.5 or later [#56539](https://github.com/pingcap/tidb/issues/56539) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that improper use of metadata locks might lead to writing anomalous data when using the plan cache under certain circumstances [#53634](https://github.com/pingcap/tidb/issues/53634) @[zimulala](https://github.com/zimulala) + - Fix the issue that executing `IMPORT INTO` is stuck when Global Sort is enabled and the Region size exceeds 96 MiB [#55374](https://github.com/pingcap/tidb/issues/55374) @[lance6716](https://github.com/lance6716) + - Fix the issue that the upper bound and lower bound of the histogram are corrupted when `DUMP STATS` is transforming statistics into JSON [#56083](https://github.com/pingcap/tidb/issues/56083) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that execution plan bindings cannot be created for the multi-table `DELETE` statement with aliases [#56726](https://github.com/pingcap/tidb/issues/56726) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue of memory leaks in TTL tables [#56934](https://github.com/pingcap/tidb/issues/56934) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that partition pruning does not work when the partition expression is `EXTRACT(YEAR FROM col)` [#54210](https://github.com/pingcap/tidb/issues/54210) @[mjonss](https://github.com/mjonss) + - Fix the issue that Plan Replayer might report an error when importing a table structure containing Placement Rules [#54961](https://github.com/pingcap/tidb/issues/54961) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the query might get stuck when terminated because the memory usage exceeds the limit set by `tidb_mem_quota_query` [#55042](https://github.com/pingcap/tidb/issues/55042) @[yibin87](https://github.com/yibin87) + - Fix the issue that TiDB queries cannot be canceled during cop task construction [#55957](https://github.com/pingcap/tidb/issues/55957) @[yibin87](https://github.com/yibin87) + - Fix the issue that reducing the value of `tidb_ttl_delete_worker_count` during TTL job execution makes the job fail to complete [#55561](https://github.com/pingcap/tidb/issues/55561) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that the `CAST` function does not support explicitly setting the character set [#55677](https://github.com/pingcap/tidb/issues/55677) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that TTL tasks cannot be canceled when there is a write conflict [#56422](https://github.com/pingcap/tidb/issues/56422) @[YangKeao](https://github.com/YangKeao) + - Fix the data race issue in `IndexNestedLoopHashJoin` [#49692](https://github.com/pingcap/tidb/issues/49692) @[solotzg](https://github.com/solotzg) + - Fix the issue that empty `groupOffset` in `StreamAggExec` might cause TiDB to panic [#53867](https://github.com/pingcap/tidb/issues/53867) @[xzhangxian1008](https://github.com/xzhangxian1008) + - Fix the issue that TiDB might hang or return incorrect results when executing a query containing a correlated subquery and CTE [#55551](https://github.com/pingcap/tidb/issues/55551) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue of data index inconsistency caused by retries during index addition [#55808](https://github.com/pingcap/tidb/issues/55808) @[lance6716](https://github.com/lance6716) + - Fix the issue that an `out of range` error might occur when a small display width is specified for a column of the integer type [#55837](https://github.com/pingcap/tidb/issues/55837) @[windtalker](https://github.com/windtalker) + - Fix the issue that the `LOAD DATA ... REPLACE INTO` operation causes data inconsistency [#56408](https://github.com/pingcap/tidb/issues/56408) @[fzzf678](https://github.com/fzzf678) + - Fix the issue that `columnEvaluator` cannot identify the column references in the input chunk, which leads to `runtime error: index out of range` when executing SQL statements [#53713](https://github.com/pingcap/tidb/issues/53713) @[AilinKid](https://github.com/AilinKid) + - Fix the issue of illegal memory access that might occur when a Common Table Expression (CTE) has multiple data consumers and one consumer exits without reading any data [#55881](https://github.com/pingcap/tidb/issues/55881) @[windtalker](https://github.com/windtalker) + - Fix the issue that when canceling a TTL task, the corresponding SQL is not killed forcibly [#56511](https://github.com/pingcap/tidb/issues/56511) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that TiDB panics when importing temporary tables using the `IMPORT INTO` statement [#55970](https://github.com/pingcap/tidb/issues/55970) @[D3Hunter](https://github.com/D3Hunter) + - Fix the issue that the optimizer incorrectly estimates the number of rows as 1 when accessing a unique index with the query condition `column IS NULL` [#56116](https://github.com/pingcap/tidb/issues/56116) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that the query latency of stale reads increases, caused by information schema cache misses [#53428](https://github.com/pingcap/tidb/issues/53428) @[crazycs520](https://github.com/crazycs520) + - Fix the issue that the `UPDATE` statement incorrectly updates values of the `ENUM` type [#56832](https://github.com/pingcap/tidb/issues/56832) @[xhebox](https://github.com/xhebox) + - Fix the issue that Plan Replayer might report an error when importing a table structure containing foreign keys [#56456](https://github.com/pingcap/tidb/issues/56456) @[hawkingrei](https://github.com/hawkingrei) + - Fix the issue that TTL tasks are not canceled after the `tidb_ttl_job_enable` variable is disabled [#57404](https://github.com/pingcap/tidb/issues/57404) @[YangKeao](https://github.com/YangKeao) + - Fix the issue that when an `UPDATE` or `DELETE` statement contains a recursive CTE, the statement might report an error or not take effect [#55666](https://github.com/pingcap/tidb/issues/55666) @[time-and-fate](https://github.com/time-and-fate) + - Fix the issue that the `SUB_PART` value in the `INFORMATION_SCHEMA.STATISTICS` table is `NULL` [#55812](https://github.com/pingcap/tidb/issues/55812) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that the default timeout for querying the TiFlash system table is too short [#57816](https://github.com/pingcap/tidb/issues/57816) @[JaySon-Huang](https://github.com/JaySon-Huang) + - Fix the issue that the value of the `default_collation_for_utf8mb4` variable does not work for the `SET NAMES` statement [#56439](https://github.com/pingcap/tidb/issues/56439) @[Defined2014](https://github.com/Defined2014) + - Fix the issue that TTL internal coroutine might panic when you manually delete a timer in the `mysql.tidb_timer` table [#57112](https://github.com/pingcap/tidb/issues/57112) @[lcwangchao](https://github.com/lcwangchao) + - Fix the issue that enabling acceleration of `ADD INDEX` and `CREATE INDEX` via `tidb_ddl_enable_fast_reorg` might lead to the `Duplicate entry` error [#49233](https://github.com/pingcap/tidb/issues/49233) @[lance6716](https://github.com/lance6716) + - Fix the issue that the index timestamp is set to `0` when adding indexes to large tables in a non-distributed manner [#57980](https://github.com/pingcap/tidb/issues/57980) @[lance6716](https://github.com/lance6716) + ++ TiKV + + - Fix the issue that the configuration `resolved-ts.advance-ts-interval` does not take effect, causing the replication latency of TiCDC and Point-in-time recovery (PITR) to increase dramatically when TiKV restarts [#17107](https://github.com/tikv/tikv/issues/17107) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that some tasks experience high tail latency when triggering resource control [#17589](https://github.com/tikv/tikv/issues/17589) @[glorv](https://github.com/glorv) + - Fix the issue that merging Regions might cause TiKV to panic in rare cases [#17840](https://github.com/tikv/tikv/issues/17840) @[glorv](https://github.com/glorv) + - Fix the issue that TiKV cannot report heartbeats to PD when the disk is stuck [#17939](https://github.com/tikv/tikv/issues/17939) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that when Raft and RocksDB are deployed on different disks, the slow disk detection does not work for the disk where RocksDB is located [#17884](https://github.com/tikv/tikv/issues/17884) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the issue that TiKV might panic when a stale replica processes Raft snapshots, triggered by a slow split operation and immediate removal of the new replica [#17469](https://github.com/tikv/tikv/issues/17469) @[hbisheng](https://github.com/hbisheng) + - Fix the issue that TiKV might panic when executing queries containing `RADIANS()` or `DEGREES()` functions [#17852](https://github.com/tikv/tikv/issues/17852) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that when a large number of transactions are queuing for lock release on the same key and the key is frequently updated, excessive pressure on deadlock detection might cause TiKV OOM issues [#17394](https://github.com/tikv/tikv/issues/17394) @[MyonKeminta](https://github.com/MyonKeminta) + - Fix the issue that the leader could not be quickly elected after Region split [#17602](https://github.com/tikv/tikv/issues/17602) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the panic issue that occurs when read threads access outdated indexes in the MemTable of the Raft Engine [#17383](https://github.com/tikv/tikv/issues/17383) @[LykxSassinator](https://github.com/LykxSassinator) + - Fix the flow control issue that might occur after deleting large tables or partitions [#17304](https://github.com/tikv/tikv/issues/17304) @[Connor1996](https://github.com/Connor1996) + ++ PD + + - Fix the issue that the PD HTTP client retry logic might be ineffective [#8499](https://github.com/tikv/pd/issues/8499) @[JmPotato](https://github.com/JmPotato) + - Upgrade the version of Gin Web Framework from v1.9.1 to v1.10.0 to fix potential security vulnerabilities [#8643](https://github.com/tikv/pd/issues/8643) @[JmPotato](https://github.com/JmPotato) + - Fix the issue that PD cannot quickly re-elect a leader during etcd leader transition [#8823](https://github.com/tikv/pd/issues/8823) @[rleungx](https://github.com/rleungx) + - Fix the issue that setting `replication.strictly-match-label` to `true` causes TiFlash to fail to start [#8480](https://github.com/tikv/pd/issues/8480) @[rleungx](https://github.com/rleungx) + - Fix the issue that `evict-leader-scheduler` fails to work properly when it is repeatedly created with the same Store ID [#8756](https://github.com/tikv/pd/issues/8756) @[okJiang](https://github.com/okJiang) + - Fix the performance jitter issue caused by frequent creation of random number generator [#8674](https://github.com/tikv/pd/issues/8674) @[rleungx](https://github.com/rleungx) + - Fix the memory leak issue in hotspot cache [#8698](https://github.com/tikv/pd/issues/8698) @[lhy1024](https://github.com/lhy1024) + - Fix the memory leak issue in label statistics [#8700](https://github.com/tikv/pd/issues/8700) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that deleted resource groups still appear in the monitoring panel [#8716](https://github.com/tikv/pd/issues/8716) @[AndreMouche](https://github.com/AndreMouche) + - Fix the issue that data race might occur in the scheduling server when a PD leader is switched in the microservice mode [#8538](https://github.com/tikv/pd/issues/8538) @[lhy1024](https://github.com/lhy1024) + - Fix the issue that when using a wrong parameter in `evict-leader-scheduler`, PD does not report errors correctly and some schedulers are unavailable [#8619](https://github.com/tikv/pd/issues/8619) @[rleungx](https://github.com/rleungx) + - Fix the issue that the resource group selector does not take effect on any panel [#56572](https://github.com/pingcap/tidb/issues/56572) @[glorv](https://github.com/glorv) + ++ TiFlash + + - Fix the issue that TiFlash might panic due to spurious Region overlap check failures that occur when multiple Regions are concurrently applying snapshots [#9329](https://github.com/pingcap/tiflash/issues/9329) @[CalvinNeo](https://github.com/CalvinNeo) + - Fix the issue that the `SUBSTRING()` function returns incorrect results when the second parameter is negative [#9604](https://github.com/pingcap/tiflash/issues/9604) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that some queries might report errors when late materialization is enabled [#9472](https://github.com/pingcap/tiflash/issues/9472) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that TiFlash fails to parse the table schema when the table contains Bit-type columns with a default value that contains invalid characters [#9461](https://github.com/pingcap/tiflash/issues/9461) @[Lloyd-Pottiger](https://github.com/Lloyd-Pottiger) + - Fix the issue that some JSON functions unsupported by TiFlash are pushed down to TiFlash [#9444](https://github.com/pingcap/tiflash/issues/9444) @[windtalker](https://github.com/windtalker) + - Fix the issue that the sign in the result of the `CAST AS DECIMAL` function is incorrect in certain cases [#9301](https://github.com/pingcap/tiflash/issues/9301) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that read snapshots of TiFlash write nodes are not released in a timely manner in the disaggregated storage and compute architecture [#9298](https://github.com/pingcap/tiflash/issues/9298) @[JinheLin](https://github.com/JinheLin) + - Fix the issue that the `SUBSTRING()` function does not support the `pos` and `len` arguments for certain integer types, causing query errors [#9473](https://github.com/pingcap/tiflash/issues/9473) @[gengliqi](https://github.com/gengliqi) + - Fix the issue that when using the `CAST()` function to convert a string to a datetime with a time zone or invalid characters, the result is incorrect [#8754](https://github.com/pingcap/tiflash/issues/8754) @[solotzg](https://github.com/solotzg) + - Fix the issue that `LPAD()` and `RPAD()` functions return incorrect results in some cases [#9465](https://github.com/pingcap/tiflash/issues/9465) @[guo-shaoge](https://github.com/guo-shaoge) + - Fix the issue that querying new columns might return incorrect results under the disaggregated storage and compute architecture [#9665](https://github.com/pingcap/tiflash/issues/9665) @[zimulala](https://github.com/zimulala) + ++ Tools + + + Backup & Restore (BR) + + - Fix the issue that logs might print out encrypted information [#57585](https://github.com/pingcap/tidb/issues/57585) @[kennytm](https://github.com/kennytm) + - Fix the issue that snapshot backups based on AWS EBS might fail during the preparation phase, causing the backup to get stuck [#52049](https://github.com/pingcap/tidb/issues/52049) @[YuJuncen](https://github.com/YuJuncen) + - Fix the issue that the checkpoint path of backup and restore is incompatible with some external storage [#55265](https://github.com/pingcap/tidb/issues/55265) @[Leavrth](https://github.com/Leavrth) + - Fix potential security vulnerabilities by upgrading the `k8s.io/api` library version [#57790](https://github.com/pingcap/tidb/issues/57790) @[BornChanger](https://github.com/BornChanger) + - Fix the issue that PITR tasks might return the `Information schema is out of date` error when there are a large number of tables in the cluster but the actual data size is small [#57743](https://github.com/pingcap/tidb/issues/57743) @[Tristan1900](https://github.com/Tristan1900) + + + TiCDC + + - Fix the issue that the Resolved TS latency monitoring in the Puller module displays incorrect values [#11561](https://github.com/pingcap/tiflow/issues/11561) @[wlwilliamx](https://github.com/wlwilliamx) + - Fix the issue that when `enable-table-across-nodes` is enabled, some Span replication tasks for a table might be lost during Region splits [#11675](https://github.com/pingcap/tiflow/issues/11675) @[wk989898](https://github.com/wk989898) + - Fix the issue that the redo module fails to properly report errors [#11744](https://github.com/pingcap/tiflow/issues/11744) @[CharlesCheung96](https://github.com/CharlesCheung96) + - Fix the issue that TiCDC mistakenly discards DDL tasks when the schema versions of DDL tasks become non-incremental during TiDB DDL owner changes [#11714](https://github.com/pingcap/tiflow/issues/11714) @[wlwilliamx](https://github.com/wlwilliamx) + - Fix the issue that the **barrier-ts** monitoring metric for changefeed checkpoints might be inaccurate [#11553](https://github.com/pingcap/tiflow/issues/11553) @[3AceShowHand](https://github.com/3AceShowHand) + + + TiDB Data Migration (DM) + + - Fix the issue that multiple DM-master nodes might simultaneously become leaders, leading to data inconsistency [#11602](https://github.com/pingcap/tiflow/issues/11602) @[GMHDBJD](https://github.com/GMHDBJD) diff --git a/releases/release-notes.md b/releases/release-notes.md index 1480319d7884d..091e49cc50c59 100644 --- a/releases/release-notes.md +++ b/releases/release-notes.md @@ -1,11 +1,18 @@ --- title: Release Notes -aliases: ['/docs/dev/releases/release-notes/','/docs/dev/releases/rn/'] -summary: TiDB has released multiple versions, including 8.0.0-DMR, 7.6.0-DMR, 7.5.1, 7.5.0, 7.4.0-DMR, 7.3.0-DMR, 7.2.0-DMR, 7.1.4, 7.1.3, 7.1.2, 7.1.1, 7.1.0, 7.0.0-DMR, 6.6.0-DMR, 6.5.9, 6.5.8, 6.5.7, 6.5.6, 6.5.5, 6.5.4, 6.5.3, 6.5.2, 6.5.1, 6.5.0, 6.4.0-DMR, 6.3.0-DMR, 6.2.0-DMR, 6.1.7, 6.1.6, 6.1.5, 6.1.4, 6.1.3, 6.1.2, 6.1.1, 6.1.0, 6.0.0-DMR, 5.4.3, 5.4.2, 5.4.1, 5.4.0, 5.3.4, 5.3.3, 5.3.2, 5.3.1, 5.3.0, 5.2.4, 5.2.3, 5.2.2, 5.2.1, 5.2.0, 5.1.5, 5.1.4, 5.1.3, 5.1.2, 5.1.1, 5.1.0, 5.0.6, 5.0.5, 5.0.4, 5.0.3, 5.0.2, 5.0.1, 5.0.0, 5.0.0-rc, 4.0.16, 4.0.15, 4.0.14, 4.0.13, 4.0.12, 4.0.11, 4.0.10, 4.0.9, 4.0.8, 4.0.7, 4.0.6, 4.0.5, 4.0.4, 4.0.3, 4.0.2, 4.0.1, 4.0.0, 4.0.0-rc.2, 4.0.0-rc.1, 4.0.0-rc, 4.0.0-beta.2, 4.0.0-beta.1, 4.0.0-beta, 3.1.2, 3.1.1, 3.1.0, 3.1.0-rc, 3.1.0-beta.2, 3.1.0-beta.1, 3.1.0-beta, 3.0.20, 3.0.19, 3.0.18, 3.0.17, 3.0.16, 3.0.15, 3.0.14, 3.0.13, 3.0.12, 3.0.11, 3.0.10, 3.0.9, 3.0.8, 3.0.7, 3.0.6, 3.0.5, 3.0.4, 3.0.3, 3.0.2, 3.0.1, 3.0.0, 3.0.0-rc.3, 3.0.0-rc.2, 3.0.0-rc.1, 3.0.0-beta.1, 3.0.0-beta, 2.1.19, 2.1.18, 2.1.17, 2.1.16, 2.1.15, 2.1.14, 2.1.13, 2.1.12, 2.1.11, 2.1.10, 2.1.9, 2.1.8, 2.1.7, 2.1.6, 2.1.5, 2.1.4, 2.1.3, 2.1.2, 2.1.1, 2.1.0, 2.1.0-rc.5, 2.1.0-rc.4, 2.1.0-rc.3, 2.1.0-rc.2, 2.1.0-rc.1, 2.1.0-beta, 2.0.11, 2.0.10, 2.0.9, 2.0.8, 2.0.7, 2.0.6, 2.0.5, 2.0.4, 2.0.3, 2.0.2, 2.0.1, 2.0.0, 2.0.0-rc.5, 2.0.0-rc.4, 2.0.0-rc.3, 2.0.0-rc.1, 1.1.0-beta, 1.1.0-alpha, 1.0.8, 1.0.7, 1.0.6, 1.0.5, 1.0.4, 1.0.3, 1.0.2, 1.0.1, 1.0.0, Pre-GA, rc4, rc3, rc2, rc1. +summary: TiDB has released multiple versions, including 8.1.0, 8.0.0-DMR, 7.6.0-DMR, 7.5.1, 7.5.0, 7.4.0-DMR, 7.3.0-DMR, 7.2.0-DMR, 7.1.4, 7.1.3, 7.1.2, 7.1.1, 7.1.0, 7.0.0-DMR, 6.6.0-DMR, 6.5.9, 6.5.8, 6.5.7, 6.5.6, 6.5.5, 6.5.4, 6.5.3, 6.5.2, 6.5.1, 6.5.0, 6.4.0-DMR, 6.3.0-DMR, 6.2.0-DMR, 6.1.7, 6.1.6, 6.1.5, 6.1.4, 6.1.3, 6.1.2, 6.1.1, 6.1.0, 6.0.0-DMR, 5.4.3, 5.4.2, 5.4.1, 5.4.0, 5.3.4, 5.3.3, 5.3.2, 5.3.1, 5.3.0, 5.2.4, 5.2.3, 5.2.2, 5.2.1, 5.2.0, 5.1.5, 5.1.4, 5.1.3, 5.1.2, 5.1.1, 5.1.0, 5.0.6, 5.0.5, 5.0.4, 5.0.3, 5.0.2, 5.0.1, 5.0.0, 5.0.0-rc, 4.0.16, 4.0.15, 4.0.14, 4.0.13, 4.0.12, 4.0.11, 4.0.10, 4.0.9, 4.0.8, 4.0.7, 4.0.6, 4.0.5, 4.0.4, 4.0.3, 4.0.2, 4.0.1, 4.0.0, 4.0.0-rc.2, 4.0.0-rc.1, 4.0.0-rc, 4.0.0-beta.2, 4.0.0-beta.1, 4.0.0-beta, 3.1.2, 3.1.1, 3.1.0, 3.1.0-rc, 3.1.0-beta.2, 3.1.0-beta.1, 3.1.0-beta, 3.0.20, 3.0.19, 3.0.18, 3.0.17, 3.0.16, 3.0.15, 3.0.14, 3.0.13, 3.0.12, 3.0.11, 3.0.10, 3.0.9, 3.0.8, 3.0.7, 3.0.6, 3.0.5, 3.0.4, 3.0.3, 3.0.2, 3.0.1, 3.0.0, 3.0.0-rc.3, 3.0.0-rc.2, 3.0.0-rc.1, 3.0.0-beta.1, 3.0.0-beta, 2.1.19, 2.1.18, 2.1.17, 2.1.16, 2.1.15, 2.1.14, 2.1.13, 2.1.12, 2.1.11, 2.1.10, 2.1.9, 2.1.8, 2.1.7, 2.1.6, 2.1.5, 2.1.4, 2.1.3, 2.1.2, 2.1.1, 2.1.0, 2.1.0-rc.5, 2.1.0-rc.4, 2.1.0-rc.3, 2.1.0-rc.2, 2.1.0-rc.1, 2.1.0-beta, 2.0.11, 2.0.10, 2.0.9, 2.0.8, 2.0.7, 2.0.6, 2.0.5, 2.0.4, 2.0.3, 2.0.2, 2.0.1, 2.0.0, 2.0.0-rc.5, 2.0.0-rc.4, 2.0.0-rc.3, 2.0.0-rc.1, 1.1.0-beta, 1.1.0-alpha, 1.0.8, 1.0.7, 1.0.6, 1.0.5, 1.0.4, 1.0.3, 1.0.2, 1.0.1, 1.0.0, Pre-GA, rc4, rc3, rc2, rc1. --- # TiDB Release Notes + + +## 8.1 + +- [8.1.2](/releases/release-8.1.2.md): 2024-12-26 +- [8.1.1](/releases/release-8.1.1.md): 2024-08-27 +- [8.1.0](/releases/release-8.1.0.md): 2024-05-24 + ## 8.0 - [8.0.0-DMR](/releases/release-8.0.0.md): 2024-03-29 @@ -16,6 +23,11 @@ summary: TiDB has released multiple versions, including 8.0.0-DMR, 7.6.0-DMR, 7. ## 7.5 +- [7.5.6](/releases/release-7.5.6.md): 2025-03-14 +- [7.5.5](/releases/release-7.5.5.md): 2024-12-31 +- [7.5.4](/releases/release-7.5.4.md): 2024-10-15 +- [7.5.3](/releases/release-7.5.3.md): 2024-08-05 +- [7.5.2](/releases/release-7.5.2.md): 2024-06-13 - [7.5.1](/releases/release-7.5.1.md): 2024-02-29 - [7.5.0](/releases/release-7.5.0.md): 2023-12-01 @@ -33,6 +45,7 @@ summary: TiDB has released multiple versions, including 8.0.0-DMR, 7.6.0-DMR, 7. ## 7.1 +- [7.1.6](/releases/release-7.1.6.md): 2024-11-21 - [7.1.5](/releases/release-7.1.5.md): 2024-04-26 - [7.1.4](/releases/release-7.1.4.md): 2024-03-11 - [7.1.3](/releases/release-7.1.3.md): 2023-12-21 @@ -50,6 +63,9 @@ summary: TiDB has released multiple versions, including 8.0.0-DMR, 7.6.0-DMR, 7. ## 6.5 +- [6.5.12](/releases/release-6.5.12.md): 2025-02-27 +- [6.5.11](/releases/release-6.5.11.md): 2024-09-20 +- [6.5.10](/releases/release-6.5.10.md): 2024-06-20 - [6.5.9](/releases/release-6.5.9.md): 2024-04-12 - [6.5.8](/releases/release-6.5.8.md): 2024-02-02 - [6.5.7](/releases/release-6.5.7.md): 2024-01-08 diff --git a/releases/release-pre-ga.md b/releases/release-pre-ga.md index c41a06b28855e..733ae093dc762 100644 --- a/releases/release-pre-ga.md +++ b/releases/release-pre-ga.md @@ -1,6 +1,5 @@ --- title: Pre-GA release notes -aliases: ['/docs/dev/releases/release-pre-ga/','/docs/dev/releases/prega/'] summary: TiDB Pre-GA release on August 30, 2017, focuses on MySQL compatibility, SQL optimization, stability, and performance. TiDB introduces SQL query optimizer enhancements, MySQL compatibility, JSON type support, and memory consumption reduction. Placement Driver (PD) now supports manual leader change, while TiKV uses dedicated Rocksdb for Raft log storage and improves performance. TiDB Connector for Spark Beta Release implements predicates pushdown, aggregation pushdown, and range pruning, capable of running TPC+H queries. --- diff --git a/releases/release-rc.1.md b/releases/release-rc.1.md index 350430fb5aa20..bc29abdfada79 100644 --- a/releases/release-rc.1.md +++ b/releases/release-rc.1.md @@ -1,6 +1,5 @@ --- title: TiDB RC1 Release Notes -aliases: ['/docs/dev/releases/release-rc.1/','/docs/dev/releases/rc1/'] summary: TiDB RC1 was released on December 23, 2016. Updates include improved write speed and reduced disk space usage in TiKV, optimized scheduling strategy framework in PD, and added features in the SQL query optimizer and new tools in TiDB. The release also supports more built-in functions in MySQL and enhances the speed of the `add index` statement. --- @@ -20,7 +19,7 @@ On December 23, 2016, TiDB RC1 is released. See the following updates in this re + The scheduling strategy framework is optimized and now the strategy is more flexible and reasonable. + The support for `label` is added to support Cross Data Center scheduling. -+ PD Controller is provided to operate the PD cluster more easily. ++ PD Control is provided to operate the PD cluster more easily. ## TiDB diff --git a/releases/release-rc.2.md b/releases/release-rc.2.md index 1559138c22a3e..7110cbc0e4455 100644 --- a/releases/release-rc.2.md +++ b/releases/release-rc.2.md @@ -1,12 +1,11 @@ --- title: TiDB RC2 Release Notes -aliases: ['/docs/dev/releases/release-rc.2/','/docs/dev/releases/rc2/'] summary: TiDB RC2, released on March 1, 2017, focuses on MySQL compatibility, SQL query optimization, system stability, and performance. It introduces a new permission management mechanism, allowing users to control data access similar to MySQL privilege management. Key improvements include query optimizer enhancements, basic privilege management support, MySQL built-in functions, and performance optimizations. PD now supports location aware replica scheduling and fast scheduling based on region count, while TiKV introduces Async Apply for improved write performance and various optimizations for read and insert performance. Bug fixes and memory leak solutions are also included. --- # TiDB RC2 Release Notes -On March 1, 2017, TiDB RC2 is released! This release is focused on the compatibility with MySQL, SQL query optimizer, system stability and performance in this version. What’s more, a new permission management mechanism is added and users can control data access in the same way as the MySQL privilege management system. +On March 1, 2017, TiDB RC2 is released! This release is focused on the compatibility with MySQL, SQL query optimizer, system stability and performance in this version. What's more, a new permission management mechanism is added and users can control data access in the same way as the MySQL privilege management system. ## TiDB diff --git a/releases/release-rc.3.md b/releases/release-rc.3.md index f8d49e8efb291..b7dc6c5798e70 100644 --- a/releases/release-rc.3.md +++ b/releases/release-rc.3.md @@ -1,6 +1,5 @@ --- title: TiDB RC3 Release Notes -aliases: ['/docs/dev/releases/release-rc.3/','/docs/dev/releases/rc3/'] summary: TiDB RC3, released on June 16, 2017, focuses on MySQL compatibility, SQL optimization, stability, and performance. Highlights include refined privilege management, accelerated DDL, optimized load balancing, and open-sourced TiDB Ansible for easy cluster management. Detailed updates for TiDB, Placement Driver (PD), and TiKV include improved SQL query optimization, complete privilege management, support for HTTP API, system variables for query concurrency control, and more efficient data balance. PD supports gRPC, disaster recovery toolkit, and hot Region scheduling. TiKV supports gRPC, SST format snapshot, memory leak detection, and improved data importing speed. Overall, the release enhances performance, stability, and management capabilities. --- diff --git a/releases/release-rc.4.md b/releases/release-rc.4.md index eb51331cee442..f6bf730f51b39 100644 --- a/releases/release-rc.4.md +++ b/releases/release-rc.4.md @@ -1,6 +1,5 @@ --- title: TiDB RC4 Release Notes -aliases: ['/docs/dev/releases/release-rc.4/','/docs/dev/releases/rc4/'] summary: TiDB RC4 is released with a focus on MySQL compatibility, SQL optimization, stability, and performance. Highlights include improved write performance, better query cost estimating, and support for TiSpark to access data in TiKV. Detailed updates include refactoring of the SQL query optimizer, support for JSON type and operations, and optimization of the scheduler in Placement Driver. TiKV now supports RC isolation level, Document Store, and more pushdown functions in Coprocessor. TiSpark beta release includes prediction pushdown, aggregation pushdown, and range pruning, capable of running a full set of TPC-H queries. --- diff --git a/releases/release-timeline.md b/releases/release-timeline.md index caed7d62dc89e..8851cb0231379 100644 --- a/releases/release-timeline.md +++ b/releases/release-timeline.md @@ -5,10 +5,24 @@ summary: Learn about the TiDB release timeline. # TiDB Release Timeline + + This document shows all the released TiDB versions in reverse chronological order. | Version | Release Date | | :--- | :--- | +| [7.5.6](/releases/release-7.5.6.md) | 2025-03-14 | +| [6.5.12](/releases/release-6.5.12.md) | 2025-02-27 | +| [7.5.5](/releases/release-7.5.5.md) | 2024-12-31 | +| [8.1.2](/releases/release-8.1.2.md) | 2024-12-26 | +| [7.1.6](/releases/release-7.1.6.md) | 2024-11-21 | +| [7.5.4](/releases/release-7.5.4.md) | 2024-10-15 | +| [6.5.11](/releases/release-6.5.11.md) | 2024-09-20 | +| [8.1.1](/releases/release-8.1.1.md) | 2024-08-27 | +| [7.5.3](/releases/release-7.5.3.md) | 2024-08-05 | +| [6.5.10](/releases/release-6.5.10.md) | 2024-06-20 | +| [7.5.2](/releases/release-7.5.2.md) | 2024-06-13 | +| [8.1.0](/releases/release-8.1.0.md) | 2024-05-24 | | [7.1.5](/releases/release-7.1.5.md) | 2024-04-26 | | [6.5.9](/releases/release-6.5.9.md) | 2024-04-12 | | [8.0.0-DMR](/releases/release-8.0.0.md) | 2024-03-29 | diff --git a/releases/versioning.md b/releases/versioning.md index 63d933739a128..a014db7b39a77 100644 --- a/releases/versioning.md +++ b/releases/versioning.md @@ -16,7 +16,7 @@ TiDB offers two release series: * Long-Term Support Releases * Development Milestone Releases (introduced in TiDB v6.0.0) -To learn about the support policy for major releases of TiDB, see [TiDB Release Support Policy](https://en.pingcap.com/tidb-release-support-policy/). +To learn about the support policy for major releases of TiDB, see [TiDB Release Support Policy](https://www.pingcap.com/tidb-release-support-policy/). ## Release versioning diff --git a/replicate-between-primary-and-secondary-clusters.md b/replicate-between-primary-and-secondary-clusters.md index e8bd22d44e0ad..7e1a4108921e5 100644 --- a/replicate-between-primary-and-secondary-clusters.md +++ b/replicate-between-primary-and-secondary-clusters.md @@ -1,7 +1,6 @@ --- title: Replicate data between primary and secondary clusters summary: Learn how to replicate data from a primary cluster to a secondary cluster. -aliases: ['/docs/dev/incremental-replication-between-clusters/', '/tidb/dev/replicate-betwwen-primary-and-secondary-clusters/'] --- # Replicate Data Between Primary and Secondary Clusters @@ -99,7 +98,7 @@ To replicate incremental data from a running TiDB cluster to its secondary clust ## Step 2. Migrate full data -After setting up the environment, you can use the backup and restore functions of [BR](https://github.com/pingcap/tidb/tree/master/br) to migrate full data. BR can be started in [three ways](/br/br-use-overview.md#deploy-and-use-br). In this document, we use the SQL statements, `BACKUP` and `RESTORE`. +After setting up the environment, you can use the backup and restore functions of [BR](https://github.com/pingcap/tidb/tree/release-8.1/br) to migrate full data. BR can be started in [three ways](/br/br-use-overview.md#deploy-and-use-br). In this document, we use the SQL statements, `BACKUP` and `RESTORE`. > **Note:** > diff --git a/resources/doc-templates/template-concept.md b/resources/doc-templates/template-concept.md index a7de8ab98b61e..d851fbb25ad45 100644 --- a/resources/doc-templates/template-concept.md +++ b/resources/doc-templates/template-concept.md @@ -29,7 +29,7 @@ Write a more detailed description below the image. Use unordered lists (`*`/`+`/ You can also describe the basic working principles here, or integrate the principles into the component introduction above. -### L3 heading (optional, e.g. “xxx component”) +### L3 heading (optional, e.g. "xxx component") If the component is complicated, you can elaborate it in a separate section like this one. diff --git a/role-based-access-control.md b/role-based-access-control.md index 4bb9811c26e58..efcb70c4f7ae5 100644 --- a/role-based-access-control.md +++ b/role-based-access-control.md @@ -1,7 +1,6 @@ --- title: Role-Based Access Control summary: This document introduces TiDB RBAC operations and implementation. -aliases: ['/docs/dev/role-based-access-control/','/docs/dev/reference/security/role-based-access-control/'] --- # Role-Based Access Control diff --git a/scale-tidb-using-tiup.md b/scale-tidb-using-tiup.md index 3cae28cc126c2..d3c9dc2cfc01a 100644 --- a/scale-tidb-using-tiup.md +++ b/scale-tidb-using-tiup.md @@ -1,7 +1,6 @@ --- title: Scale a TiDB Cluster Using TiUP summary: Learn how to scale the TiDB cluster using TiUP. -aliases: ['/docs/dev/scale-tidb-using-tiup/','/docs/dev/how-to/scale/with-tiup/'] --- # Scale a TiDB Cluster Using TiUP @@ -131,7 +130,8 @@ This section exemplifies how to add a TiDB node to the `10.0.1.5` host. > **Note:** > - > This operation is only required after you add PD nodes. If you only add TiDB or TiKV nodes, this operation is unnecessary. + > - Refreshing cluster configuration is only required after you add PD nodes. If you only add TiDB or TiKV nodes, skip this step. + > - If you are using TiUP v1.15.0 or a later version, skip this step because TiUP does it. If you are using a TiUP version earlier than v1.15.0, perform the following sub-steps. 1. Refresh the cluster configuration: @@ -141,10 +141,6 @@ This section exemplifies how to add a TiDB node to the `10.0.1.5` host. 2. Refresh the Prometheus configuration and restart Prometheus: - > **Note:** - > - > If you are using TiUP v1.15.0 or a later version, skip this step. If you are using a TiUP version earlier than v1.15.0, execute the following command to update the Prometheus configuration and restart Prometheus. - ```shell tiup cluster reload -R prometheus ``` @@ -298,7 +294,7 @@ This section exemplifies how to remove a TiKV node from the `10.0.1.5` host. ``` Starting /root/.tiup/components/cluster/v1.12.3/cluster display TiDB Cluster: - TiDB Version: v8.0.0 + TiDB Version: v8.1.2 ID Role Host Ports Status Data Dir Deploy Dir -- ---- ---- ----- ------ -------- ---------- 10.0.1.3:8300 cdc 10.0.1.3 8300 Up data/cdc-8300 deploy/cdc-8300 @@ -333,7 +329,8 @@ This section exemplifies how to remove a TiKV node from the `10.0.1.5` host. > **Note:** > - > This operation is only required after you remove PD nodes. If you only remove TiDB or TiKV nodes, this operation is unnecessary. + > - Refreshing cluster configuration is only required after you remove PD nodes. If you only remove TiDB or TiKV nodes, skip this step. + > - If you are using TiUP v1.15.0 or a later version, skip this step because TiUP does it. If you are using a TiUP version earlier than v1.15.0, perform the following sub-steps. 1. Refresh the cluster configuration: @@ -343,10 +340,6 @@ This section exemplifies how to remove a TiKV node from the `10.0.1.5` host. 2. Refresh the Prometheus configuration and restart Prometheus: - > **Note:** - > - > If you are using TiUP v1.15.0 or a later version, skip this step. If you are using a TiUP version earlier than v1.15.0, execute the following command to update the Prometheus configuration and restart Prometheus. - ```shell tiup cluster reload -R prometheus ``` @@ -393,6 +386,8 @@ This section exemplifies how to remove a TiFlash node from the `10.0.1.4` host. ALTER TABLE . SET tiflash replica 'new_replica_num'; ``` + After executing this statement, TiDB modifies or deletes PD [placement rules](/configure-placement-rules.md) accordingly. Then, PD schedules data based on the updated placement rules. + 3. Perform step 1 again and make sure that there is no table with TiFlash replicas more than the number of TiFlash nodes after scale-in. ### 2. Perform the scale-in operation @@ -403,20 +398,28 @@ Perform the scale-in operation with one of the following solutions. 1. Confirm the name of the node to be taken down: - {{< copyable "shell-regular" >}} - ```shell tiup cluster display ``` 2. Remove the TiFlash node (assume that the node name is `10.0.1.4:9000` from Step 1): - {{< copyable "shell-regular" >}} - ```shell tiup cluster scale-in --node 10.0.1.4:9000 ``` +3. View the status of the removed TiFlash node: + + ```shell + tiup cluster display + ``` + +4. After the status of the removed TiFlash node becomes `Tombstone`, delete the information of the removed node from the TiUP topology (TiUP will automatically clean up the related data files of the `Tombstone` node): + + ```shell + tiup cluster prune + ``` + #### Solution 2. Manually remove a TiFlash node In special cases (such as when a node needs to be forcibly taken down), or if the TiUP scale-in operation fails, you can manually remove a TiFlash node with the following steps. @@ -427,8 +430,6 @@ In special cases (such as when a node needs to be forcibly taken down), or if th * If you use TiUP deployment, replace `pd-ctl` with `tiup ctl:v pd`: - {{< copyable "shell-regular" >}} - ```shell tiup ctl:v pd -u http://: store ``` @@ -443,8 +444,6 @@ In special cases (such as when a node needs to be forcibly taken down), or if th * If you use TiUP deployment, replace `pd-ctl` with `tiup ctl:v pd`: - {{< copyable "shell-regular" >}} - ```shell tiup ctl:v pd -u http://: store delete ``` @@ -455,72 +454,21 @@ In special cases (such as when a node needs to be forcibly taken down), or if th 3. Wait for the store of the TiFlash node to disappear or for the `state_name` to become `Tombstone` before you stop the TiFlash process. -4. Manually delete TiFlash data files (the location can be found in the `data_dir` directory under the TiFlash configuration of the cluster topology file). - -5. Delete information about the TiFlash node that goes down from the cluster topology using the following command: - - {{< copyable "shell-regular" >}} - - ```shell - tiup cluster scale-in --node : --force - ``` - -> **Note:** -> -> Before all TiFlash nodes in the cluster stop running, if not all tables replicated to TiFlash are canceled, you need to manually clean up the replication rules in PD, or the TiFlash node cannot be taken down successfully. - -The steps to manually clean up the replication rules in PD are below: - -1. View all data replication rules related to TiFlash in the current PD instance: - - {{< copyable "shell-regular" >}} +4. Delete the information of the removed node from the TiUP topology (TiUP will automatically clean up the related data files of the `Tombstone` node): ```shell - curl http://:/pd/api/v1/config/rules/group/tiflash + tiup cluster prune ``` - ``` - [ - { - "group_id": "tiflash", - "id": "table-45-r", - "override": true, - "start_key": "7480000000000000FF2D5F720000000000FA", - "end_key": "7480000000000000FF2E00000000000000F8", - "role": "learner", - "count": 1, - "label_constraints": [ - { - "key": "engine", - "op": "in", - "values": [ - "tiflash" - ] - } - ] - } - ] - ``` +### 3. View the cluster status -2. Remove all data replication rules related to TiFlash. Take the rule whose `id` is `table-45-r` as an example. Delete it by the following command: +```shell +tiup cluster display +``` - {{< copyable "shell-regular" >}} +Access the monitoring platform at using your browser, and view the status of the cluster and the new nodes. - ```shell - curl -v -X DELETE http://:/pd/api/v1/config/rule/tiflash/table-45-r - ``` - -3. View the cluster status: - - {{< copyable "shell-regular" >}} - - ```shell - tiup cluster display - ``` - - Access the monitoring platform at using your browser, and view the status of the cluster and the new nodes. - -After the scale-out, the cluster topology is as follows: +After the scaling, the cluster topology is as follows: | Host IP | Service | |:----|:----| diff --git a/schedule-replicas-by-topology-labels.md b/schedule-replicas-by-topology-labels.md index 9604dd74f371d..98432a86cbbe6 100644 --- a/schedule-replicas-by-topology-labels.md +++ b/schedule-replicas-by-topology-labels.md @@ -1,7 +1,6 @@ --- title: Schedule Replicas by Topology Labels summary: Learn how to schedule replicas by topology labels. -aliases: ['/docs/dev/location-awareness/','/docs/dev/how-to/deploy/geographic-redundancy/location-awareness/','/tidb/dev/location-awareness'] --- # Schedule Replicas by Topology Labels @@ -14,9 +13,156 @@ To improve the high availability and disaster recovery capability of TiDB cluste To make this mechanism effective, you need to properly configure TiKV and PD so that the topology information of the cluster, especially the TiKV location information, is reported to PD during deployment. Before you begin, see [Deploy TiDB Using TiUP](/production-deployment-using-tiup.md) first. -## Configure `labels` based on the cluster topology +## Configure `labels` for TiKV, TiFlash, and TiDB -### Configure `labels` for TiKV and TiFlash +You can configure `labels` for TiKV, TiFlash, and TiDB based on the cluster topology. + +### Configure a cluster using TiUP (recommended) + +When using TiUP to deploy a cluster, you can configure the TiKV location in the [initialization configuration file](/production-deployment-using-tiup.md#step-3-initialize-cluster-topology-file). TiUP will generate the corresponding configuration files for TiDB, TiKV, PD, and TiFlash during deployment. + +In the following example, a two-layer topology of `zone/host` is defined. The TiDB nodes, TiKV nodes, and TiFlash nodes of the cluster are distributed among three zones, z1, z2, and z3. + +- In each zone, there are two hosts that have TiDB instances deployed, and each host has a separate TiDB instance deployed. +- In each zone, there are two hosts that have TiKV instances deployed. In z1, each host has two TiKV instances deployed. In z2 and z3, each host has a separate TiKV instance deployed. +- In each zone, there are two hosts that have TiFlash instances deployed, and each host has a separate TiFlash instance deployed. + +In the following example, `tidb-host-machine-n` represents the IP address of the `n`th TiDB node, `tikv-host-machine-n` represents the IP address of the `n`th TiKV node, and `tiflash-host-machine-n` represents the IP address of the `n`th TiFlash node. + +``` +server_configs: + pd: + replication.location-labels: ["zone", "host"] +tidb_servers: +# z1 + - host: tidb-host-machine-1 + config: + labels: + zone: z1 + host: tidb-host-machine-1 + - host: tidb-host-machine-2 + config: + labels: + zone: z1 + host: tidb-host-machine-2 +# z2 + - host: tidb-host-machine-3 + config: + labels: + zone: z2 + host: tidb-host-machine-3 + - host: tikv-host-machine-4 + config: + labels: + zone: z2 + host: tidb-host-machine-4 +# z3 + - host: tidb-host-machine-5 + config: + labels: + zone: z3 + host: tidb-host-machine-5 + - host: tidb-host-machine-6 + config: + labels: + zone: z3 + host: tidb-host-machine-6 +tikv_servers: +# z1 + # machine-1 on z1 + - host: tikv-host-machine-1 + port: 20160 + config: + server.labels: + zone: z1 + host: tikv-host-machine-1 + - host: tikv-host-machine-1 + port: 20161 + config: + server.labels: + zone: z1 + host: tikv-host-machine-1 + # machine-2 on z1 + - host: tikv-host-machine-2 + port: 20160 + config: + server.labels: + zone: z1 + host: tikv-host-machine-2 + - host: tikv-host-machine-2 + port: 20161 + config: + server.labels: + zone: z1 + host: tikv-host-machine-2 +# z2 + - host: tikv-host-machine-3 + config: + server.labels: + zone: z2 + host: tikv-host-machine-3 + - host: tikv-host-machine-4 + config: + server.labels: + zone: z2 + host: tikv-host-machine-4 +# z3 + - host: tikv-host-machine-5 + config: + server.labels: + zone: z3 + host: tikv-host-machine-5 + - host: tikv-host-machine-6 + config: + server.labels: + zone: z3 + host: tikv-host-machine-6 + +tiflash_servers: +# z1 + - host: tiflash-host-machine-1 + learner_config: + server.labels: + zone: z1 + host: tiflash-host-machine-1 + - host: tiflash-host-machine-2 + learner_config: + server.labels: + zone: z1 + host: tiflash-host-machine-2 +# z2 + - host: tiflash-host-machine-3 + learner_config: + server.labels: + zone: z2 + host: tiflash-host-machine-3 + - host: tiflash-host-machine-4 + learner_config: + server.labels: + zone: z2 + host: tiflash-host-machine-4 +# z3 + - host: tiflash-host-machine-5 + learner_config: + server.labels: + zone: z3 + host: tiflash-host-machine-5 + - host: tiflash-host-machine-6 + learner_config: + server.labels: + zone: z3 + host: tiflash-host-machine-6 +``` + +For details, see [Geo-distributed Deployment topology](/geo-distributed-deployment-topology.md). + +> **Note:** +> +> If you have not configured `replication.location-labels` in the configuration file, when you deploy a cluster using this topology file, an error might occur. It is recommended that you confirm `replication.location-labels` is configured in the configuration file before deploying a cluster. + +### Configure a cluster using command lines or configuration files + +#### Configure `labels` for TiKV and TiFlash You can use the command-line flag or set the TiKV or TiFlash configuration file to bind some attributes in the form of key-value pairs. These attributes are called `labels`. After TiKV and TiFlash are started, they report their `labels` to PD so users can identify the location of TiKV and TiFlash nodes. @@ -50,7 +196,7 @@ rack = "" host = "" ``` -### (Optional) Configure `labels` for TiDB +#### (Optional) Configure `labels` for TiDB When [Follower read](/follower-read.md) is enabled, if you want TiDB to prefer to read data from the same region, you need to configure `labels` for TiDB nodes. @@ -68,7 +214,7 @@ host = "" > > Currently, TiDB depends on the `zone` label to match and select replicas that are in the same region. To use this feature, you need to include `zone` when [configuring `location-labels` for PD](#configure-location-labels-for-pd), and configure `zone` when configuring `labels` for TiDB, TiKV, and TiFlash. For more details, see [Configure `labels` for TiKV and TiFlash](#configure-labels-for-tikv-and-tiflash). -### Configure `location-labels` for PD +## Configure `location-labels` for PD According to the description above, the label can be any key-value pair used to describe TiKV attributes. But PD cannot identify the location-related labels and the layer relationship of these labels. Therefore, you need to make the following configuration for PD to understand the TiKV node topology. @@ -100,7 +246,7 @@ To configure `location-labels`, choose one of the following methods according to pd-ctl config set location-labels zone,rack,host ``` -### Configure `isolation-level` for PD +## Configure `isolation-level` for PD If `location-labels` has been configured, you can further enhance the topological isolation requirements on TiKV clusters by configuring `isolation-level` in the PD configuration file. @@ -127,115 +273,6 @@ The `location-level` configuration is an array of strings, which needs to corres > > `isolation-level` is empty by default, which means there is no mandatory restriction on the isolation level. To set it, you need to configure `location-labels` for PD and ensure that the value of `isolation-level` is one of `location-labels` names. -### Configure a cluster using TiUP (recommended) - -When using TiUP to deploy a cluster, you can configure the TiKV location in the [initialization configuration file](/production-deployment-using-tiup.md#step-3-initialize-cluster-topology-file). TiUP will generate the corresponding configuration files for TiKV, PD, and TiFlash during deployment. - -In the following example, a two-layer topology of `zone/host` is defined. The TiKV nodes and TiFlash nodes of the cluster are distributed among three zones, z1, z2, and z3. - -- In each zone, there are two hosts that have TiKV instances deployed. In z1, each host has two TiKV instances deployed. In z2 and z3, each host has a separate TiKV instance deployed. -- In each zone, there are two hosts that have TiFlash instances deployed, and each host has a separate TiFlash instance deployed. - -In the following example, `tikv-host-machine-n` represents the IP address of the `n`th TiKV node, and `tiflash-host-machine-n` represents the IP address of the `n`th TiFlash node. - -``` -server_configs: - pd: - replication.location-labels: ["zone", "host"] - -tikv_servers: -# z1 - # machine-1 on z1 - - host: tikv-host-machine-1 - port：20160 - config: - server.labels: - zone: z1 - host: tikv-host-machine-1 - - host: tikv-host-machine-1 - port：20161 - config: - server.labels: - zone: z1 - host: tikv-host-machine-1 - # machine-2 on z1 - - host: tikv-host-machine-2 - port：20160 - config: - server.labels: - zone: z1 - host: tikv-host-machine-2 - - host: tikv-host-machine-2 - port：20161 - config: - server.labels: - zone: z1 - host: tikv-host-machine-2 -# z2 - - host: tikv-host-machine-3 - config: - server.labels: - zone: z2 - host: tikv-host-machine-3 - - host: tikv-host-machine-4 - config: - server.labels: - zone: z2 - host: tikv-host-machine-4 -# z3 - - host: tikv-host-machine-5 - config: - server.labels: - zone: z3 - host: tikv-host-machine-5 - - host: tikv-host-machine-6 - config: - server.labels: - zone: z3 - host: tikv-host-machine-6 - -tiflash_servers: -# z1 - - host: tiflash-host-machine-1 - learner_config: - server.labels: - zone: z1 - host: tiflash-host-machine-1 - - host: tiflash-host-machine-2 - learner_config: - server.labels: - zone: z1 - host: tiflash-host-machine-2 -# z2 - - host: tiflash-host-machine-3 - learner_config: - server.labels: - zone: z2 - host: tiflash-host-machine-3 - - host: tiflash-host-machine-4 - learner_config: - server.labels: - zone: z2 - host: tiflash-host-machine-4 -# z3 - - host: tiflash-host-machine-5 - learner_config: - server.labels: - zone: z3 - host: tiflash-host-machine-5 - - host: tiflash-host-machine-6 - learner_config: - server.labels: - zone: z3 - host: tiflash-host-machine-6 -``` - -For details, see [Geo-distributed Deployment topology](/geo-distributed-deployment-topology.md). - -> **Note:** -> -> If you have not configured `replication.location-labels` in the configuration file, when you deploy a cluster using this topology file, an error might occur. It is recommended that you confirm `replication.location-labels` is configured in the configuration file before deploying a cluster. - ## PD schedules based on topology label PD schedules replicas according to the label layer to make sure that different replicas of the same data are scattered as much as possible. diff --git a/schema-object-names.md b/schema-object-names.md index bb216ad01c870..01ce379e9f482 100644 --- a/schema-object-names.md +++ b/schema-object-names.md @@ -1,7 +1,6 @@ --- title: Schema Object Names summary: Learn about schema object names in TiDB SQL statements. -aliases: ['/docs/dev/schema-object-names/','/docs/dev/reference/sql/language-structure/schema-object-names/'] --- # Schema Object Names diff --git a/scripts/verify-links.sh b/scripts/verify-links.sh index 70b5b38c8aba6..119eb1fec53be 100755 --- a/scripts/verify-links.sh +++ b/scripts/verify-links.sh @@ -2,7 +2,7 @@ # # This script is used to verify links in markdown docs. # -# - External links are ignored in CI because these links may go down out of our contorl. +# - External links are ignored in CI because these links may go down out of our control. # - Anchors are ignored # - Internal links conventions # - Must be absolute and start from repo root diff --git a/security-compatibility-with-mysql.md b/security-compatibility-with-mysql.md index c120a20188f6d..c62cb640b9c59 100644 --- a/security-compatibility-with-mysql.md +++ b/security-compatibility-with-mysql.md @@ -1,7 +1,6 @@ --- title: Security Compatibility with MySQL summary: Learn TiDB's security compatibilities with MySQL. -aliases: ['/docs/dev/security-compatibility-with-mysql/','/docs/dev/reference/security/compatibility/'] --- # Security Compatibility with MySQL @@ -110,7 +109,7 @@ The implementation mechanisms are consistent between TiDB and MySQL. Both use th TiDB supports multiple authentication methods. These methods can be specified on a per user basis using [`CREATE USER`](/sql-statements/sql-statement-create-user.md) and [`ALTER USER`](/sql-statements/sql-statement-alter-user.md). These methods are compatible with the authentication methods of MySQL with the same names. -You can use one of the following supported authentication methods in the table. To specify a default method that the server advertises when the client-server connection is being established, set the [`default_authentication_plugin`](/system-variables.md#default_authentication_plugin) variable. `tidb_sm3_password` is the SM3 authentication method only supported in TiDB. Therefore, to authenticate using this method, you must connect to TiDB using [TiDB-JDBC](https://github.com/pingcap/mysql-connector-j/tree/release/8.0-sm3). `tidb_auth_token` is a JSON Web Token (JWT)-based authentication method used in TiDB Cloud, and you can also configure it for use in TiDB Self-Hosted. +You can use one of the following supported authentication methods in the table. To specify a default method that the server advertises when the client-server connection is being established, set the [`default_authentication_plugin`](/system-variables.md#default_authentication_plugin) variable. `tidb_sm3_password` is the SM3 authentication method only supported in TiDB. Therefore, to authenticate using this method, you must connect to TiDB using [TiDB-JDBC](https://github.com/pingcap/mysql-connector-j/tree/release/8.0-sm3). `tidb_auth_token` is a JSON Web Token (JWT)-based authentication method used in TiDB Cloud, and you can also configure it for use in TiDB Self-Managed. @@ -143,7 +142,7 @@ The support for TLS authentication is configured differently. For detailed infor ### `tidb_auth_token` -`tidb_auth_token` is a passwordless authentication method based on [JSON Web Token (JWT)](https://datatracker.ietf.org/doc/html/rfc7519). In v6.4.0, `tidb_auth_token` is only used for user authentication in TiDB Cloud. Starting from v6.5.0, you can also configure `tidb_auth_token` as a user authentication method for TiDB Self-Hosted. Different from password-based authentication methods such as `mysql_native_password` and `caching_sha2_password`, when you create users using `tidb_auth_token`, there is no need to set or store custom passwords. To log into TiDB, users only need to use a signed token instead of a password, which simplifies the authentication process and improves security. +`tidb_auth_token` is a passwordless authentication method based on [JSON Web Token (JWT)](https://datatracker.ietf.org/doc/html/rfc7519). In v6.4.0, `tidb_auth_token` is only used for user authentication in TiDB Cloud. Starting from v6.5.0, you can also configure `tidb_auth_token` as a user authentication method for TiDB Self-Managed. Different from password-based authentication methods such as `mysql_native_password` and `caching_sha2_password`, when you create users using `tidb_auth_token`, there is no need to set or store custom passwords. To log into TiDB, users only need to use a signed token instead of a password, which simplifies the authentication process and improves security. #### JWT @@ -190,11 +189,11 @@ Signature is used to sign the Header and Payload data. > **Warning:** > > - The encoding of the Header and Payload in base64 is reversible. Do **Not** attach any sensitive information to them. -> - The `tidb_auth_token` authentication method requires clients to support the [`mysql_clear_password`](https://dev.mysql.com/doc/refman/8.0/en/cleartext-pluggable-authentication.html) plugin to send the token to TiDB in plain text. Therefore, you need to [enale TLS between clients and servers](/enable-tls-between-clients-and-servers.md) before using `tidb_auth_token`. +> - The `tidb_auth_token` authentication method requires clients to support the [`mysql_clear_password`](https://dev.mysql.com/doc/refman/8.0/en/cleartext-pluggable-authentication.html) plugin to send the token to TiDB in plain text. Therefore, you need to [enable TLS between clients and servers](/enable-tls-between-clients-and-servers.md) before using `tidb_auth_token`. #### Usage -To configure and use `tidb_auth_token` as the authentication method for TiDB Self-Hosted users, take the following steps: +To configure and use `tidb_auth_token` as the authentication method for TiDB Self-Managed users, take the following steps: 1. Configure [`auth-token-jwks`](/tidb-configuration-file.md#auth-token-jwks-new-in-v640) and [`auth-token-refresh-interval`](/tidb-configuration-file.md#auth-token-refresh-interval-new-in-v640) in the TiDB configuration file. diff --git a/smooth-upgrade-tidb.md b/smooth-upgrade-tidb.md index e429cfc69088e..be56b493e1e74 100644 --- a/smooth-upgrade-tidb.md +++ b/smooth-upgrade-tidb.md @@ -18,7 +18,7 @@ Depending on whether the feature needs to be controlled by a switch, there are t - Upgrade from v7.1.1 to v7.2.0 or v7.3.0 - Upgrade from v7.2.0 to v7.3.0 -- The feature is disabled by default, and can be enabled by sending the `/upgrade/start` request. For details, see [TiDB HTTP API](https://github.com/pingcap/tidb/blob/master/docs/tidb_http_api.md). The supported versions are as follows: +- The feature is disabled by default, and can be enabled by sending the `/upgrade/start` request. For details, see [TiDB HTTP API](https://github.com/pingcap/tidb/blob/release-8.1/docs/tidb_http_api.md). The supported versions are as follows: - Upgrade from v7.1.2 and later v7.1 versions (that is, v7.1.x, where x >= 2) to v7.4.0 and later versions - Upgrade from v7.4.0 to later versions @@ -73,12 +73,16 @@ You can take the following steps to upgrade TiDB manually or by using a script: When using the smooth upgrade feature, note the following limitations. +> **Note:** +> +> The limitations in this section apply not only to scenarios using the smooth upgrade feature, but also to [upgrading TiDB using TiUP](/upgrade-tidb-using-tiup.md#upgrade-tidb-using-tiup). + ### Limitations on user operations * Before the upgrade, consider the following restrictions: * If there is a canceling DDL job in the cluster, that is, an ongoing DDL job is being canceled by a user, because the job in the canceling state cannot be paused, TiDB will retry canceling the job. If the retry fails, an error is reported and the upgrade is exited. - * If your current TiDB version is earlier than v8.1.0 and the TiDB Distributed eXecution Framework (DXF) is enabled, disable it by setting [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) to `OFF`. Make sure that all ongoing distributed `ADD INDEX` and `IMPORT INTO` tasks are completed. Alternatively, you can cancel these tasks and wait until the upgrade is complete to restart them. Otherwise, the `ADD INDEX` operations during the upgrade might cause data index inconsistency. + * If your current TiDB version is earlier than v8.1.0 and the TiDB Distributed eXecution Framework (DXF) is enabled, disable it by setting [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) to `OFF`. Make sure that all ongoing distributed `ADD INDEX` and `IMPORT INTO` tasks are completed. Alternatively, you can cancel these tasks and wait until the upgrade is complete to restart them. Otherwise, the `ADD INDEX` operations during the upgrade might cause data index inconsistency. If your current TiDB version is v8.1.0 or later, you do not need to disable the DXF, and you can just ignore this limitation. * In scenarios of using TiUP to upgrade TiDB, because TiUP upgrade has a timeout period, if the cluster has a large number of DDL jobs (more than 300) waiting in queues before the upgrade, the upgrade might fail. diff --git a/sql-logical-optimization.md b/sql-logical-optimization.md index 92cc2bc0d0941..0e653ae5883c5 100644 --- a/sql-logical-optimization.md +++ b/sql-logical-optimization.md @@ -16,4 +16,5 @@ This chapter introduces the following key rewrites: - [Predicates Push Down](/predicate-push-down.md) - [Partition Pruning](/partition-pruning.md) - [TopN and Limit Operator Push Down](/topn-limit-push-down.md) -- [Join Reorder](/join-reorder.md) \ No newline at end of file +- [Join Reorder](/join-reorder.md) +- [Derive TopN or Limit from Window Functions](/derive-topn-from-window.md) \ No newline at end of file diff --git a/sql-mode.md b/sql-mode.md index 9cf4d2190c858..dcd0ee0bcc326 100644 --- a/sql-mode.md +++ b/sql-mode.md @@ -1,7 +1,6 @@ --- title: SQL Mode summary: Learn SQL mode. -aliases: ['/docs/dev/sql-mode/','/docs/dev/reference/sql/sql-mode/'] --- # SQL Mode diff --git a/sql-optimization-concepts.md b/sql-optimization-concepts.md index e1c8935f02eae..a01b67910f851 100644 --- a/sql-optimization-concepts.md +++ b/sql-optimization-concepts.md @@ -1,7 +1,6 @@ --- title: SQL Optimization Process summary: Learn about the logical and physical optimization of SQL in TiDB. -aliases: ['/docs/dev/sql-optimization-concepts/','/docs/dev/reference/performance/sql-optimizer-overview/'] --- # SQL Optimization Process diff --git a/sql-plan-management.md b/sql-plan-management.md index 956c283f91bf6..e92a55b43c332 100644 --- a/sql-plan-management.md +++ b/sql-plan-management.md @@ -1,7 +1,6 @@ --- title: SQL Plan Management (SPM) summary: Learn about SQL Plan Management in TiDB. -aliases: ['/docs/dev/sql-plan-management/','/docs/dev/reference/performance/execution-plan-bind/','/docs/dev/execution-plan-binding/'] --- # SQL Plan Management (SPM) @@ -43,50 +42,44 @@ CREATE [GLOBAL | SESSION] BINDING [FOR BindableStmt] USING BindableStmt; This statement binds SQL execution plans at the GLOBAL or SESSION level. Currently, supported bindable SQL statements (BindableStmt) in TiDB include `SELECT`, `DELETE`, `UPDATE`, and `INSERT` / `REPLACE` with `SELECT` subqueries. The following is an example: ```sql -CREATE GLOBAL BINDING USING SELECT * /*+ use_index(t1, a) */ FROM t1; -CREATE GLOBAL BINDING FOR SELECT * FROM t1 USING SELECT * /*+ use_index(t1, a) */ FROM t1; +CREATE GLOBAL BINDING USING SELECT /*+ use_index(orders, orders_book_id_idx) */ * FROM orders; +CREATE GLOBAL BINDING FOR SELECT * FROM orders USING SELECT /*+ use_index(orders, orders_book_id_idx) */ * FROM orders; ``` > **Note:** > > Bindings have higher priority over manually added hints. Therefore, when you execute a statement containing a hint while a corresponding binding is present, the hint controlling the behavior of the optimizer does not take effect. However, other types of hints are still effective. -Specifically, two types of these statements cannot be bound to execution plans due to syntax conflicts. See the following examples: +Specifically, two types of these statements cannot be bound to execution plans due to syntax conflicts. A syntax error will be reported during binding creation. See the following examples: ```sql -- Type one: Statements that get the Cartesian product by using the `JOIN` keyword and not specifying the associated columns with the `USING` keyword. CREATE GLOBAL BINDING for - SELECT * FROM t t1 JOIN t t2 + SELECT * FROM orders o1 JOIN orders o2 USING - SELECT * FROM t t1 JOIN t t2; + SELECT * FROM orders o1 JOIN orders o2; -- Type two: `DELETE` statements that contain the `USING` keyword. CREATE GLOBAL BINDING for - DELETE FROM t1 USING t1 JOIN t2 ON t1.a = t2.a + DELETE FROM users USING users JOIN orders ON users.id = orders.user_id USING - DELETE FROM t1 USING t1 JOIN t2 ON t1.a = t2.a; + DELETE FROM users USING users JOIN orders ON users.id = orders.user_id; ``` You can bypass syntax conflicts by using equivalent statements. For example, you can rewrite the above statements in the following ways: ```sql --- First rewrite of type one statements: Add a `USING` clause for the `JOIN` keyword. +-- Rewrite of type one statements: Delete the `JOIN` keyword. Replace it with a comma. CREATE GLOBAL BINDING for - SELECT * FROM t t1 JOIN t t2 USING (a) + SELECT * FROM orders o1, orders o2 USING - SELECT * FROM t t1 JOIN t t2 USING (a); + SELECT * FROM orders o1, orders o2; --- Second rewrite of type one statements: Delete the `JOIN` keyword. +-- Rewrite of type two statements: Remove the `USING` keyword from the `DELETE` statement. CREATE GLOBAL BINDING for - SELECT * FROM t t1, t t2 + DELETE users FROM users JOIN orders ON users.id = orders.user_id USING - SELECT * FROM t t1, t t2; - --- Rewrite of type two statements: Remove the `USING` keyword from the `delete` statement. -CREATE GLOBAL BINDING for - DELETE t1 FROM t1 JOIN t2 ON t1.a = t2.a -using - DELETE t1 FROM t1 JOIN t2 ON t1.a = t2.a; + DELETE users FROM users JOIN orders ON users.id = orders.user_id; ``` > **Note:** @@ -98,15 +91,15 @@ Here are two examples: ```sql -- The hint takes effect in the following statement. CREATE GLOBAL BINDING for - INSERT INTO t1 SELECT * FROM t2 WHERE a > 1 AND b = 1 -using - INSERT INTO t1 SELECT /*+ use_index(@sel_1 t2, a) */ * FROM t2 WHERE a > 1 AND b = 1; + INSERT INTO orders SELECT * FROM pre_orders WHERE status = 'VALID' AND created <= (NOW() - INTERVAL 1 HOUR) +USING + INSERT INTO orders SELECT /*+ use_index(@sel_1 pre_orders, idx_created) */ * FROM pre_orders WHERE status = 'VALID' AND created <= (NOW() - INTERVAL 1 HOUR); -- The hint cannot take effect in the following statement. CREATE GLOBAL BINDING for - INSERT INTO t1 SELECT * FROM t2 WHERE a > 1 AND b = 1 -using - INSERT /*+ use_index(@sel_1 t2, a) */ INTO t1 SELECT * FROM t2 WHERE a > 1 AND b = 1; + INSERT INTO orders SELECT * FROM pre_orders WHERE status = 'VALID' AND created <= (NOW() - INTERVAL 1 HOUR) +USING + INSERT /*+ use_index(@sel_1 pre_orders, idx_created) */ INTO orders SELECT * FROM pre_orders WHERE status = 'VALID' AND created <= (NOW() - INTERVAL 1 HOUR); ``` If you do not specify the scope when creating an execution plan binding, the default scope is SESSION. The TiDB optimizer normalizes bound SQL statements and stores them in the system table. When processing SQL queries, if a normalized statement matches one of the bound SQL statements in the system table and the system variable `tidb_use_plan_baselines` is set to `on` (the default value is `on`), TiDB then uses the corresponding optimizer hint for this statement. If there are multiple matchable execution plans, the optimizer chooses the least costly one to bind. @@ -114,9 +107,9 @@ If you do not specify the scope when creating an execution plan binding, the def `Normalization` is a process that converts a constant in an SQL statement to a variable parameter and explicitly specifies the database for tables referenced in the query, with standardized processing on the spaces and line breaks in the SQL statement. See the following example: ```sql -SELECT * FROM t WHERE a > 1 +SELECT * FROM users WHERE balance > 100 -- After normalization, the above statement is as follows: -SELECT * FROM test . t WHERE a > ? +SELECT * FROM bookshop . users WHERE balance > ? ``` > **Note:** @@ -126,11 +119,11 @@ SELECT * FROM test . t WHERE a > ? > For example: > > ```sql -> SELECT * FROM t WHERE a IN (1) -> SELECT * FROM t WHERE a IN (1,2,3) +> SELECT * FROM books WHERE type IN ('Novel') +> SELECT * FROM books WHERE type IN ('Novel','Life','Education') > -- After normalization, the above statements are as follows: -> SELECT * FROM test . t WHERE a IN ( ... ) -> SELECT * FROM test . t WHERE a IN ( ... ) +> SELECT * FROM bookshop . books WHERE type IN ( ... ) +> SELECT * FROM bookshop . books WHERE type IN ( ... ) > ``` > > After normalization, `IN` predicates of different lengths are recognized as the same statement, so you only need to create one binding that applies to all these predicates. @@ -139,7 +132,7 @@ SELECT * FROM test . t WHERE a > ? > > ```sql > CREATE TABLE t (a INT, KEY(a)); -> CREATE BINDING FOR SELECT * FROM t WHERE a IN (?) USING SELECT /*+ use_index(t, a) */ * FROM t WHERE a in (?); +> CREATE BINDING FOR SELECT * FROM t WHERE a IN (?) USING SELECT /*+ use_index(t, idx_a) */ * FROM t WHERE a in (?); > > SELECT * FROM t WHERE a IN (1); > SELECT @@LAST_PLAN_FROM_BINDING; @@ -164,21 +157,21 @@ SELECT * FROM test . t WHERE a > ? > > ```sql > -- Create a binding on v7.3.0 -> mysql> CREATE GLOBAL BINDING FOR SELECT * FROM t WHERE a IN (1) USING SELECT /*+ use_index(t, a) */ * FROM t WHERE a IN (1); +> mysql> CREATE GLOBAL BINDING FOR SELECT * FROM t WHERE a IN (1) USING SELECT /*+ use_index(t, idx_a) */ * FROM t WHERE a IN (1); > mysql> SHOW GLOBAL BINDINGS; -> +-----------------------------------------------+--------------------------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+-----------------+--------+------------------------------------------------------------------+-------------+ -> | Original_sql | Bind_sql | Default_db | Status | Create_time | Update_time | Charset | Collation | Source | Sql_digest | Plan_digest | -> +-----------------------------------------------+--------------------------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+-----------------+--------+------------------------------------------------------------------+-------------+ -> | select * from `test` . `t` where `a` in ( ? ) | SELECT /*+ use_index(`t` `a`)*/ * FROM `test`.`t` WHERE `a` IN (1) | test | enabled | 2023-10-20 14:28:10.093 | 2023-10-20 14:28:10.093 | utf8 | utf8_general_ci | manual | 8b9c4e6ab8fad5ba29b034311dcbfc8a8ce57dde2e2d5d5b65313b90ebcdebf7 | | -> +-----------------------------------------------+--------------------------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+-----------------+--------+------------------------------------------------------------------+-------------+ +> +-----------------------------------------------+------------------------------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+--------------------+--------+------------------------------------------------------------------+-------------+ +> | Original_sql | Bind_sql | Default_db | Status | Create_time | Update_time | Charset | Collation | Source | Sql_digest | Plan_digest | +> +-----------------------------------------------+------------------------------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+--------------------+--------+------------------------------------------------------------------+-------------+ +> | select * from `test` . `t` where `a` in ( ? ) | SELECT /*+ use_index(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` IN (1) | test | enabled | 2024-09-03 15:39:02.695 | 2024-09-03 15:39:02.695 | utf8mb4 | utf8mb4_general_ci | manual | 8b9c4e6ab8fad5ba29b034311dcbfc8a8ce57dde2e2d5d5b65313b90ebcdebf7 | | +> +-----------------------------------------------+------------------------------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+--------------------+--------+------------------------------------------------------------------+-------------+ > > -- After the upgrade to v7.4.0 or a later version > mysql> SHOW GLOBAL BINDINGS; -> +-------------------------------------------------+--------------------------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+-----------------+--------+------------------------------------------------------------------+-------------+ -> | Original_sql | Bind_sql | Default_db | Status | Create_time | Update_time | Charset | Collation | Source | Sql_digest | Plan_digest | -> +-------------------------------------------------+--------------------------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+-----------------+--------+------------------------------------------------------------------+-------------+ -> | select * from `test` . `t` where `a` in ( ... ) | SELECT /*+ use_index(`t` `a`)*/ * FROM `test`.`t` WHERE `a` IN (1) | test | enabled | 2023-10-20 14:28:10.093 | 2023-10-20 14:28:10.093 | utf8 | utf8_general_ci | manual | 8b9c4e6ab8fad5ba29b034311dcbfc8a8ce57dde2e2d5d5b65313b90ebcdebf7 | | -> +-------------------------------------------------+--------------------------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+-----------------+--------+------------------------------------------------------------------+-------------+ +> +-------------------------------------------------+------------------------------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+--------------------+--------+------------------------------------------------------------------+-------------+ +> | Original_sql | Bind_sql | Default_db | Status | Create_time | Update_time | Charset | Collation | Source | Sql_digest | Plan_digest | +> +-------------------------------------------------+------------------------------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+--------------------+--------+------------------------------------------------------------------+-------------+ +> | select * from `test` . `t` where `a` in ( ... ) | SELECT /*+ use_index(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` IN (1) | test | enabled | 2024-09-03 15:35:59.861 | 2024-09-03 15:35:59.861 | utf8mb4 | utf8mb4_general_ci | manual | da38bf216db4a53e1a1e01c79ffa42306419442ad7238480bb7ac510723c8bdf | | +> +-------------------------------------------------+------------------------------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+--------------------+--------+------------------------------------------------------------------+-------------+ > ``` When a SQL statement has bound execution plans in both GLOBAL and SESSION scopes, because the optimizer ignores the bound execution plan in the GLOBAL scope when it encounters the SESSION binding, the bound execution plan of this statement in the SESSION scope shields the execution plan in the GLOBAL scope. @@ -254,9 +247,9 @@ To use this binding method, you need to first get the `plan_digest` correspondin For example: ```sql - CREATE TABLE t(id INT PRIMARY KEY , a INT, KEY(a)); - SELECT /*+ IGNORE_INDEX(t, a) */ * FROM t WHERE a = 1; - SELECT * FROM INFORMATION_SCHEMA.STATEMENTS_SUMMARY WHERE QUERY_SAMPLE_TEXT = 'SELECT /*+ IGNORE_INDEX(t, a) */ * FROM t WHERE a = 1'\G + CREATE TABLE t(id INT PRIMARY KEY , a INT, KEY idx_a(a)); + SELECT /*+ IGNORE_INDEX(t, idx_a) */ * FROM t WHERE a = 1; + SELECT * FROM INFORMATION_SCHEMA.STATEMENTS_SUMMARY WHERE QUERY_SAMPLE_TEXT = 'SELECT /*+ IGNORE_INDEX(t, idx_a) */ * FROM t WHERE a = 1'\G ``` The following is a part of the example query result of `statements_summary`: @@ -291,7 +284,7 @@ SHOW BINDINGS\G ``` *************************** 1. row *************************** Original_sql: select * from `test` . `t` where `a` = ? - Bind_sql: SELECT /*+ use_index(@`sel_1` `test`.`t` ) ignore_index(`t` `a`)*/ * FROM `test`.`t` WHERE `a` = 1 + Bind_sql: SELECT /*+ use_index(@`sel_1` `test`.`t` ) ignore_index(`t` `idx_a`)*/ * FROM `test`.`t` WHERE `a` = 1 ........... Sql_digest: 6909a1bbce5f64ade0a532d7058dd77b6ad5d5068aee22a531304280de48349f Plan_digest: @@ -544,8 +537,8 @@ In these scenarios, cross-database binding can effectively mitigate SQL performa To create a cross-database binding, you only need to use `*` to represent the database name when creating a binding. For example: ```sql -CREATE GLOBAL BINDING USING SELECT /*+ use_index(t, a) */ * FROM t; -- Create a GLOBAL scope standard binding. -CREATE GLOBAL BINDING USING SELECT /*+ use_index(t, a) */ * FROM *.t; -- Create a GLOBAL scope cross-database binding. +CREATE GLOBAL BINDING USING SELECT /*+ use_index(t, idx_a) */ * FROM t; -- Create a GLOBAL scope standard binding. +CREATE GLOBAL BINDING USING SELECT /*+ use_index(t, idx_a) */ * FROM *.t; -- Create a GLOBAL scope cross-database binding. SHOW GLOBAL BINDINGS; ``` @@ -555,8 +548,8 @@ The output is as follows: +----------------------------+---------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+-----------------+--------+------------------------------------------------------------------+-------------+ | Original_sql | Bind_sql | Default_db | Status | Create_time | Update_time | Charset | Collation | Source | Sql_digest | Plan_digest | +----------------------------+---------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+-----------------+--------+------------------------------------------------------------------+-------------+ -| select * from `test` . `t` | SELECT /*+ use_index(`t` `a`)*/ * FROM `test`.`t` | test | enabled | 2023-12-29 14:19:01.332 | 2023-12-29 14:19:01.332 | utf8 | utf8_general_ci | manual | 8b193b00413fdb910d39073e0d494c96ebf24d1e30b131ecdd553883d0e29b42 | | -| select * from `*` . `t` | SELECT /*+ use_index(`t` `a`)*/ * FROM `*`.`t` | | enabled | 2023-12-29 14:19:02.232 | 2023-12-29 14:19:02.232 | utf8 | utf8_general_ci | manual | 8b193b00413fdb910d39073e0d494c96ebf24d1e30b131ecdd553883d0e29b42 | | +| select * from `test` . `t` | SELECT /*+ use_index(`t` `idx_a`)*/ * FROM `test`.`t` | test | enabled | 2023-12-29 14:19:01.332 | 2023-12-29 14:19:01.332 | utf8 | utf8_general_ci | manual | 8b193b00413fdb910d39073e0d494c96ebf24d1e30b131ecdd553883d0e29b42 | | +| select * from `*` . `t` | SELECT /*+ use_index(`t` `idx_a`)*/ * FROM `*`.`t` | | enabled | 2023-12-29 14:19:02.232 | 2023-12-29 14:19:02.232 | utf8 | utf8_general_ci | manual | 8b193b00413fdb910d39073e0d494c96ebf24d1e30b131ecdd553883d0e29b42 | | +----------------------------+---------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+-----------------+--------+------------------------------------------------------------------+-------------+ ``` @@ -586,7 +579,7 @@ Apart from the creation syntax, cross-database bindings share the same deletion 3. Create a cross-database binding: ```sql - CREATE GLOBAL BINDING USING SELECT /*+ use_index(t1, a), use_index(t2, a) */ * FROM *.t1, *.t2; + CREATE GLOBAL BINDING USING SELECT /*+ use_index(t1, idx_a), use_index(t2, idx_a) */ * FROM *.t1, *.t2; ``` 4. Execute queries and verify whether the binding is used: @@ -633,7 +626,7 @@ Apart from the creation syntax, cross-database bindings share the same deletion +----------------------------------------------+------------------------------------------------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+--------------------+--------+------------------------------------------------------------------+-------------+ | Original_sql | Bind_sql | Default_db | Status | Create_time | Update_time | Charset | Collation | Source | Sql_digest | Plan_digest | +----------------------------------------------+------------------------------------------------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+--------------------+--------+------------------------------------------------------------------+-------------+ - | select * from ( `*` . `t1` ) join `*` . `t2` | SELECT /*+ use_index(`t1` `a`) use_index(`t2` `a`)*/ * FROM (`*` . `t1`) JOIN `*` . `t2` | | enabled | 2023-12-29 14:22:28.144 | 2023-12-29 14:22:28.144 | utf8 | utf8_general_ci | manual | ea8720583e80644b58877663eafb3579700e5f918a748be222c5b741a696daf4 | | + | select * from ( `*` . `t1` ) join `*` . `t2` | SELECT /*+ use_index(`t1` `idx_a`) use_index(`t2` `idx_a`)*/ * FROM (`*` . `t1`) JOIN `*` . `t2` | | enabled | 2023-12-29 14:22:28.144 | 2023-12-29 14:22:28.144 | utf8 | utf8_general_ci | manual | ea8720583e80644b58877663eafb3579700e5f918a748be222c5b741a696daf4 | | +----------------------------------------------+------------------------------------------------------------------------------------------+------------+---------+-------------------------+-------------------------+---------+--------------------+--------+------------------------------------------------------------------+-------------+ ``` diff --git a/sql-plan-replayer.md b/sql-plan-replayer.md index 82d9bf703fde2..6751b705b9ac7 100644 --- a/sql-plan-replayer.md +++ b/sql-plan-replayer.md @@ -16,10 +16,8 @@ The features of `PLAN REPLAYER` are as follows: You can use `PLAN REPLAYER` to save the on-site information of a TiDB cluster. The export interface is as follows: -{{< copyable "sql" >}} - ```sql -PLAN REPLAYER DUMP EXPLAIN [ANALYZE] [WITH STATS AS OF TIMESTAMP expression] sql-statement; +PLAN REPLAYER DUMP [WITH STATS AS OF TIMESTAMP expression] EXPLAIN [ANALYZE] sql-statement; ``` Based on `sql-statement`, TiDB sorts out and exports the following on-site information: @@ -235,7 +233,7 @@ PLAN REPLAYER CAPTURE 'sql_digest' '*'; You can view the ongoing capture tasks of `PLAN REPLAYER CAPTURE` in the TiDB cluster using the following statement: ```sql -mysql> PLAN PLAYER CAPTURE 'example_sql' 'example_plan'; +mysql> PLAN REPLAYER CAPTURE 'example_sql' 'example_plan'; Query OK, 1 row affected (0.01 sec) mysql> SELECT * FROM mysql.plan_replayer_task; @@ -269,6 +267,29 @@ The method of downloading the file of `PLAN REPLAYER CAPTURE` is the same as tha > > The result file of `PLAN REPLAYER CAPTURE` is kept in the TiDB cluster for up to one week. After one week, TiDB deletes the file. +### Remove the capture tasks + +If a capture task is no longer needed, you can remove it using the `PLAN REPLAYER CAPTURE REMOVE` statement. For example: + +```sql +mysql> PLAN REPLAYER CAPTURE '077a87a576e42360c95530ccdac7a1771c4efba17619e26be50a4cfd967204a0' '4838af52c1e07fc8694761ad193d16a689b2128bc5ced9d13beb31ae27b370ce'; +Query OK, 0 rows affected (0.01 sec) + +mysql> SELECT * FROM mysql.plan_replayer_task; ++------------------------------------------------------------------+------------------------------------------------------------------+---------------------+ +| sql_digest | plan_digest | update_time | ++------------------------------------------------------------------+------------------------------------------------------------------+---------------------+ +| 077a87a576e42360c95530ccdac7a1771c4efba17619e26be50a4cfd967204a0 | 4838af52c1e07fc8694761ad193d16a689b2128bc5ced9d13beb31ae27b370ce | 2024-05-21 11:26:10 | ++------------------------------------------------------------------+------------------------------------------------------------------+---------------------+ +1 row in set (0.01 sec) + +mysql> PLAN REPLAYER CAPTURE REMOVE '077a87a576e42360c95530ccdac7a1771c4efba17619e26be50a4cfd967204a0' '4838af52c1e07fc8694761ad193d16a689b2128bc5ced9d13beb31ae27b370ce'; +Query OK, 0 rows affected (0.01 sec) + +mysql> SELECT * FROM mysql.plan_replayer_task; +Empty set (0.01 sec) +``` + ## Use `PLAN REPLAYER CONTINUOUS CAPTURE` After `PLAN REPLAYER CONTINUOUS CAPTURE` is enabled, TiDB asynchronously records the applications' SQL statements with the `PLAN REPLAYER` method according to their `SQL DIGEST` and `PLAN DIGEST`. For SQL statements and execution plans that share the same DIGEST, `PLAN REPLAYER CONTINUOUS CAPTURE` does not record them repeatedly. diff --git a/sql-prepared-plan-cache.md b/sql-prepared-plan-cache.md index 1b9194527dd1b..fbac49f558fdd 100644 --- a/sql-prepared-plan-cache.md +++ b/sql-prepared-plan-cache.md @@ -1,7 +1,6 @@ --- title: SQL Prepared Execution Plan Cache summary: Learn about SQL Prepared Execution Plan Cache in TiDB. -aliases: ['/tidb/dev/sql-prepare-plan-cache'] --- # SQL Prepared Execution Plan Cache @@ -20,7 +19,7 @@ TiDB also supports execution plan caching for some non-`PREPARE` statements, sim In the current version of TiDB, if a `Prepare` statement meets any of the following conditions, the query or the plan is not cached: - The query contains SQL statements other than `SELECT`, `UPDATE`, `INSERT`, `DELETE`, `Union`, `Intersect`, and `Except`. -- The query accesses temporary tables, or a table that contains generated columns. +- The query accesses temporary tables, or a table that contains generated columns, or uses static mode (that is, [`tidb_partition_prune_mode`](/system-variables.md#tidb_partition_prune_mode-new-in-v51) is set to `static`) to access partitioning tables. - The query contains non-correlated sub-queries, such as `SELECT * FROM t1 WHERE t1.a > (SELECT 1 FROM t2 WHERE t2.b < 1)`. - The query contains correlated sub-queries with `PhysicalApply` operators in the execution plan, such as `SELECT * FROM t1 WHERE t1.a > (SELECT a FROM t2 WHERE t1.b > t2.b)`. - The query contains the `ignore_plan_cache` or `set_var` hint, such as `SELECT /*+ ignore_plan_cache() */ * FROM t` or `SELECT /*+ set_var(max_execution_time=1) */ * FROM t`. diff --git a/sql-statements/sql-statement-add-column.md b/sql-statements/sql-statement-add-column.md index 47ae0faa329e1..1863fd73ed7cf 100644 --- a/sql-statements/sql-statement-add-column.md +++ b/sql-statements/sql-statement-add-column.md @@ -1,7 +1,6 @@ --- title: ADD COLUMN | TiDB SQL Statement Reference summary: An overview of the usage of ADD COLUMN for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-add-column/','/docs/dev/reference/sql/statements/add-column/'] --- # ADD COLUMN diff --git a/sql-statements/sql-statement-add-index.md b/sql-statements/sql-statement-add-index.md index c85653df8f98f..e7044f0dd1450 100644 --- a/sql-statements/sql-statement-add-index.md +++ b/sql-statements/sql-statement-add-index.md @@ -1,13 +1,16 @@ --- title: ADD INDEX | TiDB SQL Statement Reference summary: An overview of the usage of ADD INDEX for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-add-index/','/docs/dev/reference/sql/statements/add-index/'] --- # ADD INDEX The `ALTER TABLE.. ADD INDEX` statement adds an index to an existing table. This operation is online in TiDB, which means that neither reads or writes to the table are blocked by adding an index. +> **Tip:** +> +> The [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md) can be used to speed up the operation of this statement. + > **Warning:** @@ -81,7 +84,9 @@ mysql> EXPLAIN SELECT * FROM t1 WHERE c1 = 3; ## MySQL compatibility -* `FULLTEXT`, `HASH` and `SPATIAL` indexes are not supported. +* TiDB accepts index types such as `HASH`, `BTREE` and `RTREE` in syntax for compatibility with MySQL, but ignores them. +* `SPATIAL` indexes are not supported. +* TiDB supports parsing the `FULLTEXT` syntax but does not support using the `FULLTEXT` indexes. * Descending indexes are not supported (similar to MySQL 5.7). * Adding the primary key of the `CLUSTERED` type to a table is not supported. For more details about the primary key of the `CLUSTERED` type, refer to [clustered index](/clustered-indexes.md). @@ -96,3 +101,4 @@ mysql> EXPLAIN SELECT * FROM t1 WHERE c1 = 3; * [ADD COLUMN](/sql-statements/sql-statement-add-column.md) * [CREATE TABLE](/sql-statements/sql-statement-create-table.md) * [EXPLAIN](/sql-statements/sql-statement-explain.md) +* [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md) diff --git a/sql-statements/sql-statement-admin-cancel-ddl.md b/sql-statements/sql-statement-admin-cancel-ddl.md index 968b059a8785a..9aafb61ccb18d 100644 --- a/sql-statements/sql-statement-admin-cancel-ddl.md +++ b/sql-statements/sql-statement-admin-cancel-ddl.md @@ -13,8 +13,8 @@ The `ADMIN CANCEL DDL` statement also allows you to cancel a DDL job that is com ## Synopsis ```ebnf+diagram -AdminStmt ::= - 'ADMIN' ( 'SHOW' ( 'DDL' ( 'JOBS' Int64Num? WhereClauseOptional | 'JOB' 'QUERIES' NumList )? | TableName 'NEXT_ROW_ID' | 'SLOW' AdminShowSlow ) | 'CHECK' ( 'TABLE' TableNameList | 'INDEX' TableName Identifier ( HandleRange ( ',' HandleRange )* )? ) | 'RECOVER' 'INDEX' TableName Identifier | 'CLEANUP' ( 'INDEX' TableName Identifier | 'TABLE' 'LOCK' TableNameList ) | 'CHECKSUM' 'TABLE' TableNameList | 'CANCEL' 'DDL' 'JOBS' NumList | 'RELOAD' ( 'EXPR_PUSHDOWN_BLACKLIST' | 'OPT_RULE_BLACKLIST' | 'BINDINGS' ) | 'PLUGINS' ( 'ENABLE' | 'DISABLE' ) PluginNameList | 'REPAIR' 'TABLE' TableName CreateTableStmt | ( 'FLUSH' | 'CAPTURE' | 'EVOLVE' ) 'BINDINGS' ) +AdminCancelDDLStmt ::= + 'ADMIN' 'CANCEL' 'DDL' 'JOBS' NumList NumList ::= Int64Num ( ',' Int64Num )* @@ -24,8 +24,6 @@ NumList ::= To cancel the currently running DDL jobs and return whether the corresponding jobs are successfully cancelled, use `ADMIN CANCEL DDL JOBS`: -{{< copyable "sql" >}} - ```sql ADMIN CANCEL DDL JOBS job_id [, job_id] ...; ``` diff --git a/sql-statements/sql-statement-admin-check-table-index.md b/sql-statements/sql-statement-admin-check-table-index.md index 3cc858951b3bc..d2232e7b63a97 100644 --- a/sql-statements/sql-statement-admin-check-table-index.md +++ b/sql-statements/sql-statement-admin-check-table-index.md @@ -28,35 +28,31 @@ If you use the `ADMIN CHECK INDEX` statement, it only checks the specified index ## Synopsis ```ebnf+diagram -AdminStmt ::= - 'ADMIN' ( 'SHOW' ( 'DDL' ( 'JOBS' Int64Num? WhereClauseOptional | 'JOB' 'QUERIES' NumList )? | TableName 'NEXT_ROW_ID' | 'SLOW' AdminShowSlow ) | 'CHECK' ( 'TABLE' TableNameList | 'INDEX' TableName Identifier ( HandleRange ( ',' HandleRange )* )? ) | 'RECOVER' 'INDEX' TableName Identifier | 'CLEANUP' ( 'INDEX' TableName Identifier | 'TABLE' 'LOCK' TableNameList ) | 'CHECKSUM' 'TABLE' TableNameList | 'CANCEL' 'DDL' 'JOBS' NumList | 'RELOAD' ( 'EXPR_PUSHDOWN_BLACKLIST' | 'OPT_RULE_BLACKLIST' | 'BINDINGS' ) | 'PLUGINS' ( 'ENABLE' | 'DISABLE' ) PluginNameList | 'REPAIR' 'TABLE' TableName CreateTableStmt | ( 'FLUSH' | 'CAPTURE' | 'EVOLVE' ) 'BINDINGS' ) +AdminCheckStmt ::= + 'ADMIN' 'CHECK' ( 'TABLE' TableNameList | 'INDEX' TableName Identifier ( HandleRange ( ',' HandleRange )* )? ) TableNameList ::= TableName ( ',' TableName )* + +HandleRange ::= '(' Int64Num ',' Int64Num ')' ``` ## Examples To check the consistency of all the data and corresponding indexes in the `tbl_name` table, use `ADMIN CHECK TABLE`: -{{< copyable "sql" >}} - ```sql ADMIN CHECK TABLE tbl_name [, tbl_name] ...; ``` If the consistency check is passed, an empty result is returned. Otherwise, an error message is returned indicating that the data is inconsistent. -{{< copyable "sql" >}} - ```sql ADMIN CHECK INDEX tbl_name idx_name; ``` The above statement is used to check the consistency of the column data and index data corresponding to the `idx_name` index in the `tbl_name` table. If the consistency check is passed, an empty result is returned; otherwise, an error message is returned indicating that the data is inconsistent. -{{< copyable "sql" >}} - ```sql ADMIN CHECK INDEX tbl_name idx_name (lower_val, upper_val) [, (lower_val, upper_val)] ...; ``` diff --git a/sql-statements/sql-statement-admin-checksum-table.md b/sql-statements/sql-statement-admin-checksum-table.md index a20fd802aef83..f0e3e45d26e22 100644 --- a/sql-statements/sql-statement-admin-checksum-table.md +++ b/sql-statements/sql-statement-admin-checksum-table.md @@ -6,13 +6,29 @@ category: reference # ADMIN CHECKSUM TABLE -The `ADMIN CHECKSUM TABLE` statement calculates a CRC64 checksum for the data and indexes of a table. This statement is used by programs such as TiDB Lightning to ensure that import operations have completed successfully. +The `ADMIN CHECKSUM TABLE` statement calculates a CRC64 checksum for the data and indexes of a table. + + + +The [checksum](/tidb-lightning/tidb-lightning-glossary.md#checksum) is calculated based on table data and properties such as `table_id`. This means that two tables with the same data but different `table_id` values will get different checksums. + +After importing a table using [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md), [TiDB Data Migration](/dm/dm-overview.md), or [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md), `ADMIN CHECKSUM TABLE ` is executed by default to validate data integrity. + + + + + +The [checksum](https://docs.pingcap.com/tidb/stable/tidb-lightning-glossary#checksum) is calculated over the table data and properties like the `table_id`. This means that two tables with the same data but different `table_id` values will get different checksums. + +After importing a table using [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md), `ADMIN CHECKSUM TABLE
` is executed by default to validate data integrity. + + ## Synopsis ```ebnf+diagram -AdminStmt ::= - 'ADMIN' ( 'SHOW' ( 'DDL' ( 'JOBS' Int64Num? WhereClauseOptional | 'JOB' 'QUERIES' NumList )? | TableName 'NEXT_ROW_ID' | 'SLOW' AdminShowSlow ) | 'CHECK' ( 'TABLE' TableNameList | 'INDEX' TableName Identifier ( HandleRange ( ',' HandleRange )* )? ) | 'RECOVER' 'INDEX' TableName Identifier | 'CLEANUP' ( 'INDEX' TableName Identifier | 'TABLE' 'LOCK' TableNameList ) | 'CHECKSUM' 'TABLE' TableNameList | 'CANCEL' 'DDL' 'JOBS' NumList | 'RELOAD' ( 'EXPR_PUSHDOWN_BLACKLIST' | 'OPT_RULE_BLACKLIST' | 'BINDINGS' ) | 'PLUGINS' ( 'ENABLE' | 'DISABLE' ) PluginNameList | 'REPAIR' 'TABLE' TableName CreateTableStmt | ( 'FLUSH' | 'CAPTURE' | 'EVOLVE' ) 'BINDINGS' ) +AdminChecksumTableStmt ::= + 'ADMIN' 'CHECKSUM' 'TABLE' TableNameList TableNameList ::= TableName ( ',' TableName )* diff --git a/sql-statements/sql-statement-admin-cleanup.md b/sql-statements/sql-statement-admin-cleanup.md index a750ceee4f967..3885add02027e 100644 --- a/sql-statements/sql-statement-admin-cleanup.md +++ b/sql-statements/sql-statement-admin-cleanup.md @@ -77,7 +77,7 @@ Query OK, 0 rows affected (0.01 sec) > - There might be a loss of both row data and index data. To restore the consistency, use the `ADMIN CLEANUP INDEX` and [`ADMIN RECOVER INDEX`](/sql-statements/sql-statement-admin-recover.md) statements together. > - The `ADMIN CLEANUP INDEX` statement is always executed in a single thread. When the table data is large, it is recommended to recover the index data by rebuilding the index. > - When you execute the `ADMIN CLEANUP INDEX` statement, the corresponding table or index is not locked and TiDB allows other sessions to modify the table records at the same time. However, in this case, `ADMIN CLEANUP INDEX` might not be able to handle all table records correctly. Therefore, when you execute `ADMIN CLEANUP INDEX`, avoid modifying the table data at the same time. -> - If you use the enterprise edition of TiDB, you can [submit a request](https://support.pingcap.com/hc/en-us) to contact the support engineer for help. +> - If you use the enterprise edition of TiDB, you can [submit a request](https://tidb.support.pingcap.com/) to contact the support engineer for help. > > The `ADMIN CLEANUP INDEX` statement is not atomic: if the statement is interrupted during execution, it is recommended to execute it again until it succeeds. diff --git a/sql-statements/sql-statement-admin-pause-ddl.md b/sql-statements/sql-statement-admin-pause-ddl.md index ac21d2d3bd6ee..9e647f3a87c67 100644 --- a/sql-statements/sql-statement-admin-pause-ddl.md +++ b/sql-statements/sql-statement-admin-pause-ddl.md @@ -12,8 +12,8 @@ You can use this statement to pause a DDL job that is issued but not yet complet ## Synopsis ```ebnf+diagram -AdminStmt ::= - 'ADMIN' ( 'SHOW' ( 'DDL' ( 'JOBS' Int64Num? WhereClauseOptional | 'JOB' 'QUERIES' NumList )? | TableName 'NEXT_ROW_ID' | 'SLOW' AdminShowSlow ) | 'CHECK' ( 'TABLE' TableNameList | 'INDEX' TableName Identifier ( HandleRange ( ',' HandleRange )* )? ) | 'RECOVER' 'INDEX' TableName Identifier | 'CLEANUP' ( 'INDEX' TableName Identifier | 'TABLE' 'LOCK' TableNameList ) | 'CHECKSUM' 'TABLE' TableNameList | 'CANCEL' 'DDL' 'JOBS' NumList | 'PAUSE' 'DDL' 'JOBS' NumList | 'RESUME' 'DDL' 'JOBS' NumList | 'RELOAD' ( 'EXPR_PUSHDOWN_BLACKLIST' | 'OPT_RULE_BLACKLIST' | 'BINDINGS' ) | 'PLUGINS' ( 'ENABLE' | 'DISABLE' ) PluginNameList | 'REPAIR' 'TABLE' TableName CreateTableStmt | ( 'FLUSH' | 'CAPTURE' | 'EVOLVE' ) 'BINDINGS' ) +AdminPauseDDLStmt ::= + 'ADMIN' 'PAUSE' 'DDL' 'JOBS' NumList NumList ::= Int64Num ( ',' Int64Num )* diff --git a/sql-statements/sql-statement-admin-recover.md b/sql-statements/sql-statement-admin-recover.md index 1bcee6c87b933..552679bab7d83 100644 --- a/sql-statements/sql-statement-admin-recover.md +++ b/sql-statements/sql-statement-admin-recover.md @@ -10,7 +10,7 @@ When the row data and index data are inconsistent, you can use the `ADMIN RECOVE ## Synopsis ```ebnf+diagram -AdminCleanupStmt ::= +AdminRecoverStmt ::= 'ADMIN' 'RECOVER' 'INDEX' TableName IndexName ``` @@ -75,7 +75,7 @@ Query OK, 0 rows affected (0.01 sec) > - There might be a loss of both row data and index data. To address the issue, use the [`ADMIN CLEANUP INDEX`](/sql-statements/sql-statement-admin-cleanup.md) and `ADMIN RECOVER INDEX` statements together to recover the consistency of row data and index data. > - The `ADMIN RECOVER INDEX` statement is always executed in a single thread. When the table data is large, it is recommended to recover the index data by rebuilding the index. > - When you execute the `ADMIN RECOVER INDEX` statement, the corresponding table or index is not locked and TiDB allows other sessions to modify the table records at the same time. However, in this case, `ADMIN RECOVER INDEX` might not be able to handle all table records correctly. Therefore, when you execute `ADMIN RECOVER INDEX`, avoid modifying the table data at the same time. -> - If you use the enterprise edition of TiDB, you can [submit a request](https://support.pingcap.com/hc/en-us) to contact the support engineer for help. +> - If you use the enterprise edition of TiDB, you can [submit a request](https://tidb.support.pingcap.com/) to contact the support engineer for help. > > The `ADMIN RECOVER INDEX` statement is not atomic: if the statement is interrupted during execution, it is recommended to execute it again until it succeeds. diff --git a/sql-statements/sql-statement-admin-resume-ddl.md b/sql-statements/sql-statement-admin-resume-ddl.md index b5a7f65a5e50a..25796070bd635 100644 --- a/sql-statements/sql-statement-admin-resume-ddl.md +++ b/sql-statements/sql-statement-admin-resume-ddl.md @@ -12,8 +12,8 @@ You can use this statement to resume a paused DDL job. After the resume is compl ## Synopsis ```ebnf+diagram -AdminStmt ::= - 'ADMIN' ( 'SHOW' ( 'DDL' ( 'JOBS' Int64Num? WhereClauseOptional | 'JOB' 'QUERIES' NumList )? | TableName 'NEXT_ROW_ID' | 'SLOW' AdminShowSlow ) | 'CHECK' ( 'TABLE' TableNameList | 'INDEX' TableName Identifier ( HandleRange ( ',' HandleRange )* )? ) | 'RECOVER' 'INDEX' TableName Identifier | 'CLEANUP' ( 'INDEX' TableName Identifier | 'TABLE' 'LOCK' TableNameList ) | 'CHECKSUM' 'TABLE' TableNameList | 'CANCEL' 'DDL' 'JOBS' NumList | 'PAUSE' 'DDL' 'JOBS' NumList | 'RESUME' 'DDL' 'JOBS' NumList | 'RELOAD' ( 'EXPR_PUSHDOWN_BLACKLIST' | 'OPT_RULE_BLACKLIST' | 'BINDINGS' ) | 'PLUGINS' ( 'ENABLE' | 'DISABLE' ) PluginNameList | 'REPAIR' 'TABLE' TableName CreateTableStmt | ( 'FLUSH' | 'CAPTURE' | 'EVOLVE' ) 'BINDINGS' ) +AdminResumeDDLStmt ::= + 'ADMIN' 'RESUME' 'DDL' 'JOBS' NumList NumList ::= Int64Num ( ',' Int64Num )* diff --git a/sql-statements/sql-statement-admin-show-ddl.md b/sql-statements/sql-statement-admin-show-ddl.md index 6ba49aad13324..ca63cd5d3b792 100644 --- a/sql-statements/sql-statement-admin-show-ddl.md +++ b/sql-statements/sql-statement-admin-show-ddl.md @@ -10,13 +10,13 @@ The `ADMIN SHOW DDL [JOBS|JOB QUERIES]` statement shows information about runnin ## Synopsis ```ebnf+diagram -AdminStmt ::= +AdminShowDDLStmt ::= 'ADMIN' 'SHOW' 'DDL' ( 'JOBS' Int64Num? WhereClauseOptional | 'JOB' 'QUERIES' NumList | 'JOB' 'QUERIES' 'LIMIT' m ( ('OFFSET' | ',') n )? - ) + )? NumList ::= Int64Num ( ',' Int64Num )* @@ -36,7 +36,7 @@ To view the status of the currently running DDL jobs, use `ADMIN SHOW DDL`. The - `OWNER_ADDRESS`: the IP address of the DDL owner. - `RUNNING_JOBS`: details about the running DDL job. - `SELF_ID`: the UUID of the TiDB node to which you are currently connected. If `SELF_ID` is the same as the `OWNER_ID`, it means that you are connected to the DDL owner. -- `QUERY`: the statement of the query. +- `QUERY`: the statements of the queries. ```sql ADMIN SHOW DDL\G; @@ -80,13 +80,17 @@ The `ADMIN SHOW DDL JOBS` statement is used to view all the results in the curre - `START_TIME`: the start time of the DDL operation. - `END_TIME`: the end time of the DDL operation. - `STATE`: the state of the DDL operation. Common states include the following: - - `queueing`: indicates that the operation job has entered the DDL job queue but has not been executed because it is still waiting for an earlier DDL job to complete. Another reason might be that after executing the `DROP` operation, it will become the `none` state, but it will soon be updated to the `synced` state, indicating that all TiDB instances have been synchronized to that state. + - `none`: indicates that the operation has not started yet. + - `queueing`: indicates that the operation job has entered the DDL job queue but has not been executed because it is still waiting for an earlier DDL job to complete. Another reason might be that after executing the `DROP` operation, the `queueing` state will become the `done` state, but it will soon be updated to the `synced` state, indicating that all TiDB instances have been synchronized to that state. - `running`: indicates that the operation is being executed. - `synced`: indicates that the operation has been executed successfully and all TiDB instances have been synchronized to this state. - `rollback done`: indicates that the operation has failed and the rollback has been completed. - `rollingback`: indicates that the operation has failed and is rolling back. - `cancelling`: indicates that the operation is being canceled. This state only appears when you use the [`ADMIN CANCEL DDL JOBS`](/sql-statements/sql-statement-admin-cancel-ddl.md) command to cancel the DDL job. + - `cancelled`: indicates that the operation has been canceled. + - `pausing`: indicates that the operation is being paused. - `paused`: indicates that the operation has been paused. This state only appears when you use the [`ADMIN PAUSED DDL JOBS`](/sql-statements/sql-statement-admin-pause-ddl.md) command to pause the DDL job. You can use the [`ADMIN RESUME DDL JOBS`](/sql-statements/sql-statement-admin-resume-ddl.md) command to resume the DDL job. + - `done`: indicates that the operation has been successfully executed on the TiDB owner node, but other TiDB nodes have not yet synchronized the changes performed by this DDL job. diff --git a/sql-statements/sql-statement-admin-show-telemetry.md b/sql-statements/sql-statement-admin-show-telemetry.md deleted file mode 100644 index bc2d4c1b08c0b..0000000000000 --- a/sql-statements/sql-statement-admin-show-telemetry.md +++ /dev/null @@ -1,435 +0,0 @@ ---- -title: ADMIN SHOW TELEMETRY | TiDB SQL Statement Reference -summary: An overview of the usage of ADMIN SHOW TELEMETRY for the TiDB database. ---- - -# ADMIN SHOW TELEMETRY - -The `ADMIN SHOW TELEMETRY` statement shows the information that will be reported back to PingCAP as part of the [telemetry](/telemetry.md) feature. - -> **Note:** -> -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). - -## Synopsis - -```ebnf+diagram -AdminStmt ::= - 'ADMIN' ( 'SHOW' ( 'DDL' ( 'JOBS' Int64Num? WhereClauseOptional | 'JOB' 'QUERIES' NumList )? | TableName 'NEXT_ROW_ID' | 'SLOW' AdminShowSlow | 'TELEMETRY' ) | 'CHECK' ( 'TABLE' TableNameList | 'INDEX' TableName Identifier ( HandleRange ( ',' HandleRange )* )? ) | 'RECOVER' 'INDEX' TableName Identifier | 'CLEANUP' ( 'INDEX' TableName Identifier | 'TABLE' 'LOCK' TableNameList ) | 'CHECKSUM' 'TABLE' TableNameList | 'CANCEL' 'DDL' 'JOBS' NumList | 'RELOAD' ( 'EXPR_PUSHDOWN_BLACKLIST' | 'OPT_RULE_BLACKLIST' | 'BINDINGS' ) | 'PLUGINS' ( 'ENABLE' | 'DISABLE' ) PluginNameList | 'REPAIR' 'TABLE' TableName CreateTableStmt | ( 'FLUSH' | 'CAPTURE' | 'EVOLVE' ) 'BINDINGS' ) - -``` - -## Examples - -{{< copyable "sql" >}} - -```sql -ADMIN SHOW TELEMETRY\G -``` - -```sql -*************************** 1. row *************************** - TRACKING_ID: a1ba1d97-b940-4d5b-a9d5-ddb0f2ac29e7 - LAST_STATUS: { - "check_at": "2021-08-11T08:23:38+02:00", - "is_error": false, - "error_msg": "", - "is_request_sent": true -} -DATA_PREVIEW: { - "hardware": [ - { - "instanceType": "tidb", - "listenHostHash": "4b84b15bff6ee5796152495a230e45e3d7e947d9", - "listenPort": "4000", - "cpu": { - "cache": "8192", - "cpuFrequency": "2301.00MHz", - "cpuLogicalCores": "8", - "cpuPhysicalCores": "4" - }, - "memory": { - "capacity": "16410021888" - }, - "disk": { - "ebbca862689fa9fef7c55c3112e375c4ce575fe4": { - "deviceName": "ebbca862689fa9fef7c55c3112e375c4ce575fe4", - "free": "624438726656", - "freePercent": "0.61", - "fstype": "btrfs", - "opts": "bind,rw,relatime", - "path": "fb365c1216b59e1cfc86950425867007a60f4435", - "total": "1022488477696", - "used": "397115568128", - "usedPercent": "0.39" - }, - "nvme0n1p1": { - "deviceName": "nvme0n1p1", - "free": "582250496", - "freePercent": "0.93", - "fstype": "vfat", - "opts": "rw,relatime", - "path": "0fc8c8d71702d81a02e216fb6ef19f4dda4973df", - "total": "627900416", - "used": "45649920", - "usedPercent": "0.07" - }, - "nvme0n1p2": { - "deviceName": "nvme0n1p2", - "free": "701976576", - "freePercent": "0.74", - "fstype": "ext4", - "opts": "rw,relatime", - "path": "/boot", - "total": "1023303680", - "used": "250863616", - "usedPercent": "0.26" - } - } - }, - { - "instanceType": "pd", - "listenHostHash": "4b84b15bff6ee5796152495a230e45e3d7e947d9", - "listenPort": "2379", - "cpu": { - "cache": "8192", - "cpuFrequency": "2301.00MHz", - "cpuLogicalCores": "8", - "cpuPhysicalCores": "4" - }, - "memory": { - "capacity": "16410021888" - }, - "disk": { - "ebbca862689fa9fef7c55c3112e375c4ce575fe4": { - "deviceName": "ebbca862689fa9fef7c55c3112e375c4ce575fe4", - "free": "624438726656", - "freePercent": "0.61", - "fstype": "btrfs", - "opts": "bind,rw,relatime", - "path": "fb365c1216b59e1cfc86950425867007a60f4435", - "total": "1022488477696", - "used": "397115568128", - "usedPercent": "0.39" - }, - "nvme0n1p1": { - "deviceName": "nvme0n1p1", - "free": "582250496", - "freePercent": "0.93", - "fstype": "vfat", - "opts": "rw,relatime", - "path": "0fc8c8d71702d81a02e216fb6ef19f4dda4973df", - "total": "627900416", - "used": "45649920", - "usedPercent": "0.07" - }, - "nvme0n1p2": { - "deviceName": "nvme0n1p2", - "free": "701976576", - "freePercent": "0.74", - "fstype": "ext4", - "opts": "rw,relatime", - "path": "/boot", - "total": "1023303680", - "used": "250863616", - "usedPercent": "0.26" - } - } - }, - { - "instanceType": "tikv", - "listenHostHash": "4b84b15bff6ee5796152495a230e45e3d7e947d9", - "listenPort": "20160", - "cpu": { - "cpuFrequency": "3730MHz", - "cpuLogicalCores": "8", - "cpuPhysicalCores": "4", - "cpuVendorId": "GenuineIntel", - "l1CacheLineSize": "64", - "l1CacheSize": "32768", - "l2CacheLineSize": "64", - "l2CacheSize": "262144", - "l3CacheLineSize": "64", - "l3CacheSize": "8388608" - }, - "memory": { - "capacity": "16803861504" - }, - "disk": { - "36e7dfacbb83843f83075d78aeb4cf850a4882a1": { - "deviceName": "36e7dfacbb83843f83075d78aeb4cf850a4882a1", - "free": "624438726656", - "freePercent": "0.61", - "fstype": "btrfs", - "path": "fb365c1216b59e1cfc86950425867007a60f4435", - "total": "1022488477696", - "used": "398049751040", - "usedPercent": "0.39" - }, - "nvme0n1p1": { - "deviceName": "nvme0n1p1", - "free": "582250496", - "freePercent": "0.93", - "fstype": "vfat", - "path": "0fc8c8d71702d81a02e216fb6ef19f4dda4973df", - "total": "627900416", - "used": "45649920", - "usedPercent": "0.07" - }, - "nvme0n1p2": { - "deviceName": "nvme0n1p2", - "free": "701976576", - "freePercent": "0.69", - "fstype": "ext4", - "path": "/boot", - "total": "1023303680", - "used": "321327104", - "usedPercent": "0.31" - } - } - }, - { - "instanceType": "tiflash", - "listenHostHash": "4b84b15bff6ee5796152495a230e45e3d7e947d9", - "listenPort": "3930", - "cpu": { - "cpuFrequency": "3400MHz", - "cpuLogicalCores": "8", - "cpuPhysicalCores": "4", - "l1CacheLineSize": "64", - "l1CacheSize": "32768", - "l2CacheLineSize": "64", - "l2CacheSize": "262144", - "l3CacheLineSize": "64", - "l3CacheSize": "8388608" - }, - "memory": { - "capacity": "16410021888" - }, - "disk": { - "36e7dfacbb83843f83075d78aeb4cf850a4882a1": { - "deviceName": "36e7dfacbb83843f83075d78aeb4cf850a4882a1", - "free": "624438726656", - "freePercent": "0.61", - "fstype": "btrfs", - "path": "fb365c1216b59e1cfc86950425867007a60f4435", - "total": "1022488477696", - "used": "398049751040", - "usedPercent": "0.39" - } - } - } - ], - "instances": [ - { - "instanceType": "tidb", - "listenHostHash": "4b84b15bff6ee5796152495a230e45e3d7e947d9", - "listenPort": "4000", - "statusHostHash": "4b84b15bff6ee5796152495a230e45e3d7e947d9", - "statusPort": "10080", - "version": "5.1.1", - "gitHash": "797bddd25310ed42f0791c8eccb78be8cce2f502", - "startTime": "2021-08-11T08:23:38+02:00", - "upTime": "22.210217487s" - }, - { - "instanceType": "pd", - "listenHostHash": "4b84b15bff6ee5796152495a230e45e3d7e947d9", - "listenPort": "2379", - "statusHostHash": "4b84b15bff6ee5796152495a230e45e3d7e947d9", - "statusPort": "2379", - "version": "5.1.1", - "gitHash": "7cba1912b317a533e18b16ea2ba9a14ed2891129", - "startTime": "2021-08-11T08:23:32+02:00", - "upTime": "28.210220368s" - }, - { - "instanceType": "tikv", - "listenHostHash": "4b84b15bff6ee5796152495a230e45e3d7e947d9", - "listenPort": "20160", - "statusHostHash": "4b84b15bff6ee5796152495a230e45e3d7e947d9", - "statusPort": "20180", - "version": "5.1.1", - "gitHash": "4705d7c6e9c42d129d3309e05911ec6b08a25a38", - "startTime": "2021-08-11T08:23:33+02:00", - "upTime": "27.210221447s" - }, - { - "instanceType": "tiflash", - "listenHostHash": "4b84b15bff6ee5796152495a230e45e3d7e947d9", - "listenPort": "3930", - "statusHostHash": "4b84b15bff6ee5796152495a230e45e3d7e947d9", - "statusPort": "20292", - "version": "v5.1.1", - "gitHash": "c8fabfb50fe28db17cc5118133a69be255c40efd", - "startTime": "2021-08-11T08:23:40+02:00", - "upTime": "20.210222452s" - } - ], - "hostExtra": { - "cpuFlags": [ - "fpu", - "vme", - "de", - "pse", - "tsc", - "msr", - "pae", - "mce", - "cx8", - "apic", - "sep", - "mtrr", - "pge", - "mca", - "cmov", - "pat", - "pse36", - "clflush", - "dts", - "acpi", - "mmx", - "fxsr", - "sse", - "sse2", - "ss", - "ht", - "tm", - "pbe", - "syscall", - "nx", - "pdpe1gb", - "rdtscp", - "lm", - "constant_tsc", - "art", - "arch_perfmon", - "pebs", - "bts", - "rep_good", - "nopl", - "xtopology", - "nonstop_tsc", - "cpuid", - "aperfmperf", - "pni", - "pclmulqdq", - "dtes64", - "monitor", - "ds_cpl", - "vmx", - "est", - "tm2", - "ssse3", - "sdbg", - "fma", - "cx16", - "xtpr", - "pdcm", - "pcid", - "sse4_1", - "sse4_2", - "x2apic", - "movbe", - "popcnt", - "tsc_deadline_timer", - "aes", - "xsave", - "avx", - "f16c", - "rdrand", - "lahf_lm", - "abm", - "3dnowprefetch", - "cpuid_fault", - "epb", - "invpcid_single", - "ssbd", - "ibrs", - "ibpb", - "stibp", - "ibrs_enhanced", - "tpr_shadow", - "vnmi", - "flexpriority", - "ept", - "vpid", - "ept_ad", - "fsgsbase", - "tsc_adjust", - "sgx", - "bmi1", - "avx2", - "smep", - "bmi2", - "erms", - "invpcid", - "mpx", - "rdseed", - "adx", - "smap", - "clflushopt", - "intel_pt", - "xsaveopt", - "xsavec", - "xgetbv1", - "xsaves", - "dtherm", - "ida", - "arat", - "pln", - "pts", - "hwp", - "hwp_notify", - "hwp_act_window", - "hwp_epp", - "md_clear", - "flush_l1d", - "arch_capabilities" - ], - "cpuModelName": "Intel(R) Core(TM) i7-10510U CPU @ 1.80GHz", - "os": "linux", - "platform": "fedora", - "platformFamily": "fedora", - "platformVersion": "34", - "kernelVersion": "5.13.5-200.fc34.x86_64", - "kernelArch": "x86_64", - "virtualizationSystem": "kvm", - "virtualizationRole": "host" - }, - "reportTimestamp": 1628663040, - "trackingId": "a1ba1d97-b940-4d5b-a9d5-ddb0f2ac29e7", - "featureUsage": { - "txn": { - "asyncCommitUsed": true, - "onePCUsed": true, - "txnCommitCounter": { - "twoPC": 9, - "asyncCommit": 0, - "onePC": 0 - } - }, - "clusterIndex": {}, - "temporaryTable": false, - "cte": { - "nonRecursiveCTEUsed": 0, - "recursiveUsed": 0, - "nonCTEUsed": 13 - } - }, - "windowedStats": [], - "slowQueryStats": { - "slowQueryBucket": {} - } -} -1 row in set (0.0259 sec) -``` - -## MySQL compatibility - -The `ADMIN` statement is a TiDB extension to MySQL syntax. - -## See also - -* [Telemetry](/telemetry.md) -* [`tidb_enable_telemetry` System Variable](/system-variables.md#tidb_enable_telemetry-new-in-v402) diff --git a/sql-statements/sql-statement-admin.md b/sql-statements/sql-statement-admin.md index 155f4caf5637d..dc9d69c469ad4 100644 --- a/sql-statements/sql-statement-admin.md +++ b/sql-statements/sql-statement-admin.md @@ -1,7 +1,6 @@ --- title: ADMIN | TiDB SQL Statement Reference summary: An overview of the usage of ADMIN for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-admin/','/docs/dev/reference/sql/statements/admin/'] --- # ADMIN @@ -38,22 +37,17 @@ This statement is a TiDB extension syntax, used to view the status of TiDB and c | [`ADMIN CHECKSUM TABLE`](/sql-statements/sql-statement-admin-checksum-table.md) | Calculates the CRC64 of all rows + indexes of a table. | | [ADMIN CHECK [TABLE\|INDEX]](/sql-statements/sql-statement-admin-check-table-index.md) | Checks for consistency of a table or index. | | [ADMIN SHOW DDL [JOBS\|QUERIES]](/sql-statements/sql-statement-admin-show-ddl.md) | Shows details about currently running or recently completed DDL jobs. | -| [`ADMIN SHOW TELEMETRY`](/sql-statements/sql-statement-admin-show-telemetry.md) | Shows information that will be reported back to PingCAP as part of the telemetry feature. | ## `ADMIN RELOAD` statement -{{< copyable "sql" >}} - ```sql ADMIN RELOAD expr_pushdown_blacklist; ``` The above statement is used to reload the blocklist pushed down by the expression. -{{< copyable "sql" >}} - ```sql ADMIN RELOAD opt_rule_blacklist; ``` @@ -64,7 +58,7 @@ The above statement is used to reload the blocklist of logic optimization rules. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ```sql ADMIN PLUGINS ENABLE plugin_name [, plugin_name] ...; @@ -80,32 +74,24 @@ The above statement is used to disable the `plugin_name` plugin. ## `ADMIN BINDINGS` related statement -{{< copyable "sql" >}} - ```sql ADMIN FLUSH BINDINGS; ``` The above statement is used to persist SQL Plan binding information. -{{< copyable "sql" >}} - ```sql ADMIN CAPTURE BINDINGS; ``` The above statement can generate the binding of SQL Plan from the `SELECT` statement that occurs more than once. -{{< copyable "sql" >}} - ```sql ADMIN EVOLVE BINDINGS; ``` After the automatic binding feature is enabled, the evolution of SQL Plan binding information is triggered every `bind-info-leave` (the default value is `3s`). The above statement is used to proactively trigger this evolution. -{{< copyable "sql" >}} - ```sql ADMIN RELOAD BINDINGS; ``` @@ -124,8 +110,6 @@ The above statement is used to reload SQL Plan binding information. To overwrite the metadata of the stored table in an untrusted way in extreme cases, use `ADMIN REPAIR TABLE`: -{{< copyable "sql" >}} - ```sql ADMIN REPAIR TABLE tbl_name CREATE TABLE STATEMENT; ``` @@ -148,7 +132,7 @@ The above statement is used to view the details of some special columns of a tab > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ```sql ADMIN SHOW SLOW RECENT N; @@ -168,15 +152,57 @@ For details, refer to [`ADMIN SHOW SLOW` command](/identify-slow-queries.md#admi ```ebnf+diagram AdminStmt ::= - 'ADMIN' ( 'SHOW' ( 'DDL' ( 'JOBS' Int64Num? WhereClauseOptional | 'JOB' 'QUERIES' NumList )? | TableName 'NEXT_ROW_ID' | 'SLOW' AdminShowSlow ) | 'CHECK' ( 'TABLE' TableNameList | 'INDEX' TableName Identifier ( HandleRange ( ',' HandleRange )* )? ) | 'RECOVER' 'INDEX' TableName Identifier | 'CLEANUP' ( 'INDEX' TableName Identifier | 'TABLE' 'LOCK' TableNameList ) | 'CHECKSUM' 'TABLE' TableNameList | 'CANCEL' 'DDL' 'JOBS' NumList | 'RELOAD' ( 'EXPR_PUSHDOWN_BLACKLIST' | 'OPT_RULE_BLACKLIST' | 'BINDINGS' ) | 'PLUGINS' ( 'ENABLE' | 'DISABLE' ) PluginNameList | 'REPAIR' 'TABLE' TableName CreateTableStmt | ( 'FLUSH' | 'CAPTURE' | 'EVOLVE' ) 'BINDINGS' ) + 'ADMIN' ( + 'SHOW' ( + 'DDL' ( + 'JOBS' Int64Num? WhereClauseOptional + | 'JOB' 'QUERIES' (NumList | AdminStmtLimitOpt) + )? + | TableName 'NEXT_ROW_ID' + | 'SLOW' AdminShowSlow + | 'BDR' 'ROLE' + ) + | 'CHECK' ( + 'TABLE' TableNameList + | 'INDEX' TableName Identifier ( HandleRange ( ',' HandleRange )* )? + ) + | 'RECOVER' 'INDEX' TableName Identifier + | 'CLEANUP' ( + 'INDEX' TableName Identifier + | 'TABLE' 'LOCK' TableNameList ) + | 'CHECKSUM' 'TABLE' TableNameList | 'CANCEL' 'DDL' 'JOBS' NumList + | ( 'CANCEL' | 'PAUSE' | 'RESUME' ) 'DDL' 'JOBS' NumList + | 'RELOAD' ( + 'EXPR_PUSHDOWN_BLACKLIST' + | 'OPT_RULE_BLACKLIST' + | 'BINDINGS' + | 'STATS_EXTENDED' + | 'STATISTICS' + ) + | 'PLUGINS' ( 'ENABLE' | 'DISABLE' ) PluginNameList + | 'REPAIR' 'TABLE' TableName CreateTableStmt + | ( 'FLUSH' | 'CAPTURE' | 'EVOLVE' ) 'BINDINGS' + | 'FLUSH' ('SESSION' | 'INSTANCE') 'PLAN_CACHE' + | 'SET' 'BDR' 'ROLE' ( 'PRIMARY' | 'SECONDARY' ) + | 'UNSET' 'BDR' 'ROLE' + ) + +NumList ::= + Int64Num ( ',' Int64Num )* + +AdminStmtLimitOpt ::= + 'LIMIT' LengthNum +| 'LIMIT' LengthNum ',' LengthNum +| 'LIMIT' LengthNum 'OFFSET' LengthNum + +TableNameList ::= + TableName ( ',' TableName )* ``` ## Examples Run the following command to view the last 10 completed DDL jobs in the currently running DDL job queue. When `NUM` is not specified, only the last 10 completed DDL jobs is presented by default. -{{< copyable "sql" >}} - ```sql ADMIN SHOW DDL JOBS; ``` @@ -201,8 +227,6 @@ ADMIN SHOW DDL JOBS; Run the following command to view the last 5 completed DDL jobs in the currently running DDL job queue: -{{< copyable "sql" >}} - ```sql ADMIN SHOW DDL JOBS 5; ``` @@ -238,8 +262,6 @@ ADMIN SHOW t NEXT_ROW_ID; Run the following command to view the uncompleted DDL jobs in the test database. The results include the DDL jobs that are running and the last 5 DDL jobs that are completed but failed. -{{< copyable "sql" >}} - ```sql ADMIN SHOW DDL JOBS 5 WHERE state != 'synced' AND db_name = 'test'; ``` diff --git a/sql-statements/sql-statement-alter-database.md b/sql-statements/sql-statement-alter-database.md index c58b33a01c08f..d65ed2df8e1a7 100644 --- a/sql-statements/sql-statement-alter-database.md +++ b/sql-statements/sql-statement-alter-database.md @@ -1,7 +1,6 @@ --- title: ALTER DATABASE | TiDB SQL Statement Reference summary: An overview of the usage of ALTER DATABASE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-alter-database/','/docs/dev/reference/sql/statements/alter-database/'] --- # ALTER DATABASE diff --git a/sql-statements/sql-statement-alter-index.md b/sql-statements/sql-statement-alter-index.md index d060bc1a31a1c..f230d75a34ce2 100644 --- a/sql-statements/sql-statement-alter-index.md +++ b/sql-statements/sql-statement-alter-index.md @@ -1,7 +1,6 @@ --- title: ALTER INDEX summary: An overview of the usage of ALTER INDEX for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-alter-index/'] --- # ALTER INDEX diff --git a/sql-statements/sql-statement-alter-instance.md b/sql-statements/sql-statement-alter-instance.md index c4a5a776ea919..31b5ddc8bd698 100644 --- a/sql-statements/sql-statement-alter-instance.md +++ b/sql-statements/sql-statement-alter-instance.md @@ -1,7 +1,6 @@ --- title: ALTER INSTANCE summary: Learn the overview of the `ALTER INSTANCE` usage in TiDB. -aliases: ['/docs/dev/sql-statements/sql-statement-alter-instance/','/docs/dev/reference/sql/statements/alter-instance/'] --- # ALTER INSTANCE @@ -10,7 +9,7 @@ The `ALTER INSTANCE` statement is used to make changes to a single TiDB instance > **Note:** > -> [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) can automatically refresh the TLS certificate, so this feature is not applicable to [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) can automatically refresh the TLS certificate, so this feature is not applicable to [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## RELOAD TLS diff --git a/sql-statements/sql-statement-alter-placement-policy.md b/sql-statements/sql-statement-alter-placement-policy.md index 2f140911c5bf6..2c00954eb58b0 100644 --- a/sql-statements/sql-statement-alter-placement-policy.md +++ b/sql-statements/sql-statement-alter-placement-policy.md @@ -9,7 +9,7 @@ summary: The usage of ALTER PLACEMENT POLICY in TiDB. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. `ALTER PLACEMENT POLICY` _replaces_ the previous policy with the new definition. It does not _merge_ the old policy with the new one. In the following example, `FOLLOWERS=4` is lost when the `ALTER PLACEMENT POLICY` is executed: diff --git a/sql-statements/sql-statement-alter-range.md b/sql-statements/sql-statement-alter-range.md index 50e1fcdc5786a..92a829ccb928e 100644 --- a/sql-statements/sql-statement-alter-range.md +++ b/sql-statements/sql-statement-alter-range.md @@ -9,7 +9,7 @@ Currently, the `ALTER RANGE` statement can only be used to modify the range of a > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-alter-resource-group.md b/sql-statements/sql-statement-alter-resource-group.md index 25fb54aefdf79..8b2f521c9b1ff 100644 --- a/sql-statements/sql-statement-alter-resource-group.md +++ b/sql-statements/sql-statement-alter-resource-group.md @@ -9,7 +9,7 @@ The `ALTER RESOURCE GROUP` statement is used to modify a resource group in a dat > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-alter-table.md b/sql-statements/sql-statement-alter-table.md index a3b28c54cc46c..92078e4ddfb21 100644 --- a/sql-statements/sql-statement-alter-table.md +++ b/sql-statements/sql-statement-alter-table.md @@ -1,7 +1,6 @@ --- title: ALTER TABLE | TiDB SQL Statement Reference summary: An overview of the usage of ALTER TABLE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-alter-table/','/docs/dev/reference/sql/statements/alter-table/'] --- # ALTER TABLE @@ -39,7 +38,7 @@ AlterTableSpec ::= | 'ORDER' 'BY' AlterOrderItem ( ',' AlterOrderItem )* | ( 'DISABLE' | 'ENABLE' ) 'KEYS' | ( 'MODIFY' ColumnKeywordOpt IfExists | 'CHANGE' ColumnKeywordOpt IfExists ColumnName ) ColumnDef ColumnPosition -| 'ALTER' ( ColumnKeywordOpt ColumnName ( 'SET' 'DEFAULT' ( SignedLiteral | '(' Expression ')' ) | 'DROP' 'DEFAULT' ) | 'CHECK' Identifier EnforcedOrNot | 'INDEX' Identifier IndexInvisible ) +| 'ALTER' ( ColumnKeywordOpt ColumnName ( 'SET' 'DEFAULT' ( SignedLiteral | '(' Expression ')' ) | 'DROP' 'DEFAULT' ) | 'CHECK' Identifier EnforcedOrNot | 'INDEX' Identifier ("VISIBLE" | "INVISIBLE") ) | 'RENAME' ( ( 'COLUMN' | KeyOrIndex ) Identifier 'TO' Identifier | ( 'TO' | '='? | 'AS' ) TableName ) | LockClause | AlgorithmClause diff --git a/sql-statements/sql-statement-alter-user.md b/sql-statements/sql-statement-alter-user.md index 3adddf1d8af80..c3ce7145c0350 100644 --- a/sql-statements/sql-statement-alter-user.md +++ b/sql-statements/sql-statement-alter-user.md @@ -1,7 +1,6 @@ --- title: ALTER USER | TiDB SQL Statement Reference summary: An overview of the usage of ALTER USER for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-alter-user/','/docs/dev/reference/sql/statements/alter-user/'] --- # ALTER USER diff --git a/sql-statements/sql-statement-analyze-table.md b/sql-statements/sql-statement-analyze-table.md index 1c1f1e469a4fe..b277481c3f634 100644 --- a/sql-statements/sql-statement-analyze-table.md +++ b/sql-statements/sql-statement-analyze-table.md @@ -1,7 +1,6 @@ --- title: ANALYZE | TiDB SQL Statement Reference summary: An overview of the usage of ANALYZE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-analyze-table/','/docs/dev/reference/sql/statements/analyze-table/'] --- # ANALYZE @@ -16,7 +15,7 @@ Currently, TiDB collects statistical information as a full collection by using t ```ebnf+diagram AnalyzeTableStmt ::= - 'ANALYZE' ( 'TABLE' ( TableNameList ( 'ALL COLUMNS' | 'PREDICATE COLUMNS' ) | TableName ( 'INDEX' IndexNameList? | AnalyzeColumnOption | 'PARTITION' PartitionNameList ( 'INDEX' IndexNameList? | AnalyzeColumnOption )? )? ) 'INDEX' IndexNameList? ) AnalyzeOptionListOpt + 'ANALYZE' ( 'TABLE' ( TableNameList ( 'ALL COLUMNS' | 'PREDICATE COLUMNS' ) | TableName ( 'INDEX' IndexNameList? | AnalyzeColumnOption | 'PARTITION' PartitionNameList ( 'INDEX' IndexNameList? | AnalyzeColumnOption )? )? ) ) AnalyzeOptionListOpt AnalyzeOptionListOpt ::= ( WITH AnalyzeOptionList )? diff --git a/sql-statements/sql-statement-backup.md b/sql-statements/sql-statement-backup.md index 612b4a3d57b1b..f7316eb078f1b 100644 --- a/sql-statements/sql-statement-backup.md +++ b/sql-statements/sql-statement-backup.md @@ -1,7 +1,6 @@ --- title: BACKUP | TiDB SQL Statement Reference summary: An overview of the usage of BACKUP for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-backup/'] --- # BACKUP @@ -11,7 +10,7 @@ This statement is used to perform a distributed backup of the TiDB cluster. > **Warning:** > > - This feature is experimental. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. -> - This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> - This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. The `BACKUP` statement uses the same engine as the [BR tool](https://docs.pingcap.com/tidb/stable/backup-and-restore-overview) does, except that the backup process is driven by TiDB itself rather than a separate BR tool. All benefits and warnings of BR also apply to this statement. diff --git a/sql-statements/sql-statement-begin.md b/sql-statements/sql-statement-begin.md index 15ef1c9d3e771..ad78ec7921e0c 100644 --- a/sql-statements/sql-statement-begin.md +++ b/sql-statements/sql-statement-begin.md @@ -1,7 +1,6 @@ --- title: BEGIN | TiDB SQL Statement Reference summary: An overview of the usage of BEGIN for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-begin/','/docs/dev/reference/sql/statements/begin/'] --- # BEGIN diff --git a/sql-statements/sql-statement-calibrate-resource.md b/sql-statements/sql-statement-calibrate-resource.md index db8c8af8e7a1b..2081409d55aa4 100644 --- a/sql-statements/sql-statement-calibrate-resource.md +++ b/sql-statements/sql-statement-calibrate-resource.md @@ -9,7 +9,7 @@ The `CALIBRATE RESOURCE` statement is used to estimate and output the ['Request > **Note:** > -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This feature is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). ## Synopsis @@ -28,7 +28,7 @@ To execute this command, make sure that the following requirements are met: - You have enabled [`tidb_enable_resource_control`](/system-variables.md#tidb_enable_resource_control-new-in-v660). - The user has `SUPER` or `RESOURCE_GROUP_ADMIN` privilege. -- The user has the `SELECT` privilege for all tables in the `METRICS_SCHEMA` schema. +- To [estimate capacity based on actual workload](#estimate-capacity-based-on-actual-workload), the user must have the `SELECT` privilege for all tables in the `METRICS_SCHEMA` schema. ## Methods for estimating capacity diff --git a/sql-statements/sql-statement-cancel-import-job.md b/sql-statements/sql-statement-cancel-import-job.md index bde048adbb8a5..d4b911d846384 100644 --- a/sql-statements/sql-statement-cancel-import-job.md +++ b/sql-statements/sql-statement-cancel-import-job.md @@ -7,10 +7,6 @@ summary: An overview of the usage of CANCEL IMPORT in TiDB. The `CANCEL IMPORT` statement is used to cancel a data import job created in TiDB. -> **Note:** -> -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. - ## Required privileges To cancel a data import job, you need to be the creator of the import job or have the `SUPER` privilege. diff --git a/sql-statements/sql-statement-change-column.md b/sql-statements/sql-statement-change-column.md index 41d857ad79bfe..5edc05af5129d 100644 --- a/sql-statements/sql-statement-change-column.md +++ b/sql-statements/sql-statement-change-column.md @@ -1,7 +1,6 @@ --- title: CHANGE COLUMN | TiDB SQL Statement Reference summary: An overview of the usage of CHANGE COLUMN for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-change-column/','/docs/dev/reference/sql/statements/change-column/'] --- # CHANGE COLUMN @@ -152,7 +151,7 @@ ERROR 8200 (HY000): Unsupported modify column: change from original type decimal * Changes of [Reorg-Data](/sql-statements/sql-statement-modify-column.md#reorg-data-change) types on primary key columns are not supported. * Changes of column types on partitioned tables are not supported. * Changes of column types on generated columns are not supported. -* Changes of some data types (for example, some TIME, Bit, Set, Enum, and JSON types) are not supported due to the compatibility issues of the `CAST` function's behavior between TiDB and MySQL. +* Changes from some data types (for example, TIME, BIT, SET, ENUM, and JSON types) to some other types are not supported due to the compatibility issues of the `CAST` function's behavior between TiDB and MySQL. ## See also diff --git a/sql-statements/sql-statement-change-drainer.md b/sql-statements/sql-statement-change-drainer.md index 23562f711c42e..7a2887f6e2e3c 100644 --- a/sql-statements/sql-statement-change-drainer.md +++ b/sql-statements/sql-statement-change-drainer.md @@ -1,7 +1,6 @@ --- title: CHANGE DRAINER summary: An overview of the usage of CHANGE DRAINER for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-change-drainer/'] --- # CHANGE DRAINER @@ -10,7 +9,7 @@ The `CHANGE DRAINER` statement modifies the status information for Drainer in th > **Note:** > -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This feature is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). > **Tip:** > diff --git a/sql-statements/sql-statement-change-pump.md b/sql-statements/sql-statement-change-pump.md index 2b2fe66db3260..e76276c35a104 100644 --- a/sql-statements/sql-statement-change-pump.md +++ b/sql-statements/sql-statement-change-pump.md @@ -1,7 +1,6 @@ --- title: CHANGE PUMP summary: An overview of the usage of CHANGE PUMP for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-change-pump/'] --- # CHANGE PUMP @@ -10,7 +9,7 @@ The `CHANGE PUMP` statement modifies the status information for Pump in the clus > **Note:** > -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This feature is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). > **Tip:** > diff --git a/sql-statements/sql-statement-commit.md b/sql-statements/sql-statement-commit.md index 8543ab83a54bf..eb5279d8bf62b 100644 --- a/sql-statements/sql-statement-commit.md +++ b/sql-statements/sql-statement-commit.md @@ -1,12 +1,11 @@ --- title: COMMIT | TiDB SQL Statement Reference summary: An overview of the usage of COMMIT for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-commit/','/docs/dev/reference/sql/statements/commit/'] --- # COMMIT -This statement commits a transaction inside of the TIDB server. +This statement commits a transaction inside of the TiDB server. In the absence of a `BEGIN` or `START TRANSACTION` statement, the default behavior of TiDB is that every statement will be its own transaction and autocommit. This behavior ensures MySQL compatibility. @@ -41,7 +40,7 @@ Query OK, 0 rows affected (0.01 sec) * Currently, TiDB use Metadata Locking (MDL) to prevent DDL statements from modifying tables used by transactions by default. The behavior of metadata lock is different between TiDB and MySQL. For more details, see [Metadata Lock](/metadata-lock.md). * By default, TiDB 3.0.8 and later versions use [Pessimistic Locking](/pessimistic-transaction.md). When using [Optimistic Locking](/optimistic-transaction.md), it is important to consider that a `COMMIT` statement might fail because rows have been modified by another transaction. -* When Optimistic Locking is enabled, `UNIQUE` and `PRIMARY KEY` constraint checks are deferred until statement commit. This results in additional situations where a a `COMMIT` statement might fail. This behavior can be changed by setting `tidb_constraint_check_in_place=ON`. +* When Optimistic Locking is enabled, `UNIQUE` and `PRIMARY KEY` constraint checks are deferred until statement commit. This results in additional situations where a `COMMIT` statement might fail. This behavior can be changed by setting `tidb_constraint_check_in_place=ON`. * TiDB parses but ignores the syntax `ROLLBACK AND [NO] RELEASE`. This functionality is used in MySQL to disconnect the client session immediately after committing the transaction. In TiDB, it is recommended to instead use the `mysql_close()` functionality of your client driver. * TiDB parses but ignores the syntax `ROLLBACK AND [NO] CHAIN`. This functionality is used in MySQL to immediately start a new transaction with the same isolation level while the current transaction is being committed. In TiDB, it is recommended to instead start a new transaction. diff --git a/sql-statements/sql-statement-create-binding.md b/sql-statements/sql-statement-create-binding.md index fcf3dc8ce70d8..ace22fde975ce 100644 --- a/sql-statements/sql-statement-create-binding.md +++ b/sql-statements/sql-statement-create-binding.md @@ -1,7 +1,6 @@ --- title: CREATE [GLOBAL|SESSION] BINDING summary: Use of CREATE BINDING in TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-create-binding/'] --- # CREATE [GLOBAL|SESSION] BINDING diff --git a/sql-statements/sql-statement-create-database.md b/sql-statements/sql-statement-create-database.md index 72c489bd8c8e2..1e27434343bca 100644 --- a/sql-statements/sql-statement-create-database.md +++ b/sql-statements/sql-statement-create-database.md @@ -1,7 +1,6 @@ --- title: CREATE DATABASE | TiDB SQL Statement Reference summary: An overview of the usage of CREATE DATABASE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-create-database/','/docs/dev/reference/sql/statements/create-database/'] --- # CREATE DATABASE diff --git a/sql-statements/sql-statement-create-index.md b/sql-statements/sql-statement-create-index.md index 91ca862705fdb..e776037a60afc 100644 --- a/sql-statements/sql-statement-create-index.md +++ b/sql-statements/sql-statement-create-index.md @@ -1,7 +1,6 @@ --- title: CREATE INDEX | TiDB SQL Statement Reference summary: An overview of the usage of CREATE INDEX for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-create-index/','/docs/dev/reference/sql/statements/create-index/'] --- # CREATE INDEX @@ -43,7 +42,7 @@ IndexOption ::= | IndexType | 'WITH' 'PARSER' Identifier | 'COMMENT' stringLit -| IndexInvisible +| ("VISIBLE" | "INVISIBLE") IndexTypeName ::= 'BTREE' @@ -138,11 +137,37 @@ You can drop an expression index in the same way as dropping an ordinary index: DROP INDEX idx1 ON t1; ``` -Expression index involves various kinds of expressions. To ensure correctness, only some fully tested functions are allowed for creating an expression index. This means that only these functions are allowed in expressions in a production environment. You can get these functions by querying the `tidb_allow_function_for_expression_index` variable. Currently, the allowed functions are as follows: - -``` -JSON_ARRAY, JSON_ARRAY_APPEND, JSON_ARRAY_INSERT, JSON_CONTAINS, JSON_CONTAINS_PATH, JSON_DEPTH, JSON_EXTRACT, JSON_INSERT, JSON_KEYS, JSON_LENGTH, JSON_MERGE_PATCH, JSON_MERGE_PRESERVE, JSON_OBJECT, JSON_PRETTY, JSON_QUOTE, JSON_REMOVE, JSON_REPLACE, JSON_SEARCH, JSON_SET, JSON_STORAGE_SIZE, JSON_TYPE, JSON_UNQUOTE, JSON_VALID, LOWER, MD5, REVERSE, TIDB_SHARD, UPPER, VITESS_HASH -``` +Expression index involves various kinds of expressions. To ensure correctness, only some fully tested functions are allowed for creating an expression index. This means that only these functions are allowed in expressions in a production environment. You can get these functions by querying the [`tidb_allow_function_for_expression_index`](/system-variables.md#tidb_allow_function_for_expression_index-new-in-v520) variable. Currently, the allowed functions are as follows: + +- [`JSON_ARRAY()`](/functions-and-operators/json-functions.md) +- [`JSON_ARRAY_APPEND()`](/functions-and-operators/json-functions.md) +- [`JSON_ARRAY_INSERT()`](/functions-and-operators/json-functions.md) +- [`JSON_CONTAINS()`](/functions-and-operators/json-functions.md) +- [`JSON_CONTAINS_PATH()`](/functions-and-operators/json-functions.md) +- [`JSON_DEPTH()`](/functions-and-operators/json-functions.md) +- [`JSON_EXTRACT()`](/functions-and-operators/json-functions.md) +- [`JSON_INSERT()`](/functions-and-operators/json-functions.md) +- [`JSON_KEYS()`](/functions-and-operators/json-functions.md) +- [`JSON_LENGTH()`](/functions-and-operators/json-functions.md) +- [`JSON_MERGE_PATCH()`](/functions-and-operators/json-functions.md) +- [`JSON_MERGE_PRESERVE()`](/functions-and-operators/json-functions.md) +- [`JSON_OBJECT()`](/functions-and-operators/json-functions.md) +- [`JSON_PRETTY()`](/functions-and-operators/json-functions.md) +- [`JSON_QUOTE()`](/functions-and-operators/json-functions.md) +- [`JSON_REMOVE()`](/functions-and-operators/json-functions.md) +- [`JSON_REPLACE()`](/functions-and-operators/json-functions.md) +- [`JSON_SEARCH()`](/functions-and-operators/json-functions.md) +- [`JSON_SET()`](/functions-and-operators/json-functions.md) +- [`JSON_STORAGE_SIZE()`](/functions-and-operators/json-functions.md) +- [`JSON_TYPE()`](/functions-and-operators/json-functions.md) +- [`JSON_UNQUOTE()`](/functions-and-operators/json-functions.md) +- [`JSON_VALID()`](/functions-and-operators/json-functions.md) +- [`LOWER()`](/functions-and-operators/string-functions.md#lower) +- [`MD5()`](/functions-and-operators/encryption-and-compression-functions.md) +- [`REVERSE()`](/functions-and-operators/string-functions.md#reverse) +- [`TIDB_SHARD()`](/functions-and-operators/tidb-functions.md#tidb_shard) +- [`UPPER()`](/functions-and-operators/string-functions.md#upper) +- [`VITESS_HASH()`](/functions-and-operators/tidb-functions.md) For the functions that are not included in the above list, those functions are not fully tested and not recommended for a production environment, which can be seen as experimental. Other expressions such as operators, `CAST`, and `CASE WHEN` are also seen as experimental and not recommended for production. @@ -232,7 +257,7 @@ Multi-valued indexes are a kind of secondary index defined on an array column. I ### Create multi-valued indexes -You can create multi-valued indexes by using the [`CAST(... AS ... ARRAY)`](/functions-and-operators/cast-functions-and-operators.md) expression in the index definition, as creating an expression index. +You can create multi-valued indexes by using the [`CAST(... AS ... ARRAY)`](/functions-and-operators/cast-functions-and-operators.md#cast) function in the index definition, as creating an expression index. ```sql mysql> CREATE TABLE customers ( @@ -349,7 +374,8 @@ The system variables associated with the `CREATE INDEX` statement are `tidb_ddl_ ## MySQL compatibility -* TiDB supports parsing the `FULLTEXT` and `SPATIAL` syntax but does not support using the `FULLTEXT`, `HASH`, and `SPATIAL` indexes. +* TiDB supports parsing the `FULLTEXT` syntax but does not support using the `FULLTEXT`, `HASH`, and `SPATIAL` indexes. +* TiDB accepts index types such as `HASH`, `BTREE` and `RTREE` in syntax for compatibility with MySQL, but ignores them. * Descending indexes are not supported (similar to MySQL 5.7). * Adding the primary key of the `CLUSTERED` type to a table is not supported. For more details about the primary key of the `CLUSTERED` type, refer to [clustered index](/clustered-indexes.md). * Expression indexes are incompatible with views. When a query is executed using a view, the expression index cannot be used at the same time. diff --git a/sql-statements/sql-statement-create-placement-policy.md b/sql-statements/sql-statement-create-placement-policy.md index 7d0c7c0d77235..3cf749d39468d 100644 --- a/sql-statements/sql-statement-create-placement-policy.md +++ b/sql-statements/sql-statement-create-placement-policy.md @@ -9,7 +9,7 @@ summary: The usage of CREATE PLACEMENT POLICY in TiDB. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-create-resource-group.md b/sql-statements/sql-statement-create-resource-group.md index 6d34e7c1b464e..edaa3a5a592b3 100644 --- a/sql-statements/sql-statement-create-resource-group.md +++ b/sql-statements/sql-statement-create-resource-group.md @@ -9,7 +9,7 @@ You can use the `CREATE RESOURCE GROUP` statement to create a resource group. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-create-sequence.md b/sql-statements/sql-statement-create-sequence.md index f73f273435cd7..9d482399a82b5 100644 --- a/sql-statements/sql-statement-create-sequence.md +++ b/sql-statements/sql-statement-create-sequence.md @@ -1,7 +1,6 @@ --- title: CREATE SEQUENCE summary: An overview of the usage of CREATE SEQUENCE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-create-sequence/','/docs/dev/reference/sql/statements/create-sequence/'] --- # CREATE SEQUENCE diff --git a/sql-statements/sql-statement-create-table-like.md b/sql-statements/sql-statement-create-table-like.md index 912af59fbd0cb..331abd305abbb 100644 --- a/sql-statements/sql-statement-create-table-like.md +++ b/sql-statements/sql-statement-create-table-like.md @@ -1,7 +1,6 @@ --- title: CREATE TABLE LIKE | TiDB SQL Statement Reference summary: An overview of the usage of CREATE TABLE LIKE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-create-table-like/','/docs/dev/reference/sql/statements/create-table-like/'] --- # CREATE TABLE LIKE diff --git a/sql-statements/sql-statement-create-table.md b/sql-statements/sql-statement-create-table.md index cc0106328bf86..ee01c824d2c6a 100644 --- a/sql-statements/sql-statement-create-table.md +++ b/sql-statements/sql-statement-create-table.md @@ -1,7 +1,6 @@ --- title: CREATE TABLE | TiDB SQL Statement Reference summary: An overview of the usage of CREATE TABLE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-create-table/','/docs/dev/reference/sql/statements/create-table/'] --- # CREATE TABLE @@ -76,6 +75,7 @@ KeyPart ::= IndexOption ::= 'COMMENT' String | ( 'VISIBLE' | 'INVISIBLE' ) +| ('USING' | 'TYPE') ('BTREE' | 'RTREE' | 'HASH') ForeignKeyDef ::= ( 'CONSTRAINT' Identifier )? 'FOREIGN' 'KEY' @@ -124,6 +124,37 @@ OnCommitOpt ::= PlacementPolicyOption ::= "PLACEMENT" "POLICY" EqOpt PolicyName | "PLACEMENT" "POLICY" (EqOpt | "SET") "DEFAULT" + +DefaultValueExpr ::= + NowSymOptionFractionParentheses +| SignedLiteral +| NextValueForSequenceParentheses +| BuiltinFunction + +BuiltinFunction ::= + '(' BuiltinFunction ')' +| identifier '(' ')' +| identifier '(' ExpressionList ')' +| "REPLACE" '(' ExpressionList ')' + +NowSymOptionFractionParentheses ::= + '(' NowSymOptionFractionParentheses ')' +| NowSymOptionFraction + +NowSymOptionFraction ::= + NowSym +| NowSymFunc '(' ')' +| NowSymFunc '(' NUM ')' +| CurdateSym '(' ')' +| "CURRENT_DATE" + +NextValueForSequenceParentheses ::= + '(' NextValueForSequenceParentheses ')' +| NextValueForSequence + +NextValueForSequence ::= + "NEXT" "VALUE" forKwd TableName +| "NEXTVAL" '(' TableName ')' ``` The following *table_options* are supported. Other options such as `AVG_ROW_LENGTH`, `CHECKSUM`, `COMPRESSION`, `CONNECTION`, `DELAY_KEY_WRITE`, `ENGINE`, `KEY_BLOCK_SIZE`, `MAX_ROWS`, `MIN_ROWS`, `ROW_FORMAT` and `STATS_PERSISTENT` are parsed but ignored. @@ -239,7 +270,8 @@ mysql> DESC t1; ## MySQL compatibility * All of the data types except spatial types are supported. -* `FULLTEXT`, `HASH` and `SPATIAL` indexes are not supported. +* TiDB accepts index types such as `HASH`, `BTREE` and `RTREE` in syntax for compatibility with MySQL, but ignores them. +* TiDB supports parsing the `FULLTEXT` syntax but does not support using the `FULLTEXT` indexes. @@ -256,7 +288,7 @@ mysql> DESC t1; * The `[ASC | DESC]` in `index_col_name` is currently parsed but ignored (MySQL 5.7 compatible behavior). * The `COMMENT` attribute does not support the `WITH PARSER` option. * TiDB supports 1017 columns in a single table by default and 4096 columns at most. The corresponding number limit in InnoDB is 1017 columns, and the hard limit in MySQL is 4096 columns. For details, see [TiDB Limitations](/tidb-limitations.md). -* For partitioned tables, only Range, Hash and Range Columns (single column) are supported. For details, see [partitioned table](/partitioned-table.md). +* TiDB supports `HASH`, `RANGE`, `LIST`, and `KEY` [partitioning types](/partitioned-table.md#partitioning-types). For an unsupported partition type, TiDB returns `Warning: Unsupported partition type %s, treat as normal table`, where `%s` is the specific unsupported partition type. ## See also diff --git a/sql-statements/sql-statement-create-user.md b/sql-statements/sql-statement-create-user.md index 39ef112943a66..241740a8d5a5d 100644 --- a/sql-statements/sql-statement-create-user.md +++ b/sql-statements/sql-statement-create-user.md @@ -1,7 +1,6 @@ --- title: CREATE USER | TiDB SQL Statement Reference summary: An overview of the usage of CREATE USER for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-create-user/','/docs/dev/reference/sql/statements/create-user/'] --- # CREATE USER diff --git a/sql-statements/sql-statement-create-view.md b/sql-statements/sql-statement-create-view.md index 6a4b907678d42..64a3678af8ca1 100644 --- a/sql-statements/sql-statement-create-view.md +++ b/sql-statements/sql-statement-create-view.md @@ -1,7 +1,6 @@ --- title: CREATE VIEW | TiDB SQL Statement Reference summary: An overview of the usage of CREATE VIEW for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-create-view/','/docs/dev/reference/sql/statements/create-view/'] --- # CREATE VIEW diff --git a/sql-statements/sql-statement-deallocate.md b/sql-statements/sql-statement-deallocate.md index d88f9edb374fd..0c0728d1a42cf 100644 --- a/sql-statements/sql-statement-deallocate.md +++ b/sql-statements/sql-statement-deallocate.md @@ -1,7 +1,6 @@ --- title: DEALLOCATE | TiDB SQL Statement Reference summary: An overview of the usage of DEALLOCATE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-deallocate/','/docs/dev/reference/sql/statements/deallocate/'] --- # DEALLOCATE diff --git a/sql-statements/sql-statement-delete.md b/sql-statements/sql-statement-delete.md index 7bb1e491eaae2..140ddf397cfd8 100644 --- a/sql-statements/sql-statement-delete.md +++ b/sql-statements/sql-statement-delete.md @@ -1,7 +1,6 @@ --- title: DELETE | TiDB SQL Statement Reference summary: An overview of the usage of DELETE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-delete/','/docs/dev/reference/sql/statements/delete/'] --- # DELETE diff --git a/sql-statements/sql-statement-desc.md b/sql-statements/sql-statement-desc.md index 541f6eee2e0f3..bdc55b29da8ad 100644 --- a/sql-statements/sql-statement-desc.md +++ b/sql-statements/sql-statement-desc.md @@ -1,7 +1,6 @@ --- title: DESC | TiDB SQL Statement Reference summary: An overview of the usage of `DESC` for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-desc/','/docs/dev/reference/sql/statements/desc/'] --- # DESC diff --git a/sql-statements/sql-statement-describe.md b/sql-statements/sql-statement-describe.md index 009b9d6ea8db9..785b7d2372bb1 100644 --- a/sql-statements/sql-statement-describe.md +++ b/sql-statements/sql-statement-describe.md @@ -1,7 +1,6 @@ --- title: DESCRIBE | TiDB SQL Statement Reference summary: An overview of the usage of DESCRIBE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-describe/','/docs/dev/reference/sql/statements/describe/'] --- # DESCRIBE diff --git a/sql-statements/sql-statement-do.md b/sql-statements/sql-statement-do.md index 3bf40115429d4..a40447e3569bf 100644 --- a/sql-statements/sql-statement-do.md +++ b/sql-statements/sql-statement-do.md @@ -1,7 +1,6 @@ --- title: DO | TiDB SQL Statement Reference summary: An overview of the usage of DO for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-do/','/docs/dev/reference/sql/statements/do/'] --- # DO diff --git a/sql-statements/sql-statement-drop-binding.md b/sql-statements/sql-statement-drop-binding.md index 6413ccf5fc4d2..dd5d997a40e45 100644 --- a/sql-statements/sql-statement-drop-binding.md +++ b/sql-statements/sql-statement-drop-binding.md @@ -1,7 +1,6 @@ --- title: DROP [GLOBAL|SESSION] BINDING summary: Use of DROP BINDING in TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-binding/'] --- # DROP [GLOBAL|SESSION] BINDING diff --git a/sql-statements/sql-statement-drop-column.md b/sql-statements/sql-statement-drop-column.md index bf89165c62382..3c87062e5d354 100644 --- a/sql-statements/sql-statement-drop-column.md +++ b/sql-statements/sql-statement-drop-column.md @@ -1,7 +1,6 @@ --- title: DROP COLUMN | TiDB SQL Statement Reference summary: An overview of the usage of DROP COLUMN for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-column/','/docs/dev/reference/sql/statements/drop-column/'] --- # DROP COLUMN diff --git a/sql-statements/sql-statement-drop-database.md b/sql-statements/sql-statement-drop-database.md index acb205a34a830..4a3ff9598a5e4 100644 --- a/sql-statements/sql-statement-drop-database.md +++ b/sql-statements/sql-statement-drop-database.md @@ -1,7 +1,6 @@ --- title: DROP DATABASE | TiDB SQL Statement Reference summary: An overview of the usage of DROP DATABASE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-database/','/docs/dev/reference/sql/statements/drop-database/'] --- # DROP DATABASE diff --git a/sql-statements/sql-statement-drop-index.md b/sql-statements/sql-statement-drop-index.md index aa18c34063005..982401b42109d 100644 --- a/sql-statements/sql-statement-drop-index.md +++ b/sql-statements/sql-statement-drop-index.md @@ -1,7 +1,6 @@ --- title: DROP INDEX | TiDB SQL Statement Reference summary: An overview of the usage of DROP INDEX for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-index/','/docs/dev/reference/sql/statements/drop-index/'] --- # DROP INDEX diff --git a/sql-statements/sql-statement-drop-placement-policy.md b/sql-statements/sql-statement-drop-placement-policy.md index 2e3f9e610dd8f..fe5799d18e907 100644 --- a/sql-statements/sql-statement-drop-placement-policy.md +++ b/sql-statements/sql-statement-drop-placement-policy.md @@ -9,7 +9,7 @@ summary: The usage of ALTER PLACEMENT POLICY in TiDB. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-drop-resource-group.md b/sql-statements/sql-statement-drop-resource-group.md index e42cc8e525ae6..ac4d380534293 100644 --- a/sql-statements/sql-statement-drop-resource-group.md +++ b/sql-statements/sql-statement-drop-resource-group.md @@ -9,7 +9,7 @@ You can use the `DROP RESOURCE GROUP` statement to drop a resource group. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-drop-sequence.md b/sql-statements/sql-statement-drop-sequence.md index fbc3cdec4f9f3..b5f944a6f0bce 100644 --- a/sql-statements/sql-statement-drop-sequence.md +++ b/sql-statements/sql-statement-drop-sequence.md @@ -1,7 +1,6 @@ --- title: DROP SEQUENCE summary: An overview of the usage of DROP SEQUENCE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-sequence/','/docs/dev/reference/sql/statements/drop-sequence/'] --- # DROP SEQUENCE diff --git a/sql-statements/sql-statement-drop-stats.md b/sql-statements/sql-statement-drop-stats.md index 86ddcfd01bf93..3be3431ca29fa 100644 --- a/sql-statements/sql-statement-drop-stats.md +++ b/sql-statements/sql-statement-drop-stats.md @@ -1,7 +1,6 @@ --- title: DROP STATS summary: An overview of the usage of DROP STATS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-stats/'] --- # DROP STATS @@ -12,34 +11,59 @@ The `DROP STATS` statement is used to delete the statistics of the selected tabl ```ebnf+diagram DropStatsStmt ::= - 'DROP' 'STATS' TableNameList - -TableNameList ::= - TableName ( ',' TableName )* + 'DROP' 'STATS' TableName ("PARTITION" partition | "GLOBAL")? ( ',' TableName )* TableName ::= Identifier ('.' Identifier)? ``` -## Examples +## Usage -{{< copyable "sql" >}} +The following statement deletes all statistics of `TableName`. If a partitioned table is specified, this statement deletes statistics of all partitions in this table as well as [GlobalStats generated in dynamic pruning mode](/statistics.md#collect-statistics-of-partitioned-tables-in-dynamic-pruning-mode). ```sql -CREATE TABLE t(a INT); +DROP STATS TableName +``` + ``` +Query OK, 0 rows affected (0.00 sec) +``` + +The following statement only deletes statistics of the specified partitions in `PartitionNameList`. ```sql -Query OK, 0 rows affected (0.01 sec) +DROP STATS TableName PARTITION PartitionNameList; ``` -{{< copyable "sql" >}} +``` +Query OK, 0 rows affected (0.00 sec) +``` + +The following statement only deletes GlobalStats generated in dynamic pruning mode of the specified table. ```sql -SHOW STATS_META WHERE db_name='test' and table_name='t'; +DROP STATS TableName GLOBAL; +``` + +``` +Query OK, 0 rows affected (0.00 sec) +``` + +## Examples + +```sql +CREATE TABLE t(a INT); +``` + +``` +Query OK, 0 rows affected (0.01 sec) ``` ```sql +SHOW STATS_META WHERE db_name='test' and table_name='t'; +``` + +``` +---------+------------+----------------+---------------------+--------------+-----------+ | Db_name | Table_name | Partition_name | Update_time | Modify_count | Row_count | +---------+------------+----------------+---------------------+--------------+-----------+ @@ -48,23 +72,19 @@ SHOW STATS_META WHERE db_name='test' and table_name='t'; 1 row in set (0.00 sec) ``` -{{< copyable "sql" >}} - ```sql DROP STATS t; ``` -```sql +``` Query OK, 0 rows affected (0.00 sec) ``` -{{< copyable "sql" >}} - ```sql SHOW STATS_META WHERE db_name='test' and table_name='t'; ``` -```sql +``` Empty set (0.00 sec) ``` diff --git a/sql-statements/sql-statement-drop-table.md b/sql-statements/sql-statement-drop-table.md index 8024cba7a83da..585ba050f9341 100644 --- a/sql-statements/sql-statement-drop-table.md +++ b/sql-statements/sql-statement-drop-table.md @@ -1,7 +1,6 @@ --- title: DROP TABLE | TiDB SQL Statement Reference summary: An overview of the usage of DROP TABLE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-table/','/docs/dev/reference/sql/statements/drop-table/'] --- # DROP TABLE diff --git a/sql-statements/sql-statement-drop-user.md b/sql-statements/sql-statement-drop-user.md index e87ae43ac2324..c735c5e275d49 100644 --- a/sql-statements/sql-statement-drop-user.md +++ b/sql-statements/sql-statement-drop-user.md @@ -1,7 +1,6 @@ --- title: DROP USER | TiDB SQL Statement Reference summary: An overview of the usage of DROP USER for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-user/','/docs/dev/reference/sql/statements/drop-user/'] --- # DROP USER diff --git a/sql-statements/sql-statement-drop-view.md b/sql-statements/sql-statement-drop-view.md index 6201a03ddf301..2702e3c88b92c 100644 --- a/sql-statements/sql-statement-drop-view.md +++ b/sql-statements/sql-statement-drop-view.md @@ -1,12 +1,11 @@ --- title: DROP VIEW | TiDB SQL Statement Reference summary: An overview of the usage of DROP VIEW for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-drop-view/','/docs/dev/reference/sql/statements/drop-view/'] --- # DROP VIEW -This statement drops an view object from the currently selected database. It does not effect any base tables that a view references. +This statement drops a view object from the currently selected database. It does not affect any base tables that a view references. ## Synopsis diff --git a/sql-statements/sql-statement-execute.md b/sql-statements/sql-statement-execute.md index d2366d88bd736..86dd05635665b 100644 --- a/sql-statements/sql-statement-execute.md +++ b/sql-statements/sql-statement-execute.md @@ -1,7 +1,6 @@ --- title: EXECUTE | TiDB SQL Statement Reference summary: An overview of the usage of EXECUTE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-execute/','/docs/dev/reference/sql/statements/execute/'] --- # EXECUTE diff --git a/sql-statements/sql-statement-explain-analyze.md b/sql-statements/sql-statement-explain-analyze.md index fc455434d9c09..2042d5a8a00b0 100644 --- a/sql-statements/sql-statement-explain-analyze.md +++ b/sql-statements/sql-statement-explain-analyze.md @@ -1,7 +1,6 @@ --- title: EXPLAIN ANALYZE | TiDB SQL Statement Reference summary: An overview of the usage of EXPLAIN ANALYZE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-explain-analyze/','/docs/dev/reference/sql/statements/explain-analyze/'] --- # EXPLAIN ANALYZE @@ -91,7 +90,7 @@ EXPLAIN ANALYZE SELECT * FROM t1; +-------------------+----------+---------+-----------+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------+-----------+------+ | id | estRows | actRows | task | access object | execution info | operator info | memory | disk | +-------------------+----------+---------+-----------+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------+-----------+------+ -| TableReader_5 | 10000.00 | 3 | root | | time:278.2µs, loops:2, cop_task: {num: 1, max: 437.6µs, proc_keys: 3, rpc_num: 1, rpc_time: 423.9µs, copr_cache_hit_ratio: 0.00} | data:TableFullScan_4 | 251 Bytes | N/A | +| TableReader_5 | 10000.00 | 3 | root | | time:278.2µs, loops:2, cop_task: {num: 1, max: 437.6µs, proc_keys: 3, copr_cache_hit_ratio: 0.00}, rpc_info:{Cop:{num_rpc:1, total_time:423.9µs}} | data:TableFullScan_4 | 251 Bytes | N/A | | └─TableFullScan_4 | 10000.00 | 3 | cop[tikv] | table:t1 | tikv_task:{time:0s, loops:1}, scan_detail: {total_process_keys: 3, total_process_keys_size: 111, total_keys: 4, rocksdb: {delete_skipped_count: 0, key_skipped_count: 3, block: {cache_hit_count: 0, read_count: 0, read_byte: 0 Bytes}}} | keep order:false, stats:pseudo | N/A | N/A | +-------------------+----------+---------+-----------+---------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------+-----------+------+ 2 rows in set (0.00 sec) @@ -120,15 +119,15 @@ The execution information of the `Batch_Point_Get` operator is similar to that o The execution information of a `TableReader` operator is typically as follows: ``` -cop_task: {num: 6, max: 1.07587ms, min: 844.312µs, avg: 919.601µs, p95: 1.07587ms, max_proc_keys: 16, p95_proc_keys: 16, tot_proc: 1ms, tot_wait: 1ms, rpc_num: 6, rpc_time: 5.313996 ms, copr_cache_hit_ratio: 0.00} +cop_task: {num: 6, max: 1.07587ms, min: 844.312µs, avg: 919.601µs, p95: 1.07587ms, max_proc_keys: 16, p95_proc_keys: 16, tot_proc: 1ms, tot_wait: 1ms, copr_cache_hit_ratio: 0.00}, rpc_info:{Cop:{num_rpc:6, total_time:5.313996ms}} ``` - `cop_task`: Contains the execution information of `cop` tasks. For example: - `num`: The number of cop tasks. - `max`, `min`, `avg`, `p95`: The maximum, minimum, average, and P95 values of the execution time consumed for executing cop tasks. - `max_proc_keys` and `p95_proc_keys`: The maximum and P95 key-values scanned by TiKV in all cop tasks. If the difference between the maximum value and the P95 value is large, the data distribution might be imbalanced. - - `rpc_num`, `rpc_time`: The total number and total time consumed for `Cop` RPC requests sent to TiKV. - `copr_cache_hit_ratio`: The hit rate of Coprocessor Cache for `cop` task requests. +- `rpc_info`: The total number and total time of RPC requests sent to TiKV aggregated by request type. - `backoff`: Contains different types of backoff and the total waiting time of backoff. ### Insert diff --git a/sql-statements/sql-statement-explain.md b/sql-statements/sql-statement-explain.md index 5a872c6ffc28a..615b9d301f419 100644 --- a/sql-statements/sql-statement-explain.md +++ b/sql-statements/sql-statement-explain.md @@ -1,14 +1,17 @@ --- title: EXPLAIN | TiDB SQL Statement Reference summary: An overview of the usage of EXPLAIN for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-explain/','/docs/dev/reference/sql/statements/explain/'] --- # `EXPLAIN` The `EXPLAIN` statement shows the execution plan for a query without executing it. It complements the `EXPLAIN ANALYZE` statement, which executes the query. If the output of `EXPLAIN` does not match the expected result, consider executing `ANALYZE TABLE` on each table in the query to make sure the table statistics are up to date. -The statements `DESC` and `DESCRIBE` are aliases of this statement. The alternative usage of `EXPLAIN ` is documented under [`SHOW [FULL] COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md). +> **Note:** +> +> Certain subqueries are pre-executed during the optimization phase to generate optimal execution plans, even in the `EXPLAIN` statement. For more information on this behavior and how to disable it, see [`tidb_opt_enable_non_eval_scalar_subquery`](/system-variables.md#tidb_opt_enable_non_eval_scalar_subquery-new-in-v730) and [Disable the early execution of subqueries](/explain-walkthrough.md#disable-the-early-execution-of-subqueries). + +The statements `DESC` and `DESCRIBE` are aliases of the `EXPLAIN` statement. The alternative usage of `EXPLAIN ` is documented in [`SHOW [FULL] COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md). TiDB supports the `EXPLAIN [options] FOR CONNECTION connection_id` statement. However, this statement is different from the `EXPLAIN FOR` statement in MySQL. For more details, see [`EXPLAIN FOR CONNECTION`](#explain-for-connection). diff --git a/sql-statements/sql-statement-flashback-cluster.md b/sql-statements/sql-statement-flashback-cluster.md index e57553f1a6b20..d32d44e749e23 100644 --- a/sql-statements/sql-statement-flashback-cluster.md +++ b/sql-statements/sql-statement-flashback-cluster.md @@ -1,7 +1,6 @@ --- title: FLASHBACK CLUSTER summary: Learn the usage of FLASHBACK CLUSTER in TiDB databases. -aliases: ['/tidb/dev/sql-statement-flashback-to-timestamp'] --- # FLASHBACK CLUSTER @@ -12,11 +11,12 @@ Starting from v6.5.6, v7.1.3, v7.5.1, and v7.6.0, TiDB introduces the `FLASHBACK > **Warning:** > -> The `FLASHBACK CLUSTER TO [TIMESTAMP|TSO]` syntax is not applicable to [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. To avoid unexpected results, do not execute this statement on TiDB Serverless clusters. +> The `FLASHBACK CLUSTER TO [TIMESTAMP|TSO]` syntax is not applicable to [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. To avoid unexpected results, do not execute this statement on TiDB Cloud Serverless clusters. > **Warning:** > -> When specifying a recovery point in time, make sure to check the validity of your target timestamp or TSO and avoid specifying a future time that exceeds the maximum TSO currently allocated by PD (see `Current TSO` on the Grafana PD panel). Otherwise, concurrent processing linear consistency and transaction isolation levels might be violated, leading to serious data correctness issues. +> - When specifying a recovery point in time, make sure to check the validity of your target timestamp or TSO and avoid specifying a future time that exceeds the maximum TSO currently allocated by PD (see `Current TSO` on the Grafana PD panel). Otherwise, concurrent processing linear consistency and transaction isolation levels might be violated, leading to serious data correctness issues. +> - During `FLASHBACK CLUSTER` execution, the data cleanup process does not guarantee transaction consistency. After `FLASHBACK CLUSTER` completes, if you intend to use any historical version reading features in TiDB (such as [Stale Read](/stale-read.md) or [`tidb_snapshot`](/read-historical-data.md)), make sure that the specified historical timestamp falls outside the `FLASHBACK CLUSTER` execution period. Reading a historical version that includes data not fully restored by FLASHBACK might violate concurrent processing linear consistency and transaction isolation levels, leading to serious data correctness issues. @@ -43,8 +43,7 @@ FLASHBACK CLUSTER TO TSO 445494839813079041; ```ebnf+diagram FlashbackToTimestampStmt - ::= 'FLASHBACK' 'CLUSTER' 'TO' 'TIMESTAMP' stringLit - | 'FLASHBACK' 'CLUSTER' 'TO' 'TSO' LengthNum + ::= 'FLASHBACK' 'CLUSTER' 'TO' ('TIMESTAMP' stringLit | 'TSO' LengthNum) ``` ## Notes diff --git a/sql-statements/sql-statement-flashback-table.md b/sql-statements/sql-statement-flashback-table.md index 3cdfcb373720e..12bf536b9a15a 100644 --- a/sql-statements/sql-statement-flashback-table.md +++ b/sql-statements/sql-statement-flashback-table.md @@ -1,7 +1,6 @@ --- title: FLASHBACK TABLE summary: Learn how to recover tables using the `FLASHBACK TABLE` statement. -aliases: ['/docs/dev/sql-statements/sql-statement-flashback-table/','/docs/dev/reference/sql/statements/flashback-table/'] --- # FLASHBACK TABLE diff --git a/sql-statements/sql-statement-flush-privileges.md b/sql-statements/sql-statement-flush-privileges.md index 16f0a5dffdc47..13c3bc2dcee01 100644 --- a/sql-statements/sql-statement-flush-privileges.md +++ b/sql-statements/sql-statement-flush-privileges.md @@ -1,7 +1,6 @@ --- title: FLUSH PRIVILEGES | TiDB SQL Statement Reference summary: An overview of the usage of FLUSH PRIVILEGES for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-flush-privileges/','/docs/dev/reference/sql/statements/flush-privileges/'] --- # FLUSH PRIVILEGES diff --git a/sql-statements/sql-statement-flush-status.md b/sql-statements/sql-statement-flush-status.md index 854f3160dbbb2..619e4a703ef48 100644 --- a/sql-statements/sql-statement-flush-status.md +++ b/sql-statements/sql-statement-flush-status.md @@ -1,7 +1,6 @@ --- title: FLUSH STATUS | TiDB SQL Statement Reference summary: An overview of the usage of FLUSH STATUS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-flush-status/','/docs/dev/reference/sql/statements/flush-status/'] --- # FLUSH STATUS diff --git a/sql-statements/sql-statement-flush-tables.md b/sql-statements/sql-statement-flush-tables.md index 4ce04037238fe..3d14ed1eb28fd 100644 --- a/sql-statements/sql-statement-flush-tables.md +++ b/sql-statements/sql-statement-flush-tables.md @@ -1,7 +1,6 @@ --- title: FLUSH TABLES | TiDB SQL Statement Reference summary: An overview of the usage of FLUSH TABLES for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-flush-tables/','/docs/dev/reference/sql/statements/flush-tables/'] --- # FLUSH TABLES diff --git a/sql-statements/sql-statement-grant-privileges.md b/sql-statements/sql-statement-grant-privileges.md index adedcb211f1af..83b14b0f76581 100644 --- a/sql-statements/sql-statement-grant-privileges.md +++ b/sql-statements/sql-statement-grant-privileges.md @@ -1,7 +1,6 @@ --- title: GRANT | TiDB SQL Statement Reference summary: An overview of the usage of GRANT for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-grant-privileges/','/docs/dev/reference/sql/statements/grant-privileges/'] --- # `GRANT ` diff --git a/sql-statements/sql-statement-import-into.md b/sql-statements/sql-statement-import-into.md index f0db2744f832b..b2ee611e88dcd 100644 --- a/sql-statements/sql-statement-import-into.md +++ b/sql-statements/sql-statement-import-into.md @@ -10,14 +10,24 @@ The `IMPORT INTO` statement lets you import data to TiDB via the [Physical Impor - `IMPORT INTO ... FROM FILE`: imports data files in formats such as `CSV`, `SQL`, and `PARQUET` into an empty table in TiDB. - `IMPORT INTO ... FROM SELECT`: imports the query result of a `SELECT` statement into an empty table in TiDB. You can also use it to import historical data queried with [`AS OF TIMESTAMP`](/as-of-timestamp.md). + + +> **Note:** +> +> Compared with [TiDB Lightning](/tidb-lightning/tidb-lightning-overview.md), `IMPORT INTO` can be directly executed on TiDB nodes, supports automated distributed task scheduling and [TiDB Global Sort](/tidb-global-sort.md), and offers significant improvements in deployment, resource utilization, task configuration convenience, ease of invocation and integration, high availability, and scalability. It is recommended that you consider using `IMPORT INTO` instead of TiDB Lightning in appropriate scenarios. + + + ## Restrictions - `IMPORT INTO` only supports importing data into existing empty tables in the database. +- `IMPORT INTO` does not support importing data into an empty partition if other partitions of the same table already contain data. The target table must be completely empty for import operations. +- `IMPORT INTO` does not support importing data into a [temporary table](/temporary-tables.md) or a [cached table](/cached-tables.md). - `IMPORT INTO` does not support transactions or rollback. Executing `IMPORT INTO` within an explicit transaction (`BEGIN`/`END`) will return an error. -- `IMPORT INTO` does not support working simultaneously with features such as [Backup & Restore](https://docs.pingcap.com/tidb/stable/backup-and-restore-overview), [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md), [acceleration of adding indexes](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630), data import using TiDB Lightning, data replication using TiCDC, or [Point-in-Time Recovery (PITR)](https://docs.pingcap.com/tidb/stable/br-log-architecture). +- `IMPORT INTO` does not support working simultaneously with features such as [Backup & Restore](https://docs.pingcap.com/tidb/stable/backup-and-restore-overview), [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md), [acceleration of adding indexes](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630), data import using TiDB Lightning, data replication using TiCDC, or [Point-in-Time Recovery (PITR)](https://docs.pingcap.com/tidb/stable/br-log-architecture). For more compatibility information, see [Compatibility of TiDB Lightning and `IMPORT INTO` with TiCDC and Log Backup](https://docs.pingcap.com/tidb/stable/tidb-lightning-compatibility-and-scenarios). - During the data import process, do not perform DDL or DML operations on the target table, and do not execute [`FLASHBACK DATABASE`](/sql-statements/sql-statement-flashback-database.md) for the target database. These operations can lead to import failures or data inconsistencies. In addition, it is **NOT** recommended to perform read operations during the import process, as the data being read might be inconsistent. Perform read and write operations only after the import is completed. -- The import process consumes system resources significantly. For TiDB Self-Hosted, to get better performance, it is recommended to use TiDB nodes with at least 32 cores and 64 GiB of memory. TiDB writes sorted data to the TiDB [temporary directory](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#temp-dir-new-in-v630) during import, so it is recommended to configure high-performance storage media for TiDB Self-Hosted, such as flash memory. For more information, see [Physical Import Mode limitations](https://docs.pingcap.com/tidb/stable/tidb-lightning-physical-import-mode#requirements-and-restrictions). -- For TiDB Self-Hosted, the TiDB [temporary directory](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#temp-dir-new-in-v630) is expected to have at least 90 GiB of available space. It is recommended to allocate storage space that is equal to or greater than the volume of data to be imported. +- The import process consumes system resources significantly. For TiDB Self-Managed, to get better performance, it is recommended to use TiDB nodes with at least 32 cores and 64 GiB of memory. TiDB writes sorted data to the TiDB [temporary directory](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#temp-dir-new-in-v630) during import, so it is recommended to configure high-performance storage media for TiDB Self-Managed, such as flash memory. For more information, see [Physical Import Mode limitations](https://docs.pingcap.com/tidb/stable/tidb-lightning-physical-import-mode#requirements-and-restrictions). +- For TiDB Self-Managed, the TiDB [temporary directory](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#temp-dir-new-in-v630) is expected to have at least 90 GiB of available space. It is recommended to allocate storage space that is equal to or greater than the volume of data to be imported. - One import job supports importing data into one target table only. - `IMPORT INTO` is not supported during TiDB cluster upgrades. - Ensure that the data to be imported does not contain any records with primary key or non-null unique index conflicts. Otherwise, the conflicts can result in import task failures. @@ -25,11 +35,11 @@ The `IMPORT INTO` statement lets you import data to TiDB via the [Physical Impor ### `IMPORT INTO ... FROM FILE` restrictions -- For TiDB Self-Hosted, each `IMPORT INTO` task supports importing data within 10 TiB. If you enable the [Global Sort](/tidb-global-sort.md) feature, each `IMPORT INTO` task supports importing data within 40 TiB. -- For [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated), if your data to be imported exceeds 500 GiB, it is recommended to use TiDB nodes with at least 16 cores and enable the [Global Sort](/tidb-global-sort.md) feature, then each `IMPORT INTO` task supports importing data within 40 TiB. If your data to be imported is within 500 GiB or if the cores of your TiDB nodes are less than 16, it is not recommended to enable the [Global Sort](/tidb-global-sort.md) feature. +- For TiDB Self-Managed, each `IMPORT INTO` task supports importing data within 10 TiB. If you enable the [Global Sort](/tidb-global-sort.md) feature, each `IMPORT INTO` task supports importing data within 40 TiB. +- For [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated), if your data to be imported exceeds 500 GiB, it is recommended to use TiDB nodes with at least 16 cores and enable the [Global Sort](/tidb-global-sort.md) feature, then each `IMPORT INTO` task supports importing data within 40 TiB. If your data to be imported is within 500 GiB or if the cores of your TiDB nodes are less than 16, it is not recommended to enable the [Global Sort](/tidb-global-sort.md) feature. - The execution of `IMPORT INTO ... FROM FILE` blocks the current connection until the import is completed. To execute the statement asynchronously, you can add the `DETACHED` option. - Up to 16 `IMPORT INTO` tasks can run simultaneously on each cluster (see [TiDB Distributed eXecution Framework (DXF) usage limitations](/tidb-distributed-execution-framework.md#limitation)). When a cluster lacks sufficient resources or reaches the maximum number of tasks, newly submitted import tasks are queued for execution. -- When the [Global Sort](/tidb-global-sort.md) feature is used for data import, the value of the `THREAD` option must be at least `16`. +- When the [Global Sort](/tidb-global-sort.md) feature is used for data import, the value of the `THREAD` option must be at least `8`. - When the [Global Sort](/tidb-global-sort.md) feature is used for data import, the data size of a single row after encoding must not exceed 32 MiB. - All `IMPORT INTO` tasks that are created when [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md) is not enabled run directly on the nodes where the tasks are submitted, and these tasks will not be scheduled for execution on other TiDB nodes even after DXF is enabled later. After DXF is enabled, only newly created `IMPORT INTO` tasks that import data from S3 or GCS are automatically scheduled or failed over to other TiDB nodes for execution. @@ -40,6 +50,7 @@ The `IMPORT INTO` statement lets you import data to TiDB via the [Physical Impor - `IMPORT INTO ... FROM SELECT` does not support the task management statements such as `SHOW IMPORT JOB(s)` and `CANCEL IMPORT JOB `. - The [temporary directory](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#temp-dir-new-in-v630) of TiDB requires sufficient space to store the entire query result of the `SELECT` statement (configuring the `DISK_QUOTA` option is not supported currently). - Importing historical data using [`tidb_snapshot`](/read-historical-data.md) is not supported. +- Because the syntax of the `SELECT` clause is complex, the `WITH` parameter in `IMPORT INTO` might conflict with it and cause parsing errors, such as `GROUP BY ... [WITH ROLLUP]`. It is recommended to create a view for complex `SELECT` statements and then use `IMPORT INTO ... FROM SELECT * FROM view_name` for importing. Alternatively, you can clarify the scope of the `SELECT` clause with parentheses, such as `IMPORT INTO ... FROM (SELECT ...) WITH ...`. ## Prerequisites for import @@ -47,7 +58,7 @@ Before using `IMPORT INTO` to import data, make sure the following requirements - The target table to be imported is already created in TiDB and it is empty. - The target cluster has sufficient space to store the data to be imported. -- For TiDB Self-Hosted, the [temporary directory](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#temp-dir-new-in-v630) of the TiDB node connected to the current session has at least 90 GiB of available space. If [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) is enabled and the data for import is from S3 or GCS, also make sure that the temporary directory of each TiDB node in the cluster has sufficient disk space. +- For TiDB Self-Managed, the [temporary directory](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#temp-dir-new-in-v630) of the TiDB node connected to the current session has at least 90 GiB of available space. If [`tidb_enable_dist_task`](/system-variables.md#tidb_enable_dist_task-new-in-v710) is enabled and the data for import is from S3 or GCS, also make sure that the temporary directory of each TiDB node in the cluster has sufficient disk space. ## Required privileges @@ -138,7 +149,7 @@ The supported options are described as follows: | `FIELDS_DEFINED_NULL_BY=''` | CSV | Specifies the value that represents `NULL` in the fields. The default value is `\N`. | | `LINES_TERMINATED_BY=''` | CSV | Specifies the line terminator. By default, `IMPORT INTO` automatically identifies `\n`, `\r`, or `\r\n` as line terminators. If the line terminator is one of these three, you do not need to explicitly specify this option. | | `SKIP_ROWS=` | CSV | Specifies the number of rows to skip. The default value is `0`. You can use this option to skip the header in a CSV file. If you use a wildcard to specify the source files for import, this option applies to all source files that are matched by the wildcard in `fileLocation`. | -| `SPLIT_FILE` | CSV | Splits a single CSV file into multiple smaller chunks of around 256 MiB for parallel processing to improve import efficiency. This parameter only works for **non-compressed** CSV files and has the same usage restrictions as that of TiDB Lightning [`strict-format`](https://docs.pingcap.com/tidb/stable/tidb-lightning-data-source#strict-format). | +| `SPLIT_FILE` | CSV | Splits a single CSV file into multiple smaller chunks of around 256 MiB for parallel processing to improve import efficiency. This parameter only works for **non-compressed** CSV files and has the same usage restrictions as that of TiDB Lightning [`strict-format`](https://docs.pingcap.com/tidb/stable/tidb-lightning-data-source#strict-format). Note that you need to explicitly specify `LINES_TERMINATED_BY` for this option. | | `DISK_QUOTA=''` | All file formats | Specifies the disk space threshold that can be used during data sorting. The default value is 80% of the disk space in the TiDB [temporary directory](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#temp-dir-new-in-v630). If the total disk size cannot be obtained, the default value is 50 GiB. When specifying `DISK_QUOTA` explicitly, make sure that the value does not exceed 80% of the disk space in the TiDB temporary directory. | | `DISABLE_TIKV_IMPORT_MODE` | All file formats | Specifies whether to disable switching TiKV to import mode during the import process. By default, switching TiKV to import mode is not disabled. If there are ongoing read-write operations in the cluster, you can enable this option to avoid impact from the import process. | | `THREAD=` | All file formats and query results of `SELECT` | Specifies the concurrency for import. For `IMPORT INTO ... FROM FILE`, the default value of `THREAD` is 50% of the number of CPU cores on the TiDB node, the minimum value is `1`, and the maximum value is the number of CPU cores. For `IMPORT INTO ... FROM SELECT`, the default value of `THREAD` is `2`, the minimum value is `1`, and the maximum value is two times the number of CPU cores on the TiDB node. To import data into a new cluster without any data, it is recommended to increase this concurrency appropriately to improve import performance. If the target cluster is already used in a production environment, it is recommended to adjust this concurrency according to your application requirements. | @@ -146,15 +157,11 @@ The supported options are described as follows: | `CHECKSUM_TABLE=''` | All file formats | Configures whether to perform a checksum check on the target table after the import to validate the import integrity. The supported values include `"required"` (default), `"optional"`, and `"off"`. `"required"` means performing a checksum check after the import. If the checksum check fails, TiDB will return an error and the import will exit. `"optional"` means performing a checksum check after the import. If an error occurs, TiDB will return a warning and ignore the error. `"off"` means not performing a checksum check after the import. | | `DETACHED` | All file formats | Controls whether to execute `IMPORT INTO` asynchronously. When this option is enabled, executing `IMPORT INTO` immediately returns the information of the import job (such as the `Job_ID`), and the job is executed asynchronously in the backend. | | `CLOUD_STORAGE_URI` | All file formats | Specifies the target address where encoded KV data for [Global Sort](/tidb-global-sort.md) is stored. When `CLOUD_STORAGE_URI` is not specified, `IMPORT INTO` determines whether to use Global Sort based on the value of the system variable [`tidb_cloud_storage_uri`](/system-variables.md#tidb_cloud_storage_uri-new-in-v740). If this system variable specifies a target storage address, `IMPORT INTO` uses this address for Global Sort. When `CLOUD_STORAGE_URI` is specified with a non-empty value, `IMPORT INTO` uses that value as the target storage address. When `CLOUD_STORAGE_URI` is specified with an empty value, local sorting is enforced. Currently, the target storage address only supports S3. For details about the URI configuration, see [Amazon S3 URI format](/external-storage-uri.md#amazon-s3-uri-format). When this feature is used, all TiDB nodes must have read and write access for the target S3 bucket, including at least these permissions: `s3:ListBucket`, `s3:GetObject`, `s3:DeleteObject`, `s3:PutObject`, `s3: AbortMultipartUpload`. | -| `DISABLE_PRECHECK` | All file formats and query results of `SELECT` | Setting this option disables pre-checks of non-critical itemes, such as checking whether there are CDC or PITR tasks. | +| `DISABLE_PRECHECK` | All file formats and query results of `SELECT` | Setting this option disables pre-checks of non-critical items, such as checking whether there are CDC or PITR tasks. | ## `IMPORT INTO ... FROM FILE` usage -> **Note:** -> -> `IMPORT INTO ... FROM FILE` is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. - -For TiDB Self-Hosted, `IMPORT INTO ... FROM FILE` supports importing data from files stored in Amazon S3, GCS, and the TiDB local storage. For [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated), `IMPORT INTO ... FROM FILE` supports importing data from files stored in Amazon S3 and GCS. +For TiDB Self-Managed, `IMPORT INTO ... FROM FILE` supports importing data from files stored in Amazon S3, GCS, and the TiDB local storage. For [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated), `IMPORT INTO ... FROM FILE` supports importing data from files stored in Amazon S3 and GCS. For [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless), `IMPORT INTO ... FROM FILE` supports importing data from files stored in Amazon S3 and Alibaba Cloud OSS. - For data files stored in Amazon S3 or GCS, `IMPORT INTO ... FROM FILE` supports running in the [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md). @@ -180,6 +187,10 @@ For TiDB Self-Hosted, `IMPORT INTO ... FROM FILE` supports importing data from f ### Global Sort +> **Note:** +> +> Global Sort is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. + `IMPORT INTO ... FROM FILE` splits the data import job of a source data file into multiple sub-jobs, each sub-job independently encoding and sorting data before importing. If the encoded KV ranges of these sub-jobs have significant overlap (to learn how TiDB encodes data to KV, see [TiDB computing](/tidb-computing.md)), TiKV needs to keep compaction during import, leading to a decrease in import performance and stability. In the following scenarios, there can be significant overlap in KV ranges: @@ -350,5 +361,7 @@ This statement is a TiDB extension to MySQL syntax. ## See also -* [`SHOW IMPORT JOB(s)`](/sql-statements/sql-statement-show-import-job.md) +* [`ADMIN CHECKSUM TABLE`](/sql-statements/sql-statement-admin-checksum-table.md) * [`CANCEL IMPORT JOB`](/sql-statements/sql-statement-cancel-import-job.md) +* [`SHOW IMPORT JOB(s)`](/sql-statements/sql-statement-show-import-job.md) +* [TiDB Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md) diff --git a/sql-statements/sql-statement-insert.md b/sql-statements/sql-statement-insert.md index ef36bda9e9e1f..f497a8239aef8 100644 --- a/sql-statements/sql-statement-insert.md +++ b/sql-statements/sql-statement-insert.md @@ -1,7 +1,6 @@ --- title: INSERT | TiDB SQL Statement Reference summary: An overview of the usage of INSERT for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-insert/','/docs/dev/reference/sql/statements/insert/'] --- # INSERT @@ -42,6 +41,10 @@ OnDuplicateKeyUpdate ::= ( 'ON' 'DUPLICATE' 'KEY' 'UPDATE' AssignmentList )? ``` +> **Note:** +> +> Starting from v6.6.0, TiDB supports [Resource Control](/tidb-resource-control.md). You can use this feature to execute SQL statements with different priorities in different resource groups. By configuring proper quotas and priorities for these resource groups, you can gain better scheduling control for SQL statements with different priorities. When resource control is enabled, statement priority (`PriorityOpt`) will no longer take effect. It is recommended that you use [Resource Control](/tidb-resource-control.md) to manage resource usage for different SQL statements. + ## Examples ```sql diff --git a/sql-statements/sql-statement-kill.md b/sql-statements/sql-statement-kill.md index 954abb3373df6..5b16f4166a26e 100644 --- a/sql-statements/sql-statement-kill.md +++ b/sql-statements/sql-statement-kill.md @@ -1,7 +1,6 @@ --- title: KILL summary: An overview of the usage of KILL for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-kill/','/docs/dev/reference/sql/statements/kill/'] --- # KILL diff --git a/sql-statements/sql-statement-load-data.md b/sql-statements/sql-statement-load-data.md index 4d447e87d997f..bbbcc9d192262 100644 --- a/sql-statements/sql-statement-load-data.md +++ b/sql-statements/sql-statement-load-data.md @@ -1,7 +1,6 @@ --- title: LOAD DATA | TiDB SQL Statement Reference summary: An overview of the usage of LOAD DATA for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-load-data/','/docs/dev/reference/sql/statements/load-data/','/tidb/dev/sql-statement-operate-load-data-job','/tidb/dev/sql-statement-show-load-data'] --- # LOAD DATA @@ -17,6 +16,14 @@ Starting from TiDB v7.0.0, the `LOAD DATA` SQL statement supports the following > > The new parameter `FIELDS DEFINED NULL BY` and support for importing data from S3 and GCS are experimental. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. + + +> **Note:** +> +> For the `LOAD DATA INFILE` statement, TiDB Cloud Dedicated supports `LOAD DATA LOCAL INFILE`, and `LOAD DATA INFILE` from Amazon S3 or Google Cloud Storage, while TiDB Cloud Serverless only supports `LOAD DATA LOCAL INFILE`. + + + ## Synopsis ```ebnf+diagram @@ -40,13 +47,13 @@ You can use `LOCAL` to specify data files on the client to be imported, where th If you are using TiDB Cloud, to use the `LOAD DATA` statement to load local data files, you need to add the `--local-infile` option to the connection string when you connect to TiDB Cloud. -- The following is an example connection string for TiDB Serverless: +- The following is an example connection string for TiDB Cloud Serverless: ``` mysql --connect-timeout 15 -u '' -h -P 4000 -D test --ssl-mode=VERIFY_IDENTITY --ssl-ca=/etc/ssl/cert.pem -p --local-infile ``` -- The following is an example connection string for TiDB Dedicated: +- The following is an example connection string for TiDB Cloud Dedicated: ``` mysql --connect-timeout 15 --ssl-mode=VERIFY_IDENTITY --ssl-ca= --tls-version="TLSv1.2" -u root -h -P 4000 -D test -p --local-infile @@ -145,6 +152,21 @@ LOAD DATA LOCAL INFILE '/mnt/evo970/data-sets/bikeshare-data/2017Q4-capitalbikes In the above example, `x'2c'` is the hexadecimal representation of the `,` character, and `b'100010'` is the binary representation of the `"` character. + + +The following example shows how to import data into a TiDB Cloud Dedicated cluster from Amazon S3 using the `LOAD DATA INFILE` statement: + +```sql +LOAD DATA INFILE 's3:///your-file.csv?role_arn=&external_id=' +INTO TABLE . +FIELDS TERMINATED BY ',' +ENCLOSED BY '"' +LINES TERMINATED BY '\n' +IGNORE 1 LINES; +``` + + + ## MySQL compatibility The syntax of the `LOAD DATA` statement is compatible with that of MySQL, except for character set options which are parsed but ignored. If you find any syntax compatibility difference, you can [report a bug](https://docs.pingcap.com/tidb/stable/support). diff --git a/sql-statements/sql-statement-load-stats.md b/sql-statements/sql-statement-load-stats.md index d3d8edea449e8..0335c4a2641be 100644 --- a/sql-statements/sql-statement-load-stats.md +++ b/sql-statements/sql-statement-load-stats.md @@ -1,7 +1,6 @@ --- title: LOAD STATS summary: An overview of the usage of LOAD STATS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-load-stats/'] --- # LOAD STATS @@ -10,7 +9,7 @@ The `LOAD STATS` statement is used to load the statistics into TiDB. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-lock-tables-and-unlock-tables.md b/sql-statements/sql-statement-lock-tables-and-unlock-tables.md index 1d1c0c306e365..baf5a772fff23 100644 --- a/sql-statements/sql-statement-lock-tables-and-unlock-tables.md +++ b/sql-statements/sql-statement-lock-tables-and-unlock-tables.md @@ -17,12 +17,13 @@ TiDB enables client sessions to acquire table locks for the purpose of cooperati A table lock protects against reads or writes by other sessions. A session that holds a `WRITE` lock can perform table-level operations such as `DROP TABLE` or `TRUNCATE TABLE`. -> **Note：** +> **Note:** > > The table locks feature is disabled by default. > -> - For TiDB Self-Hosted, to enable the table locks feature, you need to set [`enable-table-lock`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#enable-table-lock-new-in-v400) to `true` in the configuration files of all TiDB instances. -> - For TiDB Cloud, to enable the table locks feature, you need to contact [TiDB Cloud Support](https://docs.pingcap.com/tidbcloud/tidb-cloud-support) to set [`enable-table-lock`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#enable-table-lock-new-in-v400) to `true`. +> - For TiDB Self-Managed, to enable the table locks feature, you need to set [`enable-table-lock`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#enable-table-lock-new-in-v400) to `true` in the configuration files of all TiDB instances. +> - For TiDB Cloud Dedicated, to enable the table locks feature, you need to contact [TiDB Cloud Support](https://docs.pingcap.com/tidbcloud/tidb-cloud-support) to set [`enable-table-lock`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#enable-table-lock-new-in-v400) to `true`. +> - For TiDB Cloud Serverless, setting [`enable-table-lock`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#enable-table-lock-new-in-v400) to `true` is not supported. ## Synopsis diff --git a/sql-statements/sql-statement-modify-column.md b/sql-statements/sql-statement-modify-column.md index 55bb096c52065..2a51cf0ba3af2 100644 --- a/sql-statements/sql-statement-modify-column.md +++ b/sql-statements/sql-statement-modify-column.md @@ -1,7 +1,6 @@ --- title: MODIFY COLUMN | TiDB SQL Statement Reference summary: An overview of the usage of MODIFY COLUMN for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-modify-column/','/docs/dev/reference/sql/statements/modify-column/'] --- # MODIFY COLUMN @@ -205,7 +204,7 @@ CREATE TABLE `t1` ( ERROR 8200 (HY000): Unsupported modify column: table is partition table ``` -* Does not support modifying some data types (for example, some TIME types, Bit, Set, Enum, JSON) are not supported due to some compatibility issues of the `cast` function's behavior between TiDB and MySQL. +* Does not support modifying from some data types (for example, some TIME types, BIT, SET, ENUM, JSON) to some other types due to some compatibility issues of the `cast` function's behavior between TiDB and MySQL. ```sql CREATE TABLE t (a DECIMAL(13, 7)); diff --git a/sql-statements/sql-statement-overview.md b/sql-statements/sql-statement-overview.md new file mode 100644 index 0000000000000..8c375439923c5 --- /dev/null +++ b/sql-statements/sql-statement-overview.md @@ -0,0 +1,334 @@ +--- +title: SQL Statement Overview +summary: Learn about supported SQL statements in TiDB. +--- + +# SQL Statement Overview + +TiDB uses SQL statements that aim to follow ISO/IEC SQL standards, with extensions for MySQL and TiDB-specific statements where necessary. + +## Schema management / Data definition statements (DDL) + +| SQL Statement | Description | +|---------------|-------------| +| [`ALTER DATABASE`](/sql-statements/sql-statement-alter-database.md) | Modifies a database. | +| [`ALTER SEQUENCE`](/sql-statements/sql-statement-alter-sequence.md) | Modifies a sequence. | +| [`ALTER TABLE ... ADD COLUMN`](/sql-statements/sql-statement-add-column.md) | Adds a column to an existing table. | +| [`ALTER TABLE ... ADD INDEX`](/sql-statements/sql-statement-add-index.md) | Adds an index to an existing table. | +| [`ALTER TABLE ... ALTER INDEX`](/sql-statements/sql-statement-alter-index.md) | Changes an index definition. | +| [`ALTER TABLE ... CHANGE COLUMN`](/sql-statements/sql-statement-change-column.md) | Changes a column definition. | +| [`ALTER TABLE ... COMPACT`](/sql-statements/sql-statement-alter-table-compact.md) | Compacts a table. | +| [`ALTER TABLE ... DROP COLUMN`](/sql-statements/sql-statement-drop-column.md) | Drops a column from a table. | +| [`ALTER TABLE ... MODIFY COLUMN`](/sql-statements/sql-statement-modify-column.md) | Modifies a column definition. | +| [`ALTER TABLE ... RENAME INDEX`](/sql-statements/sql-statement-rename-index.md) | Renames an index. | +| [`ALTER TABLE`](/sql-statements/sql-statement-alter-table.md) | Changes a table definition. | +| [`CREATE DATABASE`](/sql-statements/sql-statement-create-database.md) | Creates a new database. | +| [`CREATE INDEX`](/sql-statements/sql-statement-create-index.md) | Creates a new index on a table. | +| [`CREATE SEQUENCE`](/sql-statements/sql-statement-create-sequence.md) | Creates a new sequence object. | +| [`CREATE TABLE LIKE`](/sql-statements/sql-statement-create-table-like.md) | Copies the definition of an existing table, without copying any data. | +| [`CREATE TABLE`](/sql-statements/sql-statement-create-table.md) | Creates a new table. | +| [`CREATE VIEW`](/sql-statements/sql-statement-create-view.md) | Creates a new view. | +| [`DROP DATABASE`](/sql-statements/sql-statement-drop-database.md) | Drops an existing database. | +| [`DROP INDEX`](/sql-statements/sql-statement-drop-index.md) | Drops an index from a table. | +| [`DROP SEQUENCE`](/sql-statements/sql-statement-drop-sequence.md) | Drops a sequence object. | +| [`DROP TABLE`](/sql-statements/sql-statement-drop-table.md) | Drops an existing table. | +| [`DROP VIEW`](/sql-statements/sql-statement-drop-view.md) | Drops an existing view. | +| [`RENAME TABLE`](/sql-statements/sql-statement-rename-table.md) | Renames a table. | +| [`SHOW COLUMNS FROM`](/sql-statements/sql-statement-show-columns-from.md) | Shows the columns from a table. | +| [`SHOW CREATE DATABASE`](/sql-statements/sql-statement-show-create-database.md) | Shows the CREATE statement for a database. | +| [`SHOW CREATE SEQUENCE`](/sql-statements/sql-statement-show-create-sequence.md) | Shows the CREATE statement for a sequence. | +| [`SHOW CREATE TABLE`](/sql-statements/sql-statement-show-create-table.md) | Shows the CREATE statement for a table. | +| [`SHOW DATABASES`](/sql-statements/sql-statement-show-databases.md) | Shows a list of databases that the current user has privileges to. | +| [`SHOW FIELDS FROM`](/sql-statements/sql-statement-show-fields-from.md) | Shows columns of a table. | +| [`SHOW INDEXES`](/sql-statements/sql-statement-show-indexes.md) | Shows indexes of a table. | +| [`SHOW SCHEMAS`](/sql-statements/sql-statement-show-schemas.md) | An alias to `SHOW DATABASES`, which shows a list of databases that the current user has privileges to. | +| [`SHOW TABLE NEXT_ROW_ID`](/sql-statements/sql-statement-show-table-next-rowid.md) | Shows the next row ID for a table. | +| [`SHOW TABLE REGIONS`](/sql-statements/sql-statement-show-table-regions.md) | Shows the Region information of a table in TiDB. | +| [`SHOW TABLE STATUS`](/sql-statements/sql-statement-show-table-status.md) | Shows various statistics about tables in TiDB. | +| [`SHOW TABLES`](/sql-statements/sql-statement-show-tables.md) | Shows tables in a database. | +| [`TRUNCATE`](/sql-statements/sql-statement-truncate.md) | Truncates all data from a table. | + +## Data manipulation statements (DML) + +| SQL Statement | Description | +|---------------|-------------| +| [`BATCH`](/sql-statements/sql-statement-batch.md) | Splits a DML statement into multiple statements in TiDB for execution. | +| [`DELETE`](/sql-statements/sql-statement-delete.md) | Deletes rows from a table. | +| [`INSERT`](/sql-statements/sql-statement-insert.md) | Inserts new rows into a table. | +| [`REPLACE`](/sql-statements/sql-statement-replace.md) | Replaces existing rows or inserts new rows. | +| [`SELECT`](/sql-statements/sql-statement-select.md) | Reads data from a table. | +| [`TABLE`](/sql-statements/sql-statement-table.md) | Retrieves rows from a table. | +| [`UPDATE`](/sql-statements/sql-statement-update.md) | Updates existing rows in a table. | +| [`WITH`](/sql-statements/sql-statement-with.md) | Defines common table expressions. | + +## Transaction statements + +| SQL Statement | Description | +|---------------|-------------| +| [`BEGIN`](/sql-statements/sql-statement-begin.md) | Begins a new transaction. | +| [`COMMIT`](/sql-statements/sql-statement-commit.md) | Commits the current transaction. | +| [`ROLLBACK`](/sql-statements/sql-statement-rollback.md) | Rolls back the current transaction. | +| [`SAVEPOINT`](/sql-statements/sql-statement-savepoint.md) | Sets a savepoint within a transaction. | +| [`SET TRANSACTION`](/sql-statements/sql-statement-set-transaction.md) | Changes the current isolation level on a `GLOBAL` or `SESSION` basis. | +| [`START TRANSACTION`](/sql-statements/sql-statement-start-transaction.md) | Starts a new transaction. | + +## Prepared statements + +| SQL Statement | Description | +|---------------|-------------| +| [`DEALLOCATE`](/sql-statements/sql-statement-deallocate.md) | Deallocates a prepared statement, freeing associated resources. | +| [`EXECUTE`](/sql-statements/sql-statement-execute.md) | Executes a prepared statement with specific parameter values. | +| [`PREPARE`](/sql-statements/sql-statement-prepare.md) | Creates a prepared statement with placeholders. | + +## Administrative statements + + + +| SQL Statement | Description | +|---------------|-------------| +| [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) | Cancels a DDL job. | +| [`ADMIN CHECK [TABLE\|INDEX]`](/sql-statements/sql-statement-admin-check-table-index.md) | Checks the integrity of a table or index. | +| [`ADMIN CHECKSUM TABLE`](/sql-statements/sql-statement-admin-checksum-table.md) | Computes the checksum of a table. | +| [`ADMIN CLEANUP INDEX`](/sql-statements/sql-statement-admin-cleanup.md) | Cleans up indexes from a table. | +| [`ADMIN PAUSE DDL`](/sql-statements/sql-statement-admin-pause-ddl.md) | Pauses DDL operations. | +| [`ADMIN RESUME DDL`](/sql-statements/sql-statement-admin-resume-ddl.md) | Resumes DDL operations. | +| [`ADMIN SHOW DDL [JOBS\|JOB QUERIES]`](/sql-statements/sql-statement-admin-show-ddl.md) | Shows DDL jobs or job queries. | +| [`ADMIN`](/sql-statements/sql-statement-admin.md) | Performs various administrative tasks. | +| [`FLUSH TABLES`](/sql-statements/sql-statement-flush-tables.md) | Included for [MySQL compatibility](/mysql-compatibility.md). It has no effective usage in TiDB. | +| [`SET `](/sql-statements/sql-statement-set-variable.md) | Modifies a system variable or user variable. | +| [`SET [NAMES\|CHARACTER SET]`](/sql-statements/sql-statement-set-names.md) | Set a character set and collation. | +| [`SPLIT REGION`](/sql-statements/sql-statement-split-region.md) | Splits a Region into smaller Regions. | + + + + + +| SQL Statement | Description | +|---------------|-------------| +| [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) | Cancels a DDL job. | +| [`ADMIN CHECK [TABLE\|INDEX]`](/sql-statements/sql-statement-admin-check-table-index.md) | Checks the integrity of a table or index. | +| [`ADMIN CHECKSUM TABLE`](/sql-statements/sql-statement-admin-checksum-table.md) | Computes the checksum of a table. | +| [`ADMIN CLEANUP INDEX`](/sql-statements/sql-statement-admin-cleanup.md) | Cleans up indexes from a table. | +| [`ADMIN PAUSE DDL`](/sql-statements/sql-statement-admin-pause-ddl.md) | Pauses DDL operations. | +| [`ADMIN RECOVER INDEX`](/sql-statements/sql-statement-admin-recover.md) | Recovers the consistency based on the redundant indexes. | +| [`ADMIN RESUME DDL`](/sql-statements/sql-statement-admin-resume-ddl.md) | Resumes DDL operations. | +| [`ADMIN SHOW DDL [JOBS\|JOB QUERIES]`](/sql-statements/sql-statement-admin-show-ddl.md) | Shows DDL jobs or job queries. | +| [`ADMIN`](/sql-statements/sql-statement-admin.md) | Performs various administrative tasks. | +| [`FLUSH TABLES`](/sql-statements/sql-statement-flush-tables.md) | Included for [MySQL compatibility](/mysql-compatibility.md). It has no effective usage in TiDB. | +| [`SET `](/sql-statements/sql-statement-set-variable.md) | Modifies a system variable or user variable. | +| [`SET [NAMES\|CHARACTER SET]`](/sql-statements/sql-statement-set-names.md) | Set a character set and collation. | +| [`SPLIT REGION`](/sql-statements/sql-statement-split-region.md) | Splits a Region into smaller Regions. | + + + +## Data import and export + +| SQL Statement | Description | +|---------------|-------------| +| [`CANCEL IMPORT JOB`](/sql-statements/sql-statement-cancel-import-job.md) | Cancels an ongoing import job. | +| [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) | Imports data into a table via the [Physical Import Mode](https://docs.pingcap.com/tidb/stable/tidb-lightning-physical-import-mode) of TiDB Lightning. | +| [`LOAD DATA`](/sql-statements/sql-statement-load-data.md) | Loads data into a table from Amazon S3 or Google Cloud Storage. | +| [`SHOW IMPORT JOB`](/sql-statements/sql-statement-show-import-job.md) | Shows the status of an import job. | + +## Backup & restore + +| SQL Statement | Description | +|---------------|-------------| +| [`BACKUP`](/sql-statements/sql-statement-backup.md) | Performs a distributed backup of the TiDB cluster. | +| [`FLASHBACK CLUSTER`](/sql-statements/sql-statement-flashback-cluster.md) | Restores the cluster to a specific snapshot. | +| [`FLASHBACK DATABASE`](/sql-statements/sql-statement-flashback-database.md) | Restores a database and its data deleted by the `DROP` statement. | +| [`FLASHBACK TABLE`](/sql-statements/sql-statement-flashback-table.md) | Restore the tables and data dropped by the `DROP` or `TRUNCATE` operation. | +| [`RECOVER TABLE`](/sql-statements/sql-statement-recover-table.md) | Recovers a deleted table and the data on it. | +| [`RESTORE`](/sql-statements/sql-statement-restore.md) | Restores a database from a backup. | +| [`SHOW BACKUPS`](/sql-statements/sql-statement-show-backups.md) | Shows backup tasks. | +| [`SHOW RESTORES`](/sql-statements/sql-statement-show-backups.md) | Shows restore tasks. | + +## Placement policy + +| SQL Statement | Description | +|---------------|-------------| +| [`ALTER PLACEMENT POLICY`](/sql-statements/sql-statement-alter-placement-policy.md) | Modifies a placement policy. | +| [`ALTER RANGE`](/sql-statements/sql-statement-alter-range.md) | Modifies the range of a placement policy. | +| [`CREATE PLACEMENT POLICY`](/sql-statements/sql-statement-create-placement-policy.md) | Creates a new placement policy. | +| [`DROP PLACEMENT POLICY`](/sql-statements/sql-statement-drop-placement-policy.md) | Drops an existing placement policy. | +| [`SHOW CREATE PLACEMENT POLICY`](/sql-statements/sql-statement-show-create-placement-policy.md) | Shows the `CREATE` statement for a placement policy. | +| [`SHOW PLACEMENT FOR`](/sql-statements/sql-statement-show-placement-for.md) | Shows placement rules for a specific table. | +| [`SHOW PLACEMENT LABELS`](/sql-statements/sql-statement-show-placement-labels.md) | Shows available placement labels. | +| [`SHOW PLACEMENT`](/sql-statements/sql-statement-show-placement.md) | Shows placement rules. | + +## Resource groups + + + +| SQL Statement | Description | +|---------------|-------------| +| [`ALTER RESOURCE GROUP`](/sql-statements/sql-statement-alter-resource-group.md) | Modifies a resource group. | +| [`CALIBRATE RESOURCE`](/sql-statements/sql-statement-calibrate-resource.md) | Estimates and outputs the [Request Unit (RU)](/tidb-resource-control.md#what-is-request-unit-ru) capacity of the current cluster. | +| [`CREATE RESOURCE GROUP`](/sql-statements/sql-statement-create-resource-group.md) | Creates a new resource group. | +| [`DROP RESOURCE GROUP`](/sql-statements/sql-statement-drop-resource-group.md) | Drops a resource group. | +| [`QUERY WATCH`](/sql-statements/sql-statement-query-watch.md) | Manages the runaway query watch list. | +| [`SET RESOURCE GROUP`](/sql-statements/sql-statement-set-resource-group.md) | Sets a resource group. | +| [`SHOW CREATE RESOURCE GROUP`](/sql-statements/sql-statement-show-create-resource-group.md) | Shows the `CREATE` statement for a resource group. | + + + + + +| SQL Statement | Description | +|---------------|-------------| +| [`ALTER RESOURCE GROUP`](/sql-statements/sql-statement-alter-resource-group.md) | Modifies a resource group. | +| [`CREATE RESOURCE GROUP`](/sql-statements/sql-statement-create-resource-group.md) | Creates a new resource group. | +| [`DROP RESOURCE GROUP`](/sql-statements/sql-statement-drop-resource-group.md) | Drops a resource group. | +| [`QUERY WATCH`](/sql-statements/sql-statement-query-watch.md) | Manages the runaway query watch list. | +| [`SET RESOURCE GROUP`](/sql-statements/sql-statement-set-resource-group.md) | Sets a resource group. | +| [`SHOW CREATE RESOURCE GROUP`](/sql-statements/sql-statement-show-create-resource-group.md) | Shows the `CREATE` statement for a resource group. | + + + +## Utility statements + +| SQL Statement | Description | +|---------------|-------------| +| [`DESC`](/sql-statements/sql-statement-desc.md) | An alias to `DESCRIBE`, which shows the structure of a table. | +| [`DESCRIBE`](/sql-statements/sql-statement-describe.md) | Shows the structure of a table. | +| [`DO`](/sql-statements/sql-statement-do.md) | Executes an expression but does not return any results. | +| [`EXPLAIN`](/sql-statements/sql-statement-explain.md) | Shows the execution plan of a query. | +| [`TRACE`](/sql-statements/sql-statement-trace.md) | Provides detailed information about query execution. | +| [`USE`](/sql-statements/sql-statement-use.md) | Sets the current database. | + +## Show statements + + + +| SQL Statement | Description | +|---------------|-------------| +| [`SHOW BUILTINS`](/sql-statements/sql-statement-show-builtins.md) | Lists builtin functions. | +| [`SHOW CHARACTER SET`](/sql-statements/sql-statement-show-character-set.md) | Lists character sets. | +| [`SHOW COLLATIONS`](/sql-statements/sql-statement-show-collation.md) | Lists collations. | +| [`SHOW ERRORS`](/sql-statements/sql-statement-show-errors.md) | Shows errors from previously executed statements. | +| [`SHOW STATUS`](/sql-statements/sql-statement-show-status.md) | Included for [compatibility with MySQL](/mysql-compatibility.md). TiDB uses [Prometheus and Grafana](/tidb-monitoring-framework.md) for centralized metrics collection instead of `SHOW STATUS` for most metrics. | +| [`SHOW VARIABLES`](/sql-statements/sql-statement-show-variables.md) | Shows system variables. | +| [`SHOW WARNINGS`](/sql-statements/sql-statement-show-warnings.md) | Shows warnings and notes from previously executed statements. | + + + + + +| SQL Statement | Description | +|---------------|-------------| +| [`SHOW BUILTINS`](/sql-statements/sql-statement-show-builtins.md) | Lists builtin functions. | +| [`SHOW CHARACTER SET`](/sql-statements/sql-statement-show-character-set.md) | Lists character sets. | +| [`SHOW COLLATIONS`](/sql-statements/sql-statement-show-collation.md) | Lists collations. | +| [`SHOW ERRORS`](/sql-statements/sql-statement-show-errors.md) | Shows errors from previously executed statements. | +| [`SHOW STATUS`](/sql-statements/sql-statement-show-status.md) | Included for [compatibility with MySQL](/mysql-compatibility.md). TiDB Cloud provides [Monitoring](/tidb-cloud/monitor-tidb-cluster.md) for centralized metrics collection instead of `SHOW STATUS` for most metrics. | +| [`SHOW VARIABLES`](/sql-statements/sql-statement-show-variables.md) | Shows system variables. | +| [`SHOW WARNINGS`](/sql-statements/sql-statement-show-warnings.md) | Shows warnings and notes from previously executed statements. | + + + +## Instance management + + + +| SQL Statement | Description | +|---------------|-------------| +| [`ALTER INSTANCE`](/sql-statements/sql-statement-alter-instance.md) | Modifies an instance. | +| [`FLUSH STATUS`](/sql-statements/sql-statement-flush-status.md) | Included for [compatibility with MySQL](/mysql-compatibility.md). TiDB uses [Prometheus and Grafana](/tidb-monitoring-framework.md) for centralized metrics collection instead of `SHOW STATUS` for most metrics. | +| [`KILL`](/sql-statements/sql-statement-kill.md) | Kills a connection in any TiDB instance in the current TiDB cluster. | +| [`SHOW CONFIG`](/sql-statements/sql-statement-show-config.md) | Shows the configuration of various components of TiDB. | +| [`SHOW ENGINES`](/sql-statements/sql-statement-show-engines.md) | Shows available storage engines. | +| [`SHOW PLUGINS`](/sql-statements/sql-statement-show-plugins.md) | Shows installed plugins. | +| [`SHOW PROCESSLIST`](/sql-statements/sql-statement-show-processlist.md) | Shows the current sessions connected to the same TiDB server. | +| [`SHOW PROFILES`](/sql-statements/sql-statement-show-profiles.md) | Included for [compatibility with MySQL](/mysql-compatibility.md). Currently, it only returns an empty result. | +| [`SHUTDOWN`](/sql-statements/sql-statement-shutdown.md) | Stops the client-connected TiDB instance, not the entire TiDB cluster. | + + + + + +| SQL Statement | Description | +|---------------|-------------| +| [`ALTER INSTANCE`](/sql-statements/sql-statement-alter-instance.md) | Modifies an instance. | +| [`FLUSH STATUS`](/sql-statements/sql-statement-flush-status.md) | Included for [compatibility with MySQL](/mysql-compatibility.md). TiDB Cloud provides [Monitoring](/tidb-cloud/monitor-tidb-cluster.md) for centralized metrics collection instead of `SHOW STATUS` for most metrics. | +| [`KILL`](/sql-statements/sql-statement-kill.md) | Kills a connection in any TiDB instance in the current TiDB cluster. | +| [`SHOW ENGINES`](/sql-statements/sql-statement-show-engines.md) | Shows available storage engines. | +| [`SHOW PLUGINS`](/sql-statements/sql-statement-show-plugins.md) | Shows installed plugins. | +| [`SHOW PROCESSLIST`](/sql-statements/sql-statement-show-processlist.md) | Shows the current sessions connected to the same TiDB server. | +| [`SHOW PROFILES`](/sql-statements/sql-statement-show-profiles.md) | Shows query profiles. Included for [compatibility with MySQL](/mysql-compatibility.md). Currently only returns an empty result. | + + + +## Locking statements + +| SQL Statement | Description | +|---------------|-------------| +| [`LOCK STATS`](/sql-statements/sql-statement-lock-stats.md) | Locks statistics of tables or partitions. | +| [`LOCK TABLES`](/sql-statements/sql-statement-lock-tables-and-unlock-tables.md) | Locks tables for the current session. | +| [`UNLOCK STATS`](/sql-statements/sql-statement-unlock-stats.md) | Unlocks statistics of tables or partitions. | +| [`UNLOCK TABLES`](/sql-statements/sql-statement-lock-tables-and-unlock-tables.md) | Unlocks tables. | + +## Account management / Data Control Language + +| SQL Statement | Description | +|---------------|-------------| +| [`ALTER USER`](/sql-statements/sql-statement-alter-user.md) | Modifies a user. | +| [`CREATE ROLE`](/sql-statements/sql-statement-create-role.md) | Creates a role. | +| [`CREATE USER`](/sql-statements/sql-statement-create-user.md) | Creates a new user. | +| [`DROP ROLE`](/sql-statements/sql-statement-drop-role.md) | Drops an existing role. | +| [`DROP USER`](/sql-statements/sql-statement-drop-user.md) | Drops an existing user. | +| [`FLUSH PRIVILEGES`](/sql-statements/sql-statement-flush-privileges.md) | Reloads the in-memory copy of privileges from the privilege tables. | +| [`GRANT `](/sql-statements/sql-statement-grant-privileges.md) | Grants privileges. | +| [`GRANT `](/sql-statements/sql-statement-grant-role.md) | Grants a role. | +| [`RENAME USER`](/sql-statements/sql-statement-rename-user.md) | Renames an existing user. | +| [`REVOKE `](/sql-statements/sql-statement-revoke-privileges.md) | Revokes privileges. | +| [`REVOKE `](/sql-statements/sql-statement-revoke-role.md) | Revokes a role. | +| [`SET DEFAULT ROLE`](/sql-statements/sql-statement-set-default-role.md) | Sets a default role. | +| [`SET PASSWORD`](/sql-statements/sql-statement-set-password.md) | Changes a password. | +| [`SET ROLE`](/sql-statements/sql-statement-set-role.md) | Enables roles in the current session. | +| [`SHOW CREATE USER`](/sql-statements/sql-statement-show-create-user.md) | Shows the `CREATE` statement for a user. | +| [`SHOW GRANTS`](/sql-statements/sql-statement-show-grants.md) | Shows privileges associated with a user. | +| [`SHOW PRIVILEGES`](/sql-statements/sql-statement-show-privileges.md) | Shows available privileges. | + +## TiCDC & TiDB Binlog + + + +| SQL Statement | Description | +|---------------|-------------| +| [`ADMIN [SET\|SHOW\|UNSET] BDR ROLE`](/sql-statements/sql-statement-admin-bdr-role.md) | Manages BDR roles. | +| [`CHANGE DRAINER`](/sql-statements/sql-statement-change-drainer.md) | Modifies the status information for Drainer in the cluster. | +| [`CHANGE PUMP`](/sql-statements/sql-statement-change-pump.md) | Modifies the status information for Pump in the cluster. | +| [`SHOW DRAINER STATUS`](/sql-statements/sql-statement-show-drainer-status.md) | Shows the status for all Drainer nodes in the cluster. | +| [`SHOW MASTER STATUS`](/sql-statements/sql-statement-show-master-status.md) | Shows the latest TSO in the cluster. | +| [`SHOW PUMP STATUS`](/sql-statements/sql-statement-show-pump-status.md) | Shows the status information for all Pump nodes in the cluster. | + + + + + +> **Note:** +> +> [TiCDC](https://docs.pingcap.com/tidb/stable/ticdc-overview) & [TiDB Binlog](https://docs.pingcap.com/tidb/stable/tidb-binlog-overview) are tools for replicating TiDB data to the upstream for TiDB Self-Managed. Most SQL statements for TiCDC and TiDB Binlog are not applicable to TiDB Cloud. For TiDB Cloud, you can use the [Changefeed](/tidb-cloud/changefeed-overview.md) feature in the [TiDB Cloud console](https://tidbcloud.com) instead to stream data. + +| SQL Statement | Description | +|---------------|-------------| +| [`SHOW MASTER STATUS`](/sql-statements/sql-statement-show-master-status.md) | Shows the latest TSO in the cluster. | + + + +## Statistics and plan management + +| SQL Statement | Description | +|---------------|-------------| +| [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) | Collects statistics about a table. | +| [`CREATE BINDING`](/sql-statements/sql-statement-create-binding.md) | Creates an execution plan binding for a SQL statement. | +| [`DROP BINDING`](/sql-statements/sql-statement-drop-binding.md) | Drops an execution plan binding from a SQL statement. | +| [`DROP STATS`](/sql-statements/sql-statement-drop-stats.md) | Drops statistics from a table. | +| [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) | Works similar to `EXPLAIN`, with the major difference that it will execute the statement. | +| [`LOAD STATS`](/sql-statements/sql-statement-load-stats.md) | Loads statistics into TiDB. | +| [`SHOW ANALYZE STATUS`](/sql-statements/sql-statement-show-analyze-status.md) | Shows statistics collection tasks. | +| [`SHOW BINDINGS`](/sql-statements/sql-statement-show-bindings.md) | Shows created SQL bindings. | +| [`SHOW STATS_HEALTHY`](/sql-statements/sql-statement-show-stats-healthy.md) | Shows an estimation of how accurate statistics are believed to be. | +| [`SHOW STATS_HISTOGRAMS`](/sql-statements/sql-statement-show-stats-histograms.md) | Shows the histogram information in statistics. | +| [`SHOW STATS_LOCKED`](/sql-statements/sql-statement-show-stats-locked.md) | Shows the tables whose statistics are locked. | +| [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md) | Shows how many rows are in a table and how many rows are changed in that table. | diff --git a/sql-statements/sql-statement-prepare.md b/sql-statements/sql-statement-prepare.md index d13088ae7907e..fc42e120ca33e 100644 --- a/sql-statements/sql-statement-prepare.md +++ b/sql-statements/sql-statement-prepare.md @@ -1,7 +1,6 @@ --- title: PREPARE | TiDB SQL Statement Reference summary: An overview of the usage of PREPARE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-prepare/','/docs/dev/reference/sql/statements/prepare/'] --- # PREPARE @@ -19,6 +18,12 @@ PrepareSQL ::= | UserVariable ``` +> **Note:** +> +> For each `PREPARE` statement, the maximum number of placeholders is 65535. + +To limit the number of `PREPARE` statements in the current TiDB instance, you can use the [`max_prepared_stmt_count`](/system-variables.md#max_prepared_stmt_count) system variable. + ## Examples ```sql diff --git a/sql-statements/sql-statement-query-watch.md b/sql-statements/sql-statement-query-watch.md index be7d49b00fc58..545ad1bfbb80f 100644 --- a/sql-statements/sql-statement-query-watch.md +++ b/sql-statements/sql-statement-query-watch.md @@ -9,7 +9,7 @@ The `QUERY WATCH` statement is used to manually manage the watch list of runaway > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-recover-table.md b/sql-statements/sql-statement-recover-table.md index 87f12aa90e753..e5f46d6d3315a 100644 --- a/sql-statements/sql-statement-recover-table.md +++ b/sql-statements/sql-statement-recover-table.md @@ -1,7 +1,6 @@ --- title: RECOVER TABLE summary: An overview of the usage of RECOVER TABLE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-recover-table/','/docs/dev/reference/sql/statements/recover-table/'] --- # RECOVER TABLE diff --git a/sql-statements/sql-statement-rename-index.md b/sql-statements/sql-statement-rename-index.md index 9c27c09400131..9d419ab7c0238 100644 --- a/sql-statements/sql-statement-rename-index.md +++ b/sql-statements/sql-statement-rename-index.md @@ -1,7 +1,6 @@ --- title: RENAME INDEX | TiDB SQL Statement Reference summary: An overview of the usage of RENAME INDEX for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-rename-index/','/docs/dev/reference/sql/statements/rename-index/'] --- # RENAME INDEX diff --git a/sql-statements/sql-statement-rename-table.md b/sql-statements/sql-statement-rename-table.md index 6f03a06f4ca91..d200beba61ffd 100644 --- a/sql-statements/sql-statement-rename-table.md +++ b/sql-statements/sql-statement-rename-table.md @@ -1,12 +1,11 @@ --- title: RENAME TABLE | TiDB SQL Statement Reference summary: An overview of the usage of RENAME TABLE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-rename-table/','/docs/dev/reference/sql/statements/rename-table/'] --- # RENAME TABLE -This statement renames an existing table to a new name. +This statement is used to rename existing tables and views, supporting renaming multiple tables at once and renaming across databases. ## Synopsis @@ -21,21 +20,39 @@ TableToTable ::= ## Examples ```sql -mysql> CREATE TABLE t1 (a int); +CREATE TABLE t1 (a int); +``` + +``` Query OK, 0 rows affected (0.12 sec) +``` + +```sql +SHOW TABLES; +``` -mysql> SHOW TABLES; +``` +----------------+ | Tables_in_test | +----------------+ | t1 | +----------------+ 1 row in set (0.00 sec) +``` + +```sql +RENAME TABLE t1 TO t2; +``` -mysql> RENAME TABLE t1 TO t2; +``` Query OK, 0 rows affected (0.08 sec) +``` + +```sql +SHOW TABLES; +``` -mysql> SHOW TABLES; +``` +----------------+ | Tables_in_test | +----------------+ @@ -44,6 +61,103 @@ mysql> SHOW TABLES; 1 row in set (0.00 sec) ``` +The following example demonstrates how to rename multiple tables across databases, assuming that the databases `db1`, `db2`, `db3`, and `db4` already exist, and that the tables `db1.t1` and `db3.t3` already exist: + +```sql +RENAME TABLE db1.t1 To db2.t2, db3.t3 To db4.t4; +``` + +``` +Query OK, 0 rows affected (0.08 sec) +``` + +```sql +USE db1; SHOW TABLES; +``` + +``` +Database changed +Empty set (0.00 sec) +``` + +```sql +USE db2; SHOW TABLES; +``` + +``` +Database changed ++---------------+ +| Tables_in_db2 | ++---------------+ +| t2 | ++---------------+ +1 row in set (0.00 sec) +``` + +```sql +USE db3; SHOW TABLES; +``` + +``` +Database changed +Empty set (0.00 sec) +``` + +```sql +USE db4; SHOW TABLES; +``` + +``` +Database changed ++---------------+ +| Tables_in_db4 | ++---------------+ +| t4 | ++---------------+ +1 row in set (0.00 sec) +``` + +The atomic rename can be used to swap out a table without having any moment in which the table does not exist. + +```sql +CREATE TABLE t1(id int PRIMARY KEY); +``` + +``` +Query OK, 0 rows affected (0.04 sec) +``` + +```sql +CREATE TABLE t1_new(id int PRIMARY KEY, n CHAR(0)); +```` + +``` +Query OK, 0 rows affected (0.04 sec) +``` + +```sql +RENAME TABLE t1 TO t1_old, t1_new TO t1; +``` + +``` +Query OK, 0 rows affected (0.07 sec) +``` + +```sql +SHOW CREATE TABLE t1\G +``` + +``` +*************************** 1. row *************************** + Table: t1 +Create Table: CREATE TABLE `t1` ( + `id` int NOT NULL, + `n` char(0) DEFAULT NULL, + PRIMARY KEY (`id`) /*T![clustered_index] CLUSTERED */ +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin +1 row in set (0.00 sec) +``` + ## MySQL compatibility The `RENAME TABLE` statement in TiDB is fully compatible with MySQL. If you find any compatibility differences, [report a bug](https://docs.pingcap.com/tidb/stable/support). diff --git a/sql-statements/sql-statement-replace.md b/sql-statements/sql-statement-replace.md index bb5912d8d2715..d44db793a3af0 100644 --- a/sql-statements/sql-statement-replace.md +++ b/sql-statements/sql-statement-replace.md @@ -1,7 +1,6 @@ --- title: REPLACE | TiDB SQL Statement Reference summary: An overview of the usage of REPLACE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-replace/','/docs/dev/reference/sql/statements/replace/'] --- # REPLACE @@ -33,6 +32,10 @@ InsertValues ::= | 'SET' ColumnSetValue? ( ',' ColumnSetValue )* ``` +> **Note:** +> +> Starting from v6.6.0, TiDB supports [Resource Control](/tidb-resource-control.md). You can use this feature to execute SQL statements with different priorities in different resource groups. By configuring proper quotas and priorities for these resource groups, you can gain better scheduling control for SQL statements with different priorities. When resource control is enabled, statement priority (`PriorityOpt`) will no longer take effect. It is recommended that you use [Resource Control](/tidb-resource-control.md) to manage resource usage for different SQL statements. + ## Examples ```sql diff --git a/sql-statements/sql-statement-restore.md b/sql-statements/sql-statement-restore.md index 76562c7f8914e..415c3d2efe358 100644 --- a/sql-statements/sql-statement-restore.md +++ b/sql-statements/sql-statement-restore.md @@ -1,7 +1,6 @@ --- title: RESTORE | TiDB SQL Statement Reference summary: An overview of the usage of RESTORE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-restore/'] --- # RESTORE @@ -11,7 +10,7 @@ This statement performs a distributed restore from a backup archive previously p > **Warning:** > > - This feature is experimental. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. -> - This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> - This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. The `RESTORE` statement uses the same engine as the [BR tool](https://docs.pingcap.com/tidb/stable/backup-and-restore-overview), except that the restore process is driven by TiDB itself rather than a separate BR tool. All benefits and caveats of BR also apply here. In particular, **`RESTORE` is currently not ACID-compliant**. Before running `RESTORE`, ensure that the following requirements are met: diff --git a/sql-statements/sql-statement-revoke-privileges.md b/sql-statements/sql-statement-revoke-privileges.md index 532b601db0949..9fb6f6681e9f5 100644 --- a/sql-statements/sql-statement-revoke-privileges.md +++ b/sql-statements/sql-statement-revoke-privileges.md @@ -1,7 +1,6 @@ --- title: REVOKE | TiDB SQL Statement Reference summary: An overview of the usage of REVOKE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-revoke-privileges/','/docs/dev/reference/sql/statements/revoke-privileges/'] --- # `REVOKE ` @@ -11,8 +10,8 @@ This statement removes privileges from an existing user. Executing this statemen ## Synopsis ```ebnf+diagram -GrantStmt ::= - 'GRANT' PrivElemList 'ON' ObjectType PrivLevel 'TO' UserSpecList RequireClauseOpt WithGrantOptionOpt +RevokeStmt ::= + 'REVOKE' PrivElemList 'ON' ObjectType PrivLevel 'FROM' UserSpecList PrivElemList ::= PrivElem ( ',' PrivElem )* @@ -55,10 +54,6 @@ PrivLevel ::= UserSpecList ::= UserSpec ( ',' UserSpec )* - -RequireClauseOpt ::= ('REQUIRE' ('NONE' | 'SSL' | 'X509' | RequireListElement ('AND'? RequireListElement)*))? - -RequireListElement ::= 'ISSUER' Issuer | 'SUBJECT' Subject | 'CIPHER' Cipher | 'SAN' SAN | 'TOKEN_ISSUER' TokenIssuer ``` ## Examples diff --git a/sql-statements/sql-statement-rollback.md b/sql-statements/sql-statement-rollback.md index 5a08335a17243..2f192d94fbb52 100644 --- a/sql-statements/sql-statement-rollback.md +++ b/sql-statements/sql-statement-rollback.md @@ -1,12 +1,11 @@ --- title: ROLLBACK | TiDB SQL Statement Reference summary: An overview of the usage of ROLLBACK for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-rollback/','/docs/dev/reference/sql/statements/rollback/'] --- # ROLLBACK -This statement reverts all changes in the current transaction inside of TIDB. It is the opposite of a `COMMIT` statement. +This statement reverts all changes in the current transaction inside of TiDB. It is the opposite of a `COMMIT` statement. ## Synopsis diff --git a/sql-statements/sql-statement-select.md b/sql-statements/sql-statement-select.md index 1e101c27e9a3a..873742101a6f0 100644 --- a/sql-statements/sql-statement-select.md +++ b/sql-statements/sql-statement-select.md @@ -1,7 +1,6 @@ --- title: SELECT | TiDB SQL Statement Reference summary: An overview of the usage of SELECT for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-select/','/docs/dev/reference/sql/statements/select/'] --- # SELECT @@ -76,6 +75,10 @@ TableSampleOpt ::= |`LOCK IN SHARE MODE` | To guarantee compatibility, TiDB parses these three modifiers, but will ignore them. | | `TABLESAMPLE` | To get a sample of rows from the table. | +> **Note:** +> +> Starting from v6.6.0, TiDB supports [Resource Control](/tidb-resource-control.md). You can use this feature to execute SQL statements with different priorities in different resource groups. By configuring proper quotas and priorities for these resource groups, you can gain better scheduling control for SQL statements with different priorities. When resource control is enabled, statement priority (`HIGH_PRIORITY`) will no longer take effect. It is recommended that you use [Resource Control](/tidb-resource-control.md) to manage resource usage for different SQL statements. + ## Examples ### SELECT @@ -127,7 +130,7 @@ The `SELECT ... INTO OUTFILE` statement is used to write the result of a query t > **Note:** > -> - This statement is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> - This statement is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). > - This statement does not support writing query results to any [external storages](https://docs.pingcap.com/tidb/stable/backup-and-restore-storages) such as Amazon S3 or GCS. In the statement, you can specify the format of the output file by using the following clauses: diff --git a/sql-statements/sql-statement-set-names.md b/sql-statements/sql-statement-set-names.md index dd76df13ff224..23c47db22e103 100644 --- a/sql-statements/sql-statement-set-names.md +++ b/sql-statements/sql-statement-set-names.md @@ -1,7 +1,6 @@ --- title: SET [NAMES|CHARACTER SET] | TiDB SQL Statement Reference summary: An overview of the usage of SET [NAMES|CHARACTER SET] for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-set-names/','/docs/dev/reference/sql/statements/set-names/'] --- # SET [NAMES|CHARACTER SET] diff --git a/sql-statements/sql-statement-set-password.md b/sql-statements/sql-statement-set-password.md index 2d3ae1e3b6e2e..9d861fc0b4762 100644 --- a/sql-statements/sql-statement-set-password.md +++ b/sql-statements/sql-statement-set-password.md @@ -1,7 +1,6 @@ --- title: SET PASSWORD | TiDB SQL Statement Reference summary: An overview of the usage of SET PASSWORD for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-set-password/','/docs/dev/reference/sql/statements/set-password/'] --- # SET PASSWORD diff --git a/sql-statements/sql-statement-set-resource-group.md b/sql-statements/sql-statement-set-resource-group.md index 6e43d00501965..f21126fc19288 100644 --- a/sql-statements/sql-statement-set-resource-group.md +++ b/sql-statements/sql-statement-set-resource-group.md @@ -9,7 +9,7 @@ summary: An overview of the usage of SET RESOURCE GROUP in the TiDB database. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-set-role.md b/sql-statements/sql-statement-set-role.md index 242677164cd0a..1800b6509a985 100644 --- a/sql-statements/sql-statement-set-role.md +++ b/sql-statements/sql-statement-set-role.md @@ -1,7 +1,6 @@ --- title: SET ROLE | TiDB SQL Statement Reference summary: An overview of the usage of SET ROLE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-set-role/'] --- # SET ROLE diff --git a/sql-statements/sql-statement-set-transaction.md b/sql-statements/sql-statement-set-transaction.md index 56c6354d533bb..acb08f235404a 100644 --- a/sql-statements/sql-statement-set-transaction.md +++ b/sql-statements/sql-statement-set-transaction.md @@ -1,7 +1,6 @@ --- title: SET TRANSACTION | TiDB SQL Statement Reference summary: An overview of the usage of SET TRANSACTION for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-set-transaction/','/docs/dev/reference/sql/statements/set-transaction/'] --- # SET TRANSACTION diff --git a/sql-statements/sql-statement-set-variable.md b/sql-statements/sql-statement-set-variable.md index 72f443fe160d0..b095869c2ec32 100644 --- a/sql-statements/sql-statement-set-variable.md +++ b/sql-statements/sql-statement-set-variable.md @@ -1,7 +1,6 @@ --- title: SET [GLOBAL|SESSION] | TiDB SQL Statement Reference summary: An overview of the usage of SET [GLOBAL|SESSION] for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-set-variable/','/docs/dev/reference/sql/statements/set-variable/'] --- # `SET [GLOBAL|SESSION] ` diff --git a/sql-statements/sql-statement-show-analyze-status.md b/sql-statements/sql-statement-show-analyze-status.md index 087a6e8819176..59efee4dc4483 100644 --- a/sql-statements/sql-statement-show-analyze-status.md +++ b/sql-statements/sql-statement-show-analyze-status.md @@ -1,7 +1,6 @@ --- title: SHOW ANALYZE STATUS summary: An overview of the usage of SHOW ANALYZE STATUS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-analyze-status/'] --- # SHOW ANALYZE STATUS @@ -14,6 +13,21 @@ Starting from TiDB v6.1.0, you can view the history tasks within the last 7 days Starting from TiDB v7.3.0, you can view the progress of the current `ANALYZE` task through the system table `mysql.analyze_jobs` or `SHOW ANALYZE STATUS`. +Currently, the `SHOW ANALYZE STATUS` statement returns the following columns: + +| Column name | Description | +| :--------------- | :------------- | +| `Table_schema` | The database name | +| `Table_name` | The table name | +| `Partition_name` | The partition name | +| `Job_info` | The task information. If an index is analyzed, this information will include the index name. When `tidb_analyze_version =2`, this information will include configuration items such as sample rate. | +| `Processed_rows` | The number of rows that have been analyzed | +| `Start_time` | The time at which the task starts | +| `State` | The state of a task, including `pending`, `running`, `finished`, and `failed` | +| `Fail_reason` | The reason why the task fails. If the execution is successful, the value is `NULL`. | +| `Instance` | The TiDB instance that executes the task | +| `Process_id` | The process ID that executes the task | + ## Synopsis ```ebnf+diagram @@ -72,4 +86,4 @@ This statement is a TiDB extension to MySQL syntax. ## See also -* [ANALYZE_STATUS table](/information-schema/information-schema-analyze-status.md) \ No newline at end of file +* [ANALYZE_STATUS table](/information-schema/information-schema-analyze-status.md) diff --git a/sql-statements/sql-statement-show-backups.md b/sql-statements/sql-statement-show-backups.md index e093b7a7445e2..0636060c8fdea 100644 --- a/sql-statements/sql-statement-show-backups.md +++ b/sql-statements/sql-statement-show-backups.md @@ -1,7 +1,6 @@ --- title: SHOW [BACKUPS|RESTORES] | TiDB SQL Statement Reference summary: An overview of the usage of SHOW [BACKUPS|RESTORES] for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-backups/'] --- # SHOW [BACKUPS|RESTORES] @@ -14,7 +13,7 @@ Use `SHOW BACKUPS` to query `BACKUP` tasks and use `SHOW RESTORES` to query `RES > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. Backups and restores that were started with the `br` commandline tool are not shown. diff --git a/sql-statements/sql-statement-show-bindings.md b/sql-statements/sql-statement-show-bindings.md index e7b3866bc8ad1..b31b177c0a12c 100644 --- a/sql-statements/sql-statement-show-bindings.md +++ b/sql-statements/sql-statement-show-bindings.md @@ -1,7 +1,6 @@ --- title: SHOW [GLOBAL|SESSION] BINDINGS summary: Use of SHOW BINDINGS binding in TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-bindings/'] --- # SHOW [GLOBAL|SESSION] BINDINGS diff --git a/sql-statements/sql-statement-show-builtins.md b/sql-statements/sql-statement-show-builtins.md index ea5a934582067..a3dcc803d4571 100644 --- a/sql-statements/sql-statement-show-builtins.md +++ b/sql-statements/sql-statement-show-builtins.md @@ -1,7 +1,6 @@ --- title: SHOW BUILTINS summary: The usage of SHOW BUILTINS in TiDB. -aliases: ['/docs/dev/sql-statements/sql-statement-show-builtins/'] --- # SHOW BUILTINS diff --git a/sql-statements/sql-statement-show-character-set.md b/sql-statements/sql-statement-show-character-set.md index f6b192afaca3c..8377ea122a9d7 100644 --- a/sql-statements/sql-statement-show-character-set.md +++ b/sql-statements/sql-statement-show-character-set.md @@ -1,7 +1,6 @@ --- title: SHOW CHARACTER SET | TiDB SQL Statement Reference summary: An overview of the usage of SHOW CHARACTER SET for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-character-set/','/docs/dev/reference/sql/statements/show-character-set/'] --- # SHOW CHARACTER SET @@ -26,16 +25,17 @@ SHOW CHARACTER SET; ``` ``` -+---------+---------------+-------------------+--------+ -| Charset | Description | Default collation | Maxlen | -+---------+---------------+-------------------+--------+ -| utf8 | UTF-8 Unicode | utf8_bin | 3 | -| utf8mb4 | UTF-8 Unicode | utf8mb4_bin | 4 | -| ascii | US ASCII | ascii_bin | 1 | -| latin1 | Latin1 | latin1_bin | 1 | -| binary | binary | binary | 1 | -+---------+---------------+-------------------+--------+ -5 rows in set (0.00 sec) ++---------+-------------------------------------+-------------------+--------+ +| Charset | Description | Default collation | Maxlen | ++---------+-------------------------------------+-------------------+--------+ +| ascii | US ASCII | ascii_bin | 1 | +| binary | binary | binary | 1 | +| gbk | Chinese Internal Code Specification | gbk_chinese_ci | 2 | +| latin1 | Latin1 | latin1_bin | 1 | +| utf8 | UTF-8 Unicode | utf8_bin | 3 | +| utf8mb4 | UTF-8 Unicode | utf8mb4_bin | 4 | ++---------+-------------------------------------+-------------------+--------+ +6 rows in set (0.00 sec) ``` ```sql diff --git a/sql-statements/sql-statement-show-collation.md b/sql-statements/sql-statement-show-collation.md index 1735b47ee22d1..075bb742b18ec 100644 --- a/sql-statements/sql-statement-show-collation.md +++ b/sql-statements/sql-statement-show-collation.md @@ -1,7 +1,6 @@ --- title: SHOW COLLATION | TiDB SQL Statement Reference summary: An overview of the usage of SHOW COLLATION for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-collation/','/docs/dev/reference/sql/statements/show-collation/'] --- # SHOW COLLATION diff --git a/sql-statements/sql-statement-show-column-stats-usage.md b/sql-statements/sql-statement-show-column-stats-usage.md new file mode 100644 index 0000000000000..b7d28adc4d04e --- /dev/null +++ b/sql-statements/sql-statement-show-column-stats-usage.md @@ -0,0 +1,58 @@ +--- +title: SHOW COLUMN_STATS_USAGE +summary: An overview of the usage of SHOW COLUMN_STATS_USAGE for TiDB database. +--- + +# SHOW COLUMN_STATS_USAGE + +The `SHOW COLUMN_STATS_USAGE` statement shows the last usage time and collection time of column statistics. You can also use it to locate `PREDICATE COLUMNS` and columns on which statistics have been collected. + +Currently, the `SHOW COLUMN_STATS_USAGE` statement returns the following columns: + +| Column name | Description | +| -------- | ------------- | +| `Db_name` | The database name | +| `Table_name` | The table name | +| `Partition_name` | The partition name | +| `Column_name` | The column name | +| `Last_used_at` | The last time when the column statistics were used in the query optimization | +| `Last_analyzed_at` | The last time when the column statistics were collected | + +## Synopsis + +```ebnf+diagram +ShowColumnStatsUsageStmt ::= + "SHOW" "COLUMN_STATS_USAGE" ShowLikeOrWhere? + +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` + +## Examples + +```sql +SHOW COLUMN_STATS_USAGE; +``` + +``` ++---------+------------+----------------+-------------+--------------+---------------------+ +| Db_name | Table_name | Partition_name | Column_name | Last_used_at | Last_analyzed_at | ++---------+------------+----------------+-------------+--------------+---------------------+ +| test | t1 | | id | NULL | 2024-05-10 11:04:23 | +| test | t1 | | b | NULL | 2024-05-10 11:04:23 | +| test | t1 | | pad | NULL | 2024-05-10 11:04:23 | +| test | t | | a | NULL | 2024-05-10 11:37:06 | +| test | t | | b | NULL | 2024-05-10 11:37:06 | ++---------+------------+----------------+-------------+--------------+---------------------+ +5 rows in set (0.00 sec) +``` + +## MySQL compatibility + +This statement is a TiDB extension to MySQL syntax. + +## See also + +* [`ANALYZE`](/sql-statements/sql-statement-analyze-table.md) +* [Introduction to Statistics](/statistics.md) \ No newline at end of file diff --git a/sql-statements/sql-statement-show-columns-from.md b/sql-statements/sql-statement-show-columns-from.md index 41de5fb623561..63103e6a0477c 100644 --- a/sql-statements/sql-statement-show-columns-from.md +++ b/sql-statements/sql-statement-show-columns-from.md @@ -1,7 +1,6 @@ --- title: SHOW [FULL] COLUMNS FROM | TiDB SQL Statement Reference summary: An overview of the usage of SHOW [FULL] COLUMNS FROM for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-columns-from/','/docs/dev/reference/sql/statements/show-columns-from/'] --- # SHOW [FULL] COLUMNS FROM diff --git a/sql-statements/sql-statement-show-config.md b/sql-statements/sql-statement-show-config.md index bb23df9b5f8c5..ee301ec69c60c 100644 --- a/sql-statements/sql-statement-show-config.md +++ b/sql-statements/sql-statement-show-config.md @@ -1,7 +1,6 @@ --- title: SHOW CONFIG summary: Overview of the use of SHOW CONFIG in the TiDB database -aliases: ['/docs/dev/sql-statements/sql-statement-show-config/'] --- # SHOW CONFIG @@ -10,7 +9,7 @@ The `SHOW CONFIG` statement is used to show the current configuration of various > **Note:** > -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This feature is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). ## Synopsis diff --git a/sql-statements/sql-statement-show-create-placement-policy.md b/sql-statements/sql-statement-show-create-placement-policy.md index eae33c3db8111..edd6668dcc2f1 100644 --- a/sql-statements/sql-statement-show-create-placement-policy.md +++ b/sql-statements/sql-statement-show-create-placement-policy.md @@ -9,7 +9,7 @@ summary: The usage of SHOW CREATE PLACEMENT POLICY in TiDB. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-show-create-resource-group.md b/sql-statements/sql-statement-show-create-resource-group.md index afd9a78ea2e6e..c63db125c8101 100644 --- a/sql-statements/sql-statement-show-create-resource-group.md +++ b/sql-statements/sql-statement-show-create-resource-group.md @@ -9,7 +9,7 @@ You can use the `SHOW CREATE RESOURCE GROUP` statement to view the current defin > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-show-create-sequence.md b/sql-statements/sql-statement-show-create-sequence.md index 1d9d56a840714..90f181baeb721 100644 --- a/sql-statements/sql-statement-show-create-sequence.md +++ b/sql-statements/sql-statement-show-create-sequence.md @@ -1,7 +1,6 @@ --- title: SHOW CREATE SEQUENCE summary: An overview of the usage of SHOW CREATE SEQUENCE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-create-sequence/','/docs/dev/reference/sql/statements/show-create-sequence/'] --- # SHOW CREATE SEQUENCE diff --git a/sql-statements/sql-statement-show-create-table.md b/sql-statements/sql-statement-show-create-table.md index 418eb63ba7321..d269579398934 100644 --- a/sql-statements/sql-statement-show-create-table.md +++ b/sql-statements/sql-statement-show-create-table.md @@ -1,7 +1,6 @@ --- title: SHOW CREATE TABLE | TiDB SQL Statement Reference summary: An overview of the usage of SHOW CREATE TABLE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-create-table/','/docs/dev/reference/sql/statements/show-create-table/'] --- # SHOW CREATE TABLE diff --git a/sql-statements/sql-statement-show-create-user.md b/sql-statements/sql-statement-show-create-user.md index b952c2dca3da3..7d1da873ce8c2 100644 --- a/sql-statements/sql-statement-show-create-user.md +++ b/sql-statements/sql-statement-show-create-user.md @@ -1,7 +1,6 @@ --- title: SHOW CREATE USER | TiDB SQL Statement Reference summary: An overview of the usage of SHOW CREATE USER for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-create-user/','/docs/dev/reference/sql/statements/show-create-user/'] --- # SHOW CREATE USER diff --git a/sql-statements/sql-statement-show-databases.md b/sql-statements/sql-statement-show-databases.md index ef28478c9af19..74a58ded4eb93 100644 --- a/sql-statements/sql-statement-show-databases.md +++ b/sql-statements/sql-statement-show-databases.md @@ -1,7 +1,6 @@ --- title: SHOW DATABASES | TiDB SQL Statement Reference summary: An overview of the usage of SHOW DATABASES for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-databases/','/docs/dev/reference/sql/statements/show-databases/'] --- # SHOW DATABASES diff --git a/sql-statements/sql-statement-show-drainer-status.md b/sql-statements/sql-statement-show-drainer-status.md index a0dd876e99fe3..2cb36d45c4959 100644 --- a/sql-statements/sql-statement-show-drainer-status.md +++ b/sql-statements/sql-statement-show-drainer-status.md @@ -1,7 +1,6 @@ --- title: SHOW DRAINER STATUS summary: An overview of the usage of SHOW DRAINER STATUS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-drainer-status/'] --- # SHOW DRAINER STATUS @@ -10,7 +9,7 @@ The `SHOW DRAINER STATUS` statement displays the status information for all Drai > **Note:** > -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This feature is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). ## Examples diff --git a/sql-statements/sql-statement-show-engines.md b/sql-statements/sql-statement-show-engines.md index 19ea33560cecb..259857c859660 100644 --- a/sql-statements/sql-statement-show-engines.md +++ b/sql-statements/sql-statement-show-engines.md @@ -1,7 +1,6 @@ --- title: SHOW ENGINES | TiDB SQL Statement Reference summary: An overview of the usage of SHOW ENGINES for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-engines/','/docs/dev/reference/sql/statements/show-engines/'] --- # SHOW ENGINES diff --git a/sql-statements/sql-statement-show-errors.md b/sql-statements/sql-statement-show-errors.md index 8771f4b77b7cd..0d634fe651a29 100644 --- a/sql-statements/sql-statement-show-errors.md +++ b/sql-statements/sql-statement-show-errors.md @@ -1,7 +1,6 @@ --- title: SHOW ERRORS | TiDB SQL Statement Reference summary: An overview of the usage of SHOW ERRORS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-errors/','/docs/dev/reference/sql/statements/show-errors/'] --- # SHOW ERRORS diff --git a/sql-statements/sql-statement-show-fields-from.md b/sql-statements/sql-statement-show-fields-from.md index 657e8bdf1aa89..85527ed6b3f90 100644 --- a/sql-statements/sql-statement-show-fields-from.md +++ b/sql-statements/sql-statement-show-fields-from.md @@ -1,7 +1,6 @@ --- title: SHOW [FULL] FIELDS FROM | TiDB SQL Statement Reference summary: An overview of the usage of SHOW [FULL] FIELDS FROM for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-fields-from/','/docs/dev/reference/sql/statements/show-fields-from/'] --- # SHOW [FULL] FIELDS FROM diff --git a/sql-statements/sql-statement-show-grants.md b/sql-statements/sql-statement-show-grants.md index 301d48b8056df..c1a445a412c94 100644 --- a/sql-statements/sql-statement-show-grants.md +++ b/sql-statements/sql-statement-show-grants.md @@ -1,7 +1,6 @@ --- title: SHOW GRANTS | TiDB SQL Statement Reference summary: An overview of the usage of SHOW GRANTS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-grants/','/docs/dev/reference/sql/statements/show-grants/'] --- # SHOW GRANTS diff --git a/sql-statements/sql-statement-show-import-job.md b/sql-statements/sql-statement-show-import-job.md index 1e2a2ac66702a..396f60558a137 100644 --- a/sql-statements/sql-statement-show-import-job.md +++ b/sql-statements/sql-statement-show-import-job.md @@ -7,10 +7,6 @@ summary: An overview of the usage of SHOW IMPORT in TiDB. The `SHOW IMPORT` statement is used to show the IMPORT jobs created in TiDB. This statement can only show jobs created by the current user. -> **Note:** -> -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. - ## Required privileges - `SHOW IMPORT JOBS`: if a user has the `SUPER` privilege, this statement shows all import jobs in TiDB. Otherwise, this statement only shows jobs created by the current user. diff --git a/sql-statements/sql-statement-show-indexes.md b/sql-statements/sql-statement-show-indexes.md index 9b1e8beef2c76..ea11d844ee0dd 100644 --- a/sql-statements/sql-statement-show-indexes.md +++ b/sql-statements/sql-statement-show-indexes.md @@ -1,7 +1,6 @@ --- title: SHOW INDEXES [FROM|IN] | TiDB SQL Statement Reference summary: An overview of the usage of SHOW INDEXES [FROM|IN] for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-indexes/','/docs/dev/reference/sql/statements/show-indexes/', '/tidb/dev/sql-statement-show-index/', '/tidb/dev/sql-statement-show-keys/'] --- # SHOW INDEXES [FROM|IN] @@ -53,6 +52,8 @@ mysql> SHOW KEYS FROM t1; 2 rows in set (0.00 sec) ``` +Note that TiDB accepts index types such as `HASH`, `BTREE` and `RTREE` in syntax for compatibility with MySQL, but ignores them. + ## MySQL compatibility The `SHOW INDEXES [FROM|IN]` statement in TiDB is fully compatible with MySQL. If you find any compatibility differences, [report a bug](https://docs.pingcap.com/tidb/stable/support). diff --git a/sql-statements/sql-statement-show-master-status.md b/sql-statements/sql-statement-show-master-status.md index 0ba461b6a5669..906c8539f2e34 100644 --- a/sql-statements/sql-statement-show-master-status.md +++ b/sql-statements/sql-statement-show-master-status.md @@ -1,7 +1,6 @@ --- title: SHOW MASTER STATUS summary: An overview of the usage of SHOW MASTER STATUS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-master-status/'] --- # SHOW MASTER STATUS diff --git a/sql-statements/sql-statement-show-placement-for.md b/sql-statements/sql-statement-show-placement-for.md index 7833feee80639..0e2e77762d57a 100644 --- a/sql-statements/sql-statement-show-placement-for.md +++ b/sql-statements/sql-statement-show-placement-for.md @@ -9,11 +9,11 @@ summary: The usage of SHOW PLACEMENT FOR in TiDB. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. The statement returns a result set in which the `Scheduling_State` field indicates the current progress that the Placement Driver (PD) has made in scheduling the placement: -* `PENDING`: The PD has not yet started scheduling the placement. This might indicate that that the placement rules are semantically correct, but cannot currently be satisfied by the cluster. For example, if `FOLLOWERS=4` but there are only 3 TiKV stores that are candidates for followers. +* `PENDING`: The PD has not yet started scheduling the placement. This might indicate that the placement rules are semantically correct, but cannot currently be satisfied by the cluster. For example, if `FOLLOWERS=4` but there are only 3 TiKV stores that are candidates for followers. * `INPROGRESS`: The PD is currently scheduling the placement. * `SCHEDULED`: The PD has successfully scheduled the placement. diff --git a/sql-statements/sql-statement-show-placement-labels.md b/sql-statements/sql-statement-show-placement-labels.md index 4e8fd3ab5f87b..39ca5d5c04924 100644 --- a/sql-statements/sql-statement-show-placement-labels.md +++ b/sql-statements/sql-statement-show-placement-labels.md @@ -9,7 +9,7 @@ summary: The usage of SHOW PLACEMENT LABELS in TiDB. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-show-placement.md b/sql-statements/sql-statement-show-placement.md index 94f21423fc143..289b5842aabeb 100644 --- a/sql-statements/sql-statement-show-placement.md +++ b/sql-statements/sql-statement-show-placement.md @@ -9,11 +9,11 @@ summary: The usage of SHOW PLACEMENT in TiDB. > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. The statement returns a result set in which the `Scheduling_State` field indicates the current progress that the Placement Driver (PD) has made in scheduling the placement: -* `PENDING`: The PD has not yet started scheduling the placement. This might indicate that that the placement rules are semantically correct, but cannot currently be satisfied by the cluster. For example, if `FOLLOWERS=4` but there are only 3 TiKV stores which are candidates for followers. +* `PENDING`: The PD has not yet started scheduling the placement. This might indicate that the placement rules are semantically correct, but cannot currently be satisfied by the cluster. For example, if `FOLLOWERS=4` but there are only 3 TiKV stores which are candidates for followers. * `INPROGRESS`: The PD is currently scheduling the placement. * `SCHEDULED`: The PD has successfully scheduled the placement. diff --git a/sql-statements/sql-statement-show-plugins.md b/sql-statements/sql-statement-show-plugins.md index 37025e3759504..d7afea3cbb649 100644 --- a/sql-statements/sql-statement-show-plugins.md +++ b/sql-statements/sql-statement-show-plugins.md @@ -1,7 +1,6 @@ --- title: SHOW PLUGINS summary: An overview of the usage of SHOW PLUGINS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-plugins/'] --- # SHOW PLUGINS @@ -10,7 +9,7 @@ aliases: ['/docs/dev/sql-statements/sql-statement-show-plugins/'] > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## Synopsis diff --git a/sql-statements/sql-statement-show-privileges.md b/sql-statements/sql-statement-show-privileges.md index 8e1c394185a06..7c024f2194384 100644 --- a/sql-statements/sql-statement-show-privileges.md +++ b/sql-statements/sql-statement-show-privileges.md @@ -1,7 +1,6 @@ --- title: SHOW PRIVILEGES | TiDB SQL Statement Reference summary: An overview of the usage of SHOW PRIVILEGES for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-privileges/','/docs/dev/reference/sql/statements/show-privileges/'] --- # SHOW PRIVILEGES diff --git a/sql-statements/sql-statement-show-processlist.md b/sql-statements/sql-statement-show-processlist.md index 712e269852e50..e6f806d6d2ce0 100644 --- a/sql-statements/sql-statement-show-processlist.md +++ b/sql-statements/sql-statement-show-processlist.md @@ -1,7 +1,6 @@ --- title: SHOW [FULL] PROCESSLIST | TiDB SQL Statement Reference summary: An overview of the usage of SHOW [FULL] PROCESSLIST for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-processlist/','/docs/dev/reference/sql/statements/show-processlist/'] --- # SHOW [FULL] PROCESSLIST diff --git a/sql-statements/sql-statement-show-profiles.md b/sql-statements/sql-statement-show-profiles.md index b6734b58f5583..8a7341ea72a07 100644 --- a/sql-statements/sql-statement-show-profiles.md +++ b/sql-statements/sql-statement-show-profiles.md @@ -1,7 +1,6 @@ --- title: SHOW PROFILES summary: An overview of the usage of SHOW PROFILES for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-profiles/'] --- # SHOW PROFILES diff --git a/sql-statements/sql-statement-show-pump-status.md b/sql-statements/sql-statement-show-pump-status.md index c232df19fb654..6c950f94d30f9 100644 --- a/sql-statements/sql-statement-show-pump-status.md +++ b/sql-statements/sql-statement-show-pump-status.md @@ -1,7 +1,6 @@ --- title: SHOW PUMP STATUS summary: An overview of the usage of SHOW PUMP STATUS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-pump-status/'] --- # SHOW PUMP STATUS @@ -10,7 +9,7 @@ The `SHOW PUMP STATUS` statement displays the status information for all Pump no > **Note:** > -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This feature is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). ## Examples diff --git a/sql-statements/sql-statement-show-schemas.md b/sql-statements/sql-statement-show-schemas.md index 5daedb7a5f670..58ec99ad3a3d2 100644 --- a/sql-statements/sql-statement-show-schemas.md +++ b/sql-statements/sql-statement-show-schemas.md @@ -1,7 +1,6 @@ --- title: SHOW SCHEMAS | TiDB SQL Statement Reference summary: An overview of the usage of SHOW SCHEMAS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-schemas/','/docs/dev/reference/sql/statements/show-schemas/'] --- # SHOW SCHEMAS diff --git a/sql-statements/sql-statement-show-stats-buckets.md b/sql-statements/sql-statement-show-stats-buckets.md new file mode 100644 index 0000000000000..f1df822e205d3 --- /dev/null +++ b/sql-statements/sql-statement-show-stats-buckets.md @@ -0,0 +1,64 @@ +--- +title: SHOW STATS_BUCKETS +summary: An overview of the usage of SHOW STATS_BUCKETS for TiDB database. +--- + +# SHOW STATS_BUCKETS + +The `SHOW STATS_BUCKETS` statement shows the bucket information in [statistics](/statistics.md). + +Currently, the `SHOW STATS_BUCKETS` statement returns the following columns: + +| Column name | Description | +| :-------- | :------------- | +| `Db_name` | The database name | +| `Table_name` | The table name | +| `Partition_name` | The partition name | +| `Column_name` | The column name (when `is_index` is `0`) or the index name (when `is_index` is `1`) | +| `Is_index` | Whether it is an index column or not | +| `Bucket_id` | The ID of a bucket | +| `Count` | The number of all the values that falls on the bucket and the previous buckets | +| `Repeats` | The occurrence number of the maximum value | +| `Lower_bound` | The minimum value | +| `Upper_bound` | The maximum value | +| `Ndv` | The number of distinct values in the bucket. This field is deprecated and always shows `0` due to its inaccurate value. | + +## Synopsis + +```ebnf+diagram +ShowStatsBucketsStmt ::= + "SHOW" "STATS_BUCKETS" ShowLikeOrWhere? + +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` + +## Examples + +```sql +SHOW STATS_BUCKETS WHERE Table_name='t'; +``` + +``` ++---------+------------+----------------+-------------+----------+-----------+-------+---------+--------------------------+--------------------------+------+ +| Db_name | Table_name | Partition_name | Column_name | Is_index | Bucket_id | Count | Repeats | Lower_Bound | Upper_Bound | Ndv | ++---------+------------+----------------+-------------+----------+-----------+-------+---------+--------------------------+--------------------------+------+ +| test | t | | a | 0 | 0 | 1 | 1 | 2023-12-27 00:00:00 | 2023-12-27 00:00:00 | 0 | +| test | t | | a | 0 | 1 | 2 | 1 | 2023-12-28 00:00:00 | 2023-12-28 00:00:00 | 0 | +| test | t | | ia | 1 | 0 | 1 | 1 | (NULL, 2) | (NULL, 2) | 0 | +| test | t | | ia | 1 | 1 | 2 | 1 | (NULL, 4) | (NULL, 4) | 0 | +| test | t | | ia | 1 | 2 | 3 | 1 | (2023-12-27 00:00:00, 1) | (2023-12-27 00:00:00, 1) | 0 | +| test | t | | ia | 1 | 3 | 4 | 1 | (2023-12-28 00:00:00, 3) | (2023-12-28 00:00:00, 3) | 0 | ++---------+------------+----------------+-------------+----------+-----------+-------+---------+--------------------------+--------------------------+------+ +6 rows in set (0.00 sec) +``` + +## MySQL compatibility + +This statement is a TiDB extension to MySQL syntax. + +## See also + +* [`ANALYZE`](/sql-statements/sql-statement-analyze-table.md) +* [Introduction to Statistics](/statistics.md) diff --git a/sql-statements/sql-statement-show-stats-healthy.md b/sql-statements/sql-statement-show-stats-healthy.md index 631e2d97c1d28..0339aac26fdd7 100644 --- a/sql-statements/sql-statement-show-stats-healthy.md +++ b/sql-statements/sql-statement-show-stats-healthy.md @@ -9,6 +9,15 @@ The `SHOW STATS_HEALTHY` statement shows an estimation of how accurate statistic The health of a table can be improved by running the [`ANALYZE`](/sql-statements/sql-statement-analyze-table.md) statement. `ANALYZE` runs automatically when the health drops below the [`tidb_auto_analyze_ratio`](/system-variables.md#tidb_auto_analyze_ratio) threshold. +Currently, the `SHOW STATS_HEALTHY` statement returns the following columns: + +| Column name | Description | +| -------- | ------------- | +| `Db_name` | The database name | +| `Table_name` | The table name | +| `Partition_name` | The partition name | +| `Healthy` | The healthy percentage between 0 and 100 | + ## Synopsis ```ebnf+diagram diff --git a/sql-statements/sql-statement-show-stats-histograms.md b/sql-statements/sql-statement-show-stats-histograms.md index 3bc9cfe0b17d8..8c343a955186d 100644 --- a/sql-statements/sql-statement-show-stats-histograms.md +++ b/sql-statements/sql-statement-show-stats-histograms.md @@ -1,13 +1,33 @@ --- title: SHOW STATS_HISTOGRAMS +aliases: ['/tidb/stable/sql-statement-show-histograms'] summary: An overview of the usage of SHOW HISTOGRAMS for TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-histograms/','/tidb/dev/sql-statement-show-histograms'] --- # SHOW STATS_HISTOGRAMS This statement shows the histogram information collected by the [`ANALYZE` statement](/sql-statements/sql-statement-analyze-table.md) as part of database [statistics](/statistics.md). +Currently, the `SHOW STATS_HISTOGRAMS` statement returns the following columns: + +| Column name | Description | +| -------- | ------------- | +| `Db_name` | Database name | +| `Table_name` | The table name | +| `Partition_name` | The partition name | +| `Column_name` | The column name (when `is_index` is `0`) or the index name (when `is_index` is `1`) | +| `Is_index` | Whether it is an index column or not | +| `Update_time` | The update time | +| `Distinct_count` | The distinct count | +| `Null_count` | NULL count | +| `Avg_col_size` | The average col size | +| `Correlation` | Pearson correlation coefficient between this column and the integer primary key column, indicating the degree of association between the two columns | +| `Load_status` | Load status, such as `allEvicted` and `allLoaded` | +| `Total_mem_usage` | The total memory usage | +| `Hist_mem_usage` | The historical memory usage | +| `Topn_mem_usage` | The TopN memory usage | +| `Cms_mem_usage` | The CMS memory usage | + ## Synopsis ```ebnf+diagram diff --git a/sql-statements/sql-statement-show-stats-locked.md b/sql-statements/sql-statement-show-stats-locked.md index a3f9d66e41273..165578f5dfcd8 100644 --- a/sql-statements/sql-statement-show-stats-locked.md +++ b/sql-statements/sql-statement-show-stats-locked.md @@ -7,6 +7,15 @@ summary: An overview of the usage of SHOW STATS_LOCKED for the TiDB database. `SHOW STATS_LOCKED` shows the tables whose statistics are locked. +Currently, the `SHOW STATS_LOCKED` statement returns the following columns: + +| Column name | Description | +| -------- | ------------- | +| `Db_name` | The database name | +| `Table_name` | The table name | +| `Partition_name` | The partition name | +| `Status` | The statistics status, such as `locked` | + ## Synopsis ```ebnf+diagram diff --git a/sql-statements/sql-statement-show-stats-meta.md b/sql-statements/sql-statement-show-stats-meta.md index b2c8f239d5db7..af102b4d3d99a 100644 --- a/sql-statements/sql-statement-show-stats-meta.md +++ b/sql-statements/sql-statement-show-stats-meta.md @@ -1,7 +1,6 @@ --- title: SHOW STATS_META summary: An overview of the usage of SHOW STATS_META for TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-stats-meta/'] --- # SHOW STATS_META diff --git a/sql-statements/sql-statement-show-stats-topn.md b/sql-statements/sql-statement-show-stats-topn.md new file mode 100644 index 0000000000000..767c4de4daa08 --- /dev/null +++ b/sql-statements/sql-statement-show-stats-topn.md @@ -0,0 +1,60 @@ +--- +title: SHOW STATS_TOPN +summary: An overview of the usage of SHOW STATS_TOPN for TiDB database. +--- + +# SHOW STATS_TOPN + +The `SHOW STATS_TOPN` statement shows the Top-N information in [statistics](/statistics.md). + +Currently, the `SHOW STATS_TOPN` statement returns the following columns: + +| Column name | Description | +| ---- | ----| +| `Db_name` | The database name | +| `Table_name` | The table name | +| `Partition_name` | The partition name | +| `Column_name` | The column name (when `is_index` is `0`) or the index name (when `is_index` is `1`) | +| `Is_index` | Whether it is an index column or not | +| `Value` | The value of this column | +| `Count` | How many times the value appears | + +## Synopsis + +```ebnf+diagram +ShowStatsTopnStmt ::= + "SHOW" "STATS_TOPN" ShowLikeOrWhere? + +ShowLikeOrWhere ::= + "LIKE" SimpleExpr +| "WHERE" Expression +``` + +## Example + +```sql +SHOW STATS_TOPN WHERE Table_name='t'; +``` + +``` ++---------+------------+----------------+-------------+----------+--------------------------+-------+ +| Db_name | Table_name | Partition_name | Column_name | Is_index | Value | Count | ++---------+------------+----------------+-------------+----------+--------------------------+-------+ +| test | t | | a | 0 | 2023-12-27 00:00:00 | 1 | +| test | t | | a | 0 | 2023-12-28 00:00:00 | 1 | +| test | t | | ia | 1 | (NULL, 2) | 1 | +| test | t | | ia | 1 | (NULL, 4) | 1 | +| test | t | | ia | 1 | (2023-12-27 00:00:00, 1) | 1 | +| test | t | | ia | 1 | (2023-12-28 00:00:00, 3) | 1 | ++---------+------------+----------------+-------------+----------+--------------------------+-------+ +6 rows in set (0.00 sec) +``` + +## MySQL compatibility + +This statement is a TiDB extension to MySQL syntax. + +## See also + +* [`ANALYZE`](/sql-statements/sql-statement-analyze-table.md) +* [Introduction to Statistics](/statistics.md) \ No newline at end of file diff --git a/sql-statements/sql-statement-show-status.md b/sql-statements/sql-statement-show-status.md index b478ec5da1598..2be6d34f47680 100644 --- a/sql-statements/sql-statement-show-status.md +++ b/sql-statements/sql-statement-show-status.md @@ -1,7 +1,6 @@ --- title: SHOW [GLOBAL|SESSION] STATUS | TiDB SQL Statement Reference summary: An overview of the usage of SHOW [GLOBAL|SESSION] STATUS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-status/','/docs/dev/reference/sql/statements/show-status/'] --- # SHOW [GLOBAL|SESSION] STATUS diff --git a/sql-statements/sql-statement-show-table-next-rowid.md b/sql-statements/sql-statement-show-table-next-rowid.md index 73ee64ae0fb49..d145df7b8ecd6 100644 --- a/sql-statements/sql-statement-show-table-next-rowid.md +++ b/sql-statements/sql-statement-show-table-next-rowid.md @@ -1,7 +1,6 @@ --- title: SHOW TABLE NEXT_ROW_ID summary: Learn the usage of `SHOW TABLE NEXT_ROW_ID` in TiDB. -aliases: ['/docs/dev/sql-statements/sql-statement-show-table-next-rowid/'] --- # SHOW TABLE NEXT_ROW_ID diff --git a/sql-statements/sql-statement-show-table-regions.md b/sql-statements/sql-statement-show-table-regions.md index b3ed1f3441750..caa6291b108cd 100644 --- a/sql-statements/sql-statement-show-table-regions.md +++ b/sql-statements/sql-statement-show-table-regions.md @@ -1,7 +1,6 @@ --- title: SHOW TABLE REGIONS summary: Learn how to use SHOW TABLE REGIONS in TiDB. -aliases: ['/docs/dev/sql-statements/sql-statement-show-table-regions/','/docs/dev/reference/sql/statements/show-table-regions/'] --- # SHOW TABLE REGIONS @@ -10,7 +9,7 @@ The `SHOW TABLE REGIONS` statement is used to show the Region information of a t > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## Syntax diff --git a/sql-statements/sql-statement-show-table-status.md b/sql-statements/sql-statement-show-table-status.md index 967196674bb22..372e2f50bf937 100644 --- a/sql-statements/sql-statement-show-table-status.md +++ b/sql-statements/sql-statement-show-table-status.md @@ -1,7 +1,6 @@ --- title: SHOW TABLE STATUS | TiDB SQL Statement Reference summary: An overview of the usage of SHOW TABLE STATUS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-table-status/','/docs/dev/reference/sql/statements/show-table-status/'] --- # SHOW TABLE STATUS diff --git a/sql-statements/sql-statement-show-tables.md b/sql-statements/sql-statement-show-tables.md index f41f6bc44d6ee..f2cc8fa015687 100644 --- a/sql-statements/sql-statement-show-tables.md +++ b/sql-statements/sql-statement-show-tables.md @@ -1,7 +1,6 @@ --- title: SHOW [FULL] TABLES | TiDB SQL Statement Reference summary: An overview of the usage of SHOW [FULL] TABLES for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-tables/','/docs/dev/reference/sql/statements/show-tables/'] --- # SHOW [FULL] TABLES diff --git a/sql-statements/sql-statement-show-variables.md b/sql-statements/sql-statement-show-variables.md index 5affcb2047b11..f2c535db00def 100644 --- a/sql-statements/sql-statement-show-variables.md +++ b/sql-statements/sql-statement-show-variables.md @@ -1,7 +1,6 @@ --- title: SHOW [GLOBAL|SESSION] VARIABLES | TiDB SQL Statement Reference summary: An overview of the usage of SHOW [GLOBAL|SESSION] VARIABLES for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-variables/','/docs/dev/reference/sql/statements/show-variables/'] --- # SHOW [GLOBAL|SESSION] VARIABLES diff --git a/sql-statements/sql-statement-show-warnings.md b/sql-statements/sql-statement-show-warnings.md index f9f14aedba8f4..a61a02401b576 100644 --- a/sql-statements/sql-statement-show-warnings.md +++ b/sql-statements/sql-statement-show-warnings.md @@ -1,7 +1,6 @@ --- title: SHOW WARNINGS | TiDB SQL Statement Reference summary: An overview of the usage of SHOW WARNINGS for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-show-warnings/','/docs/dev/reference/sql/statements/show-warnings/'] --- # SHOW WARNINGS diff --git a/sql-statements/sql-statement-shutdown.md b/sql-statements/sql-statement-shutdown.md index a0455f2245388..e55143883d0ff 100644 --- a/sql-statements/sql-statement-shutdown.md +++ b/sql-statements/sql-statement-shutdown.md @@ -1,7 +1,6 @@ --- title: SHUTDOWN summary: An overview of the usage of SHUTDOWN for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-shutdown/'] --- # SHUTDOWN @@ -10,7 +9,7 @@ The `SHUTDOWN` statement is used to perform a shutdown operation in TiDB. Execut > **Note:** > -> This feature is only applicable to TiDB Self-Hosted and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). +> This feature is only applicable to TiDB Self-Managed and not available on [TiDB Cloud](https://docs.pingcap.com/tidbcloud/). ## Synopsis diff --git a/sql-statements/sql-statement-split-region.md b/sql-statements/sql-statement-split-region.md index c5a10cdacb017..8aeaa7858d2e0 100644 --- a/sql-statements/sql-statement-split-region.md +++ b/sql-statements/sql-statement-split-region.md @@ -1,7 +1,6 @@ --- title: Split Region summary: An overview of the usage of Split Region for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-split-region/','/docs/dev/reference/sql/statements/split-region/'] --- # Split Region @@ -14,7 +13,7 @@ To solve the hotspot problem in the above scenario, TiDB introduces the pre-spli > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## Synopsis @@ -355,11 +354,11 @@ You can specify the partition to be split. ## pre_split_regions -To have evenly split Regions when a table is created, it is recommended you use `SHARD_ROW_ID_BITS` together with `PRE_SPLIT_REGIONS`. When a table is created successfully, `PRE_SPLIT_REGIONS` pre-spilts tables into the number of Regions as specified by `2^(PRE_SPLIT_REGIONS)`. +When creating a table with the `AUTO_RANDOM` or `SHARD_ROW_ID_BITS` attribute, you can also specify the `PRE_SPLIT_REGIONS` option if you want to evenly pre-split the table into Regions immediately after the table is created. The number of pre-split Regions for a table is `2^(PRE_SPLIT_REGIONS)`. > **Note:** > -> The value of `PRE_SPLIT_REGIONS` must be less than or equal to that of `SHARD_ROW_ID_BITS`. +> The value of `PRE_SPLIT_REGIONS` must be less than or equal to that of `SHARD_ROW_ID_BITS` or `AUTO_RANDOM`. The `tidb_scatter_region` global variable affects the behavior of `PRE_SPLIT_REGIONS`. This variable controls whether to wait for Regions to be pre-split and scattered before returning results after the table creation. If there are intensive writes after creating the table, you need to set the value of this variable to `1`, then TiDB will not return the results to the client until all the Regions are split and scattered. Otherwise, TiDB writes the data before the scattering is completed, which will have a significant impact on write performance. @@ -384,7 +383,7 @@ region4: [ 3<<61 , +inf ) > **Note:** > -> The Region split by the Split Region statement is controlled by the [Region merge](/best-practices/pd-scheduling-best-practices.md#region-merge) scheduler in PD. To avoid PD re-merging the newly split Region soon after, you need to [dynamically modify](/pd-control.md) configuration items related to the Region merge feature. +> The Region split by the Split Region statement is controlled by the [Region merge](/best-practices/pd-scheduling-best-practices.md#region-merge) scheduler in PD. To avoid PD re-merging the newly split Region soon after, you need to use [table attributes](/table-attributes.md) or [dynamically modify](/pd-control.md) configuration items related to the Region merge feature. diff --git a/sql-statements/sql-statement-start-transaction.md b/sql-statements/sql-statement-start-transaction.md index b943d8f2c69dd..a885dc24912b2 100644 --- a/sql-statements/sql-statement-start-transaction.md +++ b/sql-statements/sql-statement-start-transaction.md @@ -1,7 +1,6 @@ --- title: START TRANSACTION | TiDB SQL Statement Reference summary: An overview of the usage of START TRANSACTION for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-start-transaction/','/docs/dev/reference/sql/statements/start-transaction/'] --- # START TRANSACTION diff --git a/sql-statements/sql-statement-trace.md b/sql-statements/sql-statement-trace.md index 06797e7312c77..5fcd74e03645a 100644 --- a/sql-statements/sql-statement-trace.md +++ b/sql-statements/sql-statement-trace.md @@ -1,7 +1,6 @@ --- title: TRACE | TiDB SQL Statement Reference summary: An overview of the usage of TRACE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-trace/','/docs/dev/reference/sql/statements/trace/'] --- # TRACE diff --git a/sql-statements/sql-statement-truncate.md b/sql-statements/sql-statement-truncate.md index 6f5def7e46840..5375b8cff74ea 100644 --- a/sql-statements/sql-statement-truncate.md +++ b/sql-statements/sql-statement-truncate.md @@ -1,7 +1,6 @@ --- title: TRUNCATE | TiDB SQL Statement Reference summary: An overview of the usage of TRUNCATE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-truncate/','/docs/dev/reference/sql/statements/truncate/'] --- # TRUNCATE diff --git a/sql-statements/sql-statement-update.md b/sql-statements/sql-statement-update.md index 74e73671b84c3..730f741b5400b 100644 --- a/sql-statements/sql-statement-update.md +++ b/sql-statements/sql-statement-update.md @@ -1,7 +1,6 @@ --- title: UPDATE | TiDB SQL Statement Reference summary: An overview of the usage of UPDATE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-update/','/docs/dev/reference/sql/statements/update/'] --- # UPDATE @@ -27,6 +26,10 @@ TableRefs ::= EscapedTableRef ("," EscapedTableRef)* ``` +> **Note:** +> +> Starting from v6.6.0, TiDB supports [Resource Control](/tidb-resource-control.md). You can use this feature to execute SQL statements with different priorities in different resource groups. By configuring proper quotas and priorities for these resource groups, you can gain better scheduling control for SQL statements with different priorities. When resource control is enabled, statement priority (`LOW_PRIORITY` and `HIGH_PRIORITY`) will no longer take effect. It is recommended that you use [Resource Control](/tidb-resource-control.md) to manage resource usage for different SQL statements. + ## Examples ```sql diff --git a/sql-statements/sql-statement-use.md b/sql-statements/sql-statement-use.md index c9081abaa659f..57e9830f09dd8 100644 --- a/sql-statements/sql-statement-use.md +++ b/sql-statements/sql-statement-use.md @@ -1,7 +1,6 @@ --- title: USE | TiDB SQL Statement Reference summary: An overview of the usage of USE for the TiDB database. -aliases: ['/docs/dev/sql-statements/sql-statement-use/','/docs/dev/reference/sql/statements/use/'] --- # USE diff --git a/sql-tuning-best-practice.md b/sql-tuning-best-practice.md new file mode 100644 index 0000000000000..09647b79532bb --- /dev/null +++ b/sql-tuning-best-practice.md @@ -0,0 +1,984 @@ +--- +title: A Practical Guide for SQL Tuning +summary: Learn how to optimize SQL queries for better performance. +--- + +# A Practical Guide for SQL Tuning + +This guide is designed for TiDB SQL tuning beginners. It focuses on the following key principles: + +- Low entry barrier: introducing tuning concepts and methods step by step. +- Practice-oriented: providing specific steps and examples for each optimization tip. +- Quick start: prioritizing the most common and effective optimization methods. +- Gentle learning curve: emphasizing practical techniques rather than complex theory. +- Scenario-based: using real-world business cases to demonstrate optimization effects. + +## Introduction to SQL tuning + +SQL tuning is crucial for optimizing database performance. It involves systematically improving the efficiency of SQL queries through the following typical steps: + +1. Identify high-impact SQL statements: + + - Review the SQL execution history to find statements that consume significant system resources or contribute heavily to the application workload. + - Use monitoring tools and performance metrics to identify resource-intensive queries. + +2. Analyze execution plans: + + - Examine the execution plans generated by the query optimizer for the identified statements. + - Verify whether these plans are reasonably efficient and use appropriate indexes and join methods. + +3. Implement optimizations: + + Implement optimizations to inefficient SQL statements. Optimizations might include rewriting SQL statements, adding or modifying indexes, updating statistics, or adjusting database parameters. + +Repeat these steps until: + +- The system performance meets target requirements. +- No further improvements can be made to the remaining statements. + +SQL tuning is an ongoing process. As your data volume grows and query patterns change, you should: + +- Regularly monitor query performance. +- Re-evaluate your optimization strategies. +- Adapt your approach to address new performance challenges. + +By consistently applying these practices, you can maintain optimal database performance over time. + +## Goals of SQL tuning + +The primary goals of SQL tuning are the following: + +- Reduce response time for end users. +- Minimize resource consumption during query execution. + +To achieve these goals, you can use the following strategies. + +### Optimize query execution + +SQL tuning focuses on finding more efficient ways to process the same workload without changing query functionality. You can optimize query execution as follows: + +1. Improving execution plans: + - Analyze and modify query structures for more efficient processing. + - Use appropriate indexes to reduce data access and processing time. + - Enable TiFlash for analytical queries on large datasets and leverage the Massively Parallel Processing (MPP) engine for complex aggregations and joins. + +2. Enhancing data access methods: + - Use covering indexes to satisfy queries directly from the index, avoiding full table scans. + - Implement partitioning strategies to limit data scans to relevant partitions. + +Examples: + +- Create indexes on frequently queried columns to significantly reduce resource usage, particularly for queries that access small portions of the table. +- Use index-only scans for queries that return a limited number of sorted results to avoid full table scans and sorting operations. + +### Balance workload distribution + +In a distributed architecture like TiDB, balancing workloads across TiKV nodes is essential for optimal performance. To identify and resolve read and write hotspots, see [Troubleshoot hotspot issues](/troubleshoot-hot-spot-issues.md#optimization-of-small-table-hotspots). + +By implementing these strategies, you can ensure that your TiDB cluster efficiently utilizes all available resources and avoids bottlenecks caused by uneven workload distribution or serialization on individual TiKV nodes. + +## Identify high-load SQL + +The most efficient way to identify resource-intensive SQL statements is by using [TiDB Dashboard](/dashboard/dashboard-overview.md). You can also use other tools, such as views and logs, to identify high-load SQL statements. + +### Monitor SQL statements using TiDB Dashboard + +#### SQL Statements page + +In [TiDB Dashboard](/dashboard/dashboard-overview.md), navigate to the [**SQL Statements** page](/dashboard/dashboard-statement-list.md) to identify the following: + +- The SQL statement with the highest total latency, which is the statement that takes the longest time to execute across multiple executions. +- The number of times each SQL statement has been executed, which helps identify statements with the highest execution frequency. +- Execution details of each SQL statement by clicking it to view `EXPLAIN ANALYZE` results. + +TiDB normalizes SQL statements into templates by replacing literals and bind variables with `?`. This normalization and sorting process helps you quickly identify the most resource-intensive queries that might require optimization. + +![sql-statements-default](/media/sql-tuning/sql-statements-default.png) + +#### Slow Queries page + +In [TiDB Dashboard](/dashboard/dashboard-overview.md), navigate to the [**Slow Queries** page](/dashboard/dashboard-slow-query.md) to find the following: + +- The slowest SQL queries. +- The SQL query that reads the most data from TiKV. +- The `EXPLAIN ANALYZE` output by clicking a query for detailed execution analysis. + +The **Slow Queries** page does not display SQL execution frequency. A query appears on this page if its execution time exceeds the [`tidb_slow_log_threshold`](/tidb-configuration-file.md#tidb_slow_log_threshold) configuration item for a single instance. + +![slow-query-default](/media/sql-tuning/slow-query-default.png) + +### Use other tools to identify Top SQL + +In addition to TiDB Dashboard, you can use other tools to identify resource-intensive SQL queries. Each tool offers unique insights and can be valuable for different analysis scenarios. Using a combination of these tools helps ensure comprehensive SQL performance monitoring and optimization. + +- [TiDB Dashboard Top SQL Page](/dashboard/top-sql.md) +- Logs: [slow query log](/identify-slow-queries.md) and [expensive queries in TiDB log](/identify-expensive-queries.md) +- Views: [`cluster_statements_summary` view](/statement-summary-tables.md#the-cluster-tables-for-statement-summary) and [`cluster_processlist` view](/information-schema/information-schema-processlist.md#cluster_processlist) + +### Gather data on identified SQL statements + +For the top SQL statements identified, you can use [`PLAN REPLAYER`](/sql-plan-replayer.md) to capture and save SQL execution information from a TiDB cluster. This tool helps recreate the execution environment for further analysis. To export SQL execution information, use the following syntax: + +```sql +PLAN REPLAYER DUMP EXPLAIN [ANALYZE] [WITH STATS AS OF TIMESTAMP expression] sql-statement; +``` + +Use `EXPLAIN ANALYZE` whenever possible, as it provides both the execution plan and actual performance metrics for more accurate insights into query performance. + +## SQL tuning guide + +This guide provides practical advice for beginners on optimizing SQL queries in TiDB. By following these best practices, you can improve query performance and streamline SQL tuning. This guide covers the following topics: + +- [Understand query processing](#understand-query-processing) + - [Query processing workflow](#query-processing-workflow) + - [Optimizer fundamentals](#optimizer-fundamentals) + - [Statistics management](#statistics-management) +- [Understand execution plans](#understand-execution-plans) + - [How TiDB builds an execution plan](#how-tidb-builds-an-execution-plan) + - [Generate and display execution plans](#generate-and-display-execution-plans) + - [Read execution plans: first child first](#read-execution-plans-first-child-first) + - [Identify bottlenecks in execution plans](#identify-bottlenecks-in-execution-plans) +- [Index strategy in TiDB](#index-strategy-in-tidb) + - [Composite index strategy guidelines](#composite-index-strategy-guidelines) + - [The cost of indexing](#the-cost-of-indexing) + - [SQL tuning with a covering index](#sql-tuning-with-a-covering-index) + - [SQL tuning with a composite index involving sorting](#sql-tuning-with-a-composite-index-involving-sorting) + - [SQL tuning with composite indexes for efficient filtering and sorting]#sql-tuning-with-composite-indexes-for-efficient-filtering-and-sorting +- [When to use TiFlash](#when-to-use-tiflash) + - [Analytical query](#analytical-query) + - [SaaS arbitrary filtering workloads](#saas-arbitrary-filtering-workloads) + +### Understand query processing + +This section introduces the query processing workflow, optimizer fundamentals, and statistics management. + +#### Query processing workflow + +When a client sends a SQL statement to TiDB, the statement passes through the protocol layer of the TiDB server. This layer manages the connection between the TiDB server and the client, receives SQL statements, and returns data to the client. + +![workflow](/media/sql-tuning/workflow-tiflash.png) + +In the preceding figure, to the right of the protocol layer is the optimizer of the TiDB server, which processes SQL statements as follows: + +1. The SQL statement arrives at the SQL optimizer through the protocol layer and is parsed into an abstract syntax tree (AST). +2. TiDB identifies whether it is a [Point Get](/explain-indexes.md#point_get-and-batch_point_get) statement, which involves a simple one-table lookup through a primary or unique key, such as `SELECT * FROM t WHERE pk_col = 1` or `SELECT * FROM t WHERE uk_col IN (1,2,3)`. For `Point Get` statements, TiDB skips subsequent optimization steps and proceeds directly to execution in the SQL executor. +3. If the query is not a `Point Get`, the AST undergoes logical transformation, where TiDB rewrites the SQL logically based on specific rules. +4. After logical transformation, TiDB processes the AST through cost-based optimization. +5. During cost-based optimization, the optimizer uses statistics to select appropriate operators and generates a physical execution plan. +6. The generated physical execution plan is sent to the SQL executor of the TiDB node for execution. +7. Unlike traditional single-node databases, TiDB pushes down operators or coprocessors to TiKV and/or TiFlash nodes containing the data. This approach processes parts of the execution plan where the data is stored, efficiently utilizing the distributed architecture, using resources in parallel, and reducing network data transfer. The TiDB node executor then assembles the final result and returns it to the client. + +#### Optimizer fundamentals + +TiDB uses a cost-based optimizer (CBO) to determine the most efficient execution plan for a SQL statement. The optimizer evaluates different execution strategies and selects the one with the lowest estimated cost. The cost depends on factors such as: + +- The SQL statement +- Schema design +- Statistics, including: + - Table statistics + - Index statistics + - Column statistics + +Based on these inputs, the cost model generates an execution plan that details how TiDB will execute the SQL statement, including: + +- Access method +- Join method +- Join order + +The effectiveness of the optimizer depends on the quality of the information it receives. To achieve optimal performance, ensure that statistics are up to date and indexes are well-designed. + +#### Statistics management + +Statistics are essential for the TiDB optimizer. TiDB uses statistics as the input of the optimizer to estimate the number of rows processed in each step of a SQL execution plan. + +Statistics are divided into two levels: + +- **Table-level statistics**: include the total number of rows in the table and the number of rows modified since the last statistics collection. +- **Index/column-level statistics**: include detailed information such as histograms, Count-Min Sketch, Top-N (values or indexes with the highest occurrences), distribution and quantity of different values, and the number of NULL values. + +To check the accuracy and health of your statistics, you can use the following SQL statements: + +- [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md): shows metadata about table statistics. + + ```sql + SHOW STATS_META WHERE table_name='T2'\G; + ``` + + ``` + *************************** 1. row *************************** + Db_name: test + Table_name: T2 + Partition_name: + Update_time: 2023-05-11 02:16:50 + Modify_count: 10000 + Row_count: 20000 + 1 row in set (0.03 sec) + ``` + +- [`SHOW STATS_HEALTHY`](/sql-statements/sql-statement-show-stats-healthy.md): shows the health status of table statistics. + + ```sql + SHOW STATS_HEALTHY WHERE table_name='T2'\G; + ``` + + ``` + *************************** 1. row *************************** + Db_name: test + Table_name: T2 + Partition_name: + Healthy: 50 + 1 row in set (0.00 sec) + ``` + +TiDB provides two methods for collecting statistics: automatic and manual collection. In most cases, automatic collection is sufficient. TiDB triggers automatic collection when certain conditions are met. Some common triggering conditions include: + +- [`tidb_auto_analyze_ratio`](/system-variables.md#tidb_auto_analyze_ratio): the healthiness trigger. +- [`tidb_auto_analyze_start_time`](/system-variables.md#tidb_auto_analyze_start_time) and [`tidb_auto_analyze_end_time`](/system-variables.md#tidb_auto_analyze_end_time): the time window for automatic statistics collection. + +```sql +SHOW VARIABLES LIKE 'tidb\_auto\_analyze%'; +``` + +``` ++-----------------------------------------+-------------+ +| Variable_name | Value | ++-----------------------------------------+-------------+ +| tidb_auto_analyze_ratio | 0.5 | +| tidb_auto_analyze_start_time | 00:00 +0000 | +| tidb_auto_analyze_end_time | 23:59 +0000 | ++-----------------------------------------+-------------+ +``` + +In some cases, automatic collection might not meet your needs. By default, it occurs between `00:00` and `23:59`, meaning the analyze job can run at any time. To minimize performance impact on your online business, you can set specific start and end times for statistics collection. + +You can manually collect statistics using the `ANALYZE TABLE table_name` statement. This lets you adjust settings such as the sample rate, the number of Top-N values, or collect statistics for specific columns only. + +Note that after manual collection, subsequent automatic collection jobs inherit the new settings. This means that any customizations made during manual collection will apply to future automatic analyses. + +Locking table statistics is useful in the following scenarios: + +- The statistics on the table already represent the data well. +- The table is very large, and statistics collection is time-consuming. +- You want to maintain statistics only during specific time windows. + +To lock statistics for a table, you can use the [`LOCK STATS table_name`](/sql-statements/sql-statement-lock-stats.md) statement. + +For more information, see [Statistics](/statistics.md). + +### Understand execution plans + +An execution plan details the steps that TiDB will follow to execute a SQL query. This section explains how TiDB builds an execution plan and how to generate, display, and interpret execution plans. + +#### How TiDB builds an execution plan + +A SQL statement undergoes three main optimization stages in the TiDB optimizer: + +1. [Pre-processing](#1-pre-processing) +2. [Logical transformation](#2-logical-transformation) +3. [Cost-based optimization](#3-cost-based-optimization) + +##### 1. Pre-processing + +During pre-processing, TiDB determines whether the SQL statement can be executed using [`Point_Get`](/explain-indexes.md#point_get-and-batch_point_get) or [`Batch_Point_Get`](/explain-indexes.md#point_get-and-batch_point_get). These operations use a primary or unique key to read directly from TiKV through an exact key lookup. If a plan qualifies for `Point_Get` or `Batch_Point_Get`, the optimizer skips the logical transformation and cost-based optimization steps because direct key lookup is the most efficient way to access the row. + +The following is an example of a `Point_Get` query: + +```sql +EXPLAIN SELECT id, name FROM emp WHERE id = 901; +``` + +``` ++-------------+---------+------+---------------+---------------+ +| id | estRows | task | access object | operator info | ++-------------+---------+------+---------------+---------------+ +| Point_Get_1 | 1.00 | root | table:emp | handle:901 | ++-------------+---------+------+---------------+---------------+ +``` + +##### 2. Logical transformation + +During logical transformation, TiDB optimizes SQL statements based on the `SELECT` list, `WHERE` predicates, and other conditions. It generates a logical execution plan to annotate and rewrite the query. This logical plan is used in the next stage, cost-based optimization. The transformation applies rule-based optimizations such as column pruning, partition pruning, and join reordering. Because this process is rule-based and automatic, manual adjustments are usually unnecessary. + +For more information, see [SQL Logical Optimization](/sql-logical-optimization.md). + +##### 3. Cost-based optimization + +The TiDB optimizer uses statistics to estimate the number of rows processed in each step of a SQL statement and assigns a cost to each step. During cost-based optimization, the optimizer evaluates all possible plan choices, including index accesses and join methods, and calculates the total cost for each plan. The optimizer then selects the execution plan with the minimal total cost. + +The following figure illustrates various data access paths and row set operations considered during cost-based optimization. For data retrieval paths, the optimizer determines the most efficient method between index scans and full table scans, and decides whether to retrieve data from row-based TiKV storage or columnar TiFlash storage. + +The optimizer also evaluates operations that manipulate row sets, such as aggregation, join, and sorting. For example, the aggregation operator might use either `HashAgg` or `StreamAgg`, while the join method can select from `HashJoin`, `MergeJoin`, or `IndexJoin`. + +Additionally, the physical optimization phase includes pushing down expressions and operators to the physical storage engines. The physical plan is distributed to different components based on the underlying storage engines as follows: + +- Root tasks are executed on the TiDB server. +- Cop (Coprocessor) tasks are executed on TiKV. +- MPP tasks are executed on TiFlash. + +This distribution enables cross-component collaboration for efficient query processing. + +![cost-based-optimization](/media/sql-tuning/cost-based-optimization.png) + +For more information, see [SQL Physical Optimization](/sql-physical-optimization.md). + +#### Generate and display execution plans + +Besides accessing execution plan information through TiDB Dashboard, you can use the `EXPLAIN` statement to display the execution plan for a SQL query. The `EXPLAIN` output includes the following columns: + +- `id`: the operator name and a unique identifier of the step. +- `estRows`: the estimated number of rows from the particular step. +- `task`: indicates the layer where the operator is executed. For example, `root` indicates execution on the TiDB server, `cop[tikv]` indicates execution on TiKV, and `mpp[tiflash]` indicates execution on TiFlash. +- `access object`: the object where row sources are located. +- `operator info`: additional details about the operator in the step. + +```sql +EXPLAIN SELECT COUNT(*) FROM trips WHERE start_date BETWEEN '2017-07-01 00:00:00' AND '2017-07-01 23:59:59'; +``` + +``` ++--------------------------+-------------+--------------+-------------------+----------------------------------------------------------------------------------------------------+ +| id | estRows | task | access object | operator info | ++--------------------------+-------------+--------------+-------------------+----------------------------------------------------------------------------------------------------+ +| StreamAgg_20 | 1.00 | root | | funcs:count(Column#13)->Column#11 | +| └─TableReader_21 | 1.00 | root | | data:StreamAgg_9 | +| └─StreamAgg_9 | 1.00 | cop[tikv] | | funcs:count(1)->Column#13 | +| └─Selection_19 | 250.00 | cop[tikv] | | ge(trips.start_date, 2017-07-01 00:00:00.000000), le(trips.start_date, 2017-07-01 23:59:59.000000) | +| └─TableFullScan_18 | 10000.00 | cop[tikv] | table:trips | keep order:false, stats:pseudo | ++--------------------------+-------------+--------------+-------------------+----------------------------------------------------------------------------------------------------+ +5 rows in set (0.00 sec) +``` + +Different from `EXPLAIN`, `EXPLAIN ANALYZE` executes the corresponding SQL statement, records its runtime information, and returns the runtime information together with the execution plan. This runtime information is crucial for debugging query execution. For more information, see [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md). + +The `EXPLAIN ANALYZE` output includes: + +- `actRows`: the number of rows output by the operator. +- `execution info`: detailed execution information of the operator. `time` represents the total `wall time`, including the total execution time of all sub-operators. If the operator is called many times by the parent operator, then the time refers to the accumulated time. +- `memory`: the memory used by the operator. +- `disk`: the disk space used by the operator. + +The following is an example. Some attributes and table columns are omitted to improve formatting. + +```sql +EXPLAIN ANALYZE +SELECT SUM(pm.m_count) / COUNT(*) +FROM ( + SELECT COUNT(m.name) m_count + FROM universe.moons m + RIGHT JOIN ( + SELECT p.id, p.name + FROM universe.planet_categories c + JOIN universe.planets p + ON c.id = p.category_id + AND c.name = 'Jovian' + ) pc ON m.planet_id = pc.id + GROUP BY pc.name +) pm; +``` + +``` ++-----------------------------------------+.+---------+-----------+---------------------------+----------------------------------------------------------------+.+-----------+---------+ +| id |.| actRows | task | access object | execution info |.| memory | disk | ++-----------------------------------------+.+---------+-----------+---------------------------+----------------------------------------------------------------+.+-----------+---------+ +| Projection_14 |.| 1 | root | | time:1.39ms, loops:2, RU:1.561975, Concurrency:OFF |.| 9.64 KB | N/A | +| └─StreamAgg_16 |.| 1 | root | | time:1.39ms, loops:2 |.| 1.46 KB | N/A | +| └─Projection_40 |.| 4 | root | | time:1.38ms, loops:4, Concurrency:OFF |.| 8.24 KB | N/A | +| └─HashAgg_17 |.| 4 | root | | time:1.36ms, loops:4, partial_worker:{...}, final_worker:{...} |.| 82.1 KB | N/A | +| └─HashJoin_19 |.| 25 | root | | time:1.29ms, loops:2, build_hash_table:{...}, probe:{...} |.| 2.25 KB | 0 Bytes | +| ├─HashJoin_35(Build) |.| 4 | root | | time:1.08ms, loops:2, build_hash_table:{...}, probe:{...} |.| 25.7 KB | 0 Bytes | +| │ ├─IndexReader_39(Build) |.| 1 | root | | time:888.5µs, loops:2, cop_task: {...} |.| 286 Bytes | N/A | +| │ │ └─IndexRangeScan_38 |.| 1 | cop[tikv] | table:c, index:name(name) | tikv_task:{time:0s, loops:1}, scan_detail: {...} |.| N/A | N/A | +| │ └─TableReader_37(Probe) |.| 10 | root | | time:543.7µs, loops:2, cop_task: {...} |.| 577 Bytes | N/A | +| │ └─TableFullScan_36 |.| 10 | cop[tikv] | table:p | tikv_task:{time:0s, loops:1}, scan_detail: {...} |.| N/A | N/A | +| └─TableReader_22(Probe) |.| 28 | root | | time:671.7µs, loops:2, cop_task: {...} |.| 876 Bytes | N/A | +| └─TableFullScan_21 |.| 28 | cop[tikv] | table:m | tikv_task:{time:0s, loops:1}, scan_detail: {...} |.| N/A | N/A | ++-----------------------------------------+.+---------+-----------+---------------------------+----------------------------------------------------------------+.+-----------+---------+ +``` + +#### Read execution plans: first child first + +To diagnose slow SQL queries, you need to understand how to read execution plans. The key principle is **"first child first – recursive descent"**. Each operator in the plan generates a set of rows, and the execution order determines how these rows flow through the plan tree. + +The "first child first" rule means that an operator must retrieve rows from all its child operators before producing output. For example, a join operator requires rows from both its child operators to perform the join. The "recursive descent" rule means you analyze the plan from top to bottom, although actual data flows from bottom to top, as each operator depends on its children's output. + +Consider these two important concepts when reading execution plans: + +- Parent-child interaction: a parent operator calls its child operators sequentially but might cycle through them multiple times. For example, in an index lookup or nested loop join, the parent fetches a batch of rows from the first child, then retrieves zero or more rows from the second child. This process repeats until the first child's result set is fully processed. + +- Blocking vs. non-blocking operators: operators can be either blocking or non-blocking: + - Blocking operators, such as `TopN` and `HashAgg`, must create their entire result set before passing data to their parent. + - Non-blocking operators, such as `IndexLookup` and `IndexJoin`, produce and pass rows incrementally as needed. + +When reading an execution plan, start from the top and work downward. In the following example, the leaf node of the plan tree is `TableFullScan_18`, which performs a full table scan. The rows from this scan are consumed by the `Selection_19` operator, which filters the data based on `ge(trips.start_date, 2017-07-01 00:00:00.000000), le(trips.start_date, 2017-07-01 23:59:59.000000)`. Then, the group-by operator `StreamAgg_9` performs the final aggregation `COUNT(*)`. + +These three operators (`TableFullScan_18`, `Selection_19`, and `StreamAgg_9`) are pushed down to TiKV (marked as `cop[tikv]`), enabling early filtering and aggregation in TiKV to reduce data transfer between TiKV and TiDB. Finally, `TableReader_21` reads the data from `StreamAgg_9`, and `StreamAgg_20` performs the final aggregation `count(*)`. + +```sql +EXPLAIN SELECT COUNT(*) FROM trips WHERE start_date BETWEEN '2017-07-01 00:00:00' AND '2017-07-01 23:59:59'; +``` + +``` ++--------------------------+-------------+--------------+-------------------+----------------------------------------------------------------------------------------------------+ +| id | estRows | task | access object | operator info | ++--------------------------+-------------+--------------+-------------------+----------------------------------------------------------------------------------------------------+ +| StreamAgg_20 | 1.00 | root | | funcs:count(Column#13)->Column#11 | +| └─TableReader_21 | 1.00 | root | | data:StreamAgg_9 | +| └─StreamAgg_9 | 1.00 | cop[tikv] | | funcs:count(1)->Column#13 | +| └─Selection_19 | 250.00 | cop[tikv] | | ge(trips.start_date, 2017-07-01 00:00:00.000000), le(trips.start_date, 2017-07-01 23:59:59.000000) | +| └─TableFullScan_18 | 10000.00 | cop[tikv] | table:trips | keep order:false, stats:pseudo | ++--------------------------+-------------+--------------+-------------------+----------------------------------------------------------------------------------------------------+ +5 rows in set (0.00 sec) +``` + +In the following example, start by examining `IndexRangeScan_47`, the first leaf node of the plan tree. The optimizer selects only the `name` and `id` columns from the `stars` table, which can be retrieved from the `name(name)` index. As a result, the root reader for `stars` is `IndexReader_48`, not `TableReader`. + +The join between `stars` and `planets` is a hash join (`HashJoin_44`). The `planets` table is accessed using a full table scan (`TableFullScan_45`). After the join, `TopN_26` and `TopN_19` apply the `ORDER BY` and `LIMIT` clauses, respectively. The final operator, `Projection_16`, selects the final column `t5.name`. + +```sql +EXPLAIN +SELECT t5.name +FROM ( + SELECT p.name, p.gravity, p.distance_from_sun + FROM universe.planets p + JOIN universe.stars s + ON s.id = p.sun_id + AND s.name = 'Sun' + ORDER BY p.distance_from_sun ASC + LIMIT 5 +) t5 +ORDER BY t5.gravity DESC +LIMIT 3; +``` + +``` ++-----------------------------------+----------+-----------+---------------------------+ +| id | estRows | task | access object | ++-----------------------------------+----------+-----------+---------------------------+ +| Projection_16 | 3.00 | root | | +| └─TopN_19 | 3.00 | root | | +| └─TopN_26 | 5.00 | root | | +| └─HashJoin_44 | 5.00 | root | | +| ├─IndexReader_48(Build) | 1.00 | root | | +| │ └─IndexRangeScan_47 | 1.00 | cop[tikv] | table:s, index:name(name) | +| └─TableReader_46(Probe) | 10.00 | root | | +| └─TableFullScan_45 | 10.00 | cop[tikv] | table:p | ++-----------------------------------+----------+-----------+---------------------------+ +``` + +The following figure illustrates the plan tree for the second execution plan: + +![execution-plan-traverse](/media/sql-tuning/execution-plan-traverse.png) + +The execution plan follows a top-to-bottom, first-child-first traversal, corresponding to a postorder traversal (Left, Right, Root) of the plan tree. + +You can read this plan in the following order: + +1. Start at the top with `Projection_16`. +2. Move to its child, `TopN_19`. +3. Continue to `TopN_26`. +4. Proceed to `HashJoin_44`. +5. For `HashJoin_44`, process its left (Build) child first: + - `IndexReader_48` + - `IndexRangeScan_47` +6. For `HashJoin_44`, process its right (Probe) child: + - `TableReader_46` + - `TableFullScan_45` + +This traversal ensures that each operator's inputs are processed before the operator itself, enabling efficient query execution. + +#### Identify bottlenecks in execution plans + +When analyzing execution plans, compare `actRows` (actual rows) with `estRows` (estimated rows) to evaluate the accuracy of the optimizer's estimations. A significant difference between these values might indicate outdated or inaccurate statistics, which can lead to suboptimal query plans. + +To identify bottlenecks in a slow query, perform the following steps: + +1. Review the `execution info` section from top to bottom, focusing on operators with high execution time. +2. For the first child operator that consumes significant time: + - Compare `actRows` with `estRows` to assess estimation accuracy. + - Analyze detailed metrics in `execution info`, such as high execution time or other metrics. + - Check `memory` and `disk` usage for potential resource constraints. +3. Correlate these factors to determine the root cause of the performance issue. For example, if a `TableFullScan` operation has a high `actRows` count and long execution time in `execution info`, consider creating an index. If a `HashJoin` operation shows high memory usage and execution time, consider optimizing the join order or using an alternative join method. + +In the following execution plan, the query runs for 5 minutes and 51 seconds before being canceled. The key issues include: + +1. Severe underestimation: The first leaf node `IndexReader_76` reads data from the `index_orders_on_adjustment_id(adjustment_id)` index. The actual number of rows (`actRows`) is 256,811,189, which is significantly higher than the estimated 1 row (`estRows`). +2. Memory overflow: This underestimation causes the hash join operator `HashJoin_69` to build a hash table with far more data than expected, consuming excessive memory (22.6 GB) and disk space (7.65 GB). +3. Query termination: The `actRows` value of `0` for `HashJoin_69` and operators above it indicate either no matching rows or query termination due to resource constraints. In this case, the hash join consumes too much memory, triggering memory control mechanisms to terminate the query. +4. Incorrect join order: The root cause of this inefficient plan is the severe underestimation of `estRows` for `IndexRangeScan_75`, leading the optimizer to choose an incorrect join order. + +To address these issues, ensure that table statistics are up to date, particularly for the `orders` table and the `index_orders_on_adjustment_id` index. + +``` ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------...----------------------+ +| id | estRows | estCost | actRows | task | access object | execution info ...| memory | disk | ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------...----------------------+ +| TopN_19 | 1.01 | 461374372.63 | 0 | root | | time:5m51.1s, l...| 0 Bytes | 0 Bytes | +| └─IndexJoin_32 | 1.01 | 460915067.45 | 0 | root | | time:5m51.1s, l...| 0 Bytes | N/A | +| ├─HashJoin_69(Build) | 1.01 | 460913065.41 | 0 | root | | time:5m51.1s, l...| 21.6 GB | 7.65 GB | +| │ ├─IndexReader_76(Build) | 1.00 | 18.80 | 256805045 | root | | time:1m4.1s, lo...| 12.4 MB | N/A | +| │ │ └─IndexRangeScan_75 | 1.00 | 186.74 | 256811189 | cop[tikv] | table:orders, index:index_orders_on_adjustment_id(adjustment_id) | tikv_task:{proc...| N/A | N/A | +| │ └─Projection_74(Probe) | 30652.93 | 460299612.60 | 1024 | root | | time:1.08s, loo...| 413.4 KB | N/A | +| │ └─IndexLookUp_73 | 30652.93 | 460287375.95 | 6144 | root | partition:all | time:1.08s, loo...| 107.8 MB | N/A | +| │ ├─IndexRangeScan_70(Build) | 234759.64 | 53362737.50 | 390699 | cop[tikv] | table:rates, index:index_rates_on_label_id(label_id) | time:29.6ms, lo...| N/A | N/A | +| │ └─Selection_72(Probe) | 30652.93 | 110373973.91 | 187070 | cop[tikv] | | time:36.8s, loo...| N/A | N/A | +| │ └─TableRowIDScan_71 | 234759.64 | 86944962.10 | 390699 | cop[tikv] | table:rates | tikv_task:{proc...| N/A | N/A | +| └─TableReader_28(Probe) | 0.00 | 43.64 | 0 | root | | ...| N/A | N/A | +| └─Selection_27 | 0.00 | 653.96 | 0 | cop[tikv] | | ...| N/A | N/A | +| └─TableRangeScan_26 | 1.01 | 454.36 | 0 | cop[tikv] | table:labels | ...| N/A | N/A | ++-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------...----------------------+ +``` + +The following execution plan shows the expected results after fixing the incorrect estimation on the `orders` table. The query now takes 1.96 seconds to run, which is a significant improvement from the previous 5 minutes and 51 seconds: + +- Accurate estimation: The `estRows` values now closely match the `actRows`, indicating that the statistics are updated and more accurate. +- Efficient join order: The query now starts with a `TableReader` on the `labels` table, followed by an `IndexJoin` with the `rates` table, and another `IndexJoin` with the `orders` table. This join order works more efficiently with the actual data distribution. +- No memory overflow: Unlike the previous plan, this execution shows no signs of excessive memory or disk usage, indicating that the query runs within expected resource limits. + +This optimized plan demonstrates the importance of accurate statistics and proper join order in query performance. The execution time reduction (from 351 seconds to 1.96 seconds) shows the impact of addressing estimation errors. + +``` ++---------------------------------------+----------+---------+-----------+----------------------------------------------------------------------------------------+---------------...+----------+------+ +| id | estRows | actRows | task | access object | execution info...| memory | disk | ++---------------------------------------+----------+---------+-----------+----------------------------------------------------------------------------------------+---------------...+----------+------+ +| Limit_24 | 1000.00 | 1000 | root | | time:1.96s, lo...| N/A | N/A | +| └─IndexJoin_88 | 1000.00 | 1000 | root | | time:1.96s, lo...| 1.32 MB | N/A | +| ├─IndexJoin_99(Build) | 1000.00 | 2458 | root | | time:1.96s, lo...| 77.7 MB | N/A | +| │ ├─TableReader_109(Build) | 6505.62 | 158728 | root | | time:1.26s, lo...| 297.0 MB | N/A | +| │ │ └─Selection_108 | 6505.62 | 171583 | cop[tikv] | | tikv_task:{pro...| N/A | N/A | +| │ │ └─TableRangeScan_107 | 80396.43 | 179616 | cop[tikv] | table:labels | tikv_task:{pro...| N/A | N/A | +| │ └─Projection_98(Probe) | 1000.00 | 2458 | root | | time:2.13s, lo...| 59.2 KB | N/A | +| │ └─IndexLookUp_97 | 1000.00 | 2458 | root | partition:all | time:2.13s, lo...| 1.20 MB | N/A | +| │ ├─Selection_95(Build) | 6517.14 | 6481 | cop[tikv] | | time:798.6ms, ...| N/A | N/A | +| │ │ └─IndexRangeScan_93 | 6517.14 | 6481 | cop[tikv] | table:rates, index:index_rates_on_label_id(label_id) | tikv_task:{pro...| N/A | N/A | +| │ └─Selection_96(Probe) | 1000.00 | 2458 | cop[tikv] | | time:444.4ms, ...| N/A | N/A | +| │ └─TableRowIDScan_94 | 6517.14 | 6481 | cop[tikv] | table:rates | tikv_task:{pro...| N/A | N/A | +| └─TableReader_84(Probe) | 984.56 | 1998 | root | | time:207.6ms, ...| N/A | N/A | +| └─Selection_83 | 984.56 | 1998 | cop[tikv] | | tikv_task:{pro...| N/A | N/A | +| └─TableRangeScan_82 | 1000.00 | 2048 | cop[tikv] | table:orders | tikv_task:{pro...| N/A | N/A | ++---------------------------------------+----------+---------+-----------+----------------------------------------------------------------------------------------+---------------...+----------+------+ +``` + +For more information, see [TiDB Query Execution Plan Overview](/explain-overview.md) and [`EXPLAIN` Walkthrough](/explain-walkthrough.md). + +### Index strategy in TiDB + +TiDB is a distributed SQL database that separates the SQL layer (TiDB Server) from the storage layer (TiKV). Unlike traditional databases, TiDB does not use a buffer pool to cache data on compute nodes. As a result, SQL query performance and overall cluster performance depend on the number of key-value (KV) RPC requests that need to be processed. Common KV RPC requests include `Point_Get`, `Batch_Point_Get`, and Coprocessor. + +To optimize performance in TiDB, it is essential to use indexes effectively, as they can significantly reduce the number of KV RPC requests. Fewer KV RPC requests improve query performance and system efficiency. The following lists some key strategies that help optimize indexing: + +- Avoid full table scans. +- Avoid sorting operations. +- Use covering indexes or exclude unnecessary columns to reduce row lookups. + +This section explains general indexing strategies and indexing costs. It also provides three practical examples of effective indexing in TiDB, with a focus on composite and covering indexes for SQL tuning. + +#### Composite index strategy guidelines + +To create an efficient composite index, order your columns strategically. The column order directly affects how efficiently the index filters and sorts data. + +Follow these recommended column order guidelines for a composite index: + +1. Start with index prefix columns for direct access: + - Columns with equivalent conditions + - Columns with `IS NULL` conditions + +2. Add columns for sorting next: + - Let the index handle sorting operations + - Enable sort and limit pushdown to TiKV + - Preserve the sorted order + +3. Include additional filtering columns to reduce row lookups: + - Time range conditions on datetime columns + - Other non-equivalent conditions, such as `!=`, `<>`, and `IS NOT NULL` + +4. Add columns from the `SELECT` list or used in aggregation to fully utilize a covering index. + +#### The cost of indexing + +While indexes can improve query performance, they also introduce costs you should consider: + +- Performance impact on writes: + - A non-clustered index reduces the chance of single-phase commit optimization. + - Each additional index slows down write operations (such as `INSERT`, `UPDATE`, and `DELETE`). + - When data is modified, all affected indexes must be updated. + - The more indexes a table has, the greater the write performance impact. + +- Resource consumption: + - Indexes require additional disk space. + - More memory is needed to cache frequently accessed indexes. + - Backup and recovery operations take longer. + +- Write hotspot risk: + - Secondary indexes can create write hotspots. For example, a monotonically increasing datetime index will cause hotspots on table writes. + - Hotspots can lead to significant performance degradation. + +The following lists some best practices: + +- Create indexes only when they provide clear performance benefits. +- Regularly review index usage statistics using [`TIDB_INDEX_USAGE`](/information-schema/information-schema-tidb-index-usage.md). +- Consider the write/read ratio of your workload when designing indexes. + +#### SQL tuning with a covering index + +A covering index includes all columns referenced in the `WHERE` and `SELECT` clauses. Using a covering index can eliminate the need for additional index lookups, significantly improving query performance. + +The following query requires an index lookup of 2,597,411 rows and takes 46.4 seconds to execute. TiDB dispatches 67 cop tasks for the index range scan on `logs_idx` (`IndexRangeScan_11`) and 301 cop tasks for table access (`TableRowIDScan_12`). + +```sql +SELECT + SUM(`logs`.`amount`) +FROM + `logs` +WHERE + `logs`.`user_id` = 1111 + AND `logs`.`snapshot_id` IS NULL + AND `logs`.`status` IN ('complete', 'failure') + AND `logs`.`source_type` != 'online' + AND ( + `logs`.`source_type` IN ('user', 'payment') + OR `logs`.`source_type` IN ( + 'bank_account', + ) + AND `logs`.`target_type` IN ('bank_account') + ); +``` + +The original execution plan is as follows: + +``` ++-------------------------------+------------+---------+-----------+--------------------------------------------------------------------------+------------------------------------------------------------+ +| id | estRows | actRows | task | access object | execution info | ++-------------------------------+------------+---------+-----------+--------------------------------------------------------------------------+------------------------------------------------------------+ +| HashAgg_18 | 1.00 | 2571625.22 | 1 | root | | time:46.4s, loops:2, partial_worker:{wall_time:46.37, ...| +| └─IndexLookUp_19 | 1.00 | 2570096.68 | 301 | root | | time:46.4s, loops:2, index_task: {total_time: 45.8s, ...| +| ├─IndexRangeScan_11(Build) | 1309.50 | 317033.98 | 2597411 | cop[tikv] | table:logs, index:logs_idx(snapshot_id, user_id, status) | time:228ms, loops:2547, cop_task: {num: 67, max: 2.17s, ...| +| └─HashAgg_7(Probe) | 1.00 | 588434.48 | 301 | cop[tikv] | | time:3m46.7s, loops:260, cop_task: {num: 301, ...| +| └─Selection_13 | 1271.37 | 561549.27 | 2566562 | cop[tikv] | | tikv_task:{proc max:10s, min:0s, avg: 915.3ms, ...| +| └─TableRowIDScan_12 | 1309.50 | 430861.31 | 2597411 | cop[tikv] | table:logs | tikv_task:{proc max:10s, min:0s, avg: 908.7ms, ...| ++-------------------------------+------------+---------+-----------+--------------------------------------------------------------------------+------------------------------------------------------------+ +``` + +To improve query performance, you can create a covering index that includes `source_type`, `target_type`, and `amount` columns. This optimization eliminates the need for additional table lookups, reducing execution time to 90 milliseconds, and TiDB only needs to send one cop task to TiKV for data scanning. + +After creating the index, execute the `ANALYZE TABLE` statement to collect statistics. In TiDB, index creation does not automatically update statistics, so analyzing the table ensures that the optimizer selects the new index. + +```sql +CREATE INDEX logs_covered ON logs(snapshot_id, user_id, status, source_type, target_type, amount); +ANALYZE TABLE logs INDEX logs_covered; +``` + +The new execution plan is as follows: + +``` ++-------------------------------+------------+---------+-----------+---------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------+ +| id | estRows | actRows | task | access object | execution info | ++-------------------------------+------------+---------+-----------+---------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------+ +| HashAgg_13 | 1.00 | 1 | root | | time:90ms, loops:2, RU:158.885311, ...| +| └─IndexReader_14 | 1.00 | 1 | root | | time:89.8ms, loops:2, cop_task: {num: 1, ...| +| └─HashAgg_6 | 1.00 | 1 | cop[tikv] | | tikv_task:{time:88ms, loops:52}, ...| +| └─Selection_12 | 5245632.33 | 52863 | cop[tikv] | | tikv_task:{time:80ms, loops:52} ...| +| └─IndexRangeScan_11 | 5245632.33 | 52863 | cop[tikv] | table:logs, index:logs_covered(snapshot_id, user_id, status, source_type, target_type, amount) | tikv_task:{time:60ms, loops:52} ...| ++-------------------------------+------------+---------+-----------+---------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------+ +``` + +#### SQL tuning with a composite index involving sorting + +You can optimize queries with `ORDER BY` clauses by creating composite indexes that include both filtering and sorting columns. This approach helps TiDB access data efficiently while maintaining the desired order. + +For example, consider the following query that retrieves data from `test` based on specific conditions: + +```sql +EXPLAIN ANALYZE SELECT +test.* +FROM + test +WHERE + test.snapshot_id = 459840 + AND test.id > 998464 +ORDER BY + test.id ASC +LIMIT + 1000 +``` + +The execution plan shows a duration of 170 ms. TiDB uses the `test_index` to perform an `IndexRangeScan_20` with the filter `snapshot_id = 459840`. It then retrieves all columns from the table, returning 5,715 rows to TiDB after `IndexLookUp_23`. TiDB sorts these rows and returns 1,000 rows. + +The `id` column is the primary key, which means it is implicitly included in the `test_idx` index. However, `IndexRangeScan_20` does not guarantee order because `test_idx` includes two additional columns (`user_id` and `status`) after the index prefix column `snapshot_id`. As a result, the ordering of `id` is not preserved. + +The original plan is as follows: + +``` ++------------------------------+---------+---------+-----------+----------------------------------------------------------+-----------------------------------------------+--------------------------------------------+ +| id | estRows | actRows | task | access object | execution info | operator info | ++------------------------------+---------+---------+-----------+----------------------------------------------------------+-----------------------------------------------+--------------------------------------------+ +| id | estRows | actRows | task | access object | execution info ...| test.id, offset:0, count:1000 | +| TopN_10 | 19.98 | 1000 | root | | time:170.6ms, loops:2 ...| | +| └─IndexLookUp_23 | 19.98 | 5715 | root | | time:166.6ms, loops:7 ...| | +| ├─Selection_22(Build) | 19.98 | 5715 | cop[tikv] | | time:18.6ms, loops:9, cop_task: {num: 3, ...| gt(test.id, 998464) | +| │ └─IndexRangeScan_20 | 433.47 | 7715 | cop[tikv] | table:test, index:test_idx(snapshot_id, user_id, status) | tikv_task:{proc max:4ms, min:4ms, avg: 4ms ...| range:[459840,459840], keep order:false | +| └─TableRowIDScan_21(Probe) | 19.98 | 5715 | cop[tikv] | table:test | time:301.6ms, loops:10, cop_task: {num: 3, ...| keep order:false | ++------------------------------+---------+---------+-----------+----------------------------------------------------------+-----------------------------------------------+--------------------------------------------+ +``` + +To optimize the query, you can create a new index on `(snapshot_id)`. This ensures that `id` is sorted within each `snapshot_id` group. With this index, execution time drops to 96 ms. The `keep order` property becomes `true` for `IndexRangeScan_33`, and `TopN` is replaced with `Limit`. As a result, `IndexLookUp_35` returns only 1,000 rows to TiDB, eliminating the need for additional sorting operations. + +The following is the query statement with the optimized index: + +```sql +CREATE INDEX test_new ON test(snapshot_id); +ANALYZE TABLE test INDEX test_new; +``` + +The new plan is as follows: + +``` ++----------------------------------+---------+---------+-----------+----------------------------------------------+----------------------------------------------+----------------------------------------------------+ +| id | estRows | actRows | task | access object | execution info | operator info | ++----------------------------------+---------+---------+-----------+----------------------------------------------+----------------------------------------------+----------------------------------------------------+ +| Limit_14 | 17.59 | 1000 | root | | time:96.1ms, loops:2, RU:92.300155 | offset:0, count:1000 | +| └─IndexLookUp_35 | 17.59 | 1000 | root | | time:96.1ms, loops:1, index_task: ...| | +| ├─IndexRangeScan_33(Build) | 17.59 | 5715 | cop[tikv] | table:test, index:test_new(snapshot_id) | time:7.25ms, loops:8, cop_task: {num: 3, ...| range:(459840 998464,459840 +inf], keep order:true | +| └─TableRowIDScan_34(Probe) | 17.59 | 5715 | cop[tikv] | table:test | time:232.9ms, loops:9, cop_task: {num: 3, ...| keep order:false | ++----------------------------------+---------+---------+-----------+----------------------------------------------+----------------------------------------------+----------------------------------------------------+ +``` + +#### SQL tuning with composite indexes for efficient filtering and sorting + +The following query takes 11 minutes and 9 seconds to execute, which is excessively long for a query that returns only 101 rows. The slow performance is caused by several factors: + +- Inefficient index usage: The optimizer selects the index on `created_at`, resulting in a scan of 25,147,450 rows. +- Large intermediate result set: After applying the date range filter, 12,082,311 rows still require processing. +- Late filtering: The most selective predicates `(mode, user_id, and label_id)` are applied after accessing the table, resulting in 16,604 rows. +- Sorting overhead: The final sort operation on 16,604 rows adds additional processing time. + +The following is the query statement: + +```sql +SELECT `orders`.* +FROM `orders` +WHERE + `orders`.`mode` = 'production' + AND `orders`.`user_id` = 11111 + AND orders.label_id IS NOT NULL + AND orders.created_at >= '2024-04-07 18:07:52' + AND orders.created_at <= '2024-05-11 18:07:52' + AND orders.id >= 1000000000 + AND orders.id < 1500000000 +ORDER BY orders.id DESC +LIMIT 101; +``` + +The following are the existing indexes on `orders`: + +```sql +PRIMARY KEY (`id`), +UNIQUE KEY `index_orders_on_adjustment_id` (`adjustment_id`), +KEY `index_orders_on_user_id` (`user_id`), +KEY `index_orders_on_label_id` (`label_id`), +KEY `index_orders_on_created_at` (`created_at`) +``` + +The original execution plan is as follows: + +``` ++--------------------------------+-----------+---------+-----------+--------------------------------------------------------------------------------+-----------------------------------------------------+----------------------------------------------------------------------------------------+----------+------+ +| id | estRows | actRows | task | access object | execution info | operator info | memory | disk | ++--------------------------------+-----------+---------+-----------+--------------------------------------------------------------------------------+-----------------------------------------------------+----------------------------------------------------------------------------------------+----------+------+ +| TopN_10 | 101.00 | 101 | root | | time:11m9.8s, loops:2 | orders.id:desc, offset:0, count:101 | 271 KB | N/A | +| └─IndexLookUp_39 | 173.83 | 16604 | root | | time:11m9.8s, loops:19, index_task: {total_time:...}| | 20.4 MB | N/A | +| ├─Selection_37(Build) | 8296.70 | 12082311| cop[tikv] | | time:26.4ms, loops:11834, cop_task: {num: 294, m...}| ge(orders.id, 1000000000), lt(orders.id, 1500000000) | N/A | N/A | +| │ └─IndexRangeScan_35 | 6934161.90| 25147450| cop[tikv] | table:orders, index:index_orders_on_created_at(created_at) | tikv_task:{proc max:2.15s, min:0s, avg: 58.9ms, ...}| range:[2024-04-07 18:07:52,2024-05-11 18:07:52), keep order:false | N/A | N/A | +| └─Selection_38(Probe) | 173.83 | 16604 | cop[tikv] | | time:54m46.2s, loops:651, cop_task: {num: 1076, ...}| eq(orders.mode, "production"), eq(orders.user_id, 11111), not(isnull(orders.label_id)) | N/A | N/A | +| └─TableRowIDScan_36 | 8296.70 | 12082311| cop[tikv] | table:orders | tikv_task:{proc max:44.8s, min:0s, avg: 3.33s, p...}| keep order:false | N/A | N/A | ++--------------------------------+-----------+---------+-----------+--------------------------------------------------------------------------------+-----------------------------------------------------+----------------------------------------------------------------------------------------+----------+------+ +``` + +After creating a composite index `idx_composite` on `orders(user_id, mode, id, created_at, label_id)`, the query performance improves significantly. The execution time drops from 11 minutes and 9 seconds to just 5.3 milliseconds, making the query over 126,000 times faster. This massive improvement results from: + +- Efficient index usage: the new index enables an index range scan on `user_id`, `mode`, and `id`, which are the most selective predicates. This reduces the number of scanned rows from millions to just 224. +- Index-only sort: the `keep order:true` in the execution plan indicates that sorting is performed using the index structure, eliminating the need for a separate sort operation. +- Early filtering: the most selective predicates are applied first, reducing the result set to 224 rows before further filtering. +- Limit push-down: the `LIMIT` clause is pushed down to the index scan, allowing early termination of the scan once 101 rows are found. + +This case demonstrates the significant impact of a well-designed index on query performance. By aligning the index structure with the query's predicates, sort order, and required columns, the query achieves a performance improvement of over five orders of magnitude. + +```sql +CREATE INDEX idx_composite ON orders(user_id, mode, id, created_at, label_id); +ANALYZE TABLE orders index idx_composite; +``` + +The new execution plan is as follows: + +``` ++--------------------------------+-----------+---------+-----------+--------------------------------------------------------------------------------+-----------------------------------------------------+----------------------------------------------------------------------------------------------------------------------+----------+------+ +| id | estRows | actRows | task | access object | execution info | operator info | memory | disk | ++--------------------------------+-----------+---------+-----------+--------------------------------------------------------------------------------+-----------------------------------------------------+----------------------------------------------------------------------------------------------------------------------+----------+------+ +| IndexLookUp_32 | 101.00 | 101 | root | | time:5.3ms, loops:2, RU:3.435006, index_task: {t...}| limit embedded(offset:0, count:101) | 128.5 KB | N/A | +| ├─Limit_31(Build) | 101.00 | 101 | cop[tikv] | | time:1.35ms, loops:1, cop_task: {num: 1, max: 1....}| offset:0, count:101 | N/A | N/A | +| │ └─Selection_30 | 535.77 | 224 | cop[tikv] | | tikv_task:{time:0s, loops:3} | ge(orders.created_at, 2024-04-07 18:07:52), le(orders.created_at, 2024-05-11 18:07:52), not(isnull(orders.label_id)) | N/A | N/A | +| │ └─IndexRangeScan_28 | 503893.42 | 224 | cop[tikv] | table:orders, index:idx_composite(user_id, mode, id, created_at, label_id) | tikv_task:{time:0s, loops:3} | range:[11111 "production" 1000000000,11111 "production" 1500000000), keep order:true, desc | N/A | N/A | +| └─TableRowIDScan_29(Probe) | 101.00 | 101 | cop[tikv] | table:orders | time:2.9ms, loops:2, cop_task: {num: 3, max: 2.7...}| keep order:false | N/A | N/A | ++--------------------------------+-----------+---------+-----------+--------------------------------------------------------------------------------+-----------------------------------------------------+----------------------------------------------------------------------------------------------------------------------+----------+------+ +``` + +### When to use TiFlash + +This section explains when to use TiFlash in TiDB. TiFlash is optimized for analytical queries that involve complex calculations, aggregations, and large dataset scans. Its columnar storage format and MPP mode significantly improve performance in these scenarios. + +Use TiFlash in the following scenarios: + +- Large-scale data analysis: TiFlash delivers faster performance for OLAP workloads that require extensive data scans. Its columnar storage and MPP execution mode optimize query efficiency compared to TiKV. +- Complex scans, aggregations, and joins: TiFlash processes queries with heavy aggregations and joins more efficiently by reading only the necessary columns. +- Mixed workloads: in hybrid environments where both transactional (OLTP) and analytical (OLAP) workloads run simultaneously, TiFlash handles analytical queries without affecting TiKV's performance for transactional queries. +- SaaS applications with arbitrary filtering requirements: queries often involve filtering across many columns. Indexing all columns is impractical, especially when queries include a tenant ID as part of the primary key. TiFlash sorts and clusters data by primary key, making it well suited for these workloads. With the [late materialization](/tiflash/tiflash-late-materialization.md) feature, TiFlash enables efficient table range scans, improving query performance without the overhead of maintaining multiple indexes. + +Using TiFlash strategically enhances query performance and optimizes resource usage in TiDB for data-intensive analytical queries. The following sections provide examples of TiFlash use cases. + +#### Analytical query + +This section compares the execution performance of TPC-H Query 14 on TiKV and TiFlash storage engines. + +TPC-H Query 14 involves joining the `order_line` and `item` tables. The query takes **21.1 seconds** on TiKV but only **1.41 seconds** using TiFlash MPP mode, making it 15 times faster. + +- TiKV plan: TiDB fetches 3,864,397 rows from the `lineitem` table and 10 million rows from the `part` table. The hash join operation (`HashJoin_21`), along with the subsequent projection (`Projection_38`) and aggregation (`HashAgg_9`) operations, are performed in TiDB. +- TiFlash plan: The optimizer detects TiFlash replicas for both `order_line` and `item` tables. Based on cost estimation, TiDB automatically selects the MPP mode, executing the entire query within the TiFlash columnar storage engine. This includes table scans, hash joins, column projections, and aggregations, significantly improving performance compared to the TiKV plan. + +The following is the query: + +```sql +select 100.00 * sum(case when i_data like 'PR%' then ol_amount else 0 end) / (1+sum(ol_amount)) as promo_revenue +from order_line, item +where ol_i_id = i_id and ol_delivery_d >= '2007-01-02 00:00:00.000000' and ol_delivery_d < '2030-01-02 00:00:00.000000'; +``` + +The execution plan on TiKV is as follows: + +``` ++-------------------------------+--------------+-----------+-----------+----------------+----------------------------------------------+ +| ID | ESTROWS | ACTROWS | TASK | ACCESS OBJECT | EXECUTION INFO | ++-------------------------------+--------------+-----------+-----------+----------------+----------------------------------------------+ +| Projection_8 | 1.00 | 1 | root | | time:21.1s, loops:2, RU:1023225.707561, ... | +| └─HashAgg_9 | 1.00 | 1 | root | | time:21.1s, loops:2, partial_worker:{ ... | +| └─Projection_38 | 3839984.46 | 3864397 | root | | time:21.1s, loops:3776, Concurrency:5 | +| └─HashJoin_21 | 3839984.46 | 3864397 | root | | time:21.1s, loops:3776, build_hash_table:... | +| ├─TableReader_24(Build) | 3826762.62 | 3864397 | root | | time:18.4s, loops:3764, cop_task: ... | +| │ └─Selection_23 | 3826762.62 | 3864397 | cop[tikv] | | tikv_task:{proc max:717ms, min:265ms, ... | +| │ └─TableFullScan_22 | 300005811.00 | 300005811 | cop[tikv] | table:lineitem | tikv_task:{proc max:685ms, min:252ms, ... | +| └─TableReader_26(Probe) | 10000000.00 | 10000000 | root | | time:1.29s, loops:9780, cop_task: ... | +| └─TableFullScan_25 | 10000000.00 | 10000000 | cop[tikv] | table:part | tikv_task:{proc max:922ms, min:468ms, ... | ++-------------------------------+--------------+-----------+-----------+----------------+----------------------------------------------+ +``` + +The execution plan on TiFlash is as follows: + +``` ++--------------------------------------------+-------------+----------+--------------+----------------+--------------------------------------+ +| ID | ESTROWS | ACTROWS | TASK | ACCESS OBJECT | EXECUTION INFO | ++--------------------------------------------+-------------+----------+--------------+----------------+--------------------------------------+ +| Projection_8 | 1.00 | 1 | root | | time:1.41s, loops:2, RU:45879.127909 | +| └─HashAgg_52 | 1.00 | 1 | root | | time:1.41s, loops:2, ... | +| └─TableReader_54 | 1.00 | 1 | root | | time:1.41s, loops:2, ... | +| └─ExchangeSender_53 | 1.00 | 1 | mpp[tiflash] | | tiflash_task:{time:1.41s, ... | +| └─HashAgg_13 | 1.00 | 1 | mpp[tiflash] | | tiflash_task:{time:1.41s, ... | +| └─Projection_74 | 3813443.11 | 3864397 | mpp[tiflash] | | tiflash_task:{time:1.4s, ... | +| └─Projection_51 | 3813443.11 | 3864397 | mpp[tiflash] | | tiflash_task:{time:1.39s, ... | +| └─HashJoin_50 | 3813443.11 | 3864397 | mpp[tiflash] | | tiflash_task:{time:1.39s, ... | +| ├─ExchangeReceiver_31(Build) | 3800312.67 | 3864397 | mpp[tiflash] | | tiflash_task:{time:1.05s, ... | +| │ └─ExchangeSender_30 | 3800312.67 | 3864397 | mpp[tiflash] | | tiflash_task:{time:1.2s, ... | +| │ └─TableFullScan_28 | 3800312.67 | 3864397 | mpp[tiflash] | table:lineitem | tiflash_task:{time:1.15s, ... | +| └─ExchangeReceiver_34(Probe) | 10000000.00 | 10000000 | mpp[tiflash] | | tiflash_task:{time:1.24s, ... | +| └─ExchangeSender_33 | 10000000.00 | 10000000 | mpp[tiflash] | | tiflash_task:{time:1.4s, ... | +| └─TableFullScan_32 | 10000000.00 | 10000000 | mpp[tiflash] | table:part | tiflash_task:{time:59.2ms, ... | ++--------------------------------------------+-------------+----------+--------------+----------------+--------------------------------------+ +``` + +#### SaaS arbitrary filtering workloads + +In SaaS applications, tables often use composite primary keys that include tenant identification. The following example demonstrates how TiFlash can significantly improve query performance in these scenarios. + +##### Case study: multi-tenant data access + +Consider a table with a composite primary key: `(tenantId, objectTypeId, objectId)`. A typical query pattern for this table involves: + +- Retrieving the first N records for a specific tenant and object type while applying random filters across hundreds or thousands of columns. This makes creating indexes for all possible filter combinations impractical. The query might also include a sort operation after filtering. +- Calculating the total count of records that match the filter criteria. + +##### Performance comparison + +When running the same query on different storage engines, you can observe significant performance differences: + +- TiKV plan: The query takes 2 minutes 38.6 seconds on TiKV. `TableRangeScan` sends 5,121 cop tasks because the data is distributed across 5,121 Regions. +- TiFlash plan: The same query takes only 3.44 seconds on TiFlash MPP engine—nearly 46 times faster. TiFlash stores data sorted by primary key, so queries filtered by the primary key's prefix use `TableRangeScan` instead of a full table scan. TiFlash requires only 2 MPP tasks compared to TiKV's 5,121 tasks. + +The following is the query statement: + +```sql +WITH `results` AS ( + SELECT field1, field2, field3, field4 + FROM usertable + where tenantId = 1234 and objectTypeId = 6789 +), +`limited_results` AS ( + SELECT field1, field2, field3, field4 + FROM `results` LIMIT 100 +) +SELECT field1, field2, field3, field4 +FROM + ( + SELECT 100 `__total__`, field1, field2, field3, field4 + FROM `limited_results` + UNION ALL + SELECT count(*) `__total__`, field1, field2, field3, field4 + FROM `results` + ) `result_and_count`; +``` + +The execution plan on TiKV is as follows: + +``` ++--------------------------------+-----------+---------+-----------+-----------------------+-----------------------------------------------------+ +| id | estRows | actRows | task | access object | execution info | ++--------------------------------+-----------+---------+-----------+-----------------------+-----------------------------------------------------+ +| Union_18 | 101.00 | 101 | root | | time:2m38.6s, loops:3, RU:8662189.451027 | +| ├─Limit_20 | 100.00 | 100 | root | | time:23ms, loops:2 | +| │ └─TableReader_25 | 100.00 | 100 | root | | time:23ms, loops:1, cop_task: {num: 1, max: 22.8...}| +| │ └─Limit_24 | 100.00 | 100 | cop[tikv] | | tikv_task:{time:21ms, loops:3}, scan_detail: {...} | +| │ └─TableRangeScan_22 | 100.00 | 100 | cop[tikv] | table:usertable | tikv_task:{time:21ms, loops:3} | +| └─Projection_26 | 1.00 | 1 | root | | time:2m38.6s, loops:2, Concurrency:OFF | +| └─HashAgg_34 | 1.00 | 1 | root | | time:2m38.6s, loops:2, partial_worker:{...}, fin.. .| +| └─TableReader_35 | 1.00 | 5121 | root | | time:2m38.6s, loops:7, cop_task: {num: 5121, max:...| +| └─HashAgg_27 | 1.00 | 5121 | cop[tikv] | | tikv_task:{proc max:0s, min:0s, avg: 462.8ms, p...} | +| └─TableRangeScan_32 | 10000000 | 10000000| cop[tikv] | table:usertable | tikv_task:{proc max:0s, min:0s, avg: 460.5ms, p...} | ++--------------------------------+-----------+---------+-----------+-----------------------+-----------------------------------------------------+ +``` + +The execution plan on TiFlash is as follows: + +``` ++--------------------------------+-----------+---------+--------------+--------------------+-----------------------------------------------------+ +| id | estRows | actRows | task | access object | execution info | ++--------------------------------+-----------+---------+--------------+--------------------+-----------------------------------------------------+ +| Union_18 | 101.00 | 101 | root | | time:3.44s, loops:3, RU:0.000000 | +| ├─Limit_22 | 100.00 | 100 | root | | time:146.7ms, loops:2 | +| │ └─TableReader_30 | 100.00 | 100 | root | | time:146.7ms, loops:1, cop_task: {num: 1, max: 0...}| +| │ └─ExchangeSender_29 | 100.00 | 0 | mpp[tiflash] | | | +| │ └─Limit_28 | 100.00 | 0 | mpp[tiflash] | | | +| │ └─TableRangeScan_27 | 100.00 | 0 | mpp[tiflash] | table:usertable | | +| └─Projection_31 | 1.00 | 1 | root | | time:3.42s, loops:2, Concurrency:OFF | +| └─HashAgg_49 | 1.00 | 1 | root | | time:3.42s, loops:2, partial_worker:{...}, fin... | +| └─TableReader_51 | 1.00 | 2 | root | | time:3.42s, loops:2, cop_task: {num: 4, max: 0...} | +| └─ExchangeSender_50 | 1.00 | 2 | mpp[tiflash] | | tiflash_task:{proc max:3.4s, min:3.15s, avg: 3...} | +| └─HashAgg_36 | 1.00 | 2 | mpp[tiflash] | | tiflash_task:{proc max:3.4s, min:3.15s, avg: 3...} | +| └─TableRangeScan_48 | 10000000 | 10000000| mpp[tiflash] | table:usertable | tiflash_task:{proc max:3.4s, min:3.15s, avg: 3...} | ++--------------------------------+-----------+---------+--------------+--------------------+-----------------------------------------------------+ +``` + +##### Query routing between TiKV and TiFlash + +After enabling TiFlash replicas for tables with large amounts of multi-tenant data, the optimizer routes queries to either TiKV or TiFlash based on the row count: + +- Small tenants: TiKV is more suitable for tenants with small data size, as it provides high concurrency for small queries with table range scans. +- Large tenants: For tenants with large datasets (such as 10 million rows in this case), TiFlash is more efficient for the following advantages: + - TiFlash handles dynamic filtering conditions without requiring specific indexes. + - TiDB can push down `COUNT`, `SORT`, and `LIMIT` operations to TiFlash. + - TiFlash scans only the required columns using columnar storage. \ No newline at end of file diff --git a/stale-read.md b/stale-read.md index bb1d2fb33ed38..b124e3977969e 100644 --- a/stale-read.md +++ b/stale-read.md @@ -35,7 +35,7 @@ TiDB provides the methods of performing Stale Read at the statement level, the s - Session level - Specifying a time range: In a session, if you need TiDB to read the data as new as possible within a time range in subsequent queries without violating the isolation level, you can specify the time range by setting the `tidb_read_staleness` system variable. For detailed usage, refer to [`tidb_read_staleness`](/tidb-read-staleness.md). -Besides, TiDB provides a way to specify an exact point in time by setting the [`tidb_external_ts`](/system-variables.md#tidb_external_ts-new-in-v640) system variable on session or global level. For detailed usage, refer to [Perform Stale Read Using `tidb_external_ts`](/tidb-external-ts.md). +Besides, TiDB provides a way to specify an exact point in time at the session or global level by setting the [`tidb_external_ts`](/system-variables.md#tidb_external_ts-new-in-v640) and [`tidb_enable_external_ts_read`](/system-variables.md#tidb_enable_external_ts_read-new-in-v640) system variables. For detailed usage, refer to [Perform Stale Read Using `tidb_external_ts`](/tidb-external-ts.md). ### Reduce Stale Read latency diff --git a/statement-summary-tables.md b/statement-summary-tables.md index 13c1708c76198..558be411beb06 100644 --- a/statement-summary-tables.md +++ b/statement-summary-tables.md @@ -1,7 +1,6 @@ --- title: Statement Summary Tables summary: Learn about Statement Summary Table in TiDB. -aliases: ['/docs/dev/statement-summary-tables/','/docs/dev/reference/performance/statement-summary/'] --- # Statement Summary Tables @@ -18,7 +17,7 @@ Therefore, starting from v4.0.0-rc.1, TiDB provides system tables in `informatio > **Note:** > -> The preceding tables are not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> The preceding tables are not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. This document details these tables and introduces how to use them to troubleshoot SQL performance issues. @@ -210,7 +209,7 @@ To address this issue, TiDB v6.6.0 experimentally introduces the [statement summ -This section is only applicable to TiDB Self-Hosted. For TiDB Cloud, the value of the `tidb_stmt_summary_enable_persistent` parameter is `false` by default and does not support dynamic modification. +This section is only applicable to TiDB Self-Managed. For TiDB Cloud, the value of the `tidb_stmt_summary_enable_persistent` parameter is `false` by default and does not support dynamic modification. @@ -329,7 +328,7 @@ Basic fields: - `SAMPLE_USER`: The users who execute SQL statements of this category. Only one user is taken. - `PLAN_DIGEST`: The digest of the execution plan. - `PLAN`: The original execution plan. If there are multiple statements, the plan of only one statement is taken. -- `BINARY_PLAN`: The original execution plan encoded in binary format. If there are multiple statements, the plan of only one statement is taken. Execute the `SELECT tidb_decode_binary_plan('xxx...')` statement to parse the specific execution plan. +- `BINARY_PLAN`: The original execution plan encoded in binary format. If there are multiple statements, the plan of only one statement is taken. Execute the [`SELECT tidb_decode_binary_plan('xxx...')`](/functions-and-operators/tidb-functions.md#tidb_decode_binary_plan) statement to parse the specific execution plan. - `PLAN_CACHE_HITS`: The total number of times that SQL statements of this category hit the plan cache. - `PLAN_IN_CACHE`: Indicates whether the previous execution of SQL statements of this category hit the plan cache. diff --git a/statistics.md b/statistics.md index 782278d92a9c5..641c68f0cd3fc 100644 --- a/statistics.md +++ b/statistics.md @@ -1,7 +1,6 @@ --- title: Introduction to Statistics summary: Learn how the statistics collect table-level and column-level information. -aliases: ['/docs/dev/statistics/','/docs/dev/reference/performance/statistics/'] --- # Introduction to Statistics @@ -10,9 +9,11 @@ TiDB uses statistics as input to the optimizer to estimate the number of rows pr ## Collect statistics +This section describes two ways of collecting statistics: automatic update and manual collection. + ### Automatic update -For the `INSERT`, `DELETE`, or `UPDATE` statements, TiDB automatically updates the number of rows and modified rows in statistics. +For the [`INSERT`](/sql-statements/sql-statement-insert.md), [`DELETE`](/sql-statements/sql-statement-delete.md), or [`UPDATE`](/sql-statements/sql-statement-update.md) statements, TiDB automatically updates the number of rows and modified rows in statistics. @@ -26,15 +27,15 @@ TiDB persists the update information every 60 seconds. -Based upon the number of changes to a table, TiDB will automatically schedule [`ANALYZE`](/sql-statements/sql-statement-analyze-table.md) to collect statistics on those tables. This is controlled by the [`tidb_enable_auto_anlyze`](/system-variables.md#tidb_enable_auto_analyze-new-in-v610) system variable and the following `tidb_auto_analyze%` variables. +Based upon the number of changes to a table, TiDB will automatically schedule [`ANALYZE`](/sql-statements/sql-statement-analyze-table.md) to collect statistics on those tables. This is controlled by the [`tidb_enable_auto_analyze`](/system-variables.md#tidb_enable_auto_analyze-new-in-v610) system variable and the following `tidb_auto_analyze%` variables. | System Variable | Default Value | Description | |---|---|---| -| [`tidb_enable_auto_anlyze`](/system-variables.md#tidb_enable_auto_analyze-new-in-v610) | true | Controls whether TiDB automatically executes ANALYZE. | -| [`tidb_auto_analyze_ratio`](/system-variables.md#tidb_auto_analyze_ratio) | 0.5 | The threshold value of automatic update | -| [`tidb_auto_analyze_start_time`](/system-variables.md#tidb_auto_analyze_start_time) | `00:00 +0000` | The start time in a day when TiDB can perform automatic update | -| [`tidb_auto_analyze_end_time`](/system-variables.md#tidb_auto_analyze_end_time) | `23:59 +0000` | The end time in a day when TiDB can perform automatic update | -| [`tidb_auto_analyze_partition_batch_size`](/system-variables.md#tidb_auto_analyze_partition_batch_size-new-in-v640) | `128` | The number of partitions that TiDB automatically analyzes when analyzing a partitioned table (that is, when automatically updating statistics on a partitioned table) | +| [`tidb_enable_auto_analyze`](/system-variables.md#tidb_enable_auto_analyze-new-in-v610) | `ON` | Controls whether TiDB automatically executes `ANALYZE`. | +| [`tidb_auto_analyze_ratio`](/system-variables.md#tidb_auto_analyze_ratio) | `0.5` | The threshold value of automatic update. | +| [`tidb_auto_analyze_start_time`](/system-variables.md#tidb_auto_analyze_start_time) | `00:00 +0000` | The start time in a day when TiDB can perform automatic update. | +| [`tidb_auto_analyze_end_time`](/system-variables.md#tidb_auto_analyze_end_time) | `23:59 +0000` | The end time in a day when TiDB can perform automatic update. | +| [`tidb_auto_analyze_partition_batch_size`](/system-variables.md#tidb_auto_analyze_partition_batch_size-new-in-v640) | `128` | The number of partitions that TiDB automatically analyzes when analyzing a partitioned table (that is, when automatically updating statistics on a partitioned table). | | [`tidb_enable_auto_analyze_priority_queue`](/system-variables.md#tidb_enable_auto_analyze_priority_queue-new-in-v800) | `ON` | Controls whether to enable the priority queue to schedule the tasks of automatically collecting statistics. When this variable is enabled, TiDB prioritizes collecting statistics for tables that are more valuable to collect, such as newly created indexes and partitioned tables with partition changes. Additionally, TiDB prioritizes tables with lower health scores, placing them at the front of the queue. | When the ratio of the number of modified rows to the total number of rows of `tbl` in a table is greater than `tidb_auto_analyze_ratio`, and the current time is between `tidb_auto_analyze_start_time` and `tidb_auto_analyze_end_time`, TiDB executes the `ANALYZE TABLE tbl` statement in the background to automatically update the statistics on this table. @@ -43,11 +44,11 @@ To avoid the situation that modifying data on a small table frequently triggers > **Note:** > -> Currently, the automatic update does not record the configuration items input at manual `ANALYZE`. Therefore, when you use the `WITH` syntax to control the collecting behavior of `ANALYZE`, you need to manually set scheduled tasks to collect statistics. +> Currently, the automatic update does not record the configuration items input at manual `ANALYZE`. Therefore, when you use the [`WITH`](/sql-statements/sql-statement-analyze-table.md) syntax to control the collecting behavior of `ANALYZE`, you need to manually set scheduled tasks to collect statistics. ### Manual collection -Currently, TiDB collects statistical information as a full collection. You can execute the `ANALYZE TABLE` statement to collect statistics. +Currently, TiDB collects statistics as a full collection. You can execute the `ANALYZE TABLE` statement to collect statistics. You can perform full collection using the following syntax. @@ -66,12 +67,14 @@ You can perform full collection using the following syntax. `WITH NUM SAMPLES` and `WITH FLOAT_NUM SAMPLERATE` correspond to two different algorithms of collecting samples. -See [Histograms](#histogram), [Top-N](#top-n-values) and [CMSketch](#count-min-sketch) (Count-Min Sketch) for detailed explanations. For `SAMPLES`/`SAMPLERATE`, see [Improve collection performance](#improve-collection-performance). +See [Histograms](#histogram), [Top-N](#top-n) and [CMSketch](#count-min-sketch) (Count-Min Sketch) for detailed explanations. For `SAMPLES`/`SAMPLERATE`, see [Improve collection performance](#improve-collection-performance). -For information on persisting the options for easier reuse, see [Persist ANALYZE configurations](#persist-analyze-configurations). +For information on persisting the options for easier reuse, see [Persist `ANALYZE` configurations](#persist-analyze-configurations). ## Types of statistics +This section describes three types of statistics: histogram, Count-Min Sketch, and Top-N. + ### Histogram Histogram statistics are used by the optimizer to estimate selectivity of an interval or range predicate, and might also be used to determine the number of distinct values within a column for estimation of equal/IN predicates in Version 2 of statistics (refer to [Versions of Statistics](#versions-of-statistics)). @@ -88,23 +91,25 @@ For details about the parameter that determines the upper limit to the number of > **Note:** > -> Count-Min Sketch is used in statistics Version 1 only for equal/IN predicate selectivity estimation. In Version 2, other statistics are used due to challenges in managing Count-Min sketch to avoid collisions as discussed below. +> Count-Min Sketch is used in statistics Version 1 only for equal/IN predicate selectivity estimation. In Version 2, Histogram statistics are used instead due to challenges in managing Count-Min sketch to avoid collisions as discussed below. -Count-Min Sketch is a hash structure. When an equivalence query contains `a = 1` or `IN` query (for example, `a IN (1, 2, 3)`), TiDB uses this data structure for estimation. +Count-Min Sketch is a hash structure. When processing an equivalence query such as `a = 1` or an `IN` query (for example, `a IN (1, 2, 3)`), TiDB uses this data structure for estimation. A hash collision might occur since Count-Min Sketch is a hash structure. In the [`EXPLAIN`](/sql-statements/sql-statement-explain.md) statement, if the estimate of the equivalent query deviates greatly from the actual value, it can be considered that a larger value and a smaller value have been hashed together. In this case, you can take one of the following ways to avoid the hash collision: - Modify the `WITH NUM TOPN` parameter. TiDB stores the high-frequency (top x) data separately, with the other data stored in Count-Min Sketch. Therefore, to prevent a larger value and a smaller value from being hashed together, you can increase the value of `WITH NUM TOPN`. In TiDB, its default value is 20. The maximum value is 1024. For more information about this parameter, see [Manual collection](#manual-collection). - Modify two parameters `WITH NUM CMSKETCH DEPTH` and `WITH NUM CMSKETCH WIDTH`. Both affect the number of hash buckets and the collision probability. You can increase the values of the two parameters appropriately according to the actual scenario to reduce the probability of hash collision, but at the cost of higher memory usage of statistics. In TiDB, the default value of `WITH NUM CMSKETCH DEPTH` is 5, and the default value of `WITH NUM CMSKETCH WIDTH` is 2048. For more information about the two parameters, see [Manual collection](#manual-collection). -### Top-N values +### Top-N Top-N values are values with the top N occurrences in a column or index. Top-N statistics are often referred to as frequency statistics or data skew. -TiDB records the values and occurrences of Top-N values. The default value is 20, meaning the top 20 most frequent values are collected. The maximum value is 1024. For details about the parameter that determines the number of values collected, see [Manual collection](#manual-collection). +TiDB records the values and occurrences of Top-N values. Here `N` is controlled by the `WITH NUM TOPN` parameter. The default value is 20, meaning the top 20 most frequent values are collected. The maximum value is 1024. For details about the parameter, see [Manual collection](#manual-collection). ## Selective statistics collection +This section describes how to collect statistics selectively. + ### Collect statistics on indexes To collect statistics on all indexes in `IndexNameList` in `TableName`, use the following syntax: @@ -132,8 +137,6 @@ If a table has many columns, collecting statistics on all the columns can cause - To collect statistics on specific columns, use the following syntax: - {{< copyable "sql" >}} - ```sql ANALYZE TABLE TableName COLUMNS ColumnNameList [WITH NUM BUCKETS|TOPN|CMSKETCH DEPTH|CMSKETCH WIDTH]|[WITH NUM SAMPLES|WITH FLOATNUM SAMPLERATE]; ``` @@ -150,20 +153,18 @@ If a table has many columns, collecting statistics on all the columns can cause - After the setting, TiDB writes the `PREDICATE COLUMNS` information to the `mysql.column_stats_usage` system table every 100 * [`stats-lease`](/tidb-configuration-file.md#stats-lease). + After the setting, TiDB writes the `PREDICATE COLUMNS` information to the [`mysql.column_stats_usage`](/mysql-schema/mysql-schema.md#statistics-system-tables) system table every 100 * [`stats-lease`](/tidb-configuration-file.md#stats-lease). - After the setting, TiDB writes the `PREDICATE COLUMNS` information to the `mysql.column_stats_usage` system table every 300 seconds. + After the setting, TiDB writes the `PREDICATE COLUMNS` information to the [`mysql.column_stats_usage`](/mysql-schema/mysql-schema.md#statistics-system-tables) system table every 300 seconds. 2. After the query pattern of your business is relatively stable, collect statistics on `PREDICATE COLUMNS` by using the following syntax: - {{< copyable "sql" >}} - ```sql ANALYZE TABLE TableName PREDICATE COLUMNS [WITH NUM BUCKETS|TOPN|CMSKETCH DEPTH|CMSKETCH WIDTH]|[WITH NUM SAMPLES|WITH FLOATNUM SAMPLERATE]; ``` @@ -172,13 +173,11 @@ If a table has many columns, collecting statistics on all the columns can cause > **Note:** > - > - If the `mysql.column_stats_usage` system table does not contain any `PREDICATE COLUMNS` recorded for that table, the preceding syntax collects statistics on all columns and all indexes in that table. + > - If the [`mysql.column_stats_usage`](/mysql-schema/mysql-schema.md#statistics-system-tables) system table does not contain any `PREDICATE COLUMNS` recorded for that table, the preceding syntax collects statistics on all columns and all indexes in that table. > - Any columns excluded from collection (either by manually listing columns or using `PREDICATE COLUMNS`) will not have their statistics overwritten. When executing a new type of SQL query, the optimizer will use the old statistics for such columns if it exists or pseudo column statistics if columns never had statistics collected. The next ANALYZE using `PREDICATE COLUMNS` will collect the statistics on those columns. - To collect statistics on all columns and indexes, use the following syntax: - {{< copyable "sql" >}} - ```sql ANALYZE TABLE TableName ALL COLUMNS [WITH NUM BUCKETS|TOPN|CMSKETCH DEPTH|CMSKETCH WIDTH]|[WITH NUM SAMPLES|WITH FLOATNUM SAMPLERATE]; ``` @@ -187,16 +186,12 @@ If a table has many columns, collecting statistics on all the columns can cause - To collect statistics on all partitions in `PartitionNameList` in `TableName`, use the following syntax: - {{< copyable "sql" >}} - ```sql ANALYZE TABLE TableName PARTITION PartitionNameList [WITH NUM BUCKETS|TOPN|CMSKETCH DEPTH|CMSKETCH WIDTH]|[WITH NUM SAMPLES|WITH FLOATNUM SAMPLERATE]; ``` - To collect index statistics on all partitions in `PartitionNameList` in `TableName`, use the following syntax: - {{< copyable "sql" >}} - ```sql ANALYZE TABLE TableName PARTITION PartitionNameList INDEX [IndexNameList] [WITH NUM BUCKETS|TOPN|CMSKETCH DEPTH|CMSKETCH WIDTH]|[WITH NUM SAMPLES|WITH FLOATNUM SAMPLERATE]; ``` @@ -207,17 +202,15 @@ If a table has many columns, collecting statistics on all the columns can cause > > Currently, collecting statistics on `PREDICATE COLUMNS` is an experimental feature. It is not recommended that you use it in production environments. - {{< copyable "sql" >}} - ```sql ANALYZE TABLE TableName PARTITION PartitionNameList [COLUMNS ColumnNameList|PREDICATE COLUMNS|ALL COLUMNS] [WITH NUM BUCKETS|TOPN|CMSKETCH DEPTH|CMSKETCH WIDTH]|[WITH NUM SAMPLES|WITH FLOATNUM SAMPLERATE]; ``` #### Collect statistics of partitioned tables in dynamic pruning mode -When accessing partitioned tables in [dynamic pruning mode](/partitioned-table.md#dynamic-pruning-mode) (which is the default since v6.3.0), TiDB collects table-level statistics, which is called GlobalStats. Currently, GlobalStats is aggregated from statistics of all partitions. In dynamic pruning mode, a statistics update of any partitioned table can trigger the GlobalStats to be updated. +When accessing partitioned tables in [dynamic pruning mode](/partitioned-table.md#dynamic-pruning-mode) (which is the default since v6.3.0), TiDB collects table-level statistics, which is called GlobalStats. Currently, GlobalStats is aggregated from statistics of all partitions. In dynamic pruning mode, a statistics update in any partition of a table can trigger the GlobalStats of that table to be updated. -If partitions are empty, or columns for some partitions are missing, then the collection behavior is controlled by the [`tidb_skip_missing_partition_stats`](/system-variables.md#tidb_skip_missing_partition_stats-new-in-v730) variable: +If statistics of some partitions are empty, or statistics of some columns are missing in some partitions, then the collection behavior is controlled by the [`tidb_skip_missing_partition_stats`](/system-variables.md#tidb_skip_missing_partition_stats-new-in-v730) variable: - When GlobalStats update is triggered and [`tidb_skip_missing_partition_stats`](/system-variables.md#tidb_skip_missing_partition_stats-new-in-v730) is `OFF`: @@ -229,7 +222,7 @@ If partitions are empty, or columns for some partitions are missing, then the co - If statistics of all or some columns are missing for some partitions, TiDB skips these missing partition statistics when generating GlobalStats so the generation of GlobalStats is not affected. -In dynamic pruning mode, the Analyze configurations of partitions and tables should be the same. Therefore, if you specify the `COLUMNS` configuration following the `ANALYZE TABLE TableName PARTITION PartitionNameList` statement or the `OPTIONS` configuration following `WITH`, TiDB will ignore them and return a warning. +In dynamic pruning mode, the `ANALYZE` configurations of partitions and tables should be the same. Therefore, if you specify the `COLUMNS` configuration following the `ANALYZE TABLE TableName PARTITION PartitionNameList` statement or the `OPTIONS` configuration following `WITH`, TiDB will ignore them and return a warning. ## Improve collection performance @@ -251,23 +244,11 @@ Sampling is available via two separate options of the `ANALYZE` statement - with Before v5.3.0, TiDB uses the reservoir sampling method to collect statistics. Since v5.3.0, the TiDB Version 2 statistics uses the Bernoulli sampling method to collect statistics by default. To re-use the reservoir sampling method, you can use the `WITH NUM SAMPLES` statement. -The current sampling rate is calculated based on an adaptive algorithm. When you can observe the number of rows in a table using [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md), you can use this number of rows to calculate the sampling rate corresponding to 100,000 rows. If you cannot observe this number, you can use the `TABLE_KEYS` column in the [`TABLE_STORAGE_STATS`](/information-schema/information-schema-table-storage-stats.md) table as another reference to calculate the sampling rate. - - - -> **Note:** -> -> Normally, `STATS_META` is more credible than `TABLE_KEYS`. However, after importing data through the methods like [TiDB Lightning](https://docs.pingcap.com/tidb/stable/tidb-lightning-overview), the result of `STATS_META` is `0`. To handle this situation, you can use `TABLE_KEYS` to calculate the sampling rate when the result of `STATS_META` is much smaller than the result of `TABLE_KEYS`. - - - - +The current sampling rate is calculated based on an adaptive algorithm. When you can observe the number of rows in a table using [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md), you can use this number of rows to calculate the sampling rate corresponding to 100,000 rows. If you cannot observe this number, you can use the sum of all the values in the `APPROXIMATE_KEYS` column in the results of [`SHOW TABLE REGIONS`](/sql-statements/sql-statement-show-table-regions.md) of the table as another reference to calculate the sampling rate. > **Note:** > -> Normally, `STATS_META` is more credible than `TABLE_KEYS`. However, after importing data through TiDB Cloud console (see [Import Sample Data](/tidb-cloud/import-sample-data.md)), the result of `STATS_META` is `0`. To handle this situation, you can use `TABLE_KEYS` to calculate the sampling rate when the result of `STATS_META` is much smaller than the result of `TABLE_KEYS`. - - +> Normally, `STATS_META` is more credible than `APPROXIMATE_KEYS`. However, when the result of `STATS_META` is much smaller than the result of `APPROXIMATE_KEYS`, it is recommended that you use `APPROXIMATE_KEYS` to calculate the sampling rate. ### The memory quota for collecting statistics @@ -282,9 +263,9 @@ To set a proper value of `tidb_mem_quota_analyze`, consider the data size of the > **Note:** > > The following suggestions are for reference only. You need to configure the values based on the real scenario. -> -> - Minimum value: should be greater than the maximum memory usage when TiDB collects statistics from the table with the most columns. An approximate reference: when TiDB collects statistics from a table with 20 columns using the default configuration, the maximum memory usage is about 800 MiB; when TiDB collects statistics from a table with 160 columns using the default configuration, the maximum memory usage is about 5 GiB. -> - Maximum value: should be less than the available memory when TiDB is not collecting statistics. + +- Minimum value: should be greater than the maximum memory usage when TiDB collects statistics from the table with the most columns. An approximate reference: when TiDB collects statistics from a table with 20 columns using the default configuration, the maximum memory usage is about 800 MiB; when TiDB collects statistics from a table with 160 columns using the default configuration, the maximum memory usage is about 5 GiB. +- Maximum value: should be less than the available memory when TiDB is not collecting statistics. ## Persist ANALYZE configurations @@ -294,10 +275,10 @@ The following are the `ANALYZE` configurations that support persistence: | Configurations | Corresponding ANALYZE syntax | | --- | --- | -| The number of histogram buckets | WITH NUM BUCKETS | -| The number of Top-N | WITH NUM TOPN | -| The number of samples | WITH NUM SAMPLES | -| The sampling rate | WITH FLOATNUM SAMPLERATE | +| The number of histogram buckets | `WITH NUM BUCKETS` | +| The number of Top-N | `WITH NUM TOPN` | +| The number of samples | `WITH NUM SAMPLES` | +| The sampling rate | `WITH FLOATNUM SAMPLERATE` | | The `ANALYZE` column type | AnalyzeColumnOption ::= ( 'ALL COLUMNS' \| 'PREDICATE COLUMNS' \| 'COLUMNS' ColumnNameList ) | | The `ANALYZE` column | ColumnNameList ::= Identifier ( ',' Identifier )* | @@ -342,22 +323,7 @@ If you want to persist the column configuration in the `ANALYZE` statement (incl - When TiDB collects statistics automatically or when you manually collect statistics by executing the `ANALYZE` statement without specifying the column configuration, TiDB continues using the previously persisted configuration for statistics collection. - When you manually execute the `ANALYZE` statement multiple times with column configuration specified, TiDB overwrites the previously recorded persistent configuration using the new configuration specified by the latest `ANALYZE` statement. -To locate `PREDICATE COLUMNS` and columns on which statistics have been collected, use the following syntax: - -```sql -SHOW COLUMN_STATS_USAGE [ShowLikeOrWhere]; -``` - -The `SHOW COLUMN_STATS_USAGE` statement returns the following 6 columns: - -| Column name | Description | -| -------- | ------------- | -| `Db_name` | The database name | -| `Table_name` | The table name | -| `Partition_name` | The partition name | -| `Column_name` | The column name | -| `Last_used_at` | The last time when the column statistics were used in the query optimization | -| `Last_analyzed_at` | The last time when the column statistics were collected | +To locate `PREDICATE COLUMNS` and columns on which statistics have been collected, use the [`SHOW COLUMN_STATS_USAGE`](/sql-statements/sql-statement-show-column-stats-usage.md) statement. In the following example, after executing `ANALYZE TABLE t PREDICATE COLUMNS;`, TiDB collects statistics on columns `b`, `c`, and `d`, where column `b` is a `PREDICATE COLUMN` and columns `c` and `d` are index columns. @@ -401,9 +367,9 @@ WHERE db_name = 'test' AND table_name = 't' AND last_analyzed_at IS NOT NULL; ## Versions of statistics -The `tidb_analyze_version` variable controls the statistics collected by TiDB. Currently, two versions of statistics are supported: `tidb_analyze_version = 1` and `tidb_analyze_version = 2`. +The [`tidb_analyze_version`](/system-variables.md#tidb_analyze_version-new-in-v510) variable controls the statistics collected by TiDB. Currently, two versions of statistics are supported: `tidb_analyze_version = 1` and `tidb_analyze_version = 2`. -- For TiDB Self-Hosted, the default value of this variable changes from `1` to `2` starting from v5.3.0. +- For TiDB Self-Managed, the default value of this variable changes from `1` to `2` starting from v5.3.0. - For TiDB Cloud, the default value of this variable changes from `1` to `2` starting from v6.5.0. - If your cluster is upgraded from an earlier version, the default value of `tidb_analyze_version` does not change after the upgrade. @@ -413,18 +379,18 @@ The following table lists the information collected by each version for usage in | Information | Version 1 | Version 2| | --- | --- | ---| -| The total number of rows in the table | √ | √ | -| Equal/IN predicate estimation | √ (Column/Index Top-N & Count-Min Sketch) | √ (Column/Index Top-N & Histogram) | -| Range predicate estimation | √ (Column/Index Top-N & Histogram) | √ (Column/Index Top-N & Histogram) | -| `NULL` predicate estimation | √ | √ | -| The average length of columns | √ | √ | -| The average length of indexes | √ | √ | +| The total number of rows in the table | ⎷ | ⎷ | +| Equal/IN predicate estimation | ⎷ (Column/Index Top-N & Count-Min Sketch) | ⎷ (Column/Index Top-N & Histogram) | +| Range predicate estimation | ⎷ (Column/Index Top-N & Histogram) | ⎷ (Column/Index Top-N & Histogram) | +| `NULL` predicate estimation | ⎷ | ⎷ | +| The average length of columns | ⎷ | ⎷ | +| The average length of indexes | ⎷ | ⎷ | ### Switch between statistics versions It is recommended to ensure that all tables/indexes (and partitions) utilize statistics collection from the same version. Version 2 is recommended, however, it is not recommended to switch from one version to another without a justifiable reason such as an issue experienced with the version in use. A switch between versions might take a period of time when no statistics are available until all tables have been analyzed with the new version, which might negatively affect the optimizer plan choices if statistics are not available. -Examples of justifications to switch might include - with Version 1, there could be inaccuracies in equal/IN predicate estimation due to hash collisions when collecting Count-Min sketch statistics. Solutions are listed in the [Count-Min Sketch](#count-min-sketch) section. Alternatively, setting `tidb_analyze_version = 2` and rerunning `ANALYZE` on all objects is also a solution. In the early release of Version 2, there was a risk of memory overflow after `ANALYZE`. This issue is resolved, but initially, one solution was to `set tidb_analyze_version = 1` and rerun `ANALYZE` on all objects. +Examples of justifications to switch might include - with Version 1, there could be inaccuracies in equal/IN predicate estimation due to hash collisions when collecting Count-Min sketch statistics. Solutions are listed in the [Count-Min Sketch](#count-min-sketch) section. Alternatively, setting `tidb_analyze_version = 2` and rerunning `ANALYZE` on all objects is also a solution. In the early release of Version 2, there was a risk of memory overflow after `ANALYZE`. This issue is resolved, but initially, one solution was to set `tidb_analyze_version = 1` and rerun `ANALYZE` on all objects. To prepare `ANALYZE` for switching between versions: @@ -459,30 +425,7 @@ You can view the `ANALYZE` status and statistics information using the following ### `ANALYZE` state -When executing the `ANALYZE` statement, you can view the current state of `ANALYZE` using the following SQL statement: - -{{< copyable "sql" >}} - -```sql -SHOW ANALYZE STATUS [ShowLikeOrWhere] -``` - -This statement returns the state of `ANALYZE`. You can use `ShowLikeOrWhere` to filter the information you need. - -Currently, the `SHOW ANALYZE STATUS` statement returns the following 11 columns: - -| Column name | Description | -| :-------- | :------------- | -| table_schema | The database name | -| table_name | The table name | -| partition_name| The partition name | -| job_info | The task information. If an index is analyzed, this information will include the index name. When `tidb_analyze_version =2`, this information will include configuration items such as sample rate. | -| processed_rows | The number of rows that have been analyzed | -| start_time | The time at which the task starts | -| state | The state of a task, including `pending`, `running`, `finished`, and `failed` | -| fail_reason | The reason why the task fails. If the execution is successful, the value is `NULL`. | -| instance | The TiDB instance that executes the task | -| process_id | The process ID that executes the task | +When executing the `ANALYZE` statement, you can view the current state of `ANALYZE` using [`SHOW ANALYZE STATUS`](/sql-statements/sql-statement-show-analyze-status.md). Starting from TiDB v6.1.0, the `SHOW ANALYZE STATUS` statement supports showing cluster-level tasks. Even after a TiDB restart, you can still view task records before the restart using this statement. Before TiDB v6.1.0, the `SHOW ANALYZE STATUS` statement can only show instance-level tasks, and task records are cleared after a TiDB restart. @@ -503,178 +446,33 @@ mysql> SHOW ANALYZE STATUS [ShowLikeOrWhere]; ### Metadata of tables -You can use the `SHOW STATS_META` statement to view the total number of rows and the number of updated rows. - -{{< copyable "sql" >}} - -```sql -SHOW STATS_META [ShowLikeOrWhere]; -``` - -The syntax of `ShowLikeOrWhereOpt` is as follows: - -![ShowLikeOrWhereOpt](/media/sqlgram/ShowLikeOrWhereOpt.png) - -Currently, the `SHOW STATS_META` statement returns the following 6 columns: - -| Column name | Description | -| :-------- | :------------- | -| `db_name` | The database name | -| `table_name` | The table name | -| `partition_name`| The partition name | -| `update_time` | The time of the update | -| `modify_count` | The number of modified rows | -| `row_count` | The total number of rows | - -> **Note:** -> -> When TiDB automatically updates the total number of rows and the number of modified rows according to DML statements, `update_time` is also updated. Therefore, `update_time` does not necessarily indicate the last time when the `ANALYZE` statement is executed. +You can use the [`SHOW STATS_META`](/sql-statements/sql-statement-show-stats-meta.md) statement to view the total number of rows and the number of updated rows. ### Health state of tables -You can use the `SHOW STATS_HEALTHY` statement to check the health state of tables and roughly estimate the accuracy of the statistics. When `modify_count` >= `row_count`, the health state is 0; when `modify_count` < `row_count`, the health state is (1 - `modify_count`/`row_count`) * 100. - -The syntax is as follows: - -{{< copyable "sql" >}} - -```sql -SHOW STATS_HEALTHY [ShowLikeOrWhere]; -``` - -The synopsis of `SHOW STATS_HEALTHY` is: - -![ShowStatsHealthy](/media/sqlgram/ShowStatsHealthy.png) - -Currently, the `SHOW STATS_HEALTHY` statement returns the following 4 columns: - -| Column name | Description | -| :-------- | :------------- | -| `db_name` | The database name | -| `table_name` | The table name | -| `partition_name` | The partition name | -| `healthy` | The health state of tables | +You can use the [`SHOW STATS_HEALTHY`](/sql-statements/sql-statement-show-stats-healthy.md) statement to check the health state of tables and roughly estimate the accuracy of the statistics. When `modify_count` >= `row_count`, the health state is 0; when `modify_count` < `row_count`, the health state is (1 - `modify_count`/`row_count`) * 100. ### Metadata of columns -You can use the `SHOW STATS_HISTOGRAMS` statement to view the number of different values and the number of `NULL` in all the columns. - -Syntax as follows: - -{{< copyable "sql" >}} - -```sql -SHOW STATS_HISTOGRAMS [ShowLikeOrWhere] -``` - -This statement returns the number of different values and the number of `NULL` in all the columns. You can use `ShowLikeOrWhere` to filter the information you need. - -Currently, the `SHOW STATS_HISTOGRAMS` statement returns the following 10 columns: - -| Column name | Description | -| :-------- | :------------- | -| `db_name` | The database name | -| `table_name` | The table name | -| `partition_name` | The partition name | -| `column_name` | The column name (when `is_index` is `0`) or the index name (when `is_index` is `1`) | -| `is_index` | Whether it is an index column or not | -| `update_time` | The time of the update | -| `distinct_count` | The number of different values | -| `null_count` | The number of `NULL` | -| `avg_col_size` | The average length of columns | -| correlation | The Pearson correlation coefficient of the column and the integer primary key, which indicates the degree of association between the two columns| +You can use the [`SHOW STATS_HISTOGRAMS`](/sql-statements/sql-statement-show-stats-histograms.md) statement to view the number of different values and the number of `NULL` in all the columns. ### Buckets of histogram -You can use the `SHOW STATS_BUCKETS` statement to view each bucket of the histogram. - -The syntax is as follows: - -{{< copyable "sql" >}} - -```sql -SHOW STATS_BUCKETS [ShowLikeOrWhere] -``` - -The diagram is as follows: - -![SHOW STATS_BUCKETS](/media/sqlgram/SHOW_STATS_BUCKETS.png) - -This statement returns information about all the buckets. You can use `ShowLikeOrWhere` to filter the information you need. - -Currently, the `SHOW STATS_BUCKETS` statement returns the following 11 columns: - -| Column name | Description | -| :-------- | :------------- | -| `db_name` | The database name | -| `table_name` | The table name | -| `partition_name` | The partition name | -| `column_name` | The column name (when `is_index` is `0`) or the index name (when `is_index` is `1`) | -| `is_index` | Whether it is an index column or not | -| `bucket_id` | The ID of a bucket | -| `count` | The number of all the values that falls on the bucket and the previous buckets | -| `repeats` | The occurrence number of the maximum value | -| `lower_bound` | The minimum value | -| `upper_bound` | The maximum value | -| `ndv` | The number of different values in the bucket. When `tidb_analyze_version` = `1`, `ndv` is always `0`, which has no actual meaning. | +You can use the [`SHOW STATS_BUCKETS`](/sql-statements/sql-statement-show-stats-buckets.md) statement to view each bucket of the histogram. ### Top-N information -You can use the `SHOW STATS_TOPN` statement to view the Top-N information currently collected by TiDB. - -The syntax is as follows: - -{{< copyable "sql" >}} - -```sql -SHOW STATS_TOPN [ShowLikeOrWhere]; -``` - -Currently, the `SHOW STATS_TOPN` statement returns the following 7 columns: - -| Column name | Description | -| ---- | ----| -| `db_name` | The database name | -| `table_name` | The table name | -| `partition_name` | The partition name | -| `column_name` | The column name (when `is_index` is `0`) or the index name (when `is_index` is `1`) | -| `is_index` | Whether it is an index column or not | -| `value` | The value of this column | -| `count` | How many times the value appears | +You can use the [`SHOW STATS_TOPN`](/sql-statements/sql-statement-show-stats-topn.md) statement to view the Top-N information currently collected by TiDB. ## Delete statistics -You can run the `DROP STATS` statement to delete statistics. - -{{< copyable "sql" >}} - -```sql -DROP STATS TableName -``` - -The preceding statement deletes all statistics of `TableName`. If a partitioned table is specified, this statement will delete statistics of all partitions in this table as well as GlobalStats generated in dynamic pruning mode. - -{{< copyable "sql" >}} - -```sql -DROP STATS TableName PARTITION PartitionNameList; -``` - -This preceding statement only deletes statistics of the specified partitions in `PartitionNameList`. - -{{< copyable "sql" >}} - -```sql -DROP STATS TableName GLOBAL; -``` - -The preceding statement only deletes GlobalStats generated in dynamic pruning mode of the specified table. +You can run the [`DROP STATS`](/sql-statements/sql-statement-drop-stats.md) statement to delete statistics. ## Load statistics > **Note:** > -> Loading statistics is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> Loading statistics is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. By default, depending on the size of column statistics, TiDB loads statistics differently as follows: @@ -710,7 +508,9 @@ After enabling the synchronously loading statistics feature, you can control how -## Import and export statistics +## Export and import statistics + +This section describes how to export and import statistics. @@ -726,24 +526,18 @@ The interface to export statistics is as follows: + To obtain the JSON format statistics of the `${table_name}` table in the `${db_name}` database: - {{< copyable "" >}} - ``` http://${tidb-server-ip}:${tidb-server-status-port}/stats/dump/${db_name}/${table_name} ``` For example: - {{< copyable "" >}} - - ``` + ```shell curl -s http://127.0.0.1:10080/stats/dump/test/t1 -o /tmp/t1.json ``` + To obtain the JSON format statistics of the `${table_name}` table in the `${db_name}` database at specific time: - {{< copyable "" >}} - ``` http://${tidb-server-ip}:${tidb-server-status-port}/stats/dump/${db_name}/${table_name}/${yyyyMMddHHmmss} ``` @@ -756,12 +550,12 @@ The interface to export statistics is as follows: Generally, the imported statistics refer to the JSON file obtained using the export interface. -Syntax: +Loading statistics can be done with the [`LOAD STATS`](/sql-statements/sql-statement-load-stats.md) statement. -{{< copyable "sql" >}} +For example: -``` -LOAD STATS 'file_name' +```sql +LOAD STATS 'file_name'; ``` `file_name` is the file name of the statistics to be imported. @@ -837,7 +631,7 @@ mysql> SHOW WARNINGS; 1 row in set (0.00 sec) ``` -In addition, you can also lock the statistics of a partition using `LOCK STATS`. For example: +In addition, you can also lock the statistics of a partition using [`LOCK STATS`](/sql-statements/sql-statement-lock-stats.md). For example: Create a partition table `t`, and insert data into it. When the statistics of partition `p1` are not locked, the `ANALYZE` statement can be successfully executed. @@ -926,6 +720,8 @@ The following table describes the behaviors of locking statistics: ## Manage `ANALYZE` tasks and concurrency +This section describes how to terminate background `ANALYZE` tasks and control the `ANALYZE` concurrency. + ### Terminate background `ANALYZE` tasks Since TiDB v6.0, TiDB supports using the `KILL` statement to terminate an `ANALYZE` task running in the background. If you find that an `ANALYZE` task running in the background consumes a lot of resources and affects your application, you can terminate the `ANALYZE` task by taking the following steps: diff --git a/styles/PingCAP/Ambiguous.yml b/styles/PingCAP/Ambiguous.yml index 54c294a609caa..8b457cdcec2ae 100644 --- a/styles/PingCAP/Ambiguous.yml +++ b/styles/PingCAP/Ambiguous.yml @@ -1,7 +1,6 @@ extends: existence message: Consider using a clearer word than '%s' because it may cause confusion. level: suggestion -code: false ignorecase: true tokens: - a lot diff --git a/styles/Vocab/PingCAP/accept.txt b/styles/config/vocabularies/PingCAP/accept.txt similarity index 100% rename from styles/Vocab/PingCAP/accept.txt rename to styles/config/vocabularies/PingCAP/accept.txt diff --git a/styles/Vocab/PingCAP/reject.txt b/styles/config/vocabularies/PingCAP/reject.txt similarity index 100% rename from styles/Vocab/PingCAP/reject.txt rename to styles/config/vocabularies/PingCAP/reject.txt diff --git a/styles/vocab.txt b/styles/vocab.txt deleted file mode 100644 index e69de29bb2d1d..0000000000000 diff --git a/subquery-optimization.md b/subquery-optimization.md index 49dde275c4327..5a4af0c2f79fe 100644 --- a/subquery-optimization.md +++ b/subquery-optimization.md @@ -24,7 +24,7 @@ By default, subqueries use `semi join` mentioned in [Understanding TiDB Executio In this case, `ALL` and `ANY` can be replaced by `MAX` and `MIN`. When the table is empty, the result of `MAX(EXPR)` and `MIN(EXPR)` is NULL. It works the same when the result of `EXPR` contains `NULL`. Whether the result of `EXPR` contains `NULL` may affect the final result of the expression, so the complete rewrite is given in the following form: - `t.id < all (select s.id from s)` is rewritten as `t.id < min(s.id) and if(sum(s.id is null) != 0, null, true)` -- `t.id < any (select s.id from s)` is rewritten as `t.id < max(s.id) or if(sum(s.id is null) != 0, null, false)` +- `t.id > any (select s.id from s)` is rewritten as `t.id > max(s.id) or if(sum(s.id is null) != 0, null, false)` ## `... != ANY (SELECT ... FROM ...)` diff --git a/support.md b/support.md index b9f13295279f4..e10dddccff226 100644 --- a/support.md +++ b/support.md @@ -18,10 +18,10 @@ If you encounter a problem when you use TiDB, you can reach out for support from + Seek help from the TiDB community: - - The [TiDB Forum](https://ask.pingcap.com/) - [Discord channels](https://discord.gg/DQZ2dy3cuc?utm_source=doc) - Slack channels: [#everyone](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=docs) (English), [#tidb-japan](https://slack.tidb.io/invite?team=tidb-community&channel=tidb-japan&ref=docs) (Japanese) - [Stack Overflow](https://stackoverflow.com/questions/tagged/tidb) (questions tagged with #tidb) + - [TiDB Forum](https://ask.pingcap.com/) + Report a bug @@ -30,4 +30,4 @@ If you encounter a problem when you use TiDB, you can reach out for support from + Learn TiDB's implementation and design - [TiDB development guide](https://pingcap.github.io/tidb-dev-guide/) - - [TiDB Internals forum](https://internals.tidb.io/) + - [Discussions on GitHub](https://github.com/orgs/pingcap/discussions) diff --git a/sync-diff-inspector/route-diff.md b/sync-diff-inspector/route-diff.md index b048f790fbe87..e017df0f5f3b3 100644 --- a/sync-diff-inspector/route-diff.md +++ b/sync-diff-inspector/route-diff.md @@ -1,7 +1,6 @@ --- title: Data Check for Tables with Different Schema or Table Names summary: Learn the data check for different database names or table names. -aliases: ['/docs/dev/sync-diff-inspector/route-diff/','/docs/dev/reference/tools/sync-diff-inspector/route-diff/'] --- # Data Check for Tables with Different Schema or Table Names diff --git a/sync-diff-inspector/shard-diff.md b/sync-diff-inspector/shard-diff.md index a8fefc2a2bb9d..22738a3e94e27 100644 --- a/sync-diff-inspector/shard-diff.md +++ b/sync-diff-inspector/shard-diff.md @@ -1,7 +1,6 @@ --- title: Data Check in the Sharding Scenario summary: Learn the data check in the sharding scenario. -aliases: ['/docs/dev/sync-diff-inspector/shard-diff/','/docs/dev/reference/tools/sync-diff-inspector/shard-diff/'] --- # Data Check in the Sharding Scenario diff --git a/sync-diff-inspector/sync-diff-inspector-overview.md b/sync-diff-inspector/sync-diff-inspector-overview.md index 22cf71b76a7c2..07fe695691f0b 100644 --- a/sync-diff-inspector/sync-diff-inspector-overview.md +++ b/sync-diff-inspector/sync-diff-inspector-overview.md @@ -1,7 +1,6 @@ --- title: sync-diff-inspector User Guide summary: Use sync-diff-inspector to compare data and repair inconsistent data. -aliases: ['/docs/dev/sync-diff-inspector/sync-diff-inspector-overview/','/docs/dev/reference/tools/sync-diff-inspector/overview/'] --- # sync-diff-inspector User Guide @@ -73,6 +72,9 @@ check-thread-count = 4 # If enabled, SQL statements is exported to fix inconsistent tables. export-fix-sql = true +# Only compares the data instead of the table structure. This configuration item is an experimental feature. It is not recommended that you use it in the production environment. +check-data-only = false + # Only compares the table structure instead of the data. check-struct-only = false @@ -260,6 +262,8 @@ Average Speed: 113.277149MB/s - `RESULT`: Whether the check is completed. If you have configured `skip-non-existing-table = true`, the value of this column is `skipped` for tables that do not exist in the upstream or downstream - `STRUCTURE EQUALITY`: Checks whether the table structure is the same - `DATA DIFF ROWS`: `rowAdd`/`rowDelete`. Indicates the number of rows that need to be added/deleted to fix the table +- `UPCOUNT`: The number of rows in this table in the upstream data source +- `DOWNCOUNT`: The number of rows in this table in the downstream data source ### SQL statements to fix inconsistent data @@ -288,7 +292,8 @@ REPLACE INTO `sbtest`.`sbtest99`(`id`,`k`,`c`,`pad`) VALUES (3700000,2501808,'he ## Note - sync-diff-inspector consumes a certain amount of server resources when checking data. Avoid using sync-diff-inspector to check data during peak business hours. -- Before comparing the data in MySQL with that in TiDB, pay attention to the collation configuration of the tables. If the primary key or unique key is the `varchar` type and the collation configuration in MySQL differs from that in TiDB, the final check result might be incorrect because of the collation issue. You need to add collation to the sync-diff-inspector configuration file. +- Before comparing the data in MySQL with that in TiDB, check the character set and `collation` configuration of the tables. This is especially important when the primary key or unique key of a table is the `varchar` type. If collation rules differ between upstream and downstream databases, sorting issues might occur, leading to inaccurate verification results. For example, MySQL's default collation is case-insensitive, while TiDB's default collation is case-sensitive. This inconsistency might cause identical delete and insert records in the repair SQL. To avoid this issue, use the `index-fields` configuration to specify index columns that are not affected by case sensitivity. If you configure `collation` in the sync-diff-inspector configuration file and explicitly use the same collation for both upstream and downstream during chunk-based comparison, note that the order of index fields depends on the table's collation configuration. If the collations differ, one side might be unable to use the index. Additionally, if the character sets differ between upstream and downstream (for example, MySQL uses UTF-8 while TiDB uses UTF-8MB4), it is not possible to unify the collation configuration. +- If the primary key differs between upstream and downstream tables, sync-diff-inspector does not use the original primary key column to divide chunks. For example, when sharded tables in MySQL are merged into TiDB using a composite primary key that includes the original primary key and a shard key. In this case, configure the original primary key column using `index-fields` and set `check-data-only` to `true`. - sync-diff-inspector divides data into chunks first according to TiDB statistics and you need to guarantee the accuracy of the statistics. You can manually run the `analyze table {table_name}` command when the TiDB server's *workload is light*. - Pay special attention to `table-rules`. If you configure `schema-pattern="test1"`, `table-pattern = "t_1"`, `target-schema="test2"` and `target-table = "t_2"`, the `test1`.`t_1` schema in the source database and the `test2`.`t_2` schema in the target database are compared. Sharding is enabled by default in sync-diff-inspector, so if the source database has a `test2`.`t_2` table, the `test1`.`t_1` table and `test2`.`t_2` table in the source database serving as sharding are compared with the `test2`.`t_2` table in the target database. - The generated SQL file is only used as a reference for repairing data, and you need to confirm it before executing these SQL statements to repair data. diff --git a/system-variables.md b/system-variables.md index b917d29733bf2..26f2204edf142 100644 --- a/system-variables.md +++ b/system-variables.md @@ -1,7 +1,6 @@ --- title: System Variables summary: Use system variables to optimize performance or alter running behavior. -aliases: ['/tidb/dev/tidb-specific-system-variables','/docs/dev/system-variables/','/docs/dev/reference/configuration/tidb-server/mysql-variables/', '/docs/dev/tidb-specific-system-variables/','/docs/dev/reference/configuration/tidb-server/tidb-specific-variables/'] --- # System Variables @@ -312,7 +311,7 @@ mysql> SELECT * FROM t1; - Type: Enumeration - Default value: `aes-128-ecb` - Value options: `aes-128-ecb`, `aes-192-ecb`, `aes-256-ecb`, `aes-128-cbc`, `aes-192-cbc`, `aes-256-cbc`, `aes-128-ofb`, `aes-192-ofb`, `aes-256-ofb`, `aes-128-cfb`, `aes-192-cfb`, `aes-256-cfb` -- This variable sets the encryption mode for the built-in functions `AES_ENCRYPT()` and `AES_DECRYPT()`. +- This variable sets the encryption mode for the built-in functions [`AES_ENCRYPT()`](/functions-and-operators/encryption-and-compression-functions.md#aes_encrypt) and [`AES_DECRYPT()`](/functions-and-operators/encryption-and-compression-functions.md#aes_decrypt). ### character_set_client @@ -392,7 +391,7 @@ mysql> SELECT * FROM t1; > **Note:** > -> This variable is not supported on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is not supported on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). @@ -593,7 +592,7 @@ This variable is an alias for [`last_insert_id`](#last_insert_id). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -656,7 +655,7 @@ This variable is an alias for [`last_insert_id`](#last_insert_id). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -697,11 +696,12 @@ This variable is an alias for [`last_insert_id`](#last_insert_id). - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): Yes - Default value: `UNSPECIFIED` -- Value options: `UNSPECIFIED`, `0`, `1` +- Value options: `UNSPECIFIED`, `0`, `1`, `2` - This variable is used to specify different versions of the MPP execution plan. After a version is specified, TiDB selects the specified version of the MPP execution plan. The meanings of the variable values are as follows: - - `UNSPECIFIED`: means unspecified. TiDB automatically selects the latest version `1`. + - `UNSPECIFIED`: means unspecified. TiDB automatically selects the latest version `2`. - `0`: compatible with all TiDB cluster versions. Features with the MPP version greater than `0` do not take effect in this mode. - `1`: new in v6.6.0, used to enable data exchange with compression on TiFlash. For details, see [MPP version and exchange data compression](/explain-mpp.md#mpp-version-and-exchange-data-compression). + - `2`: new in v7.3.0, used to provide more accurate error messages when MPP tasks encounter errors on TiFlash. ### password_reuse_interval New in v6.5.0 @@ -737,7 +737,7 @@ This variable is an alias for [`last_insert_id`](#last_insert_id). > **Note:** > -> The `max_execution_time` system variable currently only controls the maximum execution time for read-only SQL statements. The precision of the timeout value is roughly 100ms. This means the statement might not be terminated in accurate milliseconds as you specify. +> Before v6.4.0, the `max_execution_time` system variable takes effect on all types of statements. Starting from v6.4.0, this variable only controls the maximum execution time of read-only statements. The precision of the timeout value is roughly 100ms. This means the statement might not be terminated in exact milliseconds as you specify. @@ -804,7 +804,7 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'max_prepared_stmt_count'; > **Note:** > -> This variable is not supported on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is not supported on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: No, only applicable to the current TiDB instance that you are connecting to. @@ -816,7 +816,7 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'max_prepared_stmt_count'; > **Note:** > -> This variable is not supported on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is not supported on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: No, only applicable to the current TiDB instance that you are connecting to. @@ -857,13 +857,13 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'max_prepared_stmt_count'; > **Note:** > -> Currently, this variable is not supported on [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated). DO **NOT** enable this variable for TiDB Dedicated clusters. Otherwise, you might get SQL client connection failures. This restriction is a temporary control measure and will be resolved in a future release. +> Currently, this variable is not supported on [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated). DO **NOT** enable this variable for TiDB Cloud Dedicated clusters. Otherwise, you might get SQL client connection failures. This restriction is a temporary control measure and will be resolved in a future release. - Scope: GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Boolean -- Default value: `OFF` for TiDB Self-Hosted and [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated), `ON` for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) +- Default value: `OFF` for TiDB Self-Managed and [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated), `ON` for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) @@ -885,7 +885,7 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'max_prepared_stmt_count'; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -916,7 +916,7 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'max_prepared_stmt_count'; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -1037,6 +1037,7 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'max_prepared_stmt_count'; - Scope: SESSION | GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No +- Type: Integer - Default value: `4096` - Range: `[0, 9223372036854775807]` - Unit: Bytes @@ -1079,8 +1080,8 @@ mysql> SHOW GLOBAL VARIABLES LIKE 'max_prepared_stmt_count'; - Scope: NONE - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No -- Default value: `json_array`, `json_array_append`, `json_array_insert`, `json_contains`, `json_contains_path`, `json_depth`, `json_extract`, `json_insert`, `json_keys`, `json_length`, `json_merge_patch`, `json_merge_preserve`, `json_object`, `json_pretty`, `json_quote`, `json_remove`, `json_replace`, `json_search`, `json_set`, `json_storage_size`, `json_type`, `json_unquote`, `json_valid`, `lower`, `md5`, `reverse`, `tidb_shard`, `upper`, `vitess_hash` -- This variable is used to show the functions that are allowed to be used for creating expression indexes. +- Default value: `json_array, json_array_append, json_array_insert, json_contains, json_contains_path, json_depth, json_extract, json_insert, json_keys, json_length, json_merge_patch, json_merge_preserve, json_object, json_pretty, json_quote, json_remove, json_replace, json_search, json_set, json_storage_size, json_type, json_unquote, json_valid, lower, md5, reverse, tidb_shard, upper, vitess_hash` +- This read-only variable is used to show the functions that are allowed to be used for creating [expression indexes](/sql-statements/sql-statement-create-index.md#expression-index). ### tidb_allow_mpp New in v5.0 @@ -1113,7 +1114,7 @@ MPP is a distributed computing framework provided by the TiFlash engine, which a ### tidb_analyze_distsql_scan_concurrency New in v7.6.0 -- Scope: GLOBAL +- Scope: SESSION | GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Integer @@ -1123,10 +1124,6 @@ MPP is a distributed computing framework provided by the TiFlash engine, which a ### tidb_analyze_partition_concurrency -> **Warning:** -> -> The feature controlled by this variable is not fully functional in the current TiDB version. Do not change the default value. - - Scope: SESSION | GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No @@ -1142,7 +1139,7 @@ MPP is a distributed computing framework provided by the TiFlash engine, which a - Default value: `2` - Range: `[1, 2]` - Controls how TiDB collects statistics. - - For TiDB Self-Hosted, the default value of this variable changes from `1` to `2` starting from v5.3.0. + - For TiDB Self-Managed, the default value of this variable changes from `1` to `2` starting from v5.3.0. - For TiDB Cloud, the default value of this variable changes from `1` to `2` starting from v6.5.0. - If your cluster is upgraded from an earlier version, the default value of `tidb_analyze_version` does not change after the upgrade. - For detailed introduction about this variable, see [Introduction to Statistics](/statistics.md). @@ -1363,12 +1360,12 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; ### tidb_build_sampling_stats_concurrency New in v7.5.0 -- Scope: GLOBAL +- Scope: SESSION | GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Integer - Unit: Threads -- Default value：`2` +- Default value: `2` - Range: `[1, 256]` - This variable is used to set the sampling concurrency in the `ANALYZE` process. - When the variable is set to a larger value, the execution performance of other queries is affected. @@ -1387,7 +1384,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: SESSION - Persists to cluster: No @@ -1567,7 +1564,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -1582,8 +1579,8 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> - If you are using a [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated) cluster, to improve the speed for index creation using this variable, make sure that your TiDB cluster is hosted on AWS and your TiDB node size is at least 8 vCPU. -> - For [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters, this variable is read-only. +> - If you are using a [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated) cluster, to improve the speed for index creation using this variable, make sure that your TiDB cluster is hosted on AWS and your TiDB node size is at least 8 vCPU. +> - For [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters, this variable is read-only. - Scope: GLOBAL - Persists to cluster: Yes @@ -1643,7 +1640,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -1657,7 +1654,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -1671,7 +1668,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -1681,14 +1678,14 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - Range: `[32, 10240]` - Unit: Rows - This variable is used to set the batch size during the `re-organize` phase of the DDL operation. For example, when TiDB executes the `ADD INDEX` operation, the index data needs to backfilled by `tidb_ddl_reorg_worker_cnt` (the number) concurrent workers. Each worker backfills the index data in batches. - - If many updating operations such as `UPDATE` and `REPLACE` exist during the `ADD INDEX` operation, a larger batch size indicates a larger probability of transaction conflicts. In this case, you need to adjust the batch size to a smaller value. The minimum value is 32. - - If the transaction conflict does not exist, you can set the batch size to a large value (consider the worker count. See [Interaction Test on Online Workloads and `ADD INDEX` Operations](https://docs.pingcap.com/tidb/stable/online-workloads-and-add-index-operations) for reference). This can increase the speed of the backfilling data, but the write pressure on TiKV also becomes higher. + - If `tidb_ddl_enable_fast_reorg` is set to `OFF`, `ADD INDEX` is executed as a transaction. If there are many update operations such as `UPDATE` and `REPLACE` in the target columns during the `ADD INDEX` execution, a larger batch size indicates a larger probability of transaction conflicts. In this case, it is recommended that you set the batch size to a smaller value. The minimum value is 32. + - If the transaction conflict does not exist, or if `tidb_ddl_enable_fast_reorg` is set to `ON`, you can set the batch size to a large value. This makes data backfilling faster but also increases the write pressure on TiKV. For a proper batch size, you also need to refer to the value of `tidb_ddl_reorg_worker_cnt`. See [Interaction Test on Online Workloads and `ADD INDEX` Operations](https://docs.pingcap.com/tidb/dev/online-workloads-and-add-index-operations) for reference. ### tidb_ddl_reorg_priority > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: SESSION - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No @@ -1702,7 +1699,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -1786,6 +1783,8 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - Use a bigger value in OLAP scenarios, and a smaller value in OLTP scenarios. - For OLAP scenarios, the maximum value should not exceed the number of CPU cores of all the TiKV nodes. - If a table has a lot of partitions, you can reduce the variable value appropriately (determined by the size of the data to be scanned and the frequency of the scan) to avoid TiKV becoming out of memory (OOM). +- For a simple query with only a `LIMIT` clause, if the `LIMIT` value is less than 100000, the scan operation pushed down to TiKV treats the value of this variable as `1` to enhance execution efficiency. +- For the `SELECT MAX/MIN(col) FROM ...` query, if the `col` column has an index sorted in the same order required by the `MAX(col)` or `MIN(col)` function, TiDB will rewrite the query to `SELECT col FROM ... LIMIT 1` for processing, and the value of this variable will also be processed as `1`. For example, for `SELECT MIN(col) FROM ...`, if the `col` column has an ascending index, TiDB can quickly obtain the `MIN(col)` value by rewriting the query to `SELECT col FROM ... LIMIT 1` and directly reading the first row of the index. ### tidb_dml_batch_size @@ -1831,13 +1830,14 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - `"bulk"` mode cannot be used on tables containing foreign keys and tables referenced by foreign keys when the foreign key constraint check is enabled (`foreign_key_checks = ON`). - In situations that the environment does not support or is incompatible with the `"bulk"` mode, TiDB falls back to the `"standard"` mode and returns a warning message. To verify if the `"bulk"` mode is used, you can check the `pipelined` field using [`tidb_last_txn_info`](#tidb_last_txn_info-new-in-v409). A `true` value indicates that the `"bulk"` mode is used. - When executing large transactions in the `"bulk"` mode, the transaction duration might be long. For transactions in this mode, the maximum TTL of the transaction lock is the greater value between [`max-txn-ttl`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#max-txn-ttl) and 24 hours. Additionally, if the transaction execution time exceeds the value set by [`tidb_gc_max_wait_time`](#tidb_gc_max_wait_time-new-in-v610), the GC might force a rollback of the transaction, leading to its failure. - - This mode is implemented by the Pipelined DML feature. For detailed design and GitHub issues, see [Pipelined DML](https://github.com/pingcap/tidb/blob/master/docs/design/2024-01-09-pipelined-DML.md) and [#50215](https://github.com/pingcap/tidb/issues/50215). + - When TiDB executes transactions in the `"bulk"` mode, transaction size is not limited by the TiDB configuration item [`txn-total-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-total-size-limit). + - This mode is implemented by the Pipelined DML feature. For detailed design and GitHub issues, see [Pipelined DML](https://github.com/pingcap/tidb/blob/release-8.1/docs/design/2024-01-09-pipelined-DML.md) and [#50215](https://github.com/pingcap/tidb/issues/50215). ### tidb_enable_1pc New in v5.0 > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -1870,7 +1870,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -1889,7 +1889,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -1977,7 +1977,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: No, only applicable to the current TiDB instance that you are connecting to. @@ -2012,19 +2012,6 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - Default value: `OFF` - This variable controls whether to enable TiDB to collect `PREDICATE COLUMNS`. After enabling the collection, if you disable it, the information of previously collected `PREDICATE COLUMNS` is cleared. For details, see [Collect statistics on some columns](/statistics.md#collect-statistics-on-some-columns). -### tidb_enable_concurrent_hashagg_spill New in v8.0.0 - -> **Warning:** -> -> Currently, the feature controlled by this variable is experimental. It is not recommended that you use it in production environments. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. - -- Scope: SESSION | GLOBAL -- Persists to cluster: Yes -- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No -- Type: Boolean -- Default value: `ON` -- This variable controls whether TiDB supports disk spill for the concurrent HashAgg algorithm. When it is `ON`, disk spill can be triggered for the concurrent HashAgg algorithm. This variable will be deprecated after this feature is generally available in a future release. - ### tidb_enable_enhanced_security - Scope: NONE @@ -2133,7 +2120,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -2179,7 +2166,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -2260,7 +2247,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -2410,6 +2397,19 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - Default value: `OFF` - This variable controls whether to enable concurrency for the `Apply` operator. The number of concurrencies is controlled by the `tidb_executor_concurrency` variable. The `Apply` operator processes correlated subqueries and has no concurrency by default, so the execution speed is slow. Setting this variable value to `1` can increase concurrency and speed up execution. Currently, concurrency for `Apply` is disabled by default. +### tidb_enable_parallel_hashagg_spill New in v8.0.0 + +> **Warning:** +> +> Currently, the feature controlled by this variable is experimental. It is not recommended that you use it in production environments. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. + +- Scope: SESSION | GLOBAL +- Persists to cluster: Yes +- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No +- Type: Boolean +- Default value: `ON` for v8.1.0; `OFF` for v8.1.1 and later 8.1 patch versions +- This variable controls whether TiDB supports disk spill for the parallel HashAgg algorithm. When it is `ON`, disk spill can be triggered for the parallel HashAgg algorithm. This variable will be deprecated after this feature is generally available in a future release. + ### tidb_enable_pipelined_window_function - Scope: SESSION | GLOBAL @@ -2417,7 +2417,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Boolean - Default value: `ON` -- This variable specifies whether to use the pipeline execution algorithm for window functions. +- This variable specifies whether to use the pipeline execution algorithm for [window functions](/functions-and-operators/window-functions.md). ### tidb_enable_plan_cache_for_param_limit New in v6.6.0 @@ -2445,8 +2445,8 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Boolean -- Default value: `OFF` -- This variable controls whether to enable the `PLAN REPLAYER CAPTURE` feature. The default value `OFF` means to disable the `PLAN REPLAYER CAPTURE` feature. +- Default value: `ON` +- This variable controls whether to enable the `PLAN REPLAYER CAPTURE` feature. The default value `ON` means to enable the `PLAN REPLAYER CAPTURE` feature. @@ -2553,7 +2553,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -2598,7 +2598,7 @@ mysql> SELECT job_info FROM mysql.analyze_jobs ORDER BY end_time DESC LIMIT 1; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -2647,11 +2647,11 @@ Query OK, 0 rows affected (0.09 sec) - `AUTO` functions the same way as `ON` does. - `OFF` indicates disabling the `TABLE PARTITION` feature. In this case, the syntax that creates a partition table can be executed, but the table created is not a partitioned one. -### tidb_enable_telemetry New in v4.0.2 +### tidb_enable_telemetry New in v4.0.2 and deprecated in v8.1.0 -> **Note:** +> **Warning:** > -> This TiDB variable is not applicable to TiDB Cloud. +> Starting from v8.1.0, the telemetry feature in TiDB is removed, and this variable is no longer functional. It is retained solely for compatibility with earlier versions. - Scope: GLOBAL - Persists to cluster: Yes @@ -2661,13 +2661,13 @@ Query OK, 0 rows affected (0.09 sec) -- This variable is used to dynamically control whether the telemetry collection in TiDB is enabled. In the current version, the telemetry is disabled by default. If the [`enable-telemetry`](/tidb-configuration-file.md#enable-telemetry-new-in-v402) TiDB configuration item is set to `false` on all TiDB instances, the telemetry collection is always disabled and this system variable will not take effect. See [Telemetry](/telemetry.md) for details. +- Before v8.1.0, this variable controls whether to enable the telemetry collection in TiDB. -- This variable is used to dynamically control whether the telemetry collection in TiDB is enabled. +- This TiDB variable is not applicable to TiDB Cloud. @@ -2712,7 +2712,7 @@ Query OK, 0 rows affected (0.09 sec) > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -2753,7 +2753,7 @@ Query OK, 0 rows affected (0.09 sec) - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Boolean - Default value: `ON` -- This variable is used to control whether to enable the support for window functions. Note that window functions may use reserved keywords. This might cause SQL statements that could be executed normally cannot be parsed after upgrading TiDB. In this case, you can set `tidb_enable_window_function` to `OFF`. +- This variable is used to control whether to enable the support for [window functions](/functions-and-operators/window-functions.md). Note that window functions might use reserved keywords. This might cause SQL statements that can be executed normally to fail to be parsed after TiDB is upgraded. In this case, you can set `tidb_enable_window_function` to `OFF`. ### `tidb_enable_row_level_checksum` New in v7.1.0 @@ -2775,6 +2775,8 @@ Query OK, 0 rows affected (0.09 sec) +- You can use the [`TIDB_ROW_CHECKSUM()`](/functions-and-operators/tidb-functions.md#tidb_row_checksum) function to get the checksum value of a row. + ### tidb_enforce_mpp New in v5.1 - Scope: SESSION @@ -2796,6 +2798,10 @@ MPP is a distributed computing framework provided by the TiFlash engine, which a ### tidb_evolve_plan_baselines New in v4.0 +> **Warning:** +> +> The feature controlled by this variable is experimental. It is not recommended that you use it in the production environment. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. + - Scope: SESSION | GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No @@ -2927,7 +2933,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -2942,7 +2948,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -2953,16 +2959,12 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified ### tidb_gc_life_time New in v5.0 -> **Note:** -> -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). - - Scope: GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Duration - Default value: `10m0s` -- Range: `[10m0s, 8760h0m0s]` +- Range: `[10m0s, 8760h0m0s]` for TiDB Self-Managed and [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated), `[10m0s, 168h0m0s]` for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) - The time limit during which data is retained for each GC, in the format of Go Duration. When a GC happens, the current time minus this value is the safe point. > **Note:** @@ -2976,7 +2978,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -2991,7 +2993,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -3009,7 +3011,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -3054,7 +3056,16 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified - This variable is used to set whether to record all SQL statements in the [log](/tidb-configuration-file.md#logfile). This feature is disabled by default. If maintenance personnel needs to trace all SQL statements when locating issues, they can enable this feature. +- If the [`log.general-log-file`](/tidb-configuration-file.md#general-log-file-new-in-v800) configuration item is specified, the general log is written to the specified file separately. + +- The [`log.format`](/tidb-configuration-file.md#format) configuration item enables you to configure the log message format, whether the general log is in a separate file or combined with other logs. + +- The [`tidb_redact_log`](#tidb_redact_log) variable enables you to redact SQL statements recorded in the general log. + +- Only successfully executed statements are logged in the general log. Failed statements are not recorded in the general log but are instead logged in the TiDB log with a `command dispatched failed` message. + - To see all records of this feature in the log, you need to set the TiDB configuration item [`log.level`](/tidb-configuration-file.md#level) to `"info"` or `"debug"` and then query the `"GENERAL_LOG"` string. The following information is recorded: + - `time`: The time of the event. - `conn`: The ID of the current session. - `user`: The current session user. - `schemaVersion`: The current schema version. @@ -3085,7 +3096,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -3094,7 +3105,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified - Default value: `ON` - This variable controls whether to generate binary-encoded execution plans in slow logs and statement summaries. - When this variable is set to `ON`, you can view visual execution plans in TiDB Dashboard. Note that TiDB Dashboard only provides visual display for execution plans generated after this variable is enabled. -- You can execute the `SELECT tidb_decode_binary_plan('xxx...')` statement to parse the specific plan from a binary plan. +- You can execute the [`SELECT tidb_decode_binary_plan('xxx...')`](/functions-and-operators/tidb-functions.md#tidb_decode_binary_plan) statement to parse the specific plan from a binary plan. ### tidb_gogc_tuner_max_value New in v7.5.0 @@ -3120,7 +3131,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -3133,7 +3144,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -3141,7 +3152,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified - Type: Boolean - Default value: `ON` - This variable controls the way commit TS is calculated for async commit. By default (with the `ON` value), the two-phase commit requests a new TS from the PD server and uses the TS to calculate the final commit TS. In this situation, linearizability is guaranteed for all the concurrent transactions. -- If you set this variable to `OFF`, the process of fetching TS from the PD server is skipped, with the cost that only causal consistency is guaranteed but not linearizability. For more details, see the blog post [Async Commit, the Accelerator for Transaction Commit in TiDB 5.0](https://en.pingcap.com/blog/async-commit-the-accelerator-for-transaction-commit-in-tidb-5-0/). +- If you set this variable to `OFF`, the process of fetching TS from the PD server is skipped, with the cost that only causal consistency is guaranteed but not linearizability. For more details, see the blog post [Async Commit, the Accelerator for Transaction Commit in TiDB 5.0](https://www.pingcap.com/blog/async-commit-the-accelerator-for-transaction-commit-in-tidb-5-0/). - For scenarios that require only causal consistency, you can set this variable to `OFF` to improve performance. ### tidb_hash_exchange_with_new_collation @@ -3235,6 +3246,18 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified - This variable is used to set whether to ignore the commands for closing prepared statement cache. - When this variable is set to `ON`, the `COM_STMT_CLOSE` command of the Binary protocol and the [`DEALLOCATE PREPARE`](/sql-statements/sql-statement-deallocate.md) statement of the text protocol are ignored. For details, see [Ignore the `COM_STMT_CLOSE` command and the `DEALLOCATE PREPARE` statement](/sql-prepared-plan-cache.md#ignore-the-com_stmt_close-command-and-the-deallocate-prepare-statement). +### tidb_ignore_inlist_plan_digest New in v7.6.0 + +- Scope: GLOBAL +- Persists to cluster: Yes +- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No +- Type: Boolean +- Default value: `OFF` +- This variable controls whether TiDB ignores the element differences in the `IN` list across different queries when generating Plan Digests. + + - When it is the default value `OFF`, TiDB does not ignore the element differences (including the difference in the number of elements) in the `IN` list when generating Plan Digests. The element differences in the `IN` list result in different Plan Digests. + - When it is set to `ON`, TiDB ignores the element differences (including the difference in the number of elements) in the `IN` list and uses `...` to replace elements in the `IN` list in Plan Digests. In this case, TiDB generates the same Plan Digests for `IN` queries of the same type. + ### tidb_index_join_batch_size - Scope: SESSION | GLOBAL @@ -3341,7 +3364,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: SESSION - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): Yes @@ -3442,7 +3465,7 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: No, only applicable to the current TiDB instance that you are connecting to. @@ -3474,12 +3497,12 @@ For a system upgraded to v5.0 from an earlier version, if you have not modified ### `tidb_low_resolution_tso_update_interval` New in v8.0.0 -- Scope：GLOBAL +- Scope: GLOBAL - Persists to cluster: Yes - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No -- Type：Integer -- Default value：`2000` -- Range：`[10, 60000]` +- Type: Integer +- Default value: `2000` +- Range: `[10, 60000]` - Unit: Milliseconds - This variable is used to set the update interval of the cached timestamp used in the low-precision TSO feature, in milliseconds. - This variable is only available when [`tidb_low_resolution_tso`](#tidb_low_resolution_tso) is enabled. @@ -4129,7 +4152,7 @@ mysql> desc select count(distinct a) from test.t; - Default value: `""` - This variable is used to control some internal behaviors of the optimizer. - The optimizer's behavior might vary depending on user scenarios or SQL statements. This variable provides a more fine-grained control over the optimizer and helps to prevent performance regression after upgrading caused by behavior changes in the optimizer. -- For a more detailed introduction, see [Optimizer Fix Controls](https://docs.pingcap.com/tidb/v7.2/optimizer-fix-controls). +- For a more detailed introduction, see [Optimizer Fix Controls](/optimizer-fix-controls.md). @@ -4276,7 +4299,7 @@ mysql> desc select count(distinct a) from test.t; +-----------------------------------+---------+-----------+-----------------------+---------------------------------+ ``` -- The second example uses `0`, which assumes that 0% of rows will be scanned before the qualified rows are found. +- The second example uses `0`, which assumes that 0% of rows will be scanned before the qualified rows are found. ```sql > SET SESSION tidb_opt_ordering_index_selectivity_ratio = 0; @@ -4747,7 +4770,7 @@ SHOW WARNINGS; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -4890,7 +4913,7 @@ SHOW WARNINGS; ### tidb_read_consistency New in v5.4.0 - Scope: SESSION -- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): Yes +- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): Yes (Note that if [non-transactional DML statements](/non-transactional-dml.md) exist, modifying the value of this variable using hint might not take effect.) - Type: String - Default value: `strict` - This variable is used to control the read consistency for an auto-commit read statement. @@ -4900,7 +4923,7 @@ SHOW WARNINGS; ### tidb_read_staleness New in v5.4.0 - Scope: SESSION -- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): Yes +- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Integer - Default value: `0` - Range: `[-2147483648, 0]` @@ -4954,6 +4977,8 @@ SHOW WARNINGS; - Type: Boolean - Default value: Before v7.2.0, the default value is `OFF`. Starting from v7.2.0, the default value is `ON`. - Specifies whether to remove `ORDER BY` clause in a subquery. +- In the ISO/IEC SQL standard, `ORDER BY` is mainly used to sort the results of top-level queries. For subqueries, the standard does not require that the results be sorted by `ORDER BY`. +- To sort subquery results, you can usually handle it in the outer query, such as using the window function or using `ORDER BY` again in the outer query. Doing so ensures the order of the final result set. ### tidb_replica_read New in v4.0 @@ -5024,7 +5049,7 @@ SHOW WARNINGS; - Type: Integer - Default value: `2` - Range: `[1, 2]` -- Controls the format version of the newly saved data in the table. In TiDB v4.0, the [new storage row format](https://github.com/pingcap/tidb/blob/master/docs/design/2018-07-19-row-format.md) version `2` is used by default to save new data. +- Controls the format version of the newly saved data in the table. In TiDB v4.0, the [new storage row format](https://github.com/pingcap/tidb/blob/release-8.1/docs/design/2018-07-19-row-format.md) version `2` is used by default to save new data. - If you upgrade from a TiDB version earlier than v4.0.0 to v4.0.0 or later versions, the format version is not changed, and TiDB continues to use the old format of version `1` to write data to the table, which means that **only newly created clusters use the new data format by default**. - Note that modifying this variable does not affect the old data that has been saved, but applies the corresponding version format only to the newly written data after modifying this variable. @@ -5052,7 +5077,7 @@ SHOW WARNINGS; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5066,7 +5091,7 @@ SHOW WARNINGS; > **Warning:** > -> This feature is experimental. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. +> The feature controlled by this variable is not yet effective in the current TiDB version. Do not change the default value. - Scope: GLOBAL - Persists to cluster: Yes @@ -5091,7 +5116,7 @@ SHOW WARNINGS; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5109,7 +5134,7 @@ SHOW WARNINGS; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5122,7 +5147,7 @@ SHOW WARNINGS; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5138,7 +5163,7 @@ SHOW WARNINGS; > This TiDB variable is not applicable to TiDB Cloud. - Scope: GLOBAL -- Persists to cluster: No +- Persists to cluster: No, only applicable to the current TiDB instance that you are connecting to. - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: String - Default value: "" @@ -5178,7 +5203,7 @@ SHOW WARNINGS; > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5255,7 +5280,7 @@ Query OK, 0 rows affected, 1 warning (0.00 sec) - Default value: `300` - Range: `[-1, 9223372036854775807]` - Unit: Milliseconds -- This variable is used to output the threshold value of the time consumed by the slow log. When the time consumed by a query is larger than this value, this query is considered as a slow log and its log is output to the slow query log. +- This variable outputs the threshold value of the time consumed by the slow log, and is set to 300 milliseconds by default. When the time consumed by a query is larger than this value, this query is considered as a slow query and its log is output to the slow query log. Note that when the output level of [`log.level`](https://docs.pingcap.com/tidb/v8.1/tidb-configuration-file#level) is `"debug"`, all queries are recorded in the slow query log, regardless of the setting of this variable. ### tidb_slow_query_file @@ -5274,6 +5299,16 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). +### tidb_slow_txn_log_threshold New in v7.0.0 + +- Scope: SESSION +- Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No +- Type: Unsigned integer +- Default value: `0` +- Range: `[0, 9223372036854775807]` +- Unit: Milliseconds +- This variable sets the threshold for slow transaction logging. When the execution time of a transaction exceeds this threshold, TiDB logs detailed information about the transaction. When the value is set to `0`, this feature is disabled. + ### tidb_snapshot - Scope: SESSION @@ -5326,7 +5361,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -5473,7 +5508,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5487,7 +5522,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5500,7 +5535,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5526,7 +5561,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5540,7 +5575,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5613,7 +5648,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5716,7 +5751,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5741,7 +5776,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5754,7 +5789,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5767,7 +5802,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5780,7 +5815,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5793,7 +5828,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5806,7 +5841,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5819,7 +5854,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No @@ -5832,7 +5867,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No @@ -5845,7 +5880,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -5905,13 +5940,13 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). -- This variable is used to dynamically modify the TiDB configuration item [`performance.txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50). It limits the size of a single row of data in TiDB, which is equivalent to the configuration item. The default value of this variable is `0`, which means that TiDB uses the value of the configuration item `txn-entry-size-limit` by default. When this variable is set to a non-zero value, `txn-entry-size-limit` is also set to the same value. +- This variable is used to dynamically modify the TiDB configuration item [`performance.txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500). It limits the size of a single row of data in TiDB, which is equivalent to the configuration item. The default value of this variable is `0`, which means that TiDB uses the value of the configuration item `txn-entry-size-limit` by default. When this variable is set to a non-zero value, `txn-entry-size-limit` is also set to the same value. -- This variable is used to dynamically modify the TiDB configuration item [`performance.txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v50). It limits the size of a single row of data in TiDB, which is equivalent to the configuration item. The default value of this variable is `0`, which means that TiDB uses the value of the configuration item `txn-entry-size-limit` by default. When this variable is set to a non-zero value, `txn-entry-size-limit` is also set to the same value. +- This variable is used to dynamically modify the TiDB configuration item [`performance.txn-entry-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-entry-size-limit-new-in-v4010-and-v500). It limits the size of a single row of data in TiDB, which is equivalent to the configuration item. The default value of this variable is `0`, which means that TiDB uses the value of the configuration item `txn-entry-size-limit` by default. When this variable is set to a non-zero value, `txn-entry-size-limit` is also set to the same value. @@ -5923,7 +5958,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -5948,7 +5983,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: SESSION - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No @@ -5963,7 +5998,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: SESSION - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No @@ -6004,7 +6039,7 @@ For details, see [Identify Slow Queries](/identify-slow-queries.md). - Default value: `8192` - Range: `[1, 18446744073709551615]` - When Fine Grained Shuffle is enabled, the window function pushed down to TiFlash can be executed in parallel. This variable controls the batch size of the data sent by the sender. -- Impact on performance: set a reasonable size according to your business requirements. Improper setting affects the performance. If the value is set too small, for example `1`, it causes one network transfer per Block. If the value is set too large, for example, the total number of rows of the table, it causes the receiving end to spend most of the time waiting for data, and the piplelined computation cannot work. To set a proper value, you can observe the distribution of the number of rows received by the TiFlash receiver. If most threads receive only a few rows, for example a few hundred, you can increase this value to reduce the network overhead. +- Impact on performance: set a reasonable size according to your business requirements. Improper setting affects the performance. If the value is set too small, for example `1`, it causes one network transfer per Block. If the value is set too large, for example, the total number of rows of the table, it causes the receiving end to spend most of the time waiting for data, and the pipelined computation cannot work. To set a proper value, you can observe the distribution of the number of rows received by the TiFlash receiver. If most threads receive only a few rows, for example a few hundred, you can increase this value to reduce the network overhead. ### tiflash_fine_grained_shuffle_stream_count New in v6.2.0 @@ -6143,7 +6178,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: SESSION - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No @@ -6178,7 +6213,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA > **Note:** > -> This variable is always enabled for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is always enabled for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: GLOBAL - Persists to cluster: Yes @@ -6194,7 +6229,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Integer - Default value: `8` -- Range: `[0, 2147483647]` for TiDB Self-Hosted and [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated), `[8, 2147483647]` for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) +- Range: `[0, 2147483647]` for TiDB Self-Managed and [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated), `[8, 2147483647]` for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) - This variable is a check item in the password complexity check. It checks whether the password length is sufficient. By default, the minimum password length is `8`. This variable takes effect only when [`validate_password.enable`](#validate_passwordenable-new-in-v650) is enabled. - The value of this variable must not be smaller than the expression: `validate_password.number_count + validate_password.special_char_count + (2 * validate_password.mixed_case_count)`. - If you change the value of `validate_password.number_count`, `validate_password.special_char_count`, or `validate_password.mixed_case_count` such that the expression value is larger than `validate_password.length`, the value of `validate_password.length` is automatically changed to match the expression value. @@ -6206,7 +6241,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Integer - Default value: `1` -- Range: `[0, 2147483647]` for TiDB Self-Hosted and [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated), `[1, 2147483647]` for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) +- Range: `[0, 2147483647]` for TiDB Self-Managed and [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated), `[1, 2147483647]` for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) - This variable is a check item in the password complexity check. It checks whether the password contains sufficient uppercase and lowercase letters. This variable takes effect only when [`validate_password.enable`](#validate_passwordenable-new-in-v650) is enabled and [`validate_password.policy`](#validate_passwordpolicy-new-in-v650) is set to `1` (MEDIUM) or larger. - Neither the number of uppercase letters nor the number of lowercase letters in the password can be fewer than the value of `validate_password.mixed_case_count`. For example, when the variable is set to `1`, the password must contain at least one uppercase letter and one lowercase letter. @@ -6217,7 +6252,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Integer - Default value: `1` -- Range: `[0, 2147483647]` for TiDB Self-Hosted and [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated), `[1, 2147483647]` for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) +- Range: `[0, 2147483647]` for TiDB Self-Managed and [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated), `[1, 2147483647]` for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) - This variable is a check item in the password complexity check. It checks whether the password contains sufficient numbers. This variable takes effect only when [`validate_password.enable`](#password_reuse_interval-new-in-v650) is enabled and [`validate_password.policy`](#validate_passwordpolicy-new-in-v650) is set to `1` (MEDIUM) or larger. ### validate_password.policy New in v6.5.0 @@ -6227,7 +6262,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Enumeration - Default value: `1` -- Value options: `0`, `1`, and `2` for TiDB Self-Hosted and [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated); `1` and `2` for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) +- Value options: `0`, `1`, and `2` for TiDB Self-Managed and [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated); `1` and `2` for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) - This variable controls the policy for the password complexity check. This variable takes effect only when [`validate_password.enable`](#password_reuse_interval-new-in-v650) is enabled. The value of this variable determines whether other `validate-password` variables take effect in the password complexity check, except for `validate_password.check_user_name`. - This value of this variable can be `0`, `1`, or `2` (corresponds to LOW, MEDIUM, or STRONG). Different policy levels have different checks: - 0 or LOW: password length. @@ -6241,7 +6276,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Integer - Default value: `1` -- Range: `[0, 2147483647]` for TiDB Self-Hosted and [TiDB Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-dedicated), `[1, 2147483647]` for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) +- Range: `[0, 2147483647]` for TiDB Self-Managed and [TiDB Cloud Dedicated](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-dedicated), `[1, 2147483647]` for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) - This variable is a check item in the password complexity check. It checks whether the password contains sufficient special characters. This variable takes effect only when [`validate_password.enable`](#password_reuse_interval-new-in-v650) is enabled and [`validate_password.policy`](#validate_passwordpolicy-new-in-v650) is set to `1` (MEDIUM) or larger. ### version @@ -6249,7 +6284,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA - Scope: NONE - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Default value: `8.0.11-TiDB-`(tidb version) -- This variable returns the MySQL version, followed by the TiDB version. For example '8.0.11-TiDB-v8.0.0'. +- This variable returns the MySQL version, followed by the TiDB version. For example '8.0.11-TiDB-v8.1.2'. ### version_comment @@ -6276,7 +6311,7 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA > **Note:** > -> This variable is read-only for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). +> This variable is read-only for [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless). - Scope: SESSION | GLOBAL - Persists to cluster: Yes @@ -6301,4 +6336,4 @@ Internally, the TiDB parser transforms the `SET TRANSACTION ISOLATION LEVEL [REA - Applies to hint [SET_VAR](/optimizer-hints.md#set_varvar_namevar_value): No - Type: Boolean - Default value: `ON` -- This variable controls whether to use the high precision mode when computing the window functions. +- This variable controls whether to use the high precision mode when computing the [window functions](/functions-and-operators/window-functions.md). diff --git a/table-filter.md b/table-filter.md index ec1a5cf97cab0..3b33438085673 100644 --- a/table-filter.md +++ b/table-filter.md @@ -1,7 +1,6 @@ --- title: Table Filter summary: Usage of table filter feature in TiDB tools. -aliases: ['/docs/dev/tidb-lightning/tidb-lightning-table-filter/','/docs/dev/reference/tools/tidb-lightning/table-filter/','/tidb/dev/tidb-lightning-table-filter/'] --- # Table Filter diff --git a/telemetry.md b/telemetry.md index 6c1cb5bf87112..ac10f7439bafd 100644 --- a/telemetry.md +++ b/telemetry.md @@ -1,54 +1,27 @@ --- title: Telemetry summary: Learn the telemetry feature, how to disable the feature and view its status. -aliases: ['/docs/dev/telemetry/'] +aliases: ['/tidb/stable/sql-statement-admin-show-telemetry','/tidb/v8.1/sql-statement-admin-show-telemetry'] --- # Telemetry -When the telemetry is enabled, TiDB, TiUP and TiDB Dashboard collect usage information and share the information with PingCAP to help understand how to improve the product. For example, this usage information helps prioritize new features. +When the telemetry feature is enabled, TiUP and TiSpark collect usage information and share the information with PingCAP to help understand how to improve the product. > **Note:** > -> - Starting from February 20, 2023, the telemetry feature is disabled by default in new versions of TiDB and TiDB Dashboard, including v6.6.0, and usage information is not collected and shared with PingCAP. Before upgrading to these versions, if the cluster uses the default telemetry configuration, the telemetry feature is disabled after the upgrade. See [TiDB Release Timeline](/releases/release-timeline.md) for the specific version. -> - Starting from v1.11.3, the telemetry feature is disabled by default in newly deployed TiUP, and usage information is not collected. If you upgrade from a TiUP version earlier than v1.11.3 to v1.11.3 or a later version, the telemetry feature keeps the same status as before the upgrade. +> - Starting from TiUP v1.11.3, the telemetry feature in TiUP is disabled by default, which means TiUP usage information is not collected by default. If you upgrade from a TiUP version earlier than v1.11.3 to v1.11.3 or a later version, the telemetry feature keeps the same status as before the upgrade. +> - Starting from TiSpark v3.0.3, the telemetry feature in TiSpark is disabled by default, which means TiSpark usage information is not collected by default. +> - Starting from v8.1.0, the telemetry feature in TiDB and TiDB Dashboard is removed. -## What is shared? +## What is shared when telemetry is enabled? -The following sections describe the shared usage information in detail for each component. The usage details that get shared might change over time. These changes (if any) will be announced in [release notes](/releases/release-notes.md). +The following sections describe the shared usage information in detail for TiUP and TiSpark. The usage details that get shared might change over time. These changes (if any) will be announced in [release notes](/releases/release-notes.md). > **Note:** > > In **ALL** cases, user data stored in the TiDB cluster will **NOT** be shared. You can also refer to [PingCAP Privacy Policy](https://pingcap.com/privacy-policy). -### TiDB - -When the telemetry collection feature is enabled in TiDB, the TiDB cluster collects usage details on a 6-hour basis. These usage details include but are not limited to: - -- A randomly generated telemetry ID. -- Deployment characteristics, such as the size of hardware (CPU, memory, disk), TiDB components versions, OS name. -- The status of query requests in the system, such as the number of query requests and the duration. -- Component usage, for example, whether the Async Commit feature is in use or not. -- Pseudonymized IP address of the TiDB telemetry data sender. - -To view the full content of the usage information shared to PingCAP, execute the following SQL statement: - -{{< copyable "sql" >}} - -```sql -ADMIN SHOW TELEMETRY; -``` - -### TiDB Dashboard - -When the telemetry collection feature is enabled for TiDB Dashboard, usage details of the TiDB Dashboard web UI will be shared, including (but not limited to): - -- A randomly generated telemetry ID. -- User operation information, such as the name of the TiDB Dashboard web page accessed by the user. -- Browser and OS information, such as browser name, OS name, and screen resolution. - -To view the full content of the usage information shared to PingCAP, use the [Network Activity Inspector of Chrome DevTools](https://developers.google.com/web/tools/chrome-devtools/network) or the [Network Monitor of Firefox Developer Tools](https://developer.mozilla.org/en-US/docs/Tools/Network_Monitor). - ### TiUP When the telemetry collection feature is enabled in TiUP, usage details of TiUP will be shared, including (but not limited to): @@ -59,8 +32,6 @@ When the telemetry collection feature is enabled in TiUP, usage details of TiUP To view the full content of the usage information shared to PingCAP, set the `TIUP_CLUSTER_DEBUG=enable` environment variable when executing the TiUP command. For example: -{{< copyable "shell-regular" >}} - ```shell TIUP_CLUSTER_DEBUG=enable tiup cluster list ``` @@ -83,198 +54,38 @@ You can view TiSpark usage information that is collected in Spark logs. You can cat {spark.log} | grep Telemetry report | tail -n 1 ``` -## Disable telemetry - -### Disable TiDB telemetry at deployment - -When the telemetry is enabled in existing TiDB clusters, you can configure [`enable-telemetry = false`](/tidb-configuration-file.md#enable-telemetry-new-in-v402) on each TiDB instance to disable the TiDB telemetry collection on that instance, which does not take effect until you restart the cluster. - -Detailed steps to disable telemetry in different deployment tools are listed below. - -
- Binary deployment - -Create a configuration file `tidb_config.toml` with the following content: - -{{< copyable "" >}} - -```toml -enable-telemetry = false -``` - -Specify the `--config=tidb_config.toml` command-line parameter when starting TiDB for the configuration file above to take effect. +## Enable telemetry -See [TiDB Configuration Options](/command-line-flags-for-tidb-configuration.md#--config) and [TiDB Configuration File](/tidb-configuration-file.md#enable-telemetry-new-in-v402) for details. +### Enable TiUP telemetry -
- -
- Deployment using TiUP Playground - -Create a configuration file `tidb_config.toml` with the following content: - -{{< copyable "" >}} - -```toml -enable-telemetry = false -``` - -When starting TiUP Playground, specify the `--db.config tidb_config.toml` command-line parameter for the configuration file above to take effect. For example: - -{{< copyable "shell-regular" >}} +To enable the TiUP telemetry collection, execute the following command: ```shell -tiup playground --db.config tidb_config.toml -``` - -See [Quickly Deploy a Local TiDB Cluster](/tiup/tiup-playground.md) for details. - -
- -
- Deployment using TiUP Cluster - -Modify the deployment topology file `topology.yaml` to add the following content: - -{{< copyable "" >}} - -```yaml -server_configs: - tidb: - enable-telemetry: false +tiup telemetry enable ``` -
- -
- Deployment on Kubernetes via TiDB Operator - -Configure `spec.tidb.config.enable-telemetry: false` in `tidb-cluster.yaml` or TidbCluster Custom Resource. - -See [Deploy TiDB Operator on Kubernetes](https://docs.pingcap.com/tidb-in-kubernetes/stable/deploy-tidb-operator) for details. - -> **Note:** -> -> This configuration item requires TiDB Operator v1.1.3 or later to take effect. - -
+### Enable TiSpark telemetry -### Disable TiDB telemetry for deployed TiDB clusters +To enable the TiSpark telemetry collection, configure `spark.tispark.telemetry.enable = true` in the TiSpark configuration file. -In existing TiDB clusters, you can also modify the system variable [`tidb_enable_telemetry`](/system-variables.md#tidb_enable_telemetry-new-in-v402) to dynamically disable the TiDB telemetry collection: - -{{< copyable "sql" >}} - -```sql -SET GLOBAL tidb_enable_telemetry = 0; -``` - -> **Note:** -> -> When you disable telemetry, the configuration file has a higher priority over system variable. That is, after telemetry collection is disabled by the configuration file, the value of the system variable will be ignored. - -### Disable TiDB Dashboard telemetry - -Configure [`dashboard.enable-telemetry = false`](/pd-configuration-file.md#enable-telemetry) to disable the TiDB Dashboard telemetry collection on all PD instances. You need to restart the running clusters for the configuration to take effect. - -Detailed steps to disable telemetry for different deployment tools are listed below. - -
- Binary deployment - -Create a configuration file `pd_config.toml` with the following content: - -{{< copyable "" >}} - -```toml -[dashboard] -enable-telemetry = false -``` - -Specify the `--config=pd_config.toml` command-line parameter when starting PD to take effect. - -See [PD Configuration Flags](/command-line-flags-for-pd-configuration.md#--config) and [PD Configuration File](/pd-configuration-file.md#enable-telemetry) for details. - -
- -
- Deployment using TiUP Playground - -Create a configuration file `pd_config.toml` with the following content: - -{{< copyable "" >}} - -```toml -[dashboard] -enable-telemetry = false -``` - -When starting TiUP Playground, specify the `--pd.config pd_config.toml` command-line parameter to take effect, for example: - -{{< copyable "shell-regular" >}} - -```shell -tiup playground --pd.config pd_config.toml -``` - -See [Quickly Deploy a Local TiDB Cluster](/tiup/tiup-playground.md) for details. - -
- -
- Deployment using TiUP Cluster - -Modify the deployment topology file `topology.yaml` to add the following content: - -{{< copyable "" >}} - -```yaml -server_configs: - pd: - dashboard.enable-telemetry: false -``` - -
- -
- Deployment on Kubernetes via TiDB Operator - -Configure `spec.pd.config.dashboard.enable-telemetry: false` in `tidb-cluster.yaml` or TidbCluster Custom Resource. - -See [Deploy TiDB Operator on Kubernetes](https://docs.pingcap.com/tidb-in-kubernetes/stable/deploy-tidb-operator) for details. - -> **Note:** -> -> This configuration item requires TiDB Operator v1.1.3 or later to take effect. - -
+## Disable telemetry ### Disable TiUP telemetry To disable the TiUP telemetry collection, execute the following command: -{{< copyable "shell-regular" >}} - ```shell tiup telemetry disable ``` -## Check telemetry status +### Disable TiSpark telemetry -For TiDB telemetry, execute the following SQL statement to check the telemetry status: +To disable the TiSpark telemetry collection, configure `spark.tispark.telemetry.enable = false` in the TiSpark configuration file. -{{< copyable "sql" >}} - -```sql -ADMIN SHOW TELEMETRY; -``` - -If the `DATA_PREVIEW` column in the execution result is empty, TiDB telemetry is disabled. If not, TiDB telemetry is enabled. You can also check when the usage information was shared previously according to the `LAST_STATUS` column and whether the sharing was successful or not. +## Check telemetry status For TiUP telemetry, execute the following command to check the telemetry status: -{{< copyable "shell-regular" >}} - ```shell tiup telemetry status ``` @@ -286,4 +97,4 @@ To meet compliance requirements in different countries or regions, the usage inf - For IP addresses from the Chinese mainland, usage information is sent to and stored on cloud servers in the Chinese mainland. - For IP addresses from outside of the Chinese mainland, usage information is sent to and stored on cloud servers in the US. -See [PingCAP Privacy Policy](https://en.pingcap.com/privacy-policy/) for details. +See [PingCAP Privacy Policy](https://www.pingcap.com/privacy-policy/) for details. diff --git a/templates/copyright.tex b/templates/copyright.tex index 4e6911b43eee3..82e5468c4fe93 100644 --- a/templates/copyright.tex +++ b/templates/copyright.tex @@ -1,4 +1,4 @@ \noindent \rule{\textwidth}{1pt} -© 2024 PingCAP. All Rights Reserved. +© 2025 PingCAP. All Rights Reserved. diff --git a/three-data-centers-in-two-cities-deployment.md b/three-data-centers-in-two-cities-deployment.md index d8b5e661fb2ad..ddb0137fb7679 100644 --- a/three-data-centers-in-two-cities-deployment.md +++ b/three-data-centers-in-two-cities-deployment.md @@ -1,7 +1,6 @@ --- title: Three Availability Zones in Two Regions Deployment summary: Learn the deployment solution to three availability zones in two regions. -aliases: ['/docs/dev/three-data-centers-in-two-cities-deployment/'] --- # Three Availability Zones in Two Regions Deployment @@ -181,7 +180,7 @@ In the deployment of three AZs in two regions, to optimize performance, you need > **Note:** > -> Using `raftstore.raft-min-election-timeout-ticks` and `raftstore.raft-max-election-timeout-ticks` to configure larger election timeout ticks for a TiKV node can significantly decrease the likelihood of Regions on that node becoming Leaders. However, in a disaster scenario where some TiKV nodes are offline and the remaining active TiKV nodes lag behind in Raft logs, only Regions on this TiKV node with large election timeout ticks can become Leaders. Because Regions on this TiKV node must wait for at least the duration set by `raftstore.raft-min-election-timeout-ticks' before initiating an election, it is recommended to avoid setting these values excessively large to prevent potential impact on the cluster availability in such scenarios. +> Using `raftstore.raft-min-election-timeout-ticks` and `raftstore.raft-max-election-timeout-ticks` to configure larger election timeout ticks for a TiKV node can significantly decrease the likelihood of Regions on that node becoming Leaders. However, in a disaster scenario where some TiKV nodes are offline and the remaining active TiKV nodes lag behind in Raft logs, only Regions on this TiKV node with large election timeout ticks can become Leaders. Because Regions on this TiKV node must wait for at least the duration set by `raftstore.raft-min-election-timeout-ticks` before initiating an election, it is recommended to avoid setting these values excessively large to prevent potential impact on the cluster availability in such scenarios. - Configure scheduling. After the cluster is enabled, use the `tiup ctl:v{CLUSTER_VERSION} pd` tool to modify the scheduling policy. Modify the number of TiKV Raft replicas. Configure this number as planned. In this example, the number of replicas is five. diff --git a/ticdc-deployment-topology.md b/ticdc-deployment-topology.md index 75a724821e15c..c618076648458 100644 --- a/ticdc-deployment-topology.md +++ b/ticdc-deployment-topology.md @@ -1,7 +1,6 @@ --- title: TiCDC Deployment Topology summary: Learn the deployment topology of TiCDC based on the minimal TiDB topology. -aliases: ['/docs/dev/ticdc-deployment-topology/'] --- # TiCDC Deployment Topology @@ -24,6 +23,10 @@ TiCDC is a tool for replicating the incremental data of TiDB, introduced in TiDB | CDC | 3 | 8 VCore 16GB * 1 | 10.0.1.11
10.0.1.12
10.0.1.13 | Default port
Global directory configuration | | Monitoring & Grafana | 1 | 4 VCore 8GB * 1 500GB (ssd) | 10.0.1.11 | Default port
Global directory configuration | +> **Note:** +> +> The IP addresses of the instances are given as examples only. In your actual deployment, replace the IP addresses with your actual IP addresses. + ### Topology templates - [The simple template for the TiCDC topology](https://github.com/pingcap/docs/blob/master/config-templates/simple-cdc.yaml) diff --git a/ticdc/deploy-ticdc.md b/ticdc/deploy-ticdc.md index 2fa74b732bb7f..fbd6673006081 100644 --- a/ticdc/deploy-ticdc.md +++ b/ticdc/deploy-ticdc.md @@ -95,7 +95,7 @@ tiup cluster upgrade --transfer-timeout 600 > **Note:** > -> In the preceding command, you need to replace `` and `` with the actual cluster name and cluster version. For example, the version can be v8.0.0. +> In the preceding command, you need to replace `` and `` with the actual cluster name and cluster version. For example, the version can be v8.1.2. ### Upgrade cautions @@ -152,7 +152,7 @@ See [Enable TLS Between TiDB Components](/enable-tls-between-components.md). ## View TiCDC status using the command-line tool -Run the following command to view the TiCDC cluster status. Note that you need to replace `v` with the TiCDC cluster version, such as `v8.0.0`: +Run the following command to view the TiCDC cluster status. Note that you need to replace `v` with the TiCDC cluster version, such as `v8.1.2`: ```shell tiup cdc:v cli capture list --server=http://10.0.10.25:8300 diff --git a/ticdc/ticdc-alert-rules.md b/ticdc/ticdc-alert-rules.md index 10e9f0235271a..5526eee3ddd69 100644 --- a/ticdc/ticdc-alert-rules.md +++ b/ticdc/ticdc-alert-rules.md @@ -16,7 +16,7 @@ For critical alerts, you need to pay close attention to abnormal monitoring metr - Alert rule: - `(time() - ticdc_owner_checkpoint_ts / 1000) > 600` + `ticdc_owner_checkpoint_ts_lag > 600` - Description: @@ -30,7 +30,7 @@ For critical alerts, you need to pay close attention to abnormal monitoring metr - Alert rule: - `(time() - ticdc_owner_resolved_ts / 1000) > 300` + `ticdc_owner_resolved_ts_lag > 300` - Description: @@ -54,20 +54,6 @@ For critical alerts, you need to pay close attention to abnormal monitoring metr This alert is similar to replication interruption. See [TiCDC Handles Replication Interruption](/ticdc/troubleshoot-ticdc.md#how-do-i-handle-replication-interruptions). -### `ticdc_processor_exit_with_error_count` - -- Alert rule: - - `changes(ticdc_processor_exit_with_error_count[1m]) > 0` - -- Description: - - A replication task reports an error and exits. - -- Solution: - - See [TiCDC Handles Replication Interruption](/ticdc/troubleshoot-ticdc.md#how-do-i-handle-replication-interruptions). - ## Warning alerts Warning alerts are a reminder for an issue or error. @@ -86,61 +72,47 @@ Warning alerts are a reminder for an issue or error. Collect TiCDC logs to locate the root cause. -### `cdc_sink_flush_duration_time_more_than_10s` +### `cdc_no_owner` - Alert rule: - `histogram_quantile(0.9, rate(ticdc_sink_txn_worker_flush_duration[1m])) > 10` + `sum(rate(ticdc_owner_ownership_counter[240s])) < 0.5` - Description: - It takes a replication task more than 10 seconds to write data to the downstream database. + There is no owner in the TiCDC cluster for more than 10 minutes. - Solution: - Check whether there are problems in the downstream database. + Collect TiCDC logs to identify the root cause. -### `cdc_processor_checkpoint_tso_no_change_for_1m` +### `ticdc_changefeed_meet_error` - Alert rule: - `changes(ticdc_processor_checkpoint_ts[1m]) < 1` + `(max_over_time(ticdc_owner_status[1m]) == 1 or max_over_time(ticdc_owner_status[1m]) == 6) > 0` - Description: - A replication task has not advanced for more than 1 minute. + A replication task encounters an error. - Solution: See [TiCDC Handles Replication Interruption](/ticdc/troubleshoot-ticdc.md#how-do-i-handle-replication-interruptions). -### `ticdc_puller_entry_sorter_sort_bucket` - -- Alert rule: - - `histogram_quantile(0.9, rate(ticdc_puller_entry_sorter_sort_bucket{}[1m])) > 1` - -- Description: - - The delay of TiCDC puller entry sorter is too high. - -- Solution: - - Collect TiCDC logs to locate the root cause. - -### `ticdc_puller_entry_sorter_merge_bucket` +### `ticdc_processor_exit_with_error_count` - Alert rule: - `histogram_quantile(0.9, rate(ticdc_puller_entry_sorter_merge_bucket{}[1m])) > 1` + `changes(ticdc_processor_exit_with_error_count[1m]) > 0` - Description: - The delay of TiCDC puller entry sorter merge is too high. + A replication task reports an error and exits. - Solution: - Collect TiCDC logs to locate the root cause. + See [TiCDC Handles Replication Interruption](/ticdc/troubleshoot-ticdc.md#how-do-i-handle-replication-interruptions). ### `tikv_cdc_min_resolved_ts_no_change_for_1m` @@ -170,15 +142,15 @@ Warning alerts are a reminder for an issue or error. Collect TiCDC monitoring metrics and TiKV logs to locate the root cause. -### `ticdc_sink_mysql_execution_error` +### `ticdc_sink_execution_error` - Alert rule: - `changes(ticdc_sink_mysql_execution_error[1m]) > 0` + `changes(ticdc_sink_execution_error[1m]) > 0` - Description: - An error occurs when a replication task writes data to the downstream MySQL. + An error occurs when a replication task writes data to the downstream. - Solution: diff --git a/ticdc/ticdc-avro-checksum-verification.md b/ticdc/ticdc-avro-checksum-verification.md index 1cc9e89a290c8..3721bf2b3ad7c 100644 --- a/ticdc/ticdc-avro-checksum-verification.md +++ b/ticdc/ticdc-avro-checksum-verification.md @@ -7,7 +7,7 @@ summary: Introduce the detailed implementation of TiCDC row data checksum verifi This document introduces how to consume data sent to Kafka by TiCDC and encoded by Avro protocol using Golang, and how to perform data verification using the [Single-row data checksum feature](/ticdc/ticdc-integrity-check.md). -The source code of this example is available in the [`avro-checksum-verification`](https://github.com/pingcap/tiflow/tree/master/examples/golang/avro-checksum-verification) directory. +The source code of this example is available in the [`avro-checksum-verification`](https://github.com/pingcap/tiflow/tree/release-8.1/examples/golang/avro-checksum-verification) directory. The example in this document uses [kafka-go](https://github.com/segmentio/kafka-go) to create a simple Kafka consumer program. This program continuously reads data from a specified topic, calculates the checksum, and verifies its value. diff --git a/ticdc/ticdc-avro-protocol.md b/ticdc/ticdc-avro-protocol.md index b616631729b6c..46a816fd2207e 100644 --- a/ticdc/ticdc-avro-protocol.md +++ b/ticdc/ticdc-avro-protocol.md @@ -5,7 +5,9 @@ summary: Learn the concept of TiCDC Avro Protocol and how to use it. # TiCDC Avro Protocol -Avro is a data exchange format protocol defined by [Apache Avro™](https://avro.apache.org/) and chosen by [Confluent Platform](https://docs.confluent.io/platform/current/platform.html) as the default data exchange format. This document describes the implementation of the Avro data format in TiCDC, including TiDB extension fields, definition of the Avro data format, and the interaction between Avro and [Confluent Schema Registry](https://docs.confluent.io/platform/current/schema-registry/index.html). +TiCDC Avro protocol is a third-party implementation of the [Confluent Avro](https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/serdes-avro.html) data exchange protocol defined by [Confluent Platform](https://docs.confluent.io/platform/current/platform.html). Avro is a data format defined by [Apache Avro™](https://avro.apache.org/) + +This document describes how TiCDC implements the Confluent Avro protocol, and the interaction between Avro and [Confluent Schema Registry](https://docs.confluent.io/platform/current/schema-registry/index.html). > **Warning:** > @@ -13,8 +15,6 @@ Avro is a data exchange format protocol defined by [Apache Avro™](https://avro ## Use Avro -When using Message Queue (MQ) as a downstream sink, you can specify Avro in `sink-uri`. TiCDC captures TiDB DML events, creates Avro messages from these events, and sends the messages downstream. When Avro detects a schema change, it registers the latest schema with Schema Registry. - The following is a configuration example using Avro: {{< copyable "shell-regular" >}} @@ -23,30 +23,11 @@ The following is a configuration example using Avro: cdc cli changefeed create --server=http://127.0.0.1:8300 --changefeed-id="kafka-avro" --sink-uri="kafka://127.0.0.1:9092/topic-name?protocol=avro" --schema-registry=http://127.0.0.1:8081 --config changefeed_config.toml ``` -```shell -[sink] -dispatchers = [ - {matcher = ['*.*'], topic = "tidb_{schema}_{table}"}, -] -``` - -The value of `--schema-registry` supports the `https` protocol and `username:password` authentication, for example, `--schema-registry=https://username:password@schema-registry-uri.com`. The username and password must be URL-encoded. +The value of `--schema-registry` supports the `https` protocol and `username:password` authentication. The username and password must be URL-encoded. For example, `--schema-registry=https://username:password@schema-registry-uri.com`. -## TiDB extension fields - -By default, Avro only collects data of changed rows in DML events and does not collect the type of data changes or TiDB-specific CommitTS (the unique identifiers of transactions). To address this issue, TiCDC introduces the following three TiDB extension fields to the Avro protocol message. When `enable-tidb-extension` is set to `true` (`false` by default) in `sink-uri`, TiCDC adds these three fields to the Avro messages during message generation. - -- `_tidb_op`: The DML type. "c" indicates insert and "u" indicates updates. -- `_tidb_commit_ts`: The unique identifier of a transaction. -- `_tidb_commit_physical_time`: The physical timestamp in a transaction identifier. - -The following is a configuration example: - -{{< copyable "shell-regular" >}} - -```shell -cdc cli changefeed create --server=http://127.0.0.1:8300 --changefeed-id="kafka-avro-enable-extension" --sink-uri="kafka://127.0.0.1:9092/topic-name?protocol=avro&enable-tidb-extension=true" --schema-registry=http://127.0.0.1:8081 --config changefeed_config.toml -``` +> **Note：** +> +> When using the Avro protocol, one Kafka topic can only contain data for one table. You need to configure the [Topic dispatcher](/ticdc/ticdc-sink-to-kafka.md#topic-dispatchers) in the configuration file. ```shell [sink] @@ -93,7 +74,38 @@ The `fields` in the key contains only primary key columns or unique index column } ``` -The data format of Value is the same as that of Key, by default. However, `fields` in the Value contains all columns, not just the primary key columns. +The data format of Value is the same as that of Key, by default. However, `fields` in the Value contains all columns. + +> **Note:** +> +> The Avro protocol encodes DML events as follows: +> +> - For Delete events, Avro only encodes the Key part. The Value part is empty. +> - For Insert events, Avro encodes all column data to the Value part. +> - For Update events, Avro encodes only all column data that is updated to the Value part. +> +> The Avro protocol does not encode the old values for Update and Delete events. Additionally, to be compatible with most Confluent sink connectors that rely on `null` records to identify deletions (`delete.on.null`), Delete events do not include extension information, such as `_tidb_commit_ts`, even when `enable-tidb-extension` is enabled. If you need these features, consider using other protocols such as Canal-JSON or Debezium. + +## TiDB extension fields + +By default, Avro only collects data of changed rows in DML events and does not collect the type of data changes or TiDB-specific CommitTS (the unique identifiers of transactions). To address this issue, TiCDC introduces the following three TiDB extension fields to the Avro protocol message. When `enable-tidb-extension` is set to `true` (`false` by default) in `sink-uri`, TiCDC adds these three fields to the Avro messages during message generation. + +- `_tidb_op`: The DML type. "c" indicates insert and "u" indicates updates. +- `_tidb_commit_ts`: The unique identifier of a transaction. +- `_tidb_commit_physical_time`: The physical timestamp in a transaction identifier. + +The following is a configuration example: + +```shell +cdc cli changefeed create --server=http://127.0.0.1:8300 --changefeed-id="kafka-avro-enable-extension" --sink-uri="kafka://127.0.0.1:9092/topic-name?protocol=avro&enable-tidb-extension=true" --schema-registry=http://127.0.0.1:8081 --config changefeed_config.toml +``` + +```shell +[sink] +dispatchers = [ + {matcher = ['*.*'], topic = "tidb_{schema}_{table}"}, +] +``` After you enable [`enable-tidb-extension`](#tidb-extension-fields), the data format of the Value will be as follows: @@ -159,7 +171,7 @@ If one column can be NULL, the Column data format can be: - `{{ColumnName}}` indicates the column name. - `{{TIDB_TYPE}}` indicates the type in TiDB, which is not a one-to-one mapping with the SQL type. -- `{{AVRO_TYPE}}` indicates the type in [avro spec](https://avro.apache.org/docs/current/spec.html). +- `{{AVRO_TYPE}}` indicates the type in the [Avro Specification](https://avro.apache.org/docs/++version++/specification). | SQL TYPE | TIDB_TYPE | AVRO_TYPE | Description | |------------|-----------|-----------|---------------------------------------------------------------------------------------------------------------------------| @@ -272,14 +284,27 @@ DECIMAL(10, 4) ## DDL events and schema changes -Avro does not generate DDL events downstream. It checks whether a schema changes each time a DML event occurs. If a schema changes, Avro generates a new schema and registers it with the Schema Registry. If the schema change does not pass the compatibility check, the registration fails. TiCDC does not resolve any schema compatibility issues. +Avro does not send DDL events and Watermark events to downstream. It checks whether a schema changes each time a DML event occurs. If a schema changes, Avro generates a new schema and registers it with the Schema Registry. If the schema change does not pass the compatibility check, the registration fails. TiCDC does not resolve any schema compatibility issues. -Note that, even if a schema change passes the compatibility check and a new version is registered, the data producers and consumers still need to perform an upgrade to ensure normal running of the system. +For example, the default [compatibility policy](https://docs.confluent.io/platform/current/schema-registry/fundamentals/schema-evolution.html#compatibility-types) of Confluent Schema Registry is `BACKWARD`, and you add a non-empty column to the source table. In this situation, Avro generates a new schema but fails to register it with Schema Registry due to compatibility issues. At this time, the changefeed enters an error state. -Assume that the default compatibility policy of Confluent Schema Registry is `BACKWARD` and add a non-empty column to the source table. In this situation, Avro generates a new schema but fails to register it with Schema Registry due to compatibility issues. At this time, the changefeed enters an error state. +Note that, even if a schema change passes the compatibility check and a new version is registered, the data producers and consumers still need to acquire the new schema for data encoding and decoding. For more information about schemas, refer to [Schema Registry related documents](https://docs.confluent.io/platform/current/schema-registry/avro.html). +## Consumer implementation + +The TiCDC Avro protocol can be deserialized by [`io.confluent.kafka.serializers.KafkaAvroDeserializer`](https://docs.confluent.io/platform/current/schema-registry/fundamentals/serdes-develop/serdes-avro.html#avro-deserializer). + +The consumer program can acquire the latest schema via [Schema Registry API](https://docs.confluent.io/platform/current/schema-registry/develop/api.html), and then use the schema to deserialize the data encoded by the TiCDC Avro protocol. + +### Distinguish event types + +The consumer program can distinguish DML event types by the following rules: + +* If there is only the Key part, it is a Delete event. +* If there are both Key and Value parts, it is either an Insert or an Update event. If the [TiDB extension fields](#tidb-extension-fields) are enabled, you can use the `_tidb_op` field to identify if it is an Insert or Update event. If the TiDB extension fields are not enabled, you cannot distinguish them. + ## Topic distribution Schema Registry supports three [Subject Name Strategies](https://docs.confluent.io/platform/current/schema-registry/serdes-develop/index.html#subject-name-strategy): TopicNameStrategy, RecordNameStrategy, and TopicRecordNameStrategy. Currently, TiCDC Avro only supports TopicNameStrategy, which means that a Kafka topic can only receive data in one data format. Therefore, TiCDC Avro prohibits mapping multiple tables to the same topic. When you create a changefeed, an error will be reported if the topic rule does not include the `{schema}` and `{table}` placeholders in the configured distribution rule. diff --git a/ticdc/ticdc-behavior-change.md b/ticdc/ticdc-behavior-change.md deleted file mode 100644 index 887ff14aa49f4..0000000000000 --- a/ticdc/ticdc-behavior-change.md +++ /dev/null @@ -1,54 +0,0 @@ ---- -title: TiCDC Behavior Changes -summary: Introduce the behavior changes of TiCDC changefeed, including the reasons and the impact of these changes. ---- - -# TiCDC Behavior Changes - -## Split update events into delete and insert events - -### Transactions containing a single update change - -Starting from v6.5.3, v7.1.1, and v7.2.0, when using a non-MySQL sink, for transactions that only contain a single update change, if the primary key or non-null unique index value is modified in the update event, TiCDC splits this event into delete and insert events. For more information, see GitHub issue [#9086](https://github.com/pingcap/tiflow/issues/9086). - -This change primarily addresses the following issues: - -* When using the CSV and AVRO protocols, only the new value is output without the old value. Therefore, when the primary key or non-null unique index value changes, the consumer can only receive the new value, making it impossible to process the value before the change (for example, delete the old value). -* When using the index value dispatcher to distribute data across different Kafka partitions based on the key, multiple consumer processes in the downstream consumer group consume Kafka topic partitions independently. Due to different consumption progress, data inconsistency might occur. - -Take the following SQL as an example: - -```sql -CREATE TABLE t (a INT PRIMARY KEY, b INT); -INSERT INTO t VALUES (1, 1); -UPDATE t SET a = 2 WHERE a = 1; -``` - -In this example, the primary key `a` is updated from `1` to `2`. If the update event is not split: - -* When using the CSV and AVRO protocols, the consumer only obtains the new value `a = 2` and cannot obtain the old value `a = 1`. This might cause the downstream consumer to only insert the new value `2` without deleting the old value `1`. -* When using the index value dispatcher, the event for inserting `(1, 1)` might be sent to Partition 0, and the update event `(2, 1)` might be sent to Partition 1. If the consumption progress of Partition 1 is faster than that of Partition 0, an error might occur due to the absence of corresponding data in the downstream. Therefore, TiCDC splits the update event into delete and insert events. The event for deleting `(1, 1)` is sent to Partition 0, and the event for writing `(2, 1)` is sent to Partition 1, ensuring that the events are consumed successfully regardless of the progress of the consumer. - -### Transactions containing multiple update changes - -Starting from v6.5.4, v7.1.2, and v7.4.0, for transactions containing multiple changes, if the primary key or non-null unique index value is modified in the update event, TiCDC splits the event into delete and insert events and ensures that all events follow the sequence of delete events preceding insert events. For more information, see GitHub issue [#9430](https://github.com/pingcap/tiflow/issues/9430). - -This change primarily addresses the potential issue of primary key conflicts when using the MySQL sink to directly write these two events to the downstream, leading to changefeed errors. - -Take the following SQL as an example: - -```sql -CREATE TABLE t (a INT PRIMARY KEY, b INT); -INSERT INTO t VALUES (1, 1); -INSERT INTO t VALUES (2, 2); - -BEGIN; -UPDATE t SET a = 1 WHERE a = 3; -UPDATE t SET a = 2 WHERE a = 1; -UPDATE t SET a = 3 WHERE a = 2; -COMMIT; -``` - -In this example, by executing three SQL statements to swap the primary keys of two rows, TiCDC only receives two update change events, that is, changing the primary key `a` from `1` to `2` and changing the primary key `a` from `2` to `1`. If the MySQL sink directly writes these two update events to the downstream, a primary key conflict might occur, leading to changefeed errors. - -Therefore, TiCDC splits these two events into four events, that is, deleting records `(1, 1)` and `(2, 2)` and writing records `(2, 1)` and `(1, 2)`. diff --git a/ticdc/ticdc-bidirectional-replication.md b/ticdc/ticdc-bidirectional-replication.md index 607f23dda2aae..9c9af87547fcf 100644 --- a/ticdc/ticdc-bidirectional-replication.md +++ b/ticdc/ticdc-bidirectional-replication.md @@ -101,6 +101,20 @@ When no BDR role is set, you can execute any DDL. But after you set `bdr_mode=tr ### Replication scenarios of replicable DDLs 1. Choose a TiDB cluster and execute `ADMIN SET BDR ROLE PRIMARY` to set it as the primary cluster. + + ```sql + ADMIN SET BDR ROLE PRIMARY; + Query OK, 0 rows affected + Time: 0.003s + + ADMIN SHOW BDR ROLE; + +----------+ + | BDR_ROLE | + +----------+ + | primary | + +----------+ + ``` + 2. On other TiDB clusters, execute `ADMIN SET BDR ROLE SECONDARY` to set them as the secondary clusters. 3. Execute **replicable DDLs** on the primary cluster. The successfully executed DDLs will be replicated to the secondary clusters by TiCDC. diff --git a/ticdc/ticdc-canal-json.md b/ticdc/ticdc-canal-json.md index c7c076298396e..1b5151ddea5ac 100644 --- a/ticdc/ticdc-canal-json.md +++ b/ticdc/ticdc-canal-json.md @@ -42,7 +42,7 @@ cdc cli changefeed create --server=http://127.0.0.1:8300 --changefeed-id="kafka- ## Definitions of message formats -This section describes the formats of DDL Event, DML Event and WATERMARK Event, and how the data is resolved on the consumer side. +This section describes the formats of DDL Event, DML Event and WATERMARK Event, and how the data is parsed on the consumer side. ### DDL Event @@ -64,7 +64,7 @@ TiCDC encodes a DDL Event into the following Canal-JSON format. "data": null, "old": null, "_tidb": { // TiDB extension field - "commitTs": 163963309467037594 + "commitTs": 429918007904436226 // A TiDB TSO timestamp } } ``` @@ -133,7 +133,7 @@ TiCDC encodes a row of DML data change event as follows: ], "old": null, "_tidb": { // TiDB extension field - "commitTs": 163963314122145239 + "commitTs": 429918007904436226 // A TiDB TSO timestamp } } ``` @@ -162,14 +162,14 @@ The following is an example of the WATERMARK Event. "data": null, "old": null, "_tidb": { // TiDB extension field - "watermarkTs": 429918007904436226 + "watermarkTs": 429918007904436226 // A TiDB TSO timestamp } } ``` -### Data resolution on the consumer side +### Data parsing on the consumer side -As you can see from the example above, Canal-JSON has a uniform data format, with different field filling rules for different Event types. You can use a uniform method to resolve this JSON format data, and then determine the Event type by checking the field values. +As you can see from the example above, Canal-JSON has a unified data format, with different field filling rules for different Event types. Consumers can parse data in this JSON format using a unified method, and then determine the Event type by checking the field values: * When `isDdl` is `true`, the message contains a DDL Event. * When `isDdl` is `false`, you need to further check the `type` field. If `type` is `TIDB_WATERMARK`, it is a WATERMARK Event; otherwise, it is a DML Event. diff --git a/ticdc/ticdc-changefeed-config.md b/ticdc/ticdc-changefeed-config.md index f5023ddc2f885..8a01f4436e3cf 100644 --- a/ticdc/ticdc-changefeed-config.md +++ b/ticdc/ticdc-changefeed-config.md @@ -16,7 +16,7 @@ cdc cli changefeed create --server=http://10.0.10.25:8300 --sink-uri="mysql://ro ```shell Create changefeed successfully! ID: simple-replication-task -Info: {"upstream_id":7178706266519722477,"namespace":"default","id":"simple-replication-task","sink_uri":"mysql://root:xxxxx@127.0.0.1:4000/?time-zone=","create_time":"2024-03-28T15:05:46.679218+08:00","start_ts":438156275634929669,"engine":"unified","config":{"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":3600000000000,"filter":{"rules":["test.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v8.0.0"} +Info: {"upstream_id":7178706266519722477,"namespace":"default","id":"simple-replication-task","sink_uri":"mysql://root:xxxxx@127.0.0.1:4000/?time-zone=","create_time":"2024-12-26T15:05:46.679218+08:00","start_ts":438156275634929669,"engine":"unified","config":{"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":3600000000000,"filter":{"rules":["test.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v8.1.2"} ``` - `--changefeed-id`: The ID of the replication task. The format must match the `^[a-zA-Z0-9]+(\-[a-zA-Z0-9]+)*$` regular expression. If this ID is not specified, TiCDC automatically generates a UUID (the version 4 format) as the ID. @@ -74,6 +74,11 @@ case-sensitive = false # The format of this parameter is "h m s", for example, "1h30m30s". changefeed-error-stuck-duration = "30m" +# The default value is false, indicating that bi-directional replication (BDR) mode is not enabled. +# To set up BDR clusters using TiCDC, modify this parameter to `true` and set the TiDB clusters to BDR mode. +# For more information, see https://docs.pingcap.com/tidb/stable/ticdc-bidirectional-replication. +# bdr-mode = false + [mounter] # The number of threads with which the mounter decodes KV data. The default value is 16. # worker-num = 16 @@ -100,7 +105,7 @@ rules = ['*.*', '!test.*'] # The second event filter rule. # matcher = ["test.fruit"] # matcher is an allow list, which means this rule only applies to the fruit table in the test database. -# ignore-event = ["drop table", "delete"] # Ignore the `drop table` DDL events and the `delete` DML events. +# ignore-event = ["drop table", "delete"] # Ignore the `drop table` DDL events and the `delete` DML events. Note that when a value in the clustered index column is updated in TiDB, TiCDC splits an `UPDATE` event into `DELETE` and `INSERT` events. TiCDC cannot identify such events as `UPDATE` events and thus cannot correctly filter out such events. # ignore-sql = ["^drop table", "alter table"] # Ignore DDL statements that start with `drop table` or contain `alter table`. # ignore-insert-value-expr = "price > 1000 and origin = 'no where'" # Ignore insert DMLs that contain the conditions "price > 1000" and "origin = 'no where'". @@ -128,7 +133,7 @@ enable-table-across-nodes = false # For example, if you specify the routing rule for a matcher as the string `code`, then all Pulsar messages that match that matcher will be routed with `code` as the key. # dispatchers = [ # {matcher = ['test1.*', 'test2.*'], topic = "Topic expression 1", partition = "index-value"}, -# {matcher = ['test3.*', 'test4.*'], topic = "Topic expression 2", partition = "index-value", index-name="index1"}, +# {matcher = ['test3.*', 'test4.*'], topic = "Topic expression 2", partition = "index-value", index = "index1"}, # {matcher = ['test1.*', 'test5.*'], topic = "Topic expression 3", partition = "table"}, # {matcher = ['test6.*'], partition = "columns", columns = "['a', 'b']"} # {matcher = ['test7.*'], partition = "ts"} @@ -187,6 +192,10 @@ date-separator = 'day' # Note: This configuration item only takes effect if the downstream is a storage service. enable-partition-separator = true +# Controls whether to disable the output of schema information. The default value is false, which means enabling the output of schema information. +# Note: This parameter only takes effect when the sink type is MQ and the output protocol is Debezium. +debezium-disable-schema = false + # Since v6.5.0, TiCDC supports saving data changes to storage services in CSV format. Ignore the following configurations if you replicate data to MQ or MySQL sinks. # [sink.csv] # The character used to separate fields in the CSV file. The value must be an ASCII character and defaults to `,`. @@ -204,6 +213,8 @@ enable-partition-separator = true # output-handle-key = false # Whether to output the value before the row data changes. The default value is false. # When it is enabled, the UPDATE event will output two rows of data: the first row is a DELETE event that outputs the data before the change; the second row is an INSERT event that outputs the changed data. +# When it is enabled (setting it to true), the "is-update" column will be added before the column with data changes. This added column is used to identify whether the data change of the current row comes from the UPDATE event or the original INSERT/DELETE event. +# If the data change of the current row comes from the UPDATE event, the value of the "is-update" column is true. Otherwise it is false. # output-old-value = false # Starting from v8.0.0, TiCDC supports the Simple message encoding protocol. The following are the configuration parameters for the Simple protocol. @@ -259,10 +270,10 @@ encoding-worker-num = 16 # The number of flushing workers in the redo module. # The default value is 8. flush-worker-num = 8 -# The behavior to compress redo log files. +# The behavior to compress redo log files (introduced in v6.5.6, v7.1.3, v7.5.1, and v7.6.0). # Available options are "" and "lz4". The default value is "", which means no compression. compression = "" -# The concurrency for uploading a single redo file. +# The concurrency for uploading a single redo file (introduced in v6.5.6, v7.1.3, v7.5.1, and v7.6.0). # The default value is 1, which means concurrency is disabled. flush-concurrency = 1 @@ -288,6 +299,10 @@ sasl-oauth-scopes = ["producer.kafka", "consumer.kafka"] sasl-oauth-grant-type = "client_credentials" # The audience in the Kafka SASL OAUTHBEARER authentication. The default value is empty. This parameter is optional when the OAUTHBEARER authentication is used. sasl-oauth-audience = "kafka" + +# The following configuration item controls whether to output the original data change event. The default value is false. For more information, see https://docs.pingcap.com/tidb/v8.1/ticdc-split-update-behavior#control-whether-to-split-primary-or-unique-key-update-events. +# output-raw-change-event = false + # The following configuration is only required when using Avro as the protocol and AWS Glue Schema Registry: # Please refer to the section "Integrate TiCDC with AWS Glue Schema Registry" in the document "Sync Data to Kafka": https://docs.pingcap.com/tidb/dev/ticdc-sink-to-kafka#integrate-ticdc-with-aws-glue-schema-registry # [sink.kafka-config.glue-schema-registry-config] @@ -338,6 +353,9 @@ batching-max-publish-delay=10 # The timeout for a Pulsar producer to send a message. The value is 30 seconds by default. send-timeout=30 +# The following configuration item controls whether to output the original data change event. The default value is false. For more information, see https://docs.pingcap.com/tidb/v8.1/ticdc-split-update-behavior#control-whether-to-split-primary-or-unique-key-update-events. +# output-raw-change-event = false + [sink.cloud-storage-config] # The concurrency for saving data changes to the downstream cloud storage. # The default value is 16. @@ -357,4 +375,6 @@ file-cleanup-cron-spec = "0 0 2 * * *" # The concurrency for uploading a single file. # The default value is 1, which means concurrency is disabled. flush-concurrency = 1 +# The following configuration item controls whether to output the original data change event. The default value is false. For more information, see https://docs.pingcap.com/tidb/v8.1/ticdc-split-update-behavior#control-whether-to-split-primary-or-unique-key-update-events. +output-raw-change-event = false ``` diff --git a/ticdc/ticdc-changefeed-overview.md b/ticdc/ticdc-changefeed-overview.md index 0c63f6f60f357..b7478591d7a4d 100644 --- a/ticdc/ticdc-changefeed-overview.md +++ b/ticdc/ticdc-changefeed-overview.md @@ -44,4 +44,4 @@ You can manage a TiCDC cluster and its replication tasks using the command-line You can also use the HTTP interface (the TiCDC OpenAPI feature) to manage a TiCDC cluster and its replication tasks. For details, see [TiCDC OpenAPI](/ticdc/ticdc-open-api.md). -If your TiCDC is deployed using TiUP, you can start `cdc cli` by running the `tiup cdc:v cli` command. Replace `v` with the TiCDC cluster version, such as `v8.0.0`. You can also run `cdc cli` directly. +If your TiCDC is deployed using TiUP, you can start `cdc cli` by running the `tiup cdc:v cli` command. Replace `v` with the TiCDC cluster version, such as `v8.1.2`. You can also run `cdc cli` directly. diff --git a/ticdc/ticdc-compatibility.md b/ticdc/ticdc-compatibility.md index f5e37e75a20f7..e969b1fbe3b96 100644 --- a/ticdc/ticdc-compatibility.md +++ b/ticdc/ticdc-compatibility.md @@ -7,15 +7,33 @@ summary: Learn about compatibility issues of TiCDC and how to handle them. This section describes compatibility issues related to TiCDC and how to handle them. - +1. Create a changefeed. For more information, see [Create a replication task](/ticdc/ticdc-manage-changefeed.md#create-a-replication-task). +2. Start TiDB Lightning and import data using the logical import mode. For more information, see [Use logical import mode](/tidb-lightning/tidb-lightning-logical-import-mode-usage.md). + +In the physical import mode, TiDB Lightning imports data by inserting SST files into TiKV. TiCDC is not compatible with this mode and does not support replicating data imported through physical import mode. If you need to use both TiDB Lightning's physical import mode and TiCDC, choose one of the following solutions based on your downstream system: + +- If the downstream is a TiDB cluster, perform the following steps: + + 1. Use TiDB Lightning to import data into both the upstream and downstream TiDB clusters to ensure data consistency. + 2. Create a changefeed to replicate subsequent incremental data written through SQL. For more information, see [Create a replication task](/ticdc/ticdc-manage-changefeed.md#create-a-replication-task). + +- If the downstream is not a TiDB cluster, perform the following steps: + + 1. Use the offline import tool provided by your downstream system to import TiDB Lightning's input files. + 2. Create a changefeed to replicate subsequent incremental data written through SQL. For more information, see [Create a replication task](/ticdc/ticdc-manage-changefeed.md#create-a-replication-task). + +## Compatibility with TiFlash + +Currently, when you use TiCDC to replicate tables to a downstream TiDB cluster, creating TiFlash replicas for the tables is not supported, which means that TiCDC does not support replicating TiFlash-related DDL statements, such as: + +* `ALTER TABLE table_name SET TIFLASH REPLICA count;` +* `ALTER DATABASE db_name SET TIFLASH REPLICA count;` ## CLI and configuration file compatibility diff --git a/ticdc/ticdc-csv.md b/ticdc/ticdc-csv.md index a55c78b89e693..12278eec59ca0 100644 --- a/ticdc/ticdc-csv.md +++ b/ticdc/ticdc-csv.md @@ -27,6 +27,7 @@ delimiter = ',' # Before v7.6.0, you can only set the delimiter to a single char quote = '"' null = '\N' include-commit-ts = true +output-old-value = false ``` ## Transactional constraints @@ -47,7 +48,8 @@ In the CSV file, each column is defined as follows: - Column 2: Table name. - Column 3: Schema name. - Column 4: The `commit-ts` of the source transaction. This column is optional. -- Column 5 to the last column: One or more columns that represent data to be changed. +- Column 5: The `is-update` column only exists when the value of `output-old-value` is true, which is used to identify whether the row data change comes from the UPDATE event (the value of the column is true) or the INSERT/DELETE event (the value is false). +- Column 6 to the last column: One or more columns with data changes. Assume that table `hr.employee` is defined as follows: @@ -61,7 +63,7 @@ CREATE TABLE `employee` ( ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; ``` -The DML events of this table are stored in the CSV format as follows: +When `include-commit-ts = true` and `output-old-value = false`, the DML events of this table are stored in the CSV format as follows: ```shell "I","employee","hr",433305438660591626,101,"Smith","Bob","2014-06-04","New York" @@ -71,6 +73,18 @@ The DML events of this table are stored in the CSV format as follows: "U","employee","hr",433305438660591630,102,"Alex","Alice","2018-06-15","Beijing" ``` +When `include-commit-ts = true` and `output-old-value = true`, the DML events of this table are stored in the CSV format as follows: + +``` +"I","employee","hr",433305438660591626,false,101,"Smith","Bob","2014-06-04","New York" +"D","employee","hr",433305438660591627,true,101,"Smith","Bob","2015-10-08","Shanghai" +"I","employee","hr",433305438660591627,true,101,"Smith","Bob","2015-10-08","Los Angeles" +"D","employee","hr",433305438660591629,false,101,"Smith","Bob","2017-03-13","Dallas" +"I","employee","hr",433305438660591630,false,102,"Alex","Alice","2017-03-14","Shanghai" +"D","employee","hr",433305438660591630,true,102,"Alex","Alice","2017-03-14","Beijing" +"I","employee","hr",433305438660591630,true,102,"Alex","Alice","2018-06-15","Beijing" +``` + ## Data type mapping | MySQL type | CSV type | Example | Description | diff --git a/ticdc/ticdc-ddl.md b/ticdc/ticdc-ddl.md index 6e6dbbd3ae470..e966009092a01 100644 --- a/ticdc/ticdc-ddl.md +++ b/ticdc/ticdc-ddl.md @@ -50,6 +50,8 @@ The allow list of DDL statements supported by TiCDC is as follows: When the downstream is TiDB, TiCDC executes `ADD INDEX` and `CREATE INDEX` DDL operations asynchronously to minimize the impact on changefeed replication latency. This means that, after replicating `ADD INDEX` and `CREATE INDEX` DDLs to the downstream TiDB for execution, TiCDC returns immediately without waiting for the completion of the DDL execution. This avoids blocking subsequent DML executions. +During the execution of the `ADD INDEX` or `CREATE INDEX` DDL operation in the downstream, when TiCDC executes the next DDL operation of the same table, this DDL operation might be blocked in the `queueing` state for a long time. This can cause TiCDC to repeatedly execute this DDL operation, and if retries take too long, it might lead to replication task failure. Starting from v8.1.2, if TiCDC has the `SUPER` permission of the downstream database, it periodically runs `ADMIN SHOW DDL JOBS` to check the status of asynchronously executed DDL tasks. TiCDC will wait for index creation to complete before proceeding with replication. Although this might increase replication latency, it avoids replication task failure. + > **Note:** > > - If the execution of certain downstream DMLs relies on indexes that have not completed replication, these DMLs might be executed slowly, thereby affecting TiCDC replication latency. @@ -77,12 +79,12 @@ TiCDC processes this type of DDL as follows: | `RENAME TABLE test.t1 TO test.t2` | Replicate | `test.t1` matches the filter rule | | `RENAME TABLE test.t1 TO ignore.t1` | Replicate | `test.t1` matches the filter rule | | `RENAME TABLE ignore.t1 TO ignore.t2` | Ignore | `ignore.t1` does not match the filter rule | -| `RENAME TABLE test.n1 TO test.t1` | Report an error and exit the replication | `test.n1` does not match the filter rule, but `test.t1` matches the filter rule. This operation is illegal. In this case, refer to the error message for handling. | +| `RENAME TABLE test.n1 TO test.t1` | Report an error and exit the replication | The old table name `test.n1` does not match the filter rule, but the new table name `test.t1` matches the filter rule. This operation is illegal. In this case, refer to the error message for handling. | | `RENAME TABLE ignore.t1 TO test.t1` | Report an error and exit the replication | Same reason as above. | #### Rename multiple tables in a DDL statement -If a DDL statement renames multiple tables, TiCDC only replicates the DDL statement when the old database name, old table names, and the new database name all match the filter rule. +If a DDL statement renames multiple tables, TiCDC replicates the DDL statement only when the **old database name**, **old table names**, and **new database name** all match the filter rule. In addition, TiCDC does not support the `RENAME TABLE` DDL that swaps the table names. The following is an example. @@ -106,25 +108,32 @@ TiCDC processes this type of DDL as follows: When executing cross-database DDL statements (such as `CREATE TABLE db1.t1 LIKE t2`) in the upstream, it is recommended that you explicitly specify all relevant database names in DDL statements (such as `CREATE TABLE db1.t1 LIKE db2.t2`). Otherwise, cross-database DDL statements might not be executed correctly in the downstream due to the lack of database name information. -### SQL mode +### Notes on using event filter rules to filter DDL events -By default, TiCDC uses the default SQL mode of TiDB to parse DDL statements. If your upstream TiDB cluster uses a non-default SQL mode, you must specify the SQL mode in the TiCDC configuration file. Otherwise, TiCDC might fail to parse DDL statements correctly. For more information about TiDB SQL mode, see [SQL Mode](/sql-mode.md). +If a filtered DDL statement involves table creation or deletion, TiCDC only filters out the DDL statement without affecting the replication behavior of DML statements. The following is an example. -For example, if the upstream TiDB cluster uses the `ANSI_QUOTES` mode, you must specify the SQL mode in the changefeed configuration file as follows: +Assume that the configuration file of your changefeed is as follows: ```toml -# In the value, "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION" is the default SQL mode of TiDB. -# "ANSI_QUOTES" is the SQL mode added to your upstream TiDB cluster. - -sql-mode = "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION,ANSI_QUOTES" -``` - -If the SQL mode is not configured, TiCDC might fail to parse some DDL statements correctly. For example: +[filter] +rules = ['test.t*'] -```sql -CREATE TABLE "t1" ("a" int PRIMARY KEY); +[[filter.event-filters]] +matcher = ["test.t1"] # This filter rule applies only to the t1 table in the test database. +ignore-event = ["create table", "drop table", "truncate table", "rename table"] ``` -Because in the default SQL mode of TiDB, double quotation marks are treated as strings rather than identifiers, TiCDC fails to parse the DDL statement correctly. +| DDL | DDL behavior | DML behavior | Explanation | +| --- | --- | --- | --- | +| `CREATE TABLE test.t1 (id INT, name VARCHAR(50));` | Ignore | Replicate | `test.t1` matches the event filter rule, so the `CREATE TABLE` event is ignored. The replication of DML events remains unaffected. | +| `CREATE TABLE test.t2 (id INT, name VARCHAR(50));` | Replicate | Replicate | `test.t2` does not match the event filter rule. | +| `CREATE TABLE test.ignore (id INT, name VARCHAR(50));` | Ignore | Ignore | `test.ignore` matches the event filter rule, so both DDL and DML events are ignored. | +| `DROP TABLE test.t1;` | Ignore | - | `test.t1` matches the event filter rule, so the `DROP TABLE` event is ignored. Because the table is deleted, TiCDC no longer replicates DML events for `t1`. | +| `TRUNCATE TABLE test.t1;` | Ignore | Replicate | `test.t1` matches the event filter rule, so the `TRUNCATE TABLE` event is ignored. The replication of DML events remains unaffected. | +| `RENAME TABLE test.t1 TO test.t2;` | Ignore | Replicate | `test.t1` matches the event filter rule, so the `RENAME TABLE` event is ignored. The replication of DML events remains unaffected. | +| `RENAME TABLE test.t1 TO test.ignore;` | Ignore | Ignore | `test.t1` matches the event filter rule, so the `RENAME TABLE` event is ignored. `test.ignore` matches the event filter rule, so both DDL and DML events are ignored. | -Therefore, when creating a replication task, it is recommended that you specify the SQL mode used by the upstream TiDB cluster in the configuration file. +> **Note:** +> +> - When replicating data to a database, use the event filter to filter DDL events with caution. Ensure that the upstream and downstream database schemas remain consistent during replication. Otherwise, TiCDC might report errors or cause undefined replication behavior. +> - For versions earlier than v6.5.8, v7.1.4, and v7.5.1, using the event filter to filter DDL events involving table creation or deletion affects DML replication. It is not recommended to use this feature in these versions. diff --git a/ticdc/ticdc-faq.md b/ticdc/ticdc-faq.md index 474ca3616cb7e..b6fb8e1e49baa 100644 --- a/ticdc/ticdc-faq.md +++ b/ticdc/ticdc-faq.md @@ -55,6 +55,111 @@ The expected output is as follows: > > This feature is introduced in TiCDC 4.0.3. +## How to verify if TiCDC has replicated all updates after upstream stops updating? + +After the upstream TiDB cluster stops updating, you can verify if replication is complete by comparing the latest [TSO](/glossary.md#tso) timestamp of the upstream TiDB cluster with the replication progress in TiCDC. If the TiCDC replication progress timestamp is greater than or equal to the upstream TiDB cluster's TSO, then all updates have been replicated. To verify replication completeness, perform the following steps: + +1. Get the latest TSO timestamp from the upstream TiDB cluster. + + > **Note:** + > + > Use the [`TIDB_CURRENT_TSO()`](/functions-and-operators/tidb-functions.md#tidb_current_tso) function to get the current TSO, instead of using functions like `NOW()` that return the current time. + + The following example uses [`TIDB_PARSE_TSO()`](/functions-and-operators/tidb-functions.md#tidb_parse_tso) to convert the TSO to a readable time format for further comparison: + + ```sql + BEGIN; + SELECT TIDB_PARSE_TSO(TIDB_CURRENT_TSO()); + ROLLBACK; + ``` + + The output is as follows: + + ```sql + +------------------------------------+ + | TIDB_PARSE_TSO(TIDB_CURRENT_TSO()) | + +------------------------------------+ + | 2024-11-12 20:35:34.848000 | + +------------------------------------+ + ``` + +2. Get the replication progress in TiCDC. + + You can check the replication progress in TiCDC using one of the following methods: + + * **Method 1**: query the checkpoint of the changefeed (recommended). + + Use the [TiCDC command-line tool](/ticdc/ticdc-manage-changefeed.md) `cdc cli` to view the checkpoint for all replication tasks: + + ```shell + cdc cli changefeed list --server=http://127.0.0.1:8300 + ``` + + The output is as follows: + + ```json + [ + { + "id": "syncpoint", + "namespace": "default", + "summary": { + "state": "normal", + "tso": 453880043653562372, + "checkpoint": "2024-11-12 20:36:01.447", + "error": null + } + } + ] + ``` + + In the output, `"checkpoint": "2024-11-12 20:36:01.447"` indicates that TiCDC has replicated all upstream TiDB changes before this time. If this timestamp is greater than or equal to the upstream TiDB cluster's TSO obtained in step 1, then all updates have been replicated downstream. + + * **Method 2**: query Syncpoint from the downstream TiDB. + + If the downstream is a TiDB cluster and the [TiCDC Syncpoint feature](/ticdc/ticdc-upstream-downstream-check.md) is enabled, you can get the replication progress by querying the Syncpoint in the downstream TiDB. + + > **Note:** + > + > The Syncpoint update interval is controlled by the [`sync-point-interval`](/ticdc/ticdc-upstream-downstream-check.md#enable-syncpoint) configuration item. For the most up-to-date replication progress, use method 1. + + Execute the following SQL statement in the downstream TiDB to get the upstream TSO (`primary_ts`) and downstream TSO (`secondary_ts`): + + ```sql + SELECT * FROM tidb_cdc.syncpoint_v1; + ``` + + The output is as follows: + + ```sql + +------------------+------------+--------------------+--------------------+---------------------+ + | ticdc_cluster_id | changefeed | primary_ts | secondary_ts | created_at | + +------------------+------------+--------------------+--------------------+---------------------+ + | default | syncpoint | 453879870259200000 | 453879870545461257 | 2024-11-12 20:25:01 | + | default | syncpoint | 453879948902400000 | 453879949214351361 | 2024-11-12 20:30:01 | + | default | syncpoint | 453880027545600000 | 453880027751907329 | 2024-11-12 20:35:00 | + +------------------+------------+--------------------+--------------------+---------------------+ + ``` + + In the output, each row shows the upstream TiDB snapshot at `primary_ts` matches the downstream TiDB snapshot at `secondary_ts`. + + To view the replication progress, convert the latest `primary_ts` to a readable time format: + + ```sql + SELECT TIDB_PARSE_TSO(453880027545600000); + ``` + + The output is as follows: + + ```sql + +------------------------------------+ + | TIDB_PARSE_TSO(453880027545600000) | + +------------------------------------+ + | 2024-11-12 20:35:00 | + +------------------------------------+ + ``` + + If the time corresponding to the latest `primary_ts` is greater than or equal to the upstream TiDB cluster's TSO obtained in step 1, then TiCDC has replicated all updates downstream. + ## What is `gc-ttl` in TiCDC? Since v4.0.0-rc.1, PD supports external services in setting the service-level GC safepoint. Any service can register and update its GC safepoint. PD ensures that the key-value data later than this GC safepoint is not cleaned by GC. @@ -187,7 +292,15 @@ For more information, refer to [Open protocol Row Changed Event format](/ticdc/t ## How much PD storage does TiCDC use? -TiCDC uses etcd in PD to store and regularly update the metadata. Because the time interval between the MVCC of etcd and PD's default compaction is one hour, the amount of PD storage that TiCDC uses is proportional to the amount of metadata versions generated within this hour. However, in v4.0.5, v4.0.6, and v4.0.7, TiCDC has a problem of frequent writing, so if there are 1000 tables created or scheduled in an hour, it then takes up all the etcd storage and returns the `etcdserver: mvcc: database space exceeded` error. You need to clean up the etcd storage after getting this error. See [etcd maintenance space-quota](https://etcd.io/docs/v3.4.0/op-guide/maintenance/#space-quota) for details. It is recommended to upgrade your cluster to v4.0.9 or later versions. +When using TiCDC, you might encounter the `etcdserver: mvcc: database space exceeded` error, which is primarily related to the mechanism that TiCDC uses etcd in PD to store metadata. + +etcd uses Multi-Version Concurrency Control (MVCC) to store data, and the default compaction interval in PD is 1 hour. This means that etcd retains multiple versions of all data for 1 hour before compaction. + +Before v6.0.0, TiCDC uses etcd in PD to store and update metadata for all tables in a changefeed. Therefore, the PD storage space used by TiCDC is proportional to the number of tables being replicated by the changefeed. When TiCDC is replicating a large number of tables, the etcd storage space could fill up quickly, increasing the probability of the `etcdserver: mvcc: database space exceeded` error. + +If you encounter this error, refer to [etcd maintenance space-quota](https://etcd.io/docs/v3.4.0/op-guide/maintenance/#space-quota) to clean up the etcd storage space. + +Starting from v6.0.0, TiCDC optimizes its metadata storage mechanism, effectively avoiding the etcd storage space issues caused by the preceding reasons. If your TiCDC version is earlier than v6.0.0, it is recommended to upgrade to v6.0.0 or later versions. ## Does TiCDC support replicating large transactions? Is there any risk? @@ -248,23 +361,9 @@ Since v5.0.1 or v4.0.13, for each replication to MySQL, TiCDC automatically sets TiCDC guarantees that all data is replicated at least once. When there is duplicate data in the downstream, write conflicts occur. To avoid this problem, TiCDC converts `INSERT` and `UPDATE` statements into `REPLACE INTO` statements. This behavior is controlled by the `safe-mode` parameter. -In versions earlier than v6.1.3, `safe-mode` defaults to `true`, which means all `INSERT` and `UPDATE` statements are converted into `REPLACE INTO` statements. In v6.1.3 and later versions, TiCDC can automatically determine whether the downstream has duplicate data, and the default value of `safe-mode` changes to `false`. If no duplicate data is detected, TiCDC replicates `INSERT` and `UPDATE` statements without conversion. - -## When the sink of the replication downstream is TiDB or MySQL, what permissions do users of the downstream database need? - -When the sink is TiDB or MySQL, the users of the downstream database need the following permissions: +In versions earlier than v6.1.3, the default value of `safe-mode` is `true`, which means all `INSERT` and `UPDATE` statements are converted into `REPLACE INTO` statements. -- `Select` -- `Index` -- `Insert` -- `Update` -- `Delete` -- `Create` -- `Drop` -- `Alter` -- `Create View` - -If you need to replicate `recover table` to the downstream TiDB, you should have the `Super` permission. +In v6.1.3 and later versions, the default value of `safe-mode` changes to `false`, and TiCDC can automatically determine whether the downstream has duplicate data. If no duplicate data is detected, TiCDC directly replicates `INSERT` and `UPDATE` statements without conversion; otherwise, TiCDC converts `INSERT` and `UPDATE` statements into `REPLACE INTO` statements and then replicates them. ## Why does TiCDC use disks? When does TiCDC write to disks? Does TiCDC use memory buffer to improve replication performance? @@ -298,9 +397,9 @@ For TiCDC versions earlier than v6.5.2, it is recommended that you deploy TiCDC Currently, TiCDC adopts the following order: -1. TiCDC blocks the replication progress of the tables affected by DDL statements until the DDL `CommiTs`. This ensures that DML statements executed before DDL `CommiTs` can be successfully replicated to the downstream first. +1. TiCDC blocks the replication progress of the tables affected by DDL statements until the DDL `commitTS`. This ensures that DML statements executed before DDL `commitTS` can be successfully replicated to the downstream first. 2. TiCDC continues with the replication of DDL statements. If there are multiple DDL statements, TiCDC replicates them in a serial manner. -3. After the DDL statements are executed in the downstream, TiCDC will continue with the replication of DML statements executed after DDL `CommiTs`. +3. After the DDL statements are executed in the downstream, TiCDC will continue with the replication of DML statements executed after DDL `commitTS`. ## How should I check whether the upstream and downstream data is consistent? @@ -348,3 +447,55 @@ This is because the default port number of the TiCDC cluster deployed by TiDB Op } ] ``` + +## Does TiCDC replicate generated columns of DML operations? + +Generated columns include virtual generated columns and stored generated columns. TiCDC ignores virtual generated columns and only replicates stored generated columns to the downstream. However, stored generated columns are also ignored when the downstream is MySQL or another MySQL-compatible database (rather than Kafka or other storage services). + +> **Note:** +> +> When replicating stored generated columns to Kafka or a storage service and then writing them back to MySQL, `Error 3105 (HY000): The value specified for generated column 'xx' in table 'xxx' is not allowed` might occur. To avoid this error, you can use [Open Protocol](/ticdc/ticdc-open-protocol.md#ticdc-open-protocol) for replication. The output of this protocol includes [bit flags of columns](/ticdc/ticdc-open-protocol.md#bit-flags-of-columns), which can distinguish whether a column is a generated column. + +## How do I resolve frequent `CDC:ErrMySQLDuplicateEntryCDC` errors? + +When using TiCDC to replicate data to TiDB or MySQL, you might encounter the following error if SQL statements in the upstream are executed in a specific pattern: + +`CDC:ErrMySQLDuplicateEntryCDC` + +The cause of the error: TiDB combines `DELETE + INSERT` operations on the same row within the same transaction into a single `UPDATE` row change. When TiCDC replicates these changes as updates to the downstream, the `UPDATE` operations attempting to swap unique key values might result in conflicts. + +Taking the following table as an example: + +```sql +CREATE TABLE data_table ( + id BIGINT(20) NOT NULL PRIMARY KEY, + value BINARY(16) NOT NULL, + UNIQUE KEY value_index (value) +) CHARSET=utf8mb4 COLLATE=utf8mb4_bin; +``` + +If the upstream attempts to swap the `value` field of the two rows in the table: + +```sql +DELETE FROM data_table WHERE id = 1; +DELETE FROM data_table WHERE id = 2; +INSERT INTO data_table (id, value) VALUES (1, 'v3'); +INSERT INTO data_table (id, value) VALUES (2, 'v1'); +``` + +TiDB generates two `UPDATE` row changes, so TiCDC converts them into two `UPDATE` statements for replication to the downstream: + +```sql +UPDATE data_table SET value = 'v3' WHERE id = 1; +UPDATE data_table SET value = 'v1' WHERE id = 2; +``` + +If the downstream table still contains `v1` when executing the second `UPDATE` statement, it violates the unique key constraint on the `value` column, resulting in the `CDC:ErrMySQLDuplicateEntryCDC` error. + +If the `CDC:ErrMySQLDuplicateEntryCDC` error occurs frequently, you can enable TiCDC safe mode by setting the `safe-mode=true` parameter in the [`sink-uri`](/ticdc/ticdc-sink-to-mysql.md#configure-sink-uri-for-mysql-or-tidb) configuration: + +``` +mysql://user:password@host:port/?safe-mode=true +``` + +In safe mode, TiCDC splits the `UPDATE` operation into `DELETE + REPLACE INTO` for execution, thus avoiding the unique key conflict error. \ No newline at end of file diff --git a/ticdc/ticdc-glossary.md b/ticdc/ticdc-glossary.md index b4debae89e9b9..1003a0d8b5596 100644 --- a/ticdc/ticdc-glossary.md +++ b/ticdc/ticdc-glossary.md @@ -5,7 +5,7 @@ summary: Learn the terms about TiCDC and their definitions. # TiCDC Glossary -This glossary provides TiCDC-related terms and definitions. These terms appears in TiCDC logs, monitoring metrics, configurations, and documents. +This glossary provides TiCDC-related terms and definitions. These terms appear in TiCDC logs, monitoring metrics, configurations, and documents. For TiDB-related terms and definitions, refer to [TiDB glossary](/glossary.md). diff --git a/ticdc/ticdc-manage-changefeed.md b/ticdc/ticdc-manage-changefeed.md index 8169d319b669d..ec8a033e33588 100644 --- a/ticdc/ticdc-manage-changefeed.md +++ b/ticdc/ticdc-manage-changefeed.md @@ -1,7 +1,6 @@ --- title: Manage Changefeeds summary: Learn how to manage TiCDC changefeeds. -aliases: ['/tidb/dev/manage-ticdc'] --- # Manage Changefeeds @@ -19,7 +18,7 @@ cdc cli changefeed create --server=http://10.0.10.25:8300 --sink-uri="mysql://ro ```shell Create changefeed successfully! ID: simple-replication-task -Info: {"upstream_id":7178706266519722477,"namespace":"default","id":"simple-replication-task","sink_uri":"mysql://root:xxxxx@127.0.0.1:4000/?time-zone=","create_time":"2024-03-28T15:05:46.679218+08:00","start_ts":438156275634929669,"engine":"unified","config":{"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":3600000000000,"filter":{"rules":["test.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v8.0.0"} +Info: {"upstream_id":7178706266519722477,"namespace":"default","id":"simple-replication-task","sink_uri":"mysql://root:xxxxx@127.0.0.1:4000/?time-zone=","create_time":"2024-12-26T15:05:46.679218+08:00","start_ts":438156275634929669,"engine":"unified","config":{"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":3600000000000,"filter":{"rules":["test.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v8.1.2"} ``` ## Query the replication task list @@ -136,7 +135,7 @@ cdc cli changefeed query --server=http://10.0.10.25:8300 --changefeed-id=simple- } ``` -In the preceding command and result: +In the preceding command and result: - `info` is the replication configuration of the queried changefeed. - `status` is the replication state of the queried changefeed. @@ -170,7 +169,7 @@ cdc cli changefeed resume --server=http://10.0.10.25:8300 --changefeed-id simple ``` - `--changefeed-id=uuid` represents the ID of the changefeed that corresponds to the replication task you want to resume. -- `--overwrite-checkpoint-ts`: starting from v6.2.0, you can specify the starting TSO of resuming the replication task. TiCDC starts pulling data from the specified TSO. The argument accepts `now` or a specific TSO (such as 434873584621453313). The specified TSO must be in the range of (GC safe point, CurrentTSO]. If this argument is not specified, TiCDC replicates data from the current `checkpoint-ts` by default. +- `--overwrite-checkpoint-ts`: starting from v6.2.0, you can specify the starting TSO of resuming the replication task. TiCDC starts pulling data from the specified TSO. The argument accepts `now` or a specific TSO (such as 434873584621453313). The specified TSO must be in the range of (GC safe point, CurrentTSO]. If this argument is not specified, TiCDC replicates data from the current `checkpoint-ts` by default. You can use the `cdc cli changefeed list` command to check the current value of `checkpoint-ts`. - `--no-confirm`: when the replication is resumed, you do not need to confirm the related information. Defaults to `false`. > **Note:** diff --git a/ticdc/ticdc-open-api-v2.md b/ticdc/ticdc-open-api-v2.md index a977601f078e6..43320d0961154 100644 --- a/ticdc/ticdc-open-api-v2.md +++ b/ticdc/ticdc-open-api-v2.md @@ -92,7 +92,7 @@ curl -X GET http://127.0.0.1:8300/api/v2/status ```json { - "version": "v8.0.0", + "version": "v8.1.2", "git_hash": "10413bded1bdb2850aa6d7b94eb375102e9c44dc", "id": "d2912e63-3349-447c-90ba-72a4e04b5e9e", "pid": 1447, @@ -310,7 +310,7 @@ The `sink` parameters are described as follows: | `date_separator` | `STRING` type. Indicates the date separator type of the file directory. Value options are `none`, `year`, `month`, and `day`. `none` is the default value and means that the date is not separated. (Optional) | | `dispatchers` | An configuration array for event dispatching. (Optional) | | `encoder_concurrency` | `INT` type. The number of encoder threads in the MQ sink. The default value is `16`. (Optional) | -| `protocol` | `STRING` type. For MQ sinks, you can specify the protocol format of the message. The following protocols are currently supported: `canal-json`, `open-protocol`, `avro`, and `maxwell`. | +| `protocol` | `STRING` type. For MQ sinks, you can specify the protocol format of the message. The following protocols are currently supported: `canal-json`, `open-protocol`, `avro`, `debezium`, and `simple`. | | `schema_registry` | `STRING` type. The schema registry address. (Optional) | | `terminator` | `STRING` type. The terminator is used to separate two data change events. The default value is null, which means `"\r\n"` is used as the terminator. (Optional) | | `transaction_atomicity` | `STRING` type. The atomicity level of the transaction. (Optional) | @@ -361,6 +361,7 @@ The `sink.csv` parameters are described as follows: | `file_expiration_days` | `INT` type. The duration to retain files, which takes effect only when `date-separator` is configured as `day`. | | `file_cleanup_cron_spec` | `STRING` type. The running cycle of the scheduled cleanup task, compatible with the crontab configuration, with a format of ` `. | | `flush_concurrency` | `INT` type. The concurrency for uploading a single file. | +| `output_raw_change_event` | `BOOLEAN` type. Controls whether to output the original data change event for a non-MySQL sink. | `sink.open` parameters are described as follows: @@ -772,7 +773,7 @@ This API is a synchronous interface. If the request is successful, it returns th ### Request URI -`GET /api/v2/changefeed/{changefeed_id}/synced` +`GET /api/v2/changefeeds/{changefeed_id}/synced` ### Parameter description @@ -787,7 +788,7 @@ This API is a synchronous interface. If the request is successful, it returns th The following request queries the synchronization status of the replication task with the ID `test1`. ```shell -curl -X GET http://127.0.0.1:8300/api/v2/changefeed/test1/synced +curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test1/synced ``` **Example 1: The synchronization is completed** diff --git a/ticdc/ticdc-open-api.md b/ticdc/ticdc-open-api.md index 789baa924a791..4bda851c12169 100644 --- a/ticdc/ticdc-open-api.md +++ b/ticdc/ticdc-open-api.md @@ -163,7 +163,7 @@ The configuration parameters of sink are as follows: `matcher`: The matching syntax of matcher is the same as the filter rule syntax. -`protocol`: For the sink of MQ type, you can specify the protocol format of the message. Currently the following protocols are supported: `canal-json`, `open-protocol`, `avro`, and `maxwell`. +`protocol`: For the sink of MQ type, you can specify the protocol format of the message. Currently the following protocols are supported: `canal-json`, `open-protocol`, `avro`, `debezium`, and `simple`. ### Example diff --git a/ticdc/ticdc-open-protocol.md b/ticdc/ticdc-open-protocol.md index 7e345531e09b6..94c326031aa0c 100644 --- a/ticdc/ticdc-open-protocol.md +++ b/ticdc/ticdc-open-protocol.md @@ -1,7 +1,6 @@ --- title: TiCDC Open Protocol summary: Learn the concept of TiCDC Open Protocol and how to use it. -aliases: ['/docs/dev/ticdc/ticdc-open-protocol/','/docs/dev/reference/tools/ticdc/open-protocol/','/docs/dev/ticdc/column-ddl-type-codes/'] --- # TiCDC Open Protocol @@ -241,10 +240,10 @@ COMMIT; + Log 8 is a repeated event of Log 7. Row Changed Event might be repeated, but the first Event of each version is sent orderly. ``` -5. [partition=0] [key="{\"ts\":415508878783938562,\"scm\":\"test\",\"tbl\":\"t1\",\"t\":1}"] [value="{\"u\":{\"id\":{\"t\":3,\"h\":true,\"v\":1},\"val\":{\"t\":15,\"v\":\"YWE=\"}}}"] -6. [partition=1] [key="{\"ts\":415508878783938562,\"scm\":\"test\",\"tbl\":\"t1\",\"t\":1}"] [value="{\"u\":{\"id\":{\"t\":3,\"h\":true,\"v\":2},\"val\":{\"t\":15,\"v\":\"YmI=\"}}}"] -7. [partition=0] [key="{\"ts\":415508878783938562,\"scm\":\"test\",\"tbl\":\"t1\",\"t\":1}"] [value="{\"u\":{\"id\":{\"t\":3,\"h\":true,\"v\":3},\"val\":{\"t\":15,\"v\":\"Y2M=\"}}}"] -8. [partition=0] [key="{\"ts\":415508878783938562,\"scm\":\"test\",\"tbl\":\"t1\",\"t\":1}"] [value="{\"u\":{\"id\":{\"t\":3,\"h\":true,\"v\":3},\"val\":{\"t\":15,\"v\":\"Y2M=\"}}}"] +5. [partition=0] [key="{\"ts\":415508878783938562,\"scm\":\"test\",\"tbl\":\"t1\",\"t\":1}"] [value="{\"u\":{\"id\":{\"t\":3,\"h\":true,\"v\":1},\"val\":{\"t\":15,\"v\":\"aa\"}}}"] +6. [partition=1] [key="{\"ts\":415508878783938562,\"scm\":\"test\",\"tbl\":\"t1\",\"t\":1}"] [value="{\"u\":{\"id\":{\"t\":3,\"h\":true,\"v\":2},\"val\":{\"t\":15,\"v\":\"bb\"}}}"] +7. [partition=0] [key="{\"ts\":415508878783938562,\"scm\":\"test\",\"tbl\":\"t1\",\"t\":1}"] [value="{\"u\":{\"id\":{\"t\":3,\"h\":true,\"v\":3},\"val\":{\"t\":15,\"v\":\"cc\"}}}"] +8. [partition=0] [key="{\"ts\":415508878783938562,\"scm\":\"test\",\"tbl\":\"t1\",\"t\":1}"] [value="{\"u\":{\"id\":{\"t\":3,\"h\":true,\"v\":3},\"val\":{\"t\":15,\"v\":\"cc\"}}}"] ``` Execute the following SQL statements in the upstream: @@ -275,8 +274,8 @@ COMMIT; Currently, TiCDC does not provide the standard parsing library for TiCDC Open Protocol, but the Golang version and Java version of parsing examples are provided. You can refer to the data format provided in this document and the following examples to implement the protocol parsing for consumers. -- [Golang examples](https://github.com/pingcap/tiflow/tree/master/cmd/kafka-consumer) -- [Java examples](https://github.com/pingcap/tiflow/tree/master/examples/java) +- [Golang examples](https://github.com/pingcap/tiflow/tree/release-8.1/cmd/kafka-consumer) +- [Java examples](https://github.com/pingcap/tiflow/tree/release-8.1/examples/java) ## Column type code diff --git a/ticdc/ticdc-overview.md b/ticdc/ticdc-overview.md index c9ad339ac7795..e025151bccfbc 100644 --- a/ticdc/ticdc-overview.md +++ b/ticdc/ticdc-overview.md @@ -1,12 +1,11 @@ --- title: TiCDC Overview summary: Learn what TiCDC is, what features TiCDC provides, and how to install and deploy TiCDC. -aliases: ['/docs/dev/ticdc/ticdc-overview/','/docs/dev/reference/tools/ticdc/overview/'] --- # TiCDC Overview -[TiCDC](https://github.com/pingcap/tiflow/tree/master/cdc) is a tool used to replicate incremental data from TiDB. Specifically, TiCDC pulls TiKV change logs, sorts captured data, and exports row-based incremental data to downstream databases. +[TiCDC](https://github.com/pingcap/tiflow/tree/release-8.1/cdc) is a tool used to replicate incremental data from TiDB. Specifically, TiCDC pulls TiKV change logs, sorts captured data, and exports row-based incremental data to downstream databases. ## Usage scenarios @@ -87,6 +86,61 @@ As shown in the architecture diagram, TiCDC supports replicating data to TiDB, M - To ensure eventual consistency when using TiCDC for disaster recovery, you need to configure [redo log](/ticdc/ticdc-sink-to-mysql.md#eventually-consistent-replication-in-disaster-scenarios) and ensure that the storage system where the redo log is written can be read normally when a disaster occurs in the upstream. +## Implementation of processing data changes + +This section mainly describes how TiCDC processes data changes generated by upstream DML operations. + +For data changes generated by upstream DDL operations, TiCDC obtains the complete DDL SQL statement, converts it into the corresponding format based on the sink type of the downstream, and sends it to the downstream. This section does not elaborate on this. + +> **Note:** +> +> The logic of how TiCDC processes data changes might be adjusted in subsequent versions. + +MySQL binlog directly records all DML SQL statements executed in the upstream. Unlike MySQL, TiCDC listens to the real-time information of each Region Raft Log in the upstream TiKV, and generates data change information based on the difference between the data before and after each transaction, which corresponds to multiple SQL statements. TiCDC only guarantees that the output change events are equivalent to the changes in the upstream TiDB, and does not guarantee that it can accurately restore the SQL statements that caused the data changes in the upstream TiDB. + +Data change information includes the data change types and the data values before and after the change. The difference between the data before and after the transaction can result in three types of events: + +1. The `DELETE` event: corresponds to a `DELETE` type data change message, which contains the data before the change. + +2. The `INSERT` event: corresponds to a `PUT` type data change message, which contains the data after the change. + +3. The `UPDATE` event: corresponds to a `PUT` type data change message, which contains the data both before and after the change. + +Based on the data change information, TiCDC generates data in the appropriate formats for different downstream types, and sends the data to the downstream. For example, it generates data in formats such as Canal-JSON and Avro, and writes the data to Kafka, or converts the data back into SQL statements and sends them to the downstream MySQL or TiDB. + +Currently, when TiCDC adapts data change information for the corresponding protocol, for specific `UPDATE` events, it might split them into one `DELETE` event and one `INSERT` event. For more information, see [Split `UPDATE` events for MySQL sinks](/ticdc/ticdc-split-update-behavior.md#split-update-events-for-mysql-sinks) and [Split primary or unique key `UPDATE` events for non-MySQL sinks](/ticdc/ticdc-split-update-behavior.md#split-primary-or-unique-key-update-events-for-non-mysql-sinks). + +When the downstream is MySQL or TiDB, TiCDC cannot guarantee that the SQL statements written to the downstream are exactly the same as the SQL statements executed in the upstream. This is because TiCDC does not directly obtain the original DML statements executed in the upstream, but generates SQL statements based on the data change information. However, TiCDC ensures the consistency of the final results. + +For example, the following SQL statement is executed in the upstream: + +```sql +Create Table t1 (A int Primary Key, B int); + +BEGIN; +Insert Into t1 values(1,2); +Insert Into t1 values(2,2); +Insert Into t1 values(3,3); +Commit; + +Update t1 set b = 4 where b = 2; +``` + +TiCDC generates the following two SQL statements based on the data change information, and writes them to the downstream: + +```sql +INSERT INTO `test.t1` (`A`,`B`) VALUES (1,2),(2,2),(3,3); +UPDATE `test`.`t1` +SET `A` = CASE + WHEN `A` = 1 THEN 1 + WHEN `A` = 2 THEN 2 +END, `B` = CASE + WHEN `A` = 1 THEN 4 + WHEN `A` = 2 THEN 4 +END +WHERE `A` = 1 OR `A` = 2; +``` + ## Unsupported scenarios Currently, the following scenarios are not supported: diff --git a/ticdc/ticdc-simple-protocol.md b/ticdc/ticdc-simple-protocol.md index def6e7f3f78d9..c09197ec8506c 100644 --- a/ticdc/ticdc-simple-protocol.md +++ b/ticdc/ticdc-simple-protocol.md @@ -5,7 +5,7 @@ summary: Learn how to use the TiCDC Simple protocol and the data format implemen # TiCDC Simple Protocol -TiCDC Simple protocol is a row-level data change notification protocol that provides data sources for monitoring, caching, full-text indexing, analysis engines, and primary-secondary replication between heterogeneous databases. This document describes how to use the TiCDC Simple protocol and the data format implementation. +Starting from v8.0.0, TiCDC supports the Simple protocol. This document describes how to use the TiCDC Simple protocol and the data format implementation. ## Use the TiCDC Simple protocol @@ -74,7 +74,7 @@ Other: ## Message format -In the Simple protocol, each message contains only one event. The Simple protocol supports encoding messages in JSON and Avro formats. This document uses JSON format as an example. For Avro format messages, their fields and meanings are the same as those in JSON format messages, but the encoding format is different. For details about the Avro format, see [Simple Protocol Avro Schema](https://github.com/pingcap/tiflow/blob/master/pkg/sink/codec/simple/message.json). +In the Simple protocol, each message contains only one event. The Simple protocol supports encoding messages in JSON and Avro formats. This document uses JSON format as an example. For Avro format messages, their fields and meanings are the same as those in JSON format messages, but the encoding format is different. For details about the Avro format, see [Simple Protocol Avro Schema](https://github.com/pingcap/tiflow/blob/release-8.1/pkg/sink/codec/simple/message.json). ### DDL @@ -243,7 +243,7 @@ The fields in the preceding JSON data are explained as follows: #### INSERT -TiCEC encodes an `INSERT` event in the following JSON format: +TiCDC encodes an `INSERT` event in the following JSON format: ```json { @@ -711,4 +711,4 @@ The following table describes the value range of the `mysqlType` field in the Ti ### Avro schema definition -The Simple protocol supports outputting messages in Avro format. For details about the Avro format, see [Simple Protocol Avro Schema](https://github.com/pingcap/tiflow/blob/master/pkg/sink/codec/simple/message.json). +The Simple protocol supports outputting messages in Avro format. For details about the Avro format, see [Simple Protocol Avro Schema](https://github.com/pingcap/tiflow/blob/release-8.1/pkg/sink/codec/simple/message.json). diff --git a/ticdc/ticdc-sink-to-cloud-storage.md b/ticdc/ticdc-sink-to-cloud-storage.md index 15ca22c11cf9a..ea805ec9d0dab 100644 --- a/ticdc/ticdc-sink-to-cloud-storage.md +++ b/ticdc/ticdc-sink-to-cloud-storage.md @@ -24,7 +24,7 @@ cdc cli changefeed create \ The output is as follows: ```shell -Info: {"upstream_id":7171388873935111376,"namespace":"default","id":"simple-replication-task","sink_uri":"s3://logbucket/storage_test?protocol=canal-json","create_time":"2024-03-28T18:52:05.566016967+08:00","start_ts":437706850431664129,"engine":"unified","config":{"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v8.0.0"} +Info: {"upstream_id":7171388873935111376,"namespace":"default","id":"simple-replication-task","sink_uri":"s3://logbucket/storage_test?protocol=canal-json","create_time":"2024-12-26T18:52:05.566016967+08:00","start_ts":437706850431664129,"engine":"unified","config":{"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false},"column_selectors":null,"transaction_atomicity":"none","encoder_concurrency":16,"terminator":"\r\n","date_separator":"none","enable_partition_separator":false},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"storage":""}},"state":"normal","creator_version":"v8.1.2"} ``` - `--server`: The address of any TiCDC server in the TiCDC cluster. @@ -59,24 +59,83 @@ For `[query_parameters]` in the URI, the following parameters can be configured: ### Configure sink URI for external storage +When storing data in a cloud storage system, you need to set different authentication parameters depending on the cloud service provider. This section describes the authentication methods when using Amazon S3, Google Cloud Storage (GCS), and Azure Blob Storage, and how to configure accounts to access the corresponding storage services. + + +
+ The following is an example configuration for Amazon S3: ```shell --sink-uri="s3://bucket/prefix?protocol=canal-json" ``` +Before replicating data, you need to set appropriate access permissions for the directory in Amazon S3: + +- Minimum permissions required by TiCDC: `s3:ListBucket`, `s3:PutObject`, and `s3:GetObject`. +- If the changefeed configuration item `sink.cloud-storage-config.flush-concurrency` is greater than 1, which means parallel uploading of single files is enabled, you need to additionally add permissions related to [ListParts](https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListParts.html): + - `s3:AbortMultipartUpload` + - `s3:ListMultipartUploadParts` + - `s3:ListBucketMultipartUploads` + +If you have not created a replication data storage directory, refer to [Create a bucket](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-bucket.html) to create an S3 bucket in the specified region. If necessary, you can also create a folder in the bucket by referring to [Organize objects in the Amazon S3 console by using folders](https://docs.aws.amazon.com/AmazonS3/latest/user-guide/create-folder.html). + +You can configure an account to access Amazon S3 in the following ways: + +- Method 1: Specify the access key + + If you specify an access key and a secret access key, authentication is performed according to them. In addition to specifying the key in the URI, the following methods are supported: + + - TiCDC reads the `$AWS_ACCESS_KEY_ID` and `$AWS_SECRET_ACCESS_KEY` environment variables. + - TiCDC reads the `$AWS_ACCESS_KEY` and `$AWS_SECRET_KEY` environment variables. + - TiCDC reads the shared credentials file in the path specified by the `$AWS_SHARED_CREDENTIALS_FILE` environment variable. + - TiCDC reads the shared credentials file in the `~/.aws/credentials` path. + +- Method 2: Access based on an IAM role + + Associate an [IAM role with configured permissions to access Amazon S3](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_use_switch-role-ec2.html) to the EC2 instance running the TiCDC server. After successful setup, TiCDC can directly access the corresponding directories in Amazon S3 without additional settings. + +
+
+ The following is an example configuration for GCS: ```shell --sink-uri="gcs://bucket/prefix?protocol=canal-json" ``` +You can configure the account used to access GCS by specifying an access key. Authentication is performed according to the specified `credentials-file`. In addition to specifying the key in the URI, the following methods are supported: + +- TiCDC reads the file in the path specified by the `$GOOGLE_APPLICATION_CREDENTIALS` environment variable. +- TiCDC reads the file `~/.config/gcloud/application_default_credentials.json`. +- TiCDC obtains credentials from the metadata server when the cluster is running in GCE or GAE. + +
+
+ The following is an example configuration for Azure Blob Storage: ```shell --sink-uri="azure://bucket/prefix?protocol=canal-json" ``` +You can configure an account to access Azure Blob Storage in the following ways: + +- Method 1: Specify a shared access signature + + If you configure `account-name` and `sas-token` in the URI, the storage account name and shared access signature token specified by this parameter are used. Because the shared access signature token has the `&` character, you need to encode it as `%26` before adding it to the URI. You can also directly encode the entire `sas-token` using percent-encoding. + +- Method 2: Specify the access key + + If you configure `account-name` and `account-key` in the URI, the storage account name and key specified by this parameter are used. In addition to specifying the key file in the URI, TiCDC can also read the key from the environment variable `$AZURE_STORAGE_KEY`. + +- Method 3: Use Azure AD to restore the backup + + Configure the environment variables `$AZURE_CLIENT_ID`, `$AZURE_TENANT_ID`, and `$AZURE_CLIENT_SECRET`. + +
+ + > **Tip:** > > For more information about the URI parameters of Amazon S3, GCS, and Azure Blob Storage in TiCDC, see [URI Formats of External Storage Services](/external-storage-uri.md). @@ -117,7 +176,13 @@ Data change records are saved to the following path: > **Note:** > -> The table version changes only after a DDL operation is performed on the upstream table, and the new table version is the TSO when the upstream TiDB completes the execution of the DDL. However, the change of the table version does not mean the change of the table schema. For example, adding a comment to a column does not cause the schema file content to change. +> The table version changes in the following scenarios: +> +> - The upstream TiDB performs a DDL operation on the table. +> - TiCDC schedules the table across nodes. +> - The changefeed to which the table belongs restarts. +> +> Note that the change of the table version does not mean the change of the table schema. For example, adding a comment to a column does not cause the schema file content to change. ### Index files diff --git a/ticdc/ticdc-sink-to-kafka.md b/ticdc/ticdc-sink-to-kafka.md index 3fc45116afdd0..0b2a23791f2de 100644 --- a/ticdc/ticdc-sink-to-kafka.md +++ b/ticdc/ticdc-sink-to-kafka.md @@ -14,14 +14,14 @@ Create a replication task by running the following command: ```shell cdc cli changefeed create \ --server=http://10.0.10.25:8300 \ - --sink-uri="kafka://127.0.0.1:9092/topic-name?protocol=canal-json&kafka-version=2.4.0&partition-num=6&max-message-bytes=67108864&replication-factor=1" \ + --sink-uri="kafka://127.0.0.1:9092,127.0.0.1:9093,127.0.0.1:9094/topic-name?protocol=canal-json&kafka-version=2.4.0&partition-num=6&max-message-bytes=67108864&replication-factor=1" \ --changefeed-id="simple-replication-task" ``` ```shell Create changefeed successfully! ID: simple-replication-task -Info: {"sink-uri":"kafka://127.0.0.1:9092/topic-name?protocol=canal-json&kafka-version=2.4.0&partition-num=6&max-message-bytes=67108864&replication-factor=1","opts":{},"create-time":"2023-11-28T22:04:08.103600025+08:00","start-ts":415241823337054209,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":".","config":{"case-sensitive":false,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null},"scheduler":{"type":"table-number","polling-time":-1}},"state":"normal","history":null,"error":null} +Info: {"sink-uri":"kafka://127.0.0.1:9092,127.0.0.1:9093,127.0.0.1:9094/topic-name?protocol=canal-json&kafka-version=2.4.0&partition-num=6&max-message-bytes=67108864&replication-factor=1","opts":{},"create-time":"2023-11-28T22:04:08.103600025+08:00","start-ts":415241823337054209,"target-ts":0,"admin-job-type":0,"sort-engine":"unified","sort-dir":".","config":{"case-sensitive":false,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null},"scheduler":{"type":"table-number","polling-time":-1}},"state":"normal","history":null,"error":null} ``` - `--server`: The address of any TiCDC server in the TiCDC cluster. @@ -31,14 +31,35 @@ Info: {"sink-uri":"kafka://127.0.0.1:9092/topic-name?protocol=canal-json&kafka-v - `--target-ts`: Specifies the ending TSO of the changefeed. To this TSO, the TiCDC cluster stops pulling data. The default value is empty, which means that TiCDC does not automatically stop pulling data. - `--config`: Specifies the changefeed configuration file. For details, see [TiCDC Changefeed Configuration Parameters](/ticdc/ticdc-changefeed-config.md). +## Supported Kafka versions + +The following table shows the minimum supported Kafka versions for each TiCDC version: + +| TiCDC version | Minimum supported Kafka version | +|:-------------------------|:--------------------------------| +| TiCDC >= v8.1.0 | 2.1.0 | +| v7.6.0 <= TiCDC < v8.1.0 | 2.4.0 | +| v7.5.2 <= TiCDC < v8.0.0 | 2.1.0 | +| v7.5.0 <= TiCDC < v7.5.2 | 2.4.0 | +| v6.5.0 <= TiCDC < v7.5.0 | 2.1.0 | +| v6.1.0 <= TiCDC < v6.5.0 | 2.0.0 | + ## Configure sink URI for Kafka Sink URI is used to specify the connection information of the TiCDC target system. The format is as follows: ```shell -[scheme]://[userinfo@][host]:[port][/path]?[query_parameters] +[scheme]://[host]:[port][/path]?[query_parameters] ``` +> **Tip:** +> +> If there are multiple hosts or ports for the downstream Kafka, you can configure multiple `[host]:[port]` in the sink URI. For example: +> +> ```shell +> [scheme]://[host]:[port],[host]:[port],[host]:[port][/path]?[query_parameters] +> ``` + Sample configuration: ```shell @@ -49,17 +70,17 @@ The following are descriptions of sink URI parameters and values that can be con | Parameter/Parameter value | Description | | :------------------ | :------------------------------------------------------------ | -| `127.0.0.1` | The IP address of the downstream Kafka services. | -| `9092` | The port for the downstream Kafka. | +| `host` | The IP address of the downstream Kafka services. | +| `port` | The port for the downstream Kafka. | | `topic-name` | Variable. The name of the Kafka topic. | -| `kafka-version` | The version of the downstream Kafka (optional, `2.4.0` by default. Currently, the earliest supported Kafka version is `0.11.0.2` and the latest one is `3.2.0`. This value needs to be consistent with the actual version of the downstream Kafka). | +| `protocol` | The protocol with which messages are output to Kafka. The value options are [`canal-json`](/ticdc/ticdc-canal-json.md)、[`open-protocol`](/ticdc/ticdc-open-protocol.md)、[`avro`](/ticdc/ticdc-avro-protocol.md)、[`debezium`](/ticdc/ticdc-debezium.md) and [`simple`](/ticdc/ticdc-simple-protocol.md). | +| `kafka-version` | The version of the downstream Kafka. This value needs to be consistent with the actual version of the downstream Kafka. | | `kafka-client-id` | Specifies the Kafka client ID of the replication task (optional. `TiCDC_sarama_producer_replication ID` by default). | | `partition-num` | The number of the downstream Kafka partitions (optional. The value must be **no greater than** the actual number of partitions; otherwise, the replication task cannot be created successfully. `3` by default). | | `max-message-bytes` | The maximum size of data that is sent to Kafka broker each time (optional, `10MB` by default). From v5.0.6 and v4.0.6, the default value has changed from `64MB` and `256MB` to `10MB`. | | `replication-factor` | The number of Kafka message replicas that can be saved (optional, `1` by default). This value must be greater than or equal to the value of [`min.insync.replicas`](https://kafka.apache.org/33/documentation.html#brokerconfigs_min.insync.replicas) in Kafka. | | `required-acks` | A parameter used in the `Produce` request, which notifies the broker of the number of replica acknowledgements it needs to receive before responding. Value options are `0` (`NoResponse`: no response, only `TCP ACK` is provided), `1` (`WaitForLocal`: responds only after local commits are submitted successfully), and `-1` (`WaitForAll`: responds after all replicated replicas are committed successfully. You can configure the minimum number of replicated replicas using the [`min.insync.replicas`](https://kafka.apache.org/33/documentation.html#brokerconfigs_min.insync.replicas) configuration item of the broker). (Optional, the default value is `-1`). | | `compression` | The compression algorithm used when sending messages (value options are `none`, `lz4`, `gzip`, `snappy`, and `zstd`; `none` by default). Note that the Snappy compressed file must be in the [official Snappy format](https://github.com/google/snappy). Other variants of Snappy compression are not supported.| -| `protocol` | The protocol with which messages are output to Kafka. The value options are `canal-json`, `open-protocol`, `avro` and `maxwell`. | | `auto-create-topic` | Determines whether TiCDC creates the topic automatically when the `topic-name` passed in does not exist in the Kafka cluster (optional, `true` by default). | | `enable-tidb-extension` | Optional. `false` by default. When the output protocol is `canal-json`, if the value is `true`, TiCDC sends [WATERMARK events](/ticdc/ticdc-canal-json.md#watermark-event) and adds the [TiDB extension field](/ticdc/ticdc-canal-json.md#tidb-extension-field) to Kafka messages. From v6.1.0, this parameter is also applicable to the `avro` protocol. If the value is `true`, TiCDC adds [three TiDB extension fields](/ticdc/ticdc-avro-protocol.md#tidb-extension-fields) to the Kafka message. | | `max-batch-size` | New in v4.0.9. If the message protocol supports outputting multiple data changes to one Kafka message, this parameter specifies the maximum number of data changes in one Kafka message. It currently takes effect only when Kafka's `protocol` is `open-protocol` (optional, `16` by default). | @@ -93,7 +114,8 @@ The following are descriptions of sink URI parameters and values that can be con > **Note:** > -> When `protocol` is `open-protocol`, TiCDC tries to avoid generating messages that exceed `max-message-bytes` in length. However, if a row is so large that a single change alone exceeds `max-message-bytes` in length, to avoid silent failure, TiCDC tries to output this message and prints a warning in the log. +> When `protocol` is `open-protocol`, TiCDC encodes multiple events into one Kafka message and avoids generating messages that exceed the length specified by `max-message-bytes`. +> If the encoded result of a single row change event exceeds the value of `max-message-bytes`, the changefeed reports an error and prints logs. ### TiCDC uses the authentication and authorization of Kafka @@ -136,11 +158,22 @@ The following are examples when using Kafka SASL authentication: The minimum set of permissions required for TiCDC to function properly is as follows. - The `Create`, `Write`, and `Describe` permissions for the Topic [resource type](https://docs.confluent.io/platform/current/kafka/authorization.html#resources). - - The `DescribeConfigs` permission for the Cluster resource type. + - The `DescribeConfig` permission for the Cluster resource type. + + The usage scenarios for each permission are as follows: + + | Resource type | Type of operation | Scenario | + | :-------------| :------------- | :--------------------------------| + | Cluster | `DescribeConfig`| Gets the cluster metadata while the changefeed is running | + | Topic | `Describe` | Tries to create a topic when the changefeed starts | + | Topic | `Create` | Tries to create a topic when the changefeed starts | + | Topic | `Write` | Sends data to the topic | + + When creating or starting a changefeed, you can disable the `Describe` and `Create` permissions if the specified Kafka topic already exists. ### Integrate TiCDC with Kafka Connect (Confluent Platform) -To use the [data connectors](https://docs.confluent.io/current/connect/managing/connectors.html) provided by Confluent to stream data to relational or non-relational databases, you need to use the `avro` protocol and provide a URL for [Confluent Schema Registry](https://www.confluent.io/product/confluent-platform/data-compatibility/) in `schema-registry`. +To use the [data connectors](https://docs.confluent.io/current/connect/managing/connectors.html) provided by Confluent to stream data to relational or non-relational databases, you need to use the [`avro` protocol](/ticdc/ticdc-avro-protocol.md) and provide a URL for [Confluent Schema Registry](https://www.confluent.io/product/confluent-platform/data-compatibility/) in `schema-registry`. Sample configuration: @@ -159,7 +192,7 @@ For detailed integration guide, see [Quick Start Guide on Integrating TiDB with ### Integrate TiCDC with AWS Glue Schema Registry -Starting from v7.4.0, TiCDC supports using the [AWS Glue Schema Registry](https://docs.aws.amazon.com/glue/latest/dg/schema-registry.html) as the Schema Registry when users choose the Avro protocol for data replication. The configuration example is as follows: +Starting from v7.4.0, TiCDC supports using the [AWS Glue Schema Registry](https://docs.aws.amazon.com/glue/latest/dg/schema-registry.html) as the Schema Registry when users choose the [Avro protocol](/ticdc/ticdc-avro-protocol.md) for data replication. The configuration example is as follows: ```shell ./cdc cli changefeed create --server=127.0.0.1:8300 --changefeed-id="kafka-glue-test" --sink-uri="kafka://127.0.0.1:9092/topic-name?&protocol=avro&replication-factor=3" --config changefeed_glue.toml @@ -204,10 +237,10 @@ dispatchers = [ You can use topic = "xxx" to specify a Topic dispatcher and use topic expressions to implement flexible topic dispatching policies. It is recommended that the total number of topics be less than 1000. -The format of the Topic expression is `[prefix][{schema}][middle][{table}][suffix]`. +The format of the Topic expression is `[prefix]{schema}[middle][{table}][suffix]`. - `prefix`: optional. Indicates the prefix of the Topic Name. -- `[{schema}]`: optional. Used to match the schema name. +- `{schema}`: required. Used to match the schema name. Starting from v7.1.4, this parameter is optional. - `middle`: optional. Indicates the delimiter between schema name and table name. - `{table}`: optional. Used to match the table name. - `suffix`: optional. Indicates the suffix of the Topic Name. @@ -247,7 +280,7 @@ For example, for a dispatcher like `matcher = ['test.*'], topic = {schema}_{tabl You can use `partition = "xxx"` to specify a partition dispatcher. It supports five dispatchers: `default`, `index-value`, `columns`, `table`, and `ts`. The dispatcher rules are as follows: - `default`: uses the `table` dispatcher rule by default. It calculates the partition number using the schema name and table name, ensuring data from a table is sent to the same partition. As a result, the data from a single table only exists in one partition and is guaranteed to be ordered. However, this dispatcher rule limits the send throughput, and the consumption speed cannot be improved by adding consumers. -- `index-value`: calculates the partition number using either the primary key, a unique index, or an explicitly specified index, distributing table data across multiple partitions. The data from a single table is sent to multiple partitions, and the data in each partition is ordered. You can improve the consumption speed by adding consumers. +- `index-value`: calculates the partition number using either the primary key, a unique index, or an index explicitly specified by `index`, distributing table data across multiple partitions. The data from a single table is sent to multiple partitions, and the data in each partition is ordered. You can improve the consumption speed by adding consumers. - `columns`: calculates the partition number using the values of explicitly specified columns, distributing table data across multiple partitions. The data from a single table is sent to multiple partitions, and the data in each partition is ordered. You can improve the consumption speed by adding consumers. - `table`: calculates the partition number using the schema name and table name. - `ts`: calculates the partition number using the commitTs of the row change, distributing table data across multiple partitions. The data from a single table is sent to multiple partitions, and the data in each partition is ordered. You can improve the consumption speed by adding consumers. However, multiple changes of a data item might be sent to different partitions and the consumer progress of different consumers might be different, which might cause data inconsistency. Therefore, the consumer needs to sort the data from multiple partitions by commitTs before consuming. @@ -258,14 +291,14 @@ Take the following configuration of `dispatchers` as an example: [sink] dispatchers = [ {matcher = ['test.*'], partition = "index-value"}, - {matcher = ['test1.*'], partition = "index-value", index-name = "index1"}, + {matcher = ['test1.*'], partition = "index-value", index = "index1"}, {matcher = ['test2.*'], partition = "columns", columns = ["id", "a"]}, {matcher = ['test3.*'], partition = "table"}, ] ``` - Tables in the `test` database use the `index-value` dispatcher, which calculates the partition number using the value of the primary key or unique index. If a primary key exists, the primary key is used; otherwise, the shortest unique index is used. -- Tables in the `test1` table use the `index-value` dispatcher and calculate the partition number using values of all columns in the index named `index1`. If the specified index does not exist, an error is reported. Note that the index specified by `index-name` must be a unique index. +- Tables in the `test1` table use the `index-value` dispatcher and calculate the partition number using values of all columns in the index named `index1`. If the specified index does not exist, an error is reported. Note that the index specified by `index` must be a unique index. - Tables in the `test2` database use the `columns` dispatcher and calculate the partition number using the values of columns `id` and `a`. If any of the columns does not exist, an error is reported. - Tables in the `test3` database use the `table` dispatcher. - Tables in the `test4` database use the `default` dispatcher, that is the `table` dispatcher, as they do not match any of the preceding rules. @@ -371,12 +404,16 @@ An example configuration is as follows: large-message-handle-compression = "none" ``` +When `large-message-handle-compression` is enabled, the message received by the consumer is encoded using a specific compression protocol, and the consumer application needs to use the specified compression protocol to decode the data. + This feature is different from the compression feature of the Kafka producer: * The compression algorithm specified in `large-message-handle-compression` compresses a single Kafka message. The compression is performed before comparing with the message size limit. -* You can configure the compression algorithm in `sink-uri`. The compression is applied to the entire data sending request, which contains multiple Kafka messages. The compression is performed after comparing with the message size limit. +* At the same time, you can also configure the compression algorithm by using the `compression` parameter in [`sink-uri`](#configure-sink-uri-for-kafka). This compression algorithm is applied to the entire data sending request, which contains multiple Kafka messages. -When `large-message-handle-compression` is enabled, the message received by the consumer is encoded using a specific compression protocol, and the consumer application needs to use the specified compression protocol to decode the data. +If you set `large-message-handle-compression`, when TiCDC receives a message, it first compares it to the value of the message size limit parameter, and messages larger than the size limit are compressed. If you also set `compression` in [`sink-uri`](#configure-sink-uri-for-kafka), TiCDC compresses the entire sending data request again at the sink level based on the `sink-uri` setting. + +The compression ratio for the two preceding compression methods is calculated as follows: `compression ratio = size before compression / size after compression * 100`. ### Send handle keys only @@ -425,7 +462,7 @@ The message format with handle keys only is as follows: ], "old": null, "_tidb": { // TiDB extension fields - "commitTs": 163963314122145239, + "commitTs": 429918007904436226, // A TiDB TSO timestamp "onlyHandleKey": true } } @@ -491,7 +528,7 @@ The Kafka consumer receives a message that contains the address of the large mes ], "old": null, "_tidb": { // TiDB extension fields - "commitTs": 163963314122145239, + "commitTs": 429918007904436226, // A TiDB TSO timestamp "claimCheckLocation": "s3:/claim-check-bucket/${uuid}.json" } } diff --git a/ticdc/ticdc-sink-to-mysql.md b/ticdc/ticdc-sink-to-mysql.md index 71dd6dba2ee85..edbd76df4c23f 100644 --- a/ticdc/ticdc-sink-to-mysql.md +++ b/ticdc/ticdc-sink-to-mysql.md @@ -31,6 +31,11 @@ Info: {"sink-uri":"mysql://root:123456@127.0.0.1:3306/","opts":{},"create-time": - `--target-ts`: Specifies the ending TSO of the changefeed. To this TSO, the TiCDC cluster stops pulling data. The default value is empty, which means that TiCDC does not automatically stop pulling data. - `--config`: Specifies the changefeed configuration file. For details, see [TiCDC Changefeed Configuration Parameters](/ticdc/ticdc-changefeed-config.md). +> **Note:** +> +> - TiCDC only replicates incremental data. To initialize full data, use Dumpling/TiDB Lightning or BR. +> - After the full data is initialized, you need to specify the `start-ts` as the TSO when the upstream backup is performed. For example, the `pos` value in the metadata file under the Dumpling directory, or the `backupTS` value in the log output after BR completes the backup. + ## Configure sink URI for MySQL or TiDB Sink URI is used to specify the connection information of the TiCDC target system. The format is as follows: @@ -53,18 +58,26 @@ The following are descriptions of sink URI parameters and parameter values that | Parameter/Parameter value | Description | | :------------ | :------------------------------------------------ | -| `root` | The username of the downstream database. | +| `root` | The username of the downstream database. To replicate data to TiDB or other MySQL-compatible databases, make sure that the downstream database user has [certain permissions](#permissions-required-for-the-downstream-database-user). | | `123456` | The password of the downstream database (can be encoded using Base64). | | `127.0.0.1` | The IP address of the downstream database. | -| `3306` | The port for the downstream data. | -| `worker-count` | The number of SQL statements that can be concurrently executed to the downstream (optional, `16` by default). | +| `3306` | The port for the downstream database. | +| `worker-count` | The number of SQL statements that can be concurrently executed to the downstream (optional, the default value is `16`, and the maximum value is `1024`). | | `cache-prep-stmts` | Controls whether to use prepared statements when executing SQL in the downstream and enable prepared statement cache on the client side (optional, `true` by default). | -| `max-txn-row` | The size of a transaction batch that can be executed to the downstream (optional, `256` by default). | +| `max-txn-row` | The batch size of SQL statements executed to the downstream (optional, the default value is `256`, and the maximum value is `2048`). | +| `max-multi-update-row` | The batch size of `UPDATE ROWS` SQL statements executed to the downstream when batch write (`batch-dml-enable`) is enabled, always less than `max-txn-row` (optional, the default value is `40`, and the maximum value is `256`). | +| `max-multi-update-row-size` | The size limit of `UPDATE ROWS` SQL statements executed to the downstream when batch write (`batch-dml-enable`) is enabled. If the size exceeds this limit, each row is executed as a separate SQL statement (optional, the default value is `1024`, and the maximum value is `8192`). | | `ssl-ca` | The path of the CA certificate file needed to connect to the downstream MySQL instance (optional). | | `ssl-cert` | The path of the certificate file needed to connect to the downstream MySQL instance (optional). | | `ssl-key` | The path of the certificate key file needed to connect to the downstream MySQL instance (optional). | | `time-zone` | The time zone used when connecting to the downstream MySQL instance, which is effective since v4.0.8. This is an optional parameter. If this parameter is not specified, the time zone of TiCDC service processes is used. If this parameter is set to an empty value, such as `time-zone=""`, no time zone is specified when TiCDC connects to the downstream MySQL instance and the default time zone of the downstream is used. | | `transaction-atomicity` | The atomicity level of a transaction. This is an optional parameter, with the default value of `none`. When the value is `table`, TiCDC ensures the atomicity of a single-table transaction. When the value is `none`, TiCDC splits the single-table transaction. | +| `batch-dml-enable` | Enables the batch write (batch-dml) feature (optional, the default value is `true`). | +| `read-timeout` | The go-sql-driver parameter, [I/O read timeout](https://pkg.go.dev/github.com/go-sql-driver/mysql#readme-readtimeout) (optional, the default value is `2m`). | +| `write-timeout` | The go-sql-driver parameter, [I/O write timeout](https://pkg.go.dev/github.com/go-sql-driver/mysql#readme-writetimeout) (optional, the default value is `2m`). | +| `timeout` | The go-sql-driver parameter, [timeout for establishing connections](https://pkg.go.dev/github.com/go-sql-driver/mysql#readme-timeout), also known as dial timeout (optional, the default value is `2m`). | +| `tidb-txn-mode` | Specifies the [`tidb_txn_mode`](/system-variables.md#tidb_txn_mode) environment variable (optional, the default value is `optimistic`). | +| `safe-mode` | Specifies how TiCDC handles `INSERT` and `UPDATE` statements when replicating data to the downstream. When it is `true`, TiCDC converts all upstream `INSERT` statements to `REPLACE INTO` statements, and all `UPDATE` statements to `DELETE` + `REPLACE INTO` statements. Before v6.1.3, the default value of this parameter is `true`. Starting from v6.1.3, the default value is changed to `false`. When TiCDC starts, it obtains a current timestamp `ThresholdTs`. For `INSERT` and `UPDATE` statements with `CommitTs` less than `ThresholdTs`, TiCDC converts them to `REPLACE INTO` statements and `DELETE` + `REPLACE INTO` statements respectively. For `INSERT` and `UPDATE` statements with `CommitTs` greater than or equal to `ThresholdTs`, `INSERT` statements are directly replicated to the downstream, while the behavior of `UPDATE` statements follows the [TiCDC Behavior in Splitting UPDATE Events](/ticdc/ticdc-split-update-behavior.md). | To encode the database password in the sink URI using Base64, use the following command: @@ -82,6 +95,24 @@ MTIzNDU2 > > When the sink URI contains special characters such as `! * ' ( ) ; : @ & = + $ , / ? % # [ ]`, you need to escape the special characters, for example, in [URI Encoder](https://www.urlencoder.org/). +## Permissions required for the downstream database user + +To replicate data to TiDB or other MySQL-compatible databases, the downstream database user needs the following permissions: + +- `Select` +- `Index` +- `Insert` +- `Update` +- `Delete` +- `Create` +- `Drop` +- `Alter` +- `Create View` + +To replicate [`RECOVER TABLE`](/sql-statements/sql-statement-recover-table.md) to the downstream TiDB, the downstream database user also needs the `Super` permission. + +If the downstream TiDB cluster has [read-only mode](/system-variables.md#tidb_restricted_read_only-new-in-v520) enabled, the downstream database user also needs the `RESTRICTED_REPLICA_WRITER_ADMIN` permission. + ## Eventually consistent replication in disaster scenarios Starting from v6.1.1, this feature becomes GA. Starting from v5.3.0, TiCDC supports backing up incremental data from an upstream TiDB cluster to an object storage or an NFS of the downstream cluster. When the upstream cluster encounters a disaster and becomes unavailable, TiCDC can restore the downstream data to the recent eventually consistent state. This is the eventually consistent replication capability provided by TiCDC. With this capability, you can switch applications to the downstream cluster quickly, avoiding long-time downtime and improving service continuity. @@ -138,5 +169,5 @@ cdc redo apply --tmp-dir="/tmp/cdc/redo/apply" \ In this command: - `tmp-dir`: Specifies the temporary directory for downloading TiCDC incremental data backup files. -- `storage`: Specifies the address for storing the TiCDC incremental data backup files, either an URI of object storage or an NFS directory. +- `storage`: Specifies the address for storing the TiCDC incremental data backup files, either a URI of object storage or an NFS directory. - `sink-uri`: Specifies the secondary cluster address to restore the data to. Scheme can only be `mysql`. diff --git a/ticdc/ticdc-sink-to-pulsar.md b/ticdc/ticdc-sink-to-pulsar.md index e8d62d9484c42..78d2248952eac 100644 --- a/ticdc/ticdc-sink-to-pulsar.md +++ b/ticdc/ticdc-sink-to-pulsar.md @@ -23,7 +23,7 @@ cdc cli changefeed create \ Create changefeed successfully! ID: simple-replication-task -Info: {"upstream_id":7277814241002263370,"namespace":"default","id":"simple-replication-task","sink_uri":"pulsar://127.0.0.1:6650/consumer-test?protocol=canal-json","create_time":"2024-03-28T14:42:32.000904+08:00","start_ts":444203257406423044,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["pulsar_test.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"dispatchers":[{"matcher":["pulsar_test.*"],"partition":"","topic":"test_{schema}_{table}"}],"encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"pulsar_config":{"connection-timeout":30,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"use_file_backend":false},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"}},"state":"normal","creator_version":"v8.0.0","resolved_ts":444203257406423044,"checkpoint_ts":444203257406423044,"checkpoint_time":"2024-01-25 14:42:31.410"} +Info: {"upstream_id":7277814241002263370,"namespace":"default","id":"simple-replication-task","sink_uri":"pulsar://127.0.0.1:6650/consumer-test?protocol=canal-json","create_time":"2024-12-26T14:42:32.000904+08:00","start_ts":444203257406423044,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["pulsar_test.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"dispatchers":[{"matcher":["pulsar_test.*"],"partition":"","topic":"test_{schema}_{table}"}],"encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"pulsar_config":{"connection-timeout":30,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"use_file_backend":false},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"}},"state":"normal","creator_version":"v8.1.2","resolved_ts":444203257406423044,"checkpoint_ts":444203257406423044,"checkpoint_time":"2024-12-26 14:42:31.410"} ``` The meaning of each parameter is as follows: @@ -63,6 +63,7 @@ The configurable parameters in a URI are as follows: | Parameter | Description | | :------------------ | :------------------------------------------------------------ | +| `pulsar` | The scheme for the downstream Pulsar. The value can be `pulsar` or `pulsar+ssl`. | | `127.0.0.1` | The IP address by which the downstream Pulsar provides service. | | `6650` | The connection port for the downstream Pulsar. | | `persistent://abc/def/yktest` | As shown in the preceding configuration example 1, this parameter is used to specify the tenant, namespace, and topic of Pulsar. | @@ -273,8 +274,9 @@ dispatchers = [ You can use `topic = "xxx"` to specify a topic dispatcher and use topic expressions to implement flexible topic dispatching policies. It is recommended that the total number of topics be less than 1000. -The format of a topic expression is `[prefix]{schema}[middle][{table}][suffix]`. The following are the meanings of each part: +The format of a topic expression is `[tenant_and_namespace][prefix]{schema}[middle][{table}][suffix]`. The following are the meanings of each part: +- `tenant_and_namespace`：Optional. Represents the tenant and namespace of the topic, such as `persistent://abc/def/`. If not configured, it means that the topic is in the default namespace `default` under the default tenant `public` of Pulsar. - `prefix`: Optional. Represents the prefix of the topic name. - `{schema}`: Optional. Represents the database name. - `middle`: Optional. Represents the separator between a database name and a table name. diff --git a/ticdc/ticdc-split-update-behavior.md b/ticdc/ticdc-split-update-behavior.md new file mode 100644 index 0000000000000..6f1953677b079 --- /dev/null +++ b/ticdc/ticdc-split-update-behavior.md @@ -0,0 +1,159 @@ +--- +title: TiCDC Behavior in Splitting UPDATE Events +summary: Introduce the behavior changes about whether TiCDC splits `UPDATE` events, including the reasons and the impact of these changes. +--- + +# TiCDC Behavior in Splitting UPDATE Events + +## Split `UPDATE` events for MySQL sinks + +Starting from v6.5.10, v7.1.6, v7.5.2, and v8.1.1, when using the MySQL sink, any TiCDC node that receives a request for replicating a table will fetch the current timestamp `thresholdTS` from PD before starting the replication to the downstream. Based on the value of this timestamp, TiCDC decides whether to split `UPDATE` events: + +- For transactions containing one or multiple `UPDATE` changes, if the transaction `commitTS` is less than `thresholdTS`, TiCDC splits the `UPDATE` event into a `DELETE` event and an `INSERT` event before writing them to the Sorter module. +- For `UPDATE` events with the transaction `commitTS` greater than or equal to `thresholdTS`, TiCDC does not split them. For more information, see GitHub issue [#10918](https://github.com/pingcap/tiflow/issues/10918). + +> **Note:** +> +> In v8.1.0, when using MySQL Sink, TiCDC also decides whether to split `UPDATE` events based on the value of `thresholdTS`, but `thresholdTS` is obtained differently. Specifically, in v8.1.0, `thresholdTS` is the current timestamp fetched from PD at TiCDC startup, but this way might cause data inconsistency issues in multi-node scenarios. For more information, see GitHub issue [#11219](https://github.com/pingcap/tiflow/issues/11219). + +This behavior change (that is, deciding whether to split `UPDATE` events based on `thresholdTS`) addresses the issue of downstream data inconsistencies caused by the potentially incorrect order of `UPDATE` events received by TiCDC, which can lead to an incorrect order of split `DELETE` and `INSERT` events. + +Take the following SQL statements as an example: + +```sql +CREATE TABLE t (a INT PRIMARY KEY, b INT); +INSERT INTO t VALUES (1, 1); +INSERT INTO t VALUES (2, 2); + +BEGIN; +UPDATE t SET a = 3 WHERE a = 2; +UPDATE t SET a = 2 WHERE a = 1; +COMMIT; +``` + +In this example, the two `UPDATE` statements within the transaction have a sequential dependency on execution. The primary key `a` is changed from `2` to `3`, and then the primary key `a` is changed from `1` to `2`. After this transaction is executed, the records in the upstream database are `(2, 1)` and `(3, 2)`. + +However, the order of `UPDATE` events received by TiCDC might differ from the actual execution order of the upstream transaction. For example: + +```sql +UPDATE t SET a = 2 WHERE a = 1; +UPDATE t SET a = 3 WHERE a = 2; +``` + +- Before this behavior change, TiCDC writes these `UPDATE` events to the Sorter module and then splits them into `DELETE` and `INSERT` events. After the split, the actual execution order of these events in the downstream is as follows: + + ```sql + BEGIN; + DELETE FROM t WHERE a = 1; + REPLACE INTO t VALUES (2, 1); + DELETE FROM t WHERE a = 2; + REPLACE INTO t VALUES (3, 2); + COMMIT; + ``` + + After the downstream executes the transaction, the records in the database are `(3, 2)`, which are different from the records in the upstream database (`(2, 1)` and `(3, 2)`), indicating a data inconsistency issue. + +- After this behavior change, if the transaction `commitTS` is less than the `thresholdTS` fetched from PD when TiCDC starts replicating the corresponding table to the downstream, TiCDC splits these `UPDATE` events into `DELETE` and `INSERT` events before writing them to the Sorter module. After the sorting by the Sorter module, the actual execution order of these events in the downstream is as follows: + + ```sql + BEGIN; + DELETE FROM t WHERE a = 1; + DELETE FROM t WHERE a = 2; + REPLACE INTO t VALUES (2, 1); + REPLACE INTO t VALUES (3, 2); + COMMIT; + ``` + + After the downstream executes the transaction, the records in the downstream database are the same as those in the upstream database, which are `(2, 1)` and `(3, 2)`, ensuring data consistency. + +As you can see from the preceding example, splitting the `UPDATE` event into `DELETE` and `INSERT` events before writing them to the Sorter module ensures that all `DELETE` events are executed before `INSERT` events after the split, thereby maintaining data consistency regardless of the order of `UPDATE` events received by TiCDC. + +> **Note:** +> +> After this behavior change, when using the MySQL sink, TiCDC does not split the `UPDATE` event in most cases. Consequently, there might be primary key or unique key conflicts during changefeed runtime, causing the changefeed to restart automatically. After the restart, TiCDC will split the conflicting `UPDATE` events into `DELETE` and `INSERT` events before writing them to the Sorter module. This ensures that all events within the same transaction are correctly ordered, with all `DELETE` events preceding `INSERT` events, thus correctly completing data replication. + +## Split primary or unique key `UPDATE` events for non-MySQL sinks + +### Transactions containing a single `UPDATE` change + +Starting from v6.5.3, v7.1.1, and v7.2.0, when using a non-MySQL sink, for transactions that only contain a single update change, if the primary key or non-null unique index value is modified in an `UPDATE` event, TiCDC splits this event into `DELETE` and `INSERT` events. For more information, see GitHub issue [#9086](https://github.com/pingcap/tiflow/issues/9086). + +This change primarily addresses the issue that TiCDC only outputs the new value without the old value by default when using the CSV and AVRO protocols. Due to this issue, when the primary key or non-null unique index value changes, the consumer can only receive the new value, making it impossible to process the value before the change (for example, delete the old value). Take the following SQL as an example: + +```sql +CREATE TABLE t (a INT PRIMARY KEY, b INT); +INSERT INTO t VALUES (1, 1); +UPDATE t SET a = 2 WHERE a = 1; +``` + +In this example, the primary key `a` is updated from `1` to `2`. If the `UPDATE` event is not split, the consumer can only obtain the new value `a = 2` and cannot obtain the old value `a = 1` when using the CSV and AVRO protocols. This might cause the downstream consumer to only insert the new value `2` without deleting the old value `1`. + +### Transactions containing multiple `UPDATE` changes + +Starting from v6.5.4, v7.1.2, and v7.4.0, for transactions containing multiple changes, if the primary key or non-null unique index value is modified in the `UPDATE` event, TiCDC splits the event into `DELETE` and `INSERT` events and ensures that all events follow the sequence of `DELETE` events preceding `INSERT` events. For more information, see GitHub issue [#9430](https://github.com/pingcap/tiflow/issues/9430). + +This change primarily addresses the potential issue of primary key or unique key conflicts that consumers might encounter when writing data changes from the Kafka sink or other sinks to a relational database or performing a similar operation. This issue is caused by the potentially incorrect order of `UPDATE` events received by TiCDC. + +Take the following SQL as an example: + +```sql +CREATE TABLE t (a INT PRIMARY KEY, b INT); +INSERT INTO t VALUES (1, 1); +INSERT INTO t VALUES (2, 2); + +BEGIN; +UPDATE t SET a = 3 WHERE a = 1; +UPDATE t SET a = 1 WHERE a = 2; +UPDATE t SET a = 2 WHERE a = 3; +COMMIT; +``` + +In this example, by executing three SQL statements to swap the primary keys of two rows, TiCDC only receives two update change events, that is, changing the primary key `a` from `1` to `2` and changing the primary key `a` from `2` to `1`. If consumers directly write these two `UPDATE` events to the downstream, a primary key conflict will occur, leading to changefeed errors. + +Therefore, TiCDC splits these two events into four events, that is, deleting records `(1, 1)` and `(2, 2)` and writing records `(2, 1)` and `(1, 2)`. + +### Control whether to split primary or unique key `UPDATE` events + +Starting from v6.5.10, v7.1.6, v7.5.3, and v8.1.1, when using a non-MySQL sink, TiCDC supports controlling whether to split primary or unique key `UPDATE` events via the `output-raw-change-event` parameter, as described in the GitHub issue [#11211]( https://github.com/pingcap/tiflow/issues/11211). The specific behavior of this parameter is as follows: + +- When you set `output-raw-change-event = false`, if the primary key or non-null unique index value is modified in an `UPDATE` event, TiCDC splits the event into `DELETE` and `INSERT` events and ensures that all events follow the sequence of `DELETE` events preceding `INSERT` events. +- When you set `output-raw-change-event = true`, TiCDC does not split `UPDATE` events, and the consumer side is responsible for dealing with the problems described in [Split primary or unique key `UPDATE` events for non-MySQL sinks](/ticdc/ticdc-split-update-behavior.md#split-primary-or-unique-key-update-events-for-non-mysql-sinks). Otherwise there might be a risk of data inconsistency. Note that when the primary key of a table is a clustered index, updates to the primary key are still split into `DELETE` and `INSERT` events in TiDB, and such behavior is not affected by the `output-raw-change-event` parameter. + +> **Note** +> +> In the following tables, UK/PK stands for primary key or unique key. + +#### Release 6.5 compatibility + +| Version | Protocol | Split UK/PK `UPDATE` events | Not split UK/PK `UPDATE` events | Comments | +| -- | -- | -- | -- | -- | +| <= v6.5.2 | ALL | ✗ | ✓ | | +| v6.5.3 / v6.5.4 | Canal/Open | ✗ | ✓ | | +| v6.5.3 | CSV/Avro | ✗ | ✗ | Split but does not sort. See [#9086](https://github.com/pingcap/tiflow/issues/9658) | +| v6.5.4 | Canal/Open | ✗ | ✗ | Only split and sort transactions that contain multiple changes | +| v6.5.5 ～ v6.5.9 | ALL | ✓ | ✗ | +| \>= v6.5.10 | ALL | ✓ (Default value: `output-raw-change-event = false`) | ✓ (Optional: `output-raw-change-event = true`) | | + +#### Release 7.1 compatibility + +| Version | Protocol | Split UK/PK `UPDATE` events | Not split UK/PK `UPDATE` events | Comments | +| -- | -- | -- | -- | -- | +| v7.1.0 | ALL | ✗ | ✓ | | +| v7.1.1 | Canal/Open | ✗ | ✓ | | +| v7.1.1 | CSV/Avro | ✗ | ✗ | Split but does not sort. See [#9086](https://github.com/pingcap/tiflow/issues/9658) | +| v7.1.2 ~ v7.1.5 | ALL | ✓ | ✗ | | +| \>= v7.1.6 | ALL | ✓ (Default value: `output-raw-change-event = false`) | ✓ (Optional: `output-raw-change-event = true`) | | + +#### Release 7.5 compatibility + +| Version | Protocol | Split UK/PK `UPDATE` events | Not split UK/PK `UPDATE` events | Comments | +| -- | -- | -- | -- | -- | +| <= v7.5.2 | ALL | ✓ | ✗ | +| \>= v7.5.3 | ALL | ✓ (Default value:`output-raw-change-event = false`) | ✓ (Optional: `output-raw-change-event = true`) | | + +#### Release 8.1 compatibility + +| Version | Protocol | Split UK/PK `UPDATE` events | Not split UK/PK `UPDATE` events | Comments | +| -- | -- | -- | -- | -- | +| v8.1.0 | ALL | ✓ | ✗ | +| \>= v8.1.1 | ALL | ✓ (Default value:`output-raw-change-event = false`) | ✓ (Optional: `output-raw-change-event = true`) | | diff --git a/ticdc/ticdc-storage-consumer-dev-guide.md b/ticdc/ticdc-storage-consumer-dev-guide.md index 18e1e01326d8a..7261a2476f19d 100644 --- a/ticdc/ticdc-storage-consumer-dev-guide.md +++ b/ticdc/ticdc-storage-consumer-dev-guide.md @@ -13,7 +13,7 @@ This document describes how to design and implement a TiDB data change consumer. TiCDC does not provide any standard way for implementing a consumer. This document provides a consumer example program written in Golang. This program can read data from the storage service and write the data to a MySQL-compatible database. You can refer to the data format and instructions provided in this example to implement a consumer on your own. -[Consumer program written in Golang](https://github.com/pingcap/tiflow/tree/master/cmd/storage-consumer) +[Consumer program written in Golang](https://github.com/pingcap/tiflow/tree/release-8.1/cmd/storage-consumer) ## Design a consumer diff --git a/ticdc/ticdc-upstream-downstream-check.md b/ticdc/ticdc-upstream-downstream-check.md index ad07604887292..2870c4bccdbe5 100644 --- a/ticdc/ticdc-upstream-downstream-check.md +++ b/ticdc/ticdc-upstream-downstream-check.md @@ -1,7 +1,6 @@ --- title: Upstream and Downstream Clusters Data Validation and Snapshot Read summary: Learn how to check data for TiDB upstream and downstream clusters. -aliases: ['/docs/dev/sync-diff-inspector/upstream-downstream-diff/','/docs/dev/reference/tools/sync-diff-inspector/tidb-diff/', '/tidb/dev/upstream-downstream-diff'] --- # Upstream and Downstream Clusters Data Validation and Snapshot Read diff --git a/ticdc/troubleshoot-ticdc.md b/ticdc/troubleshoot-ticdc.md index 569d29da9f794..2f52ea67a8b98 100644 --- a/ticdc/troubleshoot-ticdc.md +++ b/ticdc/troubleshoot-ticdc.md @@ -1,7 +1,6 @@ --- title: Troubleshoot TiCDC summary: Learn how to troubleshoot issues you might encounter when you use TiCDC. -aliases: ['/docs/dev/ticdc/troubleshoot-ticdc/'] --- # Troubleshoot TiCDC @@ -121,7 +120,7 @@ fetch.message.max.bytes=2147483648 ## How can I find out whether a DDL statement fails to execute in downstream during TiCDC replication? How to resume the replication? -If a DDL statement fails to execute, the replication task (changefeed) automatically stops. The checkpoint-ts is the DDL statement's finish-ts minus one. If you want TiCDC to retry executing this statement in the downstream, use `cdc cli changefeed resume` to resume the replication task. For example: +If a DDL statement fails to execute, the replication task (changefeed) automatically stops. The checkpoint-ts is the DDL statement's finish-ts. If you want TiCDC to retry executing this statement in the downstream, use `cdc cli changefeed resume` to resume the replication task. For example: {{< copyable "shell-regular" >}} @@ -131,7 +130,15 @@ cdc cli changefeed resume -c test-cf --server=http://127.0.0.1:8300 If you want to skip this DDL statement that goes wrong, set the start-ts of the changefeed to the checkpoint-ts (the timestamp at which the DDL statement goes wrong) plus one, and then run the `cdc cli changefeed create` command to create a new changefeed task. For example, if the checkpoint-ts at which the DDL statement goes wrong is `415241823337054209`, run the following commands to skip this DDL statement: -{{< copyable "shell-regular" >}} +To skip this DDL statement that goes wrong, you can configure the `ignore-txn-start-ts` parameter to skip the transactions corresponding to the specified `start-ts`. For example: + +1. Search the TiCDC log for the `apply job` field, and identify the `start-ts` of the DDLs that are taking a long time. +2. Modify the changefeed configuration. Add the above `start-ts` to the `ignore-txn-start-ts` configuration item. +3. Resume the suspended changefeed. + +> **Note:** +> +> Although setting the changefeed `start-ts` to the value of `checkpoint-ts` (at the time of the error) plus one and then recreating the task can skip the DDL statement, it can also cause TiCDC to lose the DML data change at the time of `checkpointTs+1`. Therefore, this operation is strictly prohibited in production environments. ```shell cdc cli changefeed remove --server=http://127.0.0.1:8300 --changefeed-id simple-replication-task diff --git a/tidb-architecture.md b/tidb-architecture.md index 50fb12f6b32cf..2a38039146973 100644 --- a/tidb-architecture.md +++ b/tidb-architecture.md @@ -1,7 +1,6 @@ --- title: TiDB Architecture summary: The key architecture components of the TiDB platform -aliases: ['/docs/dev/architecture/','/tidb/dev/architecture'] --- # TiDB Architecture diff --git a/tidb-binlog-deployment-topology.md b/tidb-binlog-deployment-topology.md index 3f991426c7b2b..e432fa7c1d501 100644 --- a/tidb-binlog-deployment-topology.md +++ b/tidb-binlog-deployment-topology.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Deployment Topology summary: Learn the deployment topology of TiDB Binlog based on the minimal TiDB topology. -aliases: ['/docs/dev/tidb-binlog-deployment-topology/'] --- # TiDB Binlog Deployment Topology diff --git a/tidb-binlog/bidirectional-replication-between-tidb-clusters.md b/tidb-binlog/bidirectional-replication-between-tidb-clusters.md index d8b3d37be622f..f471675b70a65 100644 --- a/tidb-binlog/bidirectional-replication-between-tidb-clusters.md +++ b/tidb-binlog/bidirectional-replication-between-tidb-clusters.md @@ -1,7 +1,6 @@ --- title: Bidirectional Replication Between TiDB Clusters summary: Learn how to perform the bidirectional replication between TiDB clusters. -aliases: ['/docs/dev/tidb-binlog/bidirectional-replication-between-tidb-clusters/','/docs/dev/reference/tidb-binlog/bidirectional-replication/'] --- # Bidirectional Replication between TiDB Clusters diff --git a/tidb-binlog/binlog-consumer-client.md b/tidb-binlog/binlog-consumer-client.md index e77f63314cc24..26dac3ab908d4 100644 --- a/tidb-binlog/binlog-consumer-client.md +++ b/tidb-binlog/binlog-consumer-client.md @@ -1,7 +1,6 @@ --- title: Binlog Consumer Client User Guide summary: Use Binlog Consumer Client to consume TiDB secondary binlog data from Kafka and output the data in a specific format. -aliases: ['/tidb/dev/binlog-slave-client','/docs/dev/tidb-binlog/binlog-slave-client/','/docs/dev/reference/tidb-binlog/binlog-slave-client/','/docs/dev/reference/tools/tidb-binlog/binlog-slave-client/'] --- # Binlog Consumer Client User Guide @@ -118,11 +117,11 @@ message Binlog { } ``` -For the definition of the data format, see [`secondary_binlog.proto`](https://github.com/pingcap/tidb/blob/master/pkg/tidb-binlog/proto/proto/secondary_binlog.proto) +For the definition of the data format, see [`secondary_binlog.proto`](https://github.com/pingcap/tidb/blob/release-8.1/pkg/tidb-binlog/proto/proto/secondary_binlog.proto) ### Driver -The [TiDB-Tools](https://github.com/pingcap/tidb-tools/) project provides [Driver](https://github.com/pingcap/tidb/tree/master/pkg/tidb-binlog/driver), which is used to read the binlog data in Kafka. It has the following features: +The [TiDB-Tools](https://github.com/pingcap/tidb-tools/) project provides [Driver](https://github.com/pingcap/tidb/tree/release-8.1/pkg/tidb-binlog/driver), which is used to read the binlog data in Kafka. It has the following features: * Read the Kafka data. * Locate the binlog stored in Kafka based on `commit ts`. diff --git a/tidb-binlog/binlog-control.md b/tidb-binlog/binlog-control.md index 94b65503a82ae..0be84899f09a3 100644 --- a/tidb-binlog/binlog-control.md +++ b/tidb-binlog/binlog-control.md @@ -1,12 +1,11 @@ --- title: binlogctl summary: Learns how to use `binlogctl`. -aliases: ['/docs/dev/tidb-binlog/binlog-control/'] --- # binlogctl -[Binlog Control](https://github.com/pingcap/tidb-binlog/tree/master/binlogctl) (`binlogctl` for short) is a command line tool for TiDB Binlog. You can use `binlogctl` to manage TiDB Binlog clusters. +[Binlog Control](https://github.com/pingcap/tidb-binlog/tree/release-8.1/binlogctl) (`binlogctl` for short) is a command line tool for TiDB Binlog. You can use `binlogctl` to manage TiDB Binlog clusters. You can use `binlogctl` to: diff --git a/tidb-binlog/deploy-tidb-binlog.md b/tidb-binlog/deploy-tidb-binlog.md index 10699145008c0..f40e240bba26c 100644 --- a/tidb-binlog/deploy-tidb-binlog.md +++ b/tidb-binlog/deploy-tidb-binlog.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Cluster Deployment summary: Learn how to deploy TiDB Binlog cluster. -aliases: ['/docs/dev/tidb-binlog/deploy-tidb-binlog/','/docs/dev/reference/tidb-binlog/deploy/','/docs/dev/how-to/deploy/tidb-binlog/'] --- # TiDB Binlog Cluster Deployment diff --git a/tidb-binlog/get-started-with-tidb-binlog.md b/tidb-binlog/get-started-with-tidb-binlog.md index b560d15684a65..d549a6821271e 100644 --- a/tidb-binlog/get-started-with-tidb-binlog.md +++ b/tidb-binlog/get-started-with-tidb-binlog.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Tutorial summary: Learn to deploy TiDB Binlog with a simple TiDB cluster. -aliases: ['/docs/dev/get-started-with-tidb-binlog/','/docs/dev/how-to/get-started/tidb-binlog/'] --- # TiDB Binlog Tutorial @@ -43,7 +42,7 @@ sudo yum install -y mariadb-server ``` ```bash -curl -L https://download.pingcap.org/tidb-community-server-v8.0.0-linux-amd64.tar.gz | tar xzf - +curl -L https://download.pingcap.org/tidb-community-server-v8.1.2-linux-amd64.tar.gz | tar xzf - cd tidb-latest-linux-amd64 ``` diff --git a/tidb-binlog/handle-tidb-binlog-errors.md b/tidb-binlog/handle-tidb-binlog-errors.md index 5ba106db49415..b10262abcdd2f 100644 --- a/tidb-binlog/handle-tidb-binlog-errors.md +++ b/tidb-binlog/handle-tidb-binlog-errors.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Error Handling summary: Learn how to handle TiDB Binlog errors. -aliases: ['/docs/dev/tidb-binlog/handle-tidb-binlog-errors/','/docs/dev/reference/tidb-binlog/troubleshoot/error-handling/'] --- # TiDB Binlog Error Handling diff --git a/tidb-binlog/maintain-tidb-binlog-cluster.md b/tidb-binlog/maintain-tidb-binlog-cluster.md index 5690e7dcf16eb..44a9ec4865033 100644 --- a/tidb-binlog/maintain-tidb-binlog-cluster.md +++ b/tidb-binlog/maintain-tidb-binlog-cluster.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Cluster Operations summary: Learn how to operate the cluster version of TiDB Binlog. -aliases: ['/docs/dev/tidb-binlog/maintain-tidb-binlog-cluster/','/docs/dev/reference/tidb-binlog/maintain/','/docs/dev/how-to/maintain/tidb-binlog/','/docs/dev/reference/tools/tidb-binlog/maintain/'] --- # TiDB Binlog Cluster Operations @@ -46,7 +45,7 @@ For how to pause, close, check, and modify the state of Drainer, see the [binlog ## Use `binlogctl` to manage Pump/Drainer -[`binlogctl`](https://github.com/pingcap/tidb-binlog/tree/master/binlogctl) is an operations tool for TiDB Binlog with the following features: +[`binlogctl`](https://github.com/pingcap/tidb-binlog/tree/release-8.1/binlogctl) is an operations tool for TiDB Binlog with the following features: * Checking the state of Pump or Drainer * Pausing or closing Pump or Drainer diff --git a/tidb-binlog/monitor-tidb-binlog-cluster.md b/tidb-binlog/monitor-tidb-binlog-cluster.md index a5bf64f19482a..19831474c3b89 100644 --- a/tidb-binlog/monitor-tidb-binlog-cluster.md +++ b/tidb-binlog/monitor-tidb-binlog-cluster.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Monitoring summary: Learn how to monitor the cluster version of TiDB Binlog. -aliases: ['/docs/dev/tidb-binlog/monitor-tidb-binlog-cluster/','/docs/dev/reference/tidb-binlog/monitor/','/docs/dev/how-to/monitor/tidb-binlog/'] --- # TiDB Binlog Monitoring diff --git a/tidb-binlog/tidb-binlog-configuration-file.md b/tidb-binlog/tidb-binlog-configuration-file.md index d825012ca9b4c..d4d8853e2c453 100644 --- a/tidb-binlog/tidb-binlog-configuration-file.md +++ b/tidb-binlog/tidb-binlog-configuration-file.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Configuration File summary: Learn the configuration items of TiDB Binlog. -aliases: ['/docs/dev/tidb-binlog/tidb-binlog-configuration-file/','/docs/dev/reference/tidb-binlog/config/'] --- # TiDB Binlog Configuration File @@ -10,7 +9,7 @@ This document introduces the configuration items of TiDB Binlog. ## Pump -This section introduces the configuration items of Pump. For the example of a complete Pump configuration file, see [Pump Configuration](https://github.com/pingcap/tidb-binlog/blob/master/cmd/pump/pump.toml). +This section introduces the configuration items of Pump. For the example of a complete Pump configuration file, see [Pump Configuration](https://github.com/pingcap/tidb-binlog/blob/release-8.1/cmd/pump/pump.toml). ### addr @@ -129,7 +128,7 @@ For the detailed description of the above items, see [GoLevelDB Document](https: ## Drainer -This section introduces the configuration items of Drainer. For the example of a complete Drainer configuration file, see [Drainer Configuration](https://github.com/pingcap/tidb-binlog/blob/master/cmd/drainer/drainer.toml) +This section introduces the configuration items of Drainer. For the example of a complete Drainer configuration file, see [Drainer Configuration](https://github.com/pingcap/tidb-binlog/blob/release-8.1/cmd/drainer/drainer.toml) ### addr @@ -306,6 +305,14 @@ If the safe mode is enabled, Drainer modifies the replication updates in the fol Default value: `false` +#### load-schema-snapshot + +- Specifies how Drainer loads table information. +- When you set it to `false`, Drainer replays all DDL operations from history to derive the table schema for each table at a specific schema version. This approach requires processing all DDL changes from the initial state to the target schema version, which might involve significant data processing and replaying. +- When you set it to `true`, Drainer directly reads the table information at the checkpoint TS. Because it directly reads the table information at a specific point in time, this method is usually more efficient. However, it is subject to the GC mechanism, because GC might delete older data versions. If the checkpoint TS is too old, the corresponding table information might have been deleted by GC, making it impossible to read directly. +- When configuring Drainer, choose whether to read the table information at the checkpoint TS based on actual needs. If data integrity and consistency are priorities and handling a large number of DDL changes is acceptable, it is recommended to set it to `false`. If efficiency and performance are more important, and the checkpoint TS is guaranteed to be after the GC safe point, it is recommended to set it to `true`. +- Default value: `false` + ### syncer.to The `syncer.to` section introduces different types of downstream configuration items according to configuration types. @@ -350,4 +357,4 @@ When the downstream is Kafka, the valid configuration items are as follows: * `host` * `user` * `password` -* `port` \ No newline at end of file +* `port` diff --git a/tidb-binlog/tidb-binlog-faq.md b/tidb-binlog/tidb-binlog-faq.md index 5218386ca2e54..6de377e22a005 100644 --- a/tidb-binlog/tidb-binlog-faq.md +++ b/tidb-binlog/tidb-binlog-faq.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog FAQs summary: Learn about the frequently asked questions (FAQs) and answers about TiDB Binlog. -aliases: ['/docs/dev/tidb-binlog/tidb-binlog-faq/','/docs/dev/reference/tidb-binlog/faq/','/docs/dev/reference/tools/tidb-binlog/faq/'] --- # TiDB Binlog FAQs diff --git a/tidb-binlog/tidb-binlog-glossary.md b/tidb-binlog/tidb-binlog-glossary.md index 4c9ae8b8d25e4..2f6351209d111 100644 --- a/tidb-binlog/tidb-binlog-glossary.md +++ b/tidb-binlog/tidb-binlog-glossary.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Glossary summary: Learn the terms used in TiDB Binlog. -aliases: ['/docs/dev/tidb-binlog/tidb-binlog-glossary/','/docs/dev/reference/tidb-binlog/glossary/'] --- # TiDB Binlog Glossary diff --git a/tidb-binlog/tidb-binlog-overview.md b/tidb-binlog/tidb-binlog-overview.md index 6ac124eddceca..e6246ee568086 100644 --- a/tidb-binlog/tidb-binlog-overview.md +++ b/tidb-binlog/tidb-binlog-overview.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Overview summary: Learn overview of the cluster version of TiDB Binlog. -aliases: ['/docs/dev/tidb-binlog/tidb-binlog-overview/','/docs/dev/reference/tidb-binlog/overview/','/docs/dev/reference/tidb-binlog-overview/','/docs/dev/reference/tools/tidb-binlog/overview/'] --- # TiDB Binlog Cluster Overview @@ -31,15 +30,15 @@ The TiDB Binlog cluster is composed of Pump and Drainer. ### Pump -[Pump](https://github.com/pingcap/tidb-binlog/blob/master/pump) is used to record the binlogs generated in TiDB, sort the binlogs based on the commit time of the transaction, and send binlogs to Drainer for consumption. +[Pump](https://github.com/pingcap/tidb-binlog/blob/release-8.1/pump) is used to record the binlogs generated in TiDB, sort the binlogs based on the commit time of the transaction, and send binlogs to Drainer for consumption. ### Drainer -[Drainer](https://github.com/pingcap/tidb-binlog/tree/master/drainer) collects and merges binlogs from each Pump, converts the binlog to SQL or data of a specific format, and replicates the data to a specific downstream platform. +[Drainer](https://github.com/pingcap/tidb-binlog/tree/release-8.1/drainer) collects and merges binlogs from each Pump, converts the binlog to SQL or data of a specific format, and replicates the data to a specific downstream platform. ### `binlogctl` guide -[`binlogctl`](https://github.com/pingcap/tidb-binlog/tree/master/binlogctl) is an operations tool for TiDB Binlog with the following features: +[`binlogctl`](https://github.com/pingcap/tidb-binlog/tree/release-8.1/binlogctl) is an operations tool for TiDB Binlog with the following features: * Obtaining the current `tso` of TiDB cluster * Checking the Pump/Drainer state @@ -69,7 +68,7 @@ The TiDB Binlog cluster is composed of Pump and Drainer. * Drainer supports replicating binlogs to MySQL, TiDB, Kafka or local files. If you need to replicate binlogs to other Drainer unsuppored destinations, you can set Drainer to replicate the binlog to Kafka and read the data in Kafka for customized processing according to binlog consumer protocol. See [Binlog Consumer Client User Guide](/tidb-binlog/binlog-consumer-client.md). -* To use TiDB Binlog for recovering incremental data, set the config `db-type` to `file` (local files in the proto buffer format). Drainer converts the binlog to data in the specified [proto buffer format](https://github.com/pingcap/tidb-binlog/blob/master/proto/pb_binlog.proto) and writes the data to local files. In this way, you can use [Reparo](/tidb-binlog/tidb-binlog-reparo.md) to recover data incrementally. +* To use TiDB Binlog for recovering incremental data, set the config `db-type` to `file` (local files in the proto buffer format). Drainer converts the binlog to data in the specified [proto buffer format](https://github.com/pingcap/tidb-binlog/blob/release-8.1/proto/pb_binlog.proto) and writes the data to local files. In this way, you can use [Reparo](/tidb-binlog/tidb-binlog-reparo.md) to recover data incrementally. Pay attention to the value of `db-type`: diff --git a/tidb-binlog/tidb-binlog-relay-log.md b/tidb-binlog/tidb-binlog-relay-log.md index 68229df5b7463..752699b1f9c27 100644 --- a/tidb-binlog/tidb-binlog-relay-log.md +++ b/tidb-binlog/tidb-binlog-relay-log.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Relay Log summary: Learn how to use relay log to maintain data consistency in extreme cases. -aliases: ['/docs/dev/tidb-binlog/tidb-binlog-relay-log/','/docs/dev/reference/tidb-binlog/relay-log/'] --- # TiDB Binlog Relay Log diff --git a/tidb-binlog/tidb-binlog-reparo.md b/tidb-binlog/tidb-binlog-reparo.md index c4f80a5015f5b..d6b8d02b505b1 100644 --- a/tidb-binlog/tidb-binlog-reparo.md +++ b/tidb-binlog/tidb-binlog-reparo.md @@ -1,7 +1,6 @@ --- title: Reparo User Guide summary: Learn to use Reparo. -aliases: ['/docs/dev/tidb-binlog/tidb-binlog-reparo/','/docs/dev/reference/tidb-binlog/reparo/'] --- # Reparo User Guide diff --git a/tidb-binlog/troubleshoot-tidb-binlog.md b/tidb-binlog/troubleshoot-tidb-binlog.md index decd1c675c2c3..b3ab2fce68e8b 100644 --- a/tidb-binlog/troubleshoot-tidb-binlog.md +++ b/tidb-binlog/troubleshoot-tidb-binlog.md @@ -1,7 +1,6 @@ --- title: TiDB Binlog Troubleshooting summary: Learn the troubleshooting process of TiDB Binlog. -aliases: ['/docs/dev/tidb-binlog/troubleshoot-tidb-binlog/','/docs/dev/reference/tidb-binlog/troubleshoot/binlog/','/docs/dev/how-to/troubleshoot/tidb-binlog/'] --- # TiDB Binlog Troubleshooting diff --git a/tidb-binlog/upgrade-tidb-binlog.md b/tidb-binlog/upgrade-tidb-binlog.md index 03829113fc26e..046fcff9ae789 100644 --- a/tidb-binlog/upgrade-tidb-binlog.md +++ b/tidb-binlog/upgrade-tidb-binlog.md @@ -1,7 +1,6 @@ --- title: Upgrade TiDB Binlog summary: Learn how to upgrade TiDB Binlog to the latest cluster version. -aliases: ['/docs/dev/tidb-binlog/upgrade-tidb-binlog/','/docs/dev/reference/tidb-binlog/upgrade/','/docs/dev/how-to/upgrade/tidb-binlog/'] --- # Upgrade TiDB Binlog diff --git a/tidb-cloud/_index.md b/tidb-cloud/_index.md index a02d39a5f99f6..e095e5e66e959 100644 --- a/tidb-cloud/_index.md +++ b/tidb-cloud/_index.md @@ -3,6 +3,7 @@ title: TiDB Cloud Documentation aliases: ['/tidbcloud/privacy-policy', '/tidbcloud/terms-of-service', '/tidbcloud/service-level-agreement'] hide_sidebar: true hide_commit: true +summary: TiDB Cloud is a fully-managed Database-as-a-Service (DBaaS) that brings everything great about TiDB to your cloud. It offers guides, samples, and references for learning, trying, developing, maintaining, migrating, monitoring, tuning, securing, billing, integrating, and referencing. --- @@ -11,7 +12,7 @@ hide_commit: true [Why TiDB Cloud](https://docs.pingcap.com/tidbcloud/tidb-cloud-intro) -[Architecture](https://docs.pingcap.com/tidbcloud/tidb-cloud-intro#architecture) +[Key Concepts](https://docs.pingcap.com/tidbcloud/key-concepts) [FAQ](https://docs.pingcap.com/tidbcloud/tidb-cloud-faq) @@ -19,10 +20,14 @@ hide_commit: true -[Try Out TiDB Cloud](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart) +[Try Out TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart) + +[Try Out TiDB + AI](https://docs.pingcap.com/tidbcloud/vector-search-get-started-using-python) [Try Out HTAP](https://docs.pingcap.com/tidbcloud/tidb-cloud-htap-quickstart) +[Try Out TiDB Cloud CLI](https://docs.pingcap.com/tidbcloud/get-started-with-cli) + [Proof of Concept](https://docs.pingcap.com/tidbcloud/tidb-cloud-poc) @@ -69,6 +74,8 @@ hide_commit: true [From Amazon S3 or GCS](https://docs.pingcap.com/tidbcloud/migrate-from-amazon-s3-or-gcs) +[From TiDB Self-Managed](https://docs.pingcap.com/tidbcloud/migrate-from-op-tidb) + [From CSV Files](https://docs.pingcap.com/tidbcloud/import-csv-files) [From Apache Parquet Files](https://docs.pingcap.com/tidbcloud/import-csv-files) @@ -111,13 +118,13 @@ hide_commit: true [Manage project access](https://docs.pingcap.com/tidbcloud/manage-user-access#manage-project-access) -[Configure Security Settings](https://docs.pingcap.com/tidbcloud/configure-security-settings) +[Configure Password Settings](https://docs.pingcap.com/tidbcloud/configure-security-settings) -[Pricing](https://en.pingcap.com/tidb-cloud-pricing/) +[Pricing](https://www.pingcap.com/pricing/) [Invoices](https://docs.pingcap.com/tidbcloud/tidb-cloud-billing#invoices) @@ -143,12 +150,12 @@ hide_commit: true +[SQL Reference](https://docs.pingcap.com/tidbcloud/basic-sql-operations) + [System Variables](https://docs.pingcap.com/tidbcloud/system-variables) [Release Notes](https://docs.pingcap.com/tidbcloud/tidb-cloud-release-notes) -[SQL Reference](https://docs.pingcap.com/tidbcloud/basic-sql-operations) - diff --git a/tidb-cloud/ai-feature-concepts.md b/tidb-cloud/ai-feature-concepts.md new file mode 100644 index 0000000000000..0560b5114f203 --- /dev/null +++ b/tidb-cloud/ai-feature-concepts.md @@ -0,0 +1,50 @@ +--- +title: AI Features +summary: Learn about AI features for TiDB Cloud. +--- + +# AI Features + +The AI features in TiDB Cloud enable you to fully leverage advanced technologies for data exploration, search, and integration. From natural language-driven SQL query generation to high-performance vector search, TiDB combines database capabilities with modern AI features to power innovative applications. With support for popular AI frameworks, embedding models, and seamless integration with ORM libraries, TiDB offers a versatile platform for use cases such as semantic search and AI-powered analytics. + +This document highlights these AI features and how they enhance the TiDB experience. + +## Chat2Query (Beta) + +Chat2Query is an AI-powered feature integrated into SQL Editor that assists users in generating, debugging, or rewriting SQL queries using natural language instructions. For more information, see [Explore your data with AI-assisted SQL Editor](/tidb-cloud/explore-data-with-chat2query.md). + +In addition, TiDB Cloud provides a Chat2Query API for TiDB Cloud Serverless clusters. After it is enabled, TiDB Cloud will automatically create a system Data App called Chat2Query and a Chat2Data endpoint in Data Service. You can call this endpoint to let AI generate and execute SQL statements by providing instructions. For more information, see [Get started with Chat2Query API](/tidb-cloud/use-chat2query-api.md). + +## Vector search (Beta) + +Vector search is a search method that prioritizes the meaning of your data to deliver relevant results. + +Unlike traditional full-text search, which relies on exact keyword matching and word frequency, vector search converts various data types (such as text, images, or audio) into high-dimensional vectors and queries based on the similarity between these vectors. This search method captures the semantic meaning and contextual information of the data, leading to a more precise understanding of user intent. + +Even when the search terms do not exactly match the content in the database, vector search can still provide results that align with the user's intent by analyzing the semantics of the data. For example, a full-text search for "a swimming animal" only returns results containing these exact keywords. In contrast, vector search can return results for other swimming animals, such as fish or ducks, even if these results do not contain the exact keywords. + +For more information, see [Vector Search (Beta) Overview](/tidb-cloud/vector-search-overview.md). + +## AI integrations + +### AI frameworks + +TiDB provides official support for several popular AI frameworks, enabling you to easily integrate AI applications developed based on these frameworks with TiDB Vector Search. + +For a list of supported AI frameworks, see [Vector Search Integration Overview](/tidb-cloud/vector-search-integration-overview.md#ai-frameworks). + +### Embedding models and services + +A vector embedding, also known as an embedding, is a sequence of numbers that represents real-world objects in a high-dimensional space. It captures the meaning and context of unstructured data, such as documents, images, audio, and videos. + +Embedding models are algorithms that transform data into [vector embeddings](/tidb-cloud/vector-search-overview.md#vector-embedding). The choice of an appropriate embedding model is crucial for ensuring the accuracy and relevance of semantic search results. + +TiDB Vector Search supports storing vectors of up to 16383 dimensions, which accommodates most embedding models. For unstructured text data, you can find top-performing text embedding models on the [Massive Text Embedding Benchmark (MTEB) Leaderboard](https://huggingface.co/spaces/mteb/leaderboard). + +### Object Relational Mapping (ORM) libraries + +Object Relational Mapping (ORM) libraries are tools that facilitate the interaction between applications and relational databases by allowing developers to work with database records as if they were objects in their programming language of choice. + +TiDB lets you integrate vector search with ORM libraries to manage vector data alongside traditional relational data. This integration is particularly useful for applications that need to store and query vector embeddings generated by AI models. By using ORM libraries, developers can seamlessly interact with vector data stored in TiDB, leveraging the database's capabilities to perform complex vector operations like nearest neighbor search. + +For a list of supported ORM libraries, see [Vector Search Integration Overview](/tidb-cloud/vector-search-integration-overview.md#object-relational-mapping-orm-libraries). \ No newline at end of file diff --git a/tidb-cloud/api-overview.md b/tidb-cloud/api-overview.md index 02bbf64bd5150..48d316a12b4da 100644 --- a/tidb-cloud/api-overview.md +++ b/tidb-cloud/api-overview.md @@ -3,13 +3,13 @@ title: TiDB Cloud API Overview summary: Learn about what is TiDB Cloud API, its features, and how to use API to manage your TiDB Cloud clusters. --- -# TiDB Cloud API Overview Beta +# TiDB Cloud API Overview (Beta) > **Note:** > > TiDB Cloud API is in beta. -The TiDB Cloud API is a [REST interface](https://en.wikipedia.org/wiki/Representational_state_transfer) that provides you with programmatic access to manage administrative objects within TiDB Cloud. Through this API, you can automatically and efficiently manage resources such as Projects, Clusters, Backups, Restores, Imports, and Billings. +The TiDB Cloud API is a [REST interface](https://en.wikipedia.org/wiki/Representational_state_transfer) that provides you with programmatic access to manage administrative objects within TiDB Cloud. Through this API, you can automatically and efficiently manage resources such as Projects, Clusters, Backups, Restores, Imports, Billings, and resources in the [Data Service](/tidb-cloud/data-service-overview.md). The API has the following features: @@ -23,6 +23,10 @@ To start using TiDB Cloud API, refer to the following resources in TiDB Cloud AP - [Authentication](https://docs.pingcap.com/tidbcloud/api/v1beta#section/Authentication) - [Rate Limiting](https://docs.pingcap.com/tidbcloud/api/v1beta#section/Rate-Limiting) - API Full References - - [v1beta1](https://docs.pingcap.com/tidbcloud/api/v1beta1) + - v1beta1 + - [Billing](https://docs.pingcap.com/tidbcloud/api/v1beta1/billing) + - [Data Service](https://docs.pingcap.com/tidbcloud/api/v1beta1/dataservice) + - [IAM](https://docs.pingcap.com/tidbcloud/api/v1beta1/iam) + - [MSP (Deprecated)](https://docs.pingcap.com/tidbcloud/api/v1beta1/msp) - [v1beta](https://docs.pingcap.com/tidbcloud/api/v1beta#tag/Project) - [Changelog](https://docs.pingcap.com/tidbcloud/api/v1beta#section/API-Changelog) diff --git a/tidb-cloud/architecture-concepts.md b/tidb-cloud/architecture-concepts.md new file mode 100644 index 0000000000000..61a5dea61f934 --- /dev/null +++ b/tidb-cloud/architecture-concepts.md @@ -0,0 +1,104 @@ +--- +title: Architecture +summary: Learn about architecture concepts for TiDB Cloud. +--- + +# Architecture + +TiDB Cloud is a fully-managed Database-as-a-Service (DBaaS) that brings the flexibility and power of [TiDB](https://docs.pingcap.com/tidb/stable/overview), an open-source HTAP (Hybrid Transactional and Analytical Processing) database, to Google Cloud and AWS. + +TiDB is MySQL-compatible, making it easy to migrate and work with existing applications, while offering seamless scalability to handle everything from small workloads to massive, high-performance clusters. It supports both transactional (OLTP) and analytical (OLAP) workloads in one system, simplifying operations and enabling real-time insights. + +TiDB Cloud provides two deployment options: **TiDB Cloud** **Serverless**, for auto-scaling, cost-efficient workloads, and **TiDB Cloud Dedicated**, for enterprise-grade applications with dedicated resources and advanced capabilities. TiDB Cloud makes it easy to scale your database, handle complex management tasks, and stay focused on developing reliable, high-performing applications. + +## TiDB Cloud Serverless + +TiDB Cloud Serverless is a fully managed serverless solution that provides HTAP capabilities similar to traditional TiDB, while offering auto-scaling to alleviate users' burdens related to capacity planning and management complexities. It includes a free tier for basic usage, with consumption-based billing for any usage that exceeds the free limits. TiDB Cloud Serverless offers two types of high availability to address varying operational requirements. + +By default, clusters utilizing the Zonal High Availability option have all components located within the same availability zone, which results in lower network latency. + +![TiDB Cloud Serverless zonal high availability](/media/tidb-cloud/serverless-zonal-high-avaliability-aws.png) + +For applications that require maximum infrastructure isolation and redundancy, the Regional High Availability option distributes nodes across multiple availability zones. + +![TiDB Cloud Serverless regional high availability](/media/tidb-cloud/serverless-regional-high-avaliability-aws.png) + +## TiDB Cloud Dedicated + +TiDB Cloud Dedicated is designed for mission-critical businesses, offering high availability across multiple availability zones, horizontal scaling, and full HTAP capabilities. + +Built on isolated cloud resources such as VPCs, VMs, managed Kubernetes services, and cloud storage, it leverages the infrastructure of major cloud providers. TiDB Cloud Dedicated clusters support the complete TiDB feature set, enabling rapid scaling, reliable backups, deployment within specific VPCs, and geographic-level disaster recovery. + +![TiDB Cloud Dedicated Architecture](/media/tidb-cloud/tidb-cloud-dedicated-architecture.png) + +## TiDB Cloud console + +The [TiDB Cloud console](https://tidbcloud.com/) is the web-based management interface for both TiDB Cloud Serverless and TiDB Cloud Dedicated. It provides tools to manage clusters, import or migrate data, monitor performance metrics, configure backups, set up security controls, and integrate with other cloud services, all from a single, user-friendly platform. + +## TiDB Cloud CLI (Beta) + +The TiDB Cloud CLI, `ticloud`, allows you to manage TiDB Cloud Serverless and TiDB Cloud Dedicated directly from your terminal with simple commands. You can perform tasks such as: + +- Creating, deleting, and listing clusters. +- Importing data into clusters. +- Exporting data from clusters. + +For more information, see [TiDB Cloud CLI Reference](/tidb-cloud/cli-reference.md). + +## TiDB Cloud API (Beta) + +The TiDB Cloud API is a REST-based interface that provides programmatic access to manage resources across TiDB Cloud Serverless and TiDB Cloud Dedicated. It enables automated and efficient handling of tasks such as managing projects, clusters, backups, restores, data imports, billing, and other resources in [TiDB Cloud Data Service](/tidb-cloud/data-service-overview.md). + +For more information, see [TiDB Cloud API Overview](/tidb-cloud/api-overview.md). + +## Nodes + +In TiDB Cloud, each cluster consists of TiDB, TiKV, and TiFlash nodes. + +- In a TiDB Cloud Dedicated cluster, you can fully manage the number and size of your dedicated TiDB, TiKV, and TiFlash nodes according to your performance requirements. For more information, see [Scalability](/tidb-cloud/scalability-concepts.md). +- In a TiDB Cloud Serverless cluster, the number and size of TiDB, TiKV, and TiFlash nodes are automatically managed. This ensures seamless scaling, eliminating the need for users to handle node configuration or management tasks. + +### TiDB node + +A [TiDB node](/tidb-computing.md) is a stateless SQL layer that connects to applications using a MySQL-compatible endpoint. It handles tasks like parsing, optimizing, and creating distributed execution plans for SQL queries. + +You can deploy multiple TiDB nodes to scale horizontally and manage higher workloads. These nodes work with load balancers, such as TiProxy or HAProxy, to provide a seamless interface. TiDB nodes do not store data themselves---they forward data requests to TiKV nodes for row-based storage or TiFlash nodes for columnar storage. + +### TiKV node + +A [TiKV node](/tikv-overview.md) is the backbone of data storage in the TiDB architecture, serving as a distributed transactional key-value storage engine that delivers reliability, scalability, and high availability. + +**Key features:** + +- **Region-based data storage** + + - Data is divided into [Regions](https://docs.pingcap.com/tidb/dev/glossary#regionpeerraft-group), each covering a specific Key Range (left-closed, right-open interval: `StartKey` to `EndKey`). + - Multiple Regions coexist within each TiKV node to ensure efficient data distribution. + +- **Transactional support** + + - TiKV nodes provide native distributed transaction support at the key-value level, ensuring Snapshot Isolation as the default isolation level. + - The TiDB node translates SQL execution plans into calls to the TiKV node API, enabling seamless SQL-level transaction support. + +- **High availability** + + - All data in TiKV nodes is replicated (default: three replicas) for durability. + - TiKV ensures native high availability and supports automatic failover, safeguarding against node failures. + +- **Scalability and reliability** + + - TiKV nodes are designed to handle expanding datasets while maintaining distributed consistency and fault tolerance. + +### TiFlash node + +A [TiFlash node](/tiflash/tiflash-overview.md) is a specialized type of storage node within the TiDB architecture. Unlike ordinary TiKV nodes, TiFlash is designed for analytical acceleration with a columnar storage model. + +**Key features:** + +- **Columnar storage** + + TiFlash nodes store data in a columnar format, making them optimized for analytical queries and significantly improving performance for read-intensive workloads. + +- **Vector search index support** + + The vector search index feature uses TiFlash replicas for tables, enabling advanced search capabilities and improving efficiency in complex analytical scenarios. \ No newline at end of file diff --git a/tidb-cloud/backup-and-restore-concepts.md b/tidb-cloud/backup-and-restore-concepts.md new file mode 100644 index 0000000000000..18feb0e0c511c --- /dev/null +++ b/tidb-cloud/backup-and-restore-concepts.md @@ -0,0 +1,42 @@ +--- +title: Backup & Restore +summary: Learn about backup & restore concepts for TiDB Cloud. +--- + +# Backup & Restore + +TiDB Cloud Backup & Restore features are designed to safeguard your data and ensure business continuity by enabling you to back up and recover cluster data. + +## Automatic backup + +For both TiDB Cloud Serverless and TiDB Cloud Dedicated clusters, snapshot backups are taken automatically by default and stored according to your backup retention policy. + +For more information, see the following: + +- [Automatic backups for TiDB Cloud Serverless clusters](/tidb-cloud/backup-and-restore-serverless.md#automatic-backups) +- [Automatic backups for TiDB Cloud Dedicated clusters](/tidb-cloud/backup-and-restore.md#turn-on-auto-backup) + +## Manual backup + +Manual backup is a feature of TiDB Cloud Dedicated that enables you to back up your data to a known state as needed, and then restore to that state at any time. + +For more information, see [Perform a manual backup](/tidb-cloud/backup-and-restore.md#perform-a-manual-backup). + +## Dual region backup + +Dual region backup is a feature of TiDB Cloud Dedicated that enables you to replicate backups from your cluster region to another different region. After it is enabled, all backups are automatically replicated to the specified region. This provides cross-region data protection and disaster recovery capabilities. It is estimated that approximately 99% of the data can be replicated to the secondary region within an hour. + +For more information, see [Turn on dual region backup](/tidb-cloud/backup-and-restore.md#turn-on-dual-region-backup). + +## Point-in-time Restore + +Point-in-time Restore is a feature that enables you to restore data of any point in time to a new cluster. You can use it to: + +- Reduce RPO in disaster recovery. +- Resolve cases of data write errors by restoring point-in-time that is before the error event. +- Audit the historical data of the business. + +If you want to perform Point-in-time Restore, note the following: + +- For TiDB Cloud Serverless clusters, Point-in-time Restore is available only for scalable clusters and not available for free clusters. For more information, see [Restore mode](/tidb-cloud/backup-and-restore-serverless.md#restore-mode). +- For TiDB Cloud Dedicated clusters, you need to [enable PITR](/tidb-cloud/backup-and-restore.md#turn-on-point-in-time-restore) in advance. diff --git a/tidb-cloud/backup-and-restore-serverless.md b/tidb-cloud/backup-and-restore-serverless.md index ba0a494131f86..f49ee32af80bf 100644 --- a/tidb-cloud/backup-and-restore-serverless.md +++ b/tidb-cloud/backup-and-restore-serverless.md @@ -1,57 +1,97 @@ --- -title: Back Up and Restore TiDB Serverless Data -summary: Learn how to back up and restore your TiDB Serverless cluster. +title: Back Up and Restore TiDB Cloud Serverless Data +summary: Learn how to back up and restore your TiDB Cloud Serverless cluster. aliases: ['/tidbcloud/restore-deleted-tidb-cluster'] --- -# Back Up and Restore TiDB Serverless Data +# Back Up and Restore TiDB Cloud Serverless Data -This document describes how to back up and restore your TiDB Serverless cluster data on TiDB Cloud. +This document describes how to back up and restore your TiDB Cloud Serverless cluster data on TiDB Cloud. > **Tip:** > -> To learn how to back up and restore TiDB Dedicated cluster data, see [Back Up and Restore TiDB Dedicated Data](/tidb-cloud/backup-and-restore.md). +> To learn how to back up and restore TiDB Cloud Dedicated cluster data, see [Back Up and Restore TiDB Cloud Dedicated Data](/tidb-cloud/backup-and-restore.md). -## Limitations +## Automatic backups -- It is important to note that TiDB Serverless clusters only support in-place restoring from backups. When a restore is performed, tables in the `mysql` schema are also impacted. Hence, any changes made to user credentials and permissions or system variables will be rolled back to the state when the backup was taken. -- Manual backup is not yet supported. -- The cluster will be unavailable during the restore process, and existing connections will be terminated. You can establish new connections once the restore is complete. -- If any TiFlash replica is enabled, the replica will be unavailable for a while after the restore because data needs to be rebuilt in TiFlash. +TiDB Cloud Serverless automatically backs up your cluster data, allowing you to restore data from a backup snapshot to minimize data loss in the event of a disaster. -## Backup +### Learn about the backup setting -Automatic backups are scheduled for your TiDB Serverless clusters according to the backup setting, which can reduce your loss in extreme disaster situations. +Automatic backup settings vary between free clusters and scalable clusters, as shown in the following table: -### Automatic backup +| Backup setting | Free clusters | Scalable clusters | +|------------------|--------------|------------------| +| Backup Cycle | Daily | Daily | +| Backup Retention | 1 day | 14 days | +| Backup Time | Fixed time | Configurable | -By the automatic backup, you can back up the TiDB Serverless cluster data every day at the backup time you have set. To set the backup time, perform the following steps: +- **Backup Cycle** is the frequency at which backups are taken. -1. Navigate to the **Backup** page of a TiDB Serverless cluster. +- **Backup Retention** is the duration for which backups are retained. Expired backups cannot be restored. + +- **Backup Time** is the time when the backup starts to be scheduled. Note that the final backup time might fall behind the configured backup time. + + - Free clusters: the backup time is a randomly fixed time. + - Scalable clusters: you can configure the backup time to every half an hour. The default value is a randomly fixed time. -2. Click **Backup Settings**. This will open the **Backup Settings** window, where you can configure the automatic backup settings according to your requirements. +### Configure the backup setting - - In **Backup Time**, schedule a start time for the daily cluster backup. +To set the backup time for a scalable cluster, perform the following steps: - If you do not specify a preferred backup time, TiDB Cloud assigns a default backup time, which is 2:00 AM in the time zone of the region where the cluster is located. +1. Navigate to the **Backup** page of a TiDB Cloud Serverless cluster. - - In **Backup Retention**, configure the minimum backup data retention period. +2. Click **Backup Setting**. This will open the **Backup Setting** window, where you can configure the automatic backup settings according to your requirements. - The backup retention period must be set within a range of 7 to 90 days. +3. In **Backup Time**, schedule a start time for the daily cluster backup. -3. Click **Confirm**. +4. Click **Confirm**. -### Delete backup files +## Restore -To delete an existing backup file, perform the following steps: +TiDB Cloud Serverless clusters offer restore functionality to help recover data in case of accidental loss or corruption. -1. Navigate to the **Backup** tab of a cluster. +### Restore mode -2. Click **Delete** for the backup file that you want to delete. +TiDB Cloud Serverless supports snapshot restore and point-in-time restore for your cluster. -## Restore +- **Snapshot Restore**: restores your cluster from a specific backup snapshot. + +- **Point-in-Time Restore (beta)**: restores your cluster to a specific time. + + - Free clusters: not supported. + - Scalable clusters: restores to any time within the last 14 days, but not before the cluster creation time or after the current time minus one minute. + +### Restore destination + +TiDB Cloud Serverless supports restoring in-place and restoring to a new cluster. + +**In-place restore** + +Restore to the current cluster will overwrite existing data. Note the following: + +- Existing connections will be terminated once the restore is started. +- The cluster will be unavailable, and new connections will be blocked during the restore process. +- Restore will affect tables in the `mysql` schema. Any changes to user credentials, permissions, or system variables will be reverted to their state at the backup time. + +**Restore to a new cluster** + +Create and restore to the new cluster. Note the following: + +- User credentials and permissions from the source cluster will not be restored to the new cluster. + +### Restore timeout + +The restore process typically completes within a few minutes. If the restore takes longer than three hours, it is automatically canceled. The outcome of a canceled restore depends on the destination: + +- **In-place restore**: the cluster status changes from **Restoring** to **Available**, and the cluster becomes accessible. +- **Restore to a new cluster**: the new cluster is deleted and the source cluster remains unchanged. + +If the data is corrupted after a canceled restore and cannot be recovered, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md) for assistance. -TiDB Serverless only supports in-place restoration. To restore your TiDB Serverless cluster from a backup, follow these steps: +### Perform the restore + +To restore your TiDB Cloud Serverless cluster, follow these steps: 1. Navigate to the **Backup** page of a cluster. @@ -60,23 +100,17 @@ TiDB Serverless only supports in-place restoration. To restore your TiDB Serverl 3. In **Restore Mode**, you can choose to restore from a specific backup or any point in time. -
+
To restore from a selected backup snapshot, take the following steps: - 1. Click **Basic Snapshot Restore**. - 2. Select the backup snapshot you want to restore to. + 1. Click **Snapshot Restore**. + 2. Select the backup snapshot you want to restore from.
- This feature lets you restore a cluster to a specific state from any time within the last 90 days. - - > **Note:** - > - > The **Point-in-Time Restore** feature is currently in beta. - - To restore from a specific point in time, take the following steps: + To restore to a specific point in time for a scalable cluster, take the following steps: 1. Click **Point-in-Time Restore**. 2. Select the date and time you want to restore to. @@ -84,6 +118,31 @@ TiDB Serverless only supports in-place restoration. To restore your TiDB Serverl
-4. Click **Restore** to begin the restoration process. +4. In **Destination**, you can choose to restore to a new cluster or restore in-place. + + +
+ + To restore to a new cluster, take the following steps: + + 1. Click **Restore to a New Cluster**. + 2. Enter a name for the new cluster. + 3. Choose the cluster plan for the new cluster. + 4. If you choose a scalable cluster, set a monthly spending limit, and then configure advanced settings as needed. Otherwise, skip this step. + +
+
+ + To restore in-place, click **In-place Restore**. + +
+ + +5. Click **Restore** to begin the restore process. + +Once the restore process begins, the cluster status changes to **Restoring**. The cluster will remain unavailable until the restore is complete and the status changes to **Available**. + +## Limitations - After initiating the restore process, the cluster status changes to **Restoring**. The cluster will be unavailable during the restore process and existing connections will be terminated. Once the restore process completes successfully, you can access the cluster as usual. +- If a TiFlash replica is enabled, it will be unavailable for a period after the restore, because data needs to be rebuilt in TiFlash. +- Manual backups are not supported for TiDB Cloud Serverless clusters. diff --git a/tidb-cloud/backup-and-restore.md b/tidb-cloud/backup-and-restore.md index 40b087d63ab68..3301cb82fd2a4 100644 --- a/tidb-cloud/backup-and-restore.md +++ b/tidb-cloud/backup-and-restore.md @@ -1,21 +1,21 @@ --- -title: Back Up and Restore TiDB Dedicated Data -summary: Learn how to back up and restore your TiDB Dedicated cluster. +title: Back Up and Restore TiDB Cloud Dedicated Data +summary: Learn how to back up and restore your TiDB Cloud Dedicated cluster. aliases: ['/tidbcloud/restore-deleted-tidb-cluster'] --- -# Back Up and Restore TiDB Dedicated Data +# Back Up and Restore TiDB Cloud Dedicated Data -This document describes how to back up and restore your TiDB Dedicated cluster data on TiDB Cloud. TiDB Dedicated supports automatic backup and manual backup. You can also restore backup data to a new cluster or restore a deleted cluster from the recycle bin. +This document describes how to back up and restore your TiDB Cloud Dedicated cluster data on TiDB Cloud. TiDB Cloud Dedicated supports automatic backup and manual backup. You can also restore backup data to a new cluster or restore a deleted cluster from the recycle bin. > **Tip** > -> To learn how to back up and restore TiDB Serverless cluster data, see [Back Up and Restore TiDB Serverless Data](/tidb-cloud/backup-and-restore-serverless.md). +> To learn how to back up and restore TiDB Cloud Serverless cluster data, see [Back Up and Restore TiDB Cloud Serverless Data](/tidb-cloud/backup-and-restore-serverless.md). ## Limitations -- For clusters of v6.2.0 or later versions, TiDB Dedicated supports restoring user accounts and SQL bindings from backups by default. -- TiDB Dedicated does not support restoring system variables stored in the `mysql` schema. +- For clusters of v6.2.0 or later versions, TiDB Cloud Dedicated supports restoring user accounts and SQL bindings from backups by default. +- TiDB Cloud Dedicated does not support restoring system variables stored in the `mysql` schema. - It is recommended that you import data first, then perform a **manual** snapshot backup, and finally enable Point-in-time Restore. Because the data imported through the TiDB Cloud console **does not** generate change logs, it cannot be automatically detected and backed up. For more information, see [Import CSV Files from Amazon S3 or GCS into TiDB Cloud](/tidb-cloud/import-csv-files.md). - If you turn on and off Point-in-time Restore multiple times, you can only choose a time point within the recoverable range after the most recent Point-in-time Restore is enabled. The earlier recoverable range is not accessible. - DO NOT modify the switches of **Point-in-time Restore** and **Dual Region Backup** at the same time. @@ -24,13 +24,13 @@ This document describes how to back up and restore your TiDB Dedicated cluster d ### Turn on auto backup -TiDB Dedicated supports both [snapshot backup](https://docs.pingcap.com/tidb/stable/br-snapshot-guide) and [log backup](https://docs.pingcap.com/tidb/stable/br-pitr-guide). Snapshot backup enables you to restore data to the backup point. By default, snapshot backups are taken automatically and stored according to your backup retention policy. You can disable auto backup at any time. +TiDB Cloud Dedicated supports both [snapshot backup](https://docs.pingcap.com/tidb/stable/br-snapshot-guide) and [log backup](https://docs.pingcap.com/tidb/stable/br-pitr-guide). Snapshot backup enables you to restore data to the backup point. By default, snapshot backups are taken automatically and stored according to your backup retention policy. You can disable auto backup at any time. #### Turn on Point-in-time Restore > **Note** > -> The Point-in-time Restore feature is supported for TiDB Dedicated clusters that are v6.4.0 or later. +> The Point-in-time Restore feature is supported for TiDB Cloud Dedicated clusters that are v6.4.0 or later. This feature supports restoring data of any point in time to a new cluster. You can use it to: @@ -42,9 +42,9 @@ It is strongly recommended to turn on this feature. The cost is the same as snap To turn on this feature, perform the following steps: -1. Navigate to the **Backup** page of a TiDB Dedicated cluster. +1. Navigate to the **Backup** page of a TiDB Cloud Dedicated cluster. -2. Click **Backup Settings**. +2. Click **Backup Setting**. 3. Toggle the **Auto Backup** switch to **On**. @@ -54,30 +54,28 @@ To turn on this feature, perform the following steps: > > Point-in-Time Restore only takes effect after the next backup task is completed. To make it take effect earlier, you can [manually perform a backup](#perform-a-manual-backup) after enabling it. -5. Click **Confirm** to preview the configuration changes. - -6. Click **Confirm** again to save changes. +5. Click **Save** to save changes. #### Configure backup schedule -TiDB Dedicated supports daily and weekly backup schedules. By default, the backup schedule is set to daily. You can choose a specific time of the day or week to start snapshot backup. +TiDB Cloud Dedicated supports daily and weekly backup schedules. By default, the backup schedule is set to daily. You can choose a specific time of the day or week to start snapshot backup. To configure the backup schedule, perform the following steps: -1. Navigate to the **Backup** page of a TiDB Dedicated cluster. +1. Navigate to the **Backup** page of a TiDB Cloud Dedicated cluster. -2. Click **Backup Settings**. +2. Click **Backup Setting**. 3. Toggle the **Auto Backup** switch to **On**. 4. Configure the backup schedule as follows: - - In **Backup Scheduler**, select either the **Daily** or **Weekly** checkbox. If you select **Weekly**, you need to specify the days of the week for the backup. + - In **Backup Cycle**, click either the **Daily Backup** or **Weekly Backup** tab. For **Weekly Backup**, you need to specify the days of the week for the backup. > **Warning** > > - When weekly backup is enabled, the Point-in-time Restore feature is enabled by default and cannot be disabled. - > - If you change the backup scheduler from weekly to daily, the Point-in-time Restore feature remains its original setting. You can manually disable it if needed. + > - If you change the backup cycle from weekly to daily, the Point-in-time Restore feature remains its original setting. You can manually disable it if needed. - In **Backup Time**, schedule a start time for the daily or weekly cluster backup. @@ -91,33 +89,30 @@ To configure the backup schedule, perform the following steps: > **Note** > - > - After you delete a cluster, the automatic backup files will be retained for a specified period, as set in backup retention. You need to delete the backup files accordingly. - > - After you delete a cluster, the existing manual backup files will be retained until you manually delete them, or your account is closed. + > - All auto-backups, except the latest one, will be deleted if their lifetime exceeds the retention period. The latest auto-backup will not be deleted unless you delete it manually. This ensures that you can restore cluster data if accidental deletion occurs. + > - After you delete a cluster, auto-backups with a lifetime within the retention period will be moved to the recycle bin. -### Turn on dual region backup (beta) +### Turn on dual region backup > **Note:** > -> - The dual region backup feature is currently in beta. -> - TiDB Dedicated clusters hosted on Google Cloud work seamlessly with Google Cloud Storage. Similar to Google Cloud Storage, **TiDB Dedicated supports dual-region pairing only within the same multi-region code as Google dual-region storage**. For example, in Asia, currently you must pair Tokyo and Osaka together for dual-region storage. For more information, refer to [Dual-regions](https://cloud.google.com/storage/docs/locations#location-dr). +> TiDB Cloud Dedicated clusters hosted on Google Cloud work seamlessly with Google Cloud Storage. Similar to Google Cloud Storage, **TiDB Cloud Dedicated supports dual-region pairing only within the same multi-region code as Google dual-region storage**. For example, in Asia, currently you must pair Tokyo and Osaka together for dual-region storage. For more information, refer to [Dual-regions](https://cloud.google.com/storage/docs/locations#location-dr). -TiDB Dedicated supports dual region backup by replicating backups from your cluster region to another different region. After you enable this feature, all backups are automatically replicated to the specified region. This provides cross-region data protection and disaster recovery capabilities. It is estimated that approximately 99% of the data can be replicated to the secondary region within an hour. +TiDB Cloud Dedicated supports dual region backup by replicating backups from your cluster region to another different region. After you enable this feature, all backups are automatically replicated to the specified region. This provides cross-region data protection and disaster recovery capabilities. It is estimated that approximately 99% of the data can be replicated to the secondary region within an hour. Dual region backup costs include both backup storage usage and cross-region data transfer fees. For more information, refer to [Data Backup Cost](https://www.pingcap.com/tidb-dedicated-pricing-details#backup-storage-cost). To turn on dual region backup, perform the following steps: -1. Navigate to the **Backup** page of a TiDB Dedicated cluster. +1. Navigate to the **Backup** page of a TiDB Cloud Dedicated cluster. -2. Click **Backup Settings**. +2. Click **Backup Setting**. 3. Toggle the **Dual Region Backup** switch to **On**. -4. From the **Dual Region** drop-down list, select a region to store the backup files. - -5. Click **Confirm** to preview the configuration changes. +4. From the **Secondary Region** drop-down list, select a region to store the backup files. -6. Click **Confirm** again to save changes. +5. Click **Save** to save changes. ### Turn off auto backup @@ -127,17 +122,15 @@ To turn on dual region backup, perform the following steps: To turn off auto backup, perform the following steps: -1. Navigate to the **Backup** page of a TiDB Dedicated cluster. +1. Navigate to the **Backup** page of a TiDB Cloud Dedicated cluster. -2. Click **Backup Settings**. +2. Click **Backup Setting**. 3. Toggle the **Auto Backup** switch to **Off**. -4. Click **Confirm** to preview the configuration changes. - -5. Click **Confirm** again to save changes. +4. Click **Save** to save changes. -### Turn off dual region backup (beta) +### Turn off dual region backup > **Tip** > @@ -145,27 +138,30 @@ To turn off auto backup, perform the following steps: To turn off dual region backup, perform the following steps: -1. Navigate to the **Backup** page of a TiDB Dedicated cluster. +1. Navigate to the **Backup** page of a TiDB Cloud Dedicated cluster. -2. Click **Backup Settings**. +2. Click **Backup Setting**. 3. Toggle the **Dual Region Backup** switch to **Off**. -4. Click **Confirm** to preview the configuration changes. - -5. Click **Confirm** again to save changes. +4. Click **Save** to save changes. ### Perform a manual backup Manual backups are user-initiated backups that enable you to back up your data to a known state as needed, and then restore to that state at any time. -To apply a manual backup to your TiDB Dedicated cluster, perform the following steps: +> **Note** +> +> - Manual backups are retained indefinitely until you choose to delete them manually or your account is closed. +> - After a TiDB Cloud Dedicated cluster is deleted, its existing manual backups will be moved to the recycle bin and kept there until manually deleted or your account is closed. + +To apply a manual backup to your TiDB Cloud Dedicated cluster, perform the following steps: -1. Navigate to the **Backup** tab of a cluster. +1. Navigate to the **Backup** page of a cluster. -2. Click **Manual Backup**. The setting window displays. +2. In the upper-right corner, click **...** > **Manual Backup**. -3. Enter a **Name**. +3. In the displayed dialog, enter a **Name**. 4. Click **Confirm**. Then your cluster data is backed up. @@ -175,17 +171,17 @@ To apply a manual backup to your TiDB Dedicated cluster, perform the following s To delete an existing backup file, perform the following steps: -1. Navigate to the **Backup** tab of a cluster. +1. Navigate to the **Backup** page of a cluster. -2. Click **Delete** for the backup file that you want to delete. +2. Locate the corresponding backup file you want to delete, and click **...** > **Delete** in the **Action** column. #### Delete a running backup job To delete a running backup job, it is similar as [**Delete backup files**](#delete-backup-files). -1. Navigate to the **Backup** tab of a cluster. +1. Navigate to the **Backup** page of a cluster. -2. Click **Delete** for the backup file that is in the **Pending** or **Running** state. +2. Locate the running backup job that is in the **Pending** or **Running** state, and click **...** > **Delete** in the **Action** column. ## Restore @@ -195,7 +191,7 @@ To delete a running backup job, it is similar as [**Delete backup files**](#dele > > When you restore a TiDB cluster from backups, the restore process retains the original time zone setting without overwriting it. -To restore your TiDB Dedicated cluster data from a backup to a new cluster, take the following steps: +To restore your TiDB Cloud Dedicated cluster data from a backup to a new cluster, take the following steps: 1. Navigate to the **Backup** tab of a cluster. @@ -212,7 +208,7 @@ To restore your TiDB Dedicated cluster data from a backup to a new cluster, take
- To restore data of any point in time within the backup retention to a new cluster, make sure that **Point-in-time Restore** in **Backup Settings** is on and then take the following steps: + To restore data of any point in time within the backup retention to a new cluster, make sure that **Point-in-time Restore** in **Backup Setting** is on and then take the following steps: - Click **Select Time Point**. - Select **Date** and **Time** you want to restore to. @@ -229,7 +225,7 @@ To restore your TiDB Dedicated cluster data from a backup to a new cluster, take
-5. In **Restore to Region**, select the same region as the **Backup Storage Region** configured in the **Backup Settings**. +5. In **Restore to Region**, select the same region as the **Primary Region** configured in the **Backup Setting**. 6. In the **Restore** window, you can also make the following changes if necessary: @@ -239,9 +235,9 @@ To restore your TiDB Dedicated cluster data from a backup to a new cluster, take 7. Click **Restore**. - The cluster restore process starts and the **Security Settings** dialog box is displayed. + The cluster restore process starts and the **Password Settings** dialog box is displayed. -8. In the **Security Settings** dialog box, set the root password and allowed IP addresses to connect to your cluster, and then click **Apply**. +8. In the **Password Settings** dialog box, set the root password to connect to your cluster, and then click **Save**. ### Restore a deleted cluster @@ -262,6 +258,6 @@ To restore a deleted cluster from recycle bin, take the following steps: 6. Click **Confirm**. - The cluster restore process starts and the **Security Settings** dialog box is displayed. + The cluster restore process starts and the **Password Settings** dialog box is displayed. -7. In the **Security Settings** dialog box, set the root password and allowed IP addresses to connect to your cluster, and then click **Apply**. +7. In the **Password Settings** dialog box, set the root password to connect to your cluster, and then click **Save**. diff --git a/tidb-cloud/branch-github-integration.md b/tidb-cloud/branch-github-integration.md index b2ef509b3ba47..ba2a4f8b23c29 100644 --- a/tidb-cloud/branch-github-integration.md +++ b/tidb-cloud/branch-github-integration.md @@ -1,21 +1,21 @@ --- -title: Integrate TiDB Serverless Branching (Beta) with GitHub -summary: Learn how to integrate the TiDB Serverless branching feature with GitHub. +title: Integrate TiDB Cloud Serverless Branching (Beta) with GitHub +summary: Learn how to integrate the TiDB Cloud Serverless branching feature with GitHub. --- -# Integrate TiDB Serverless Branching (Beta) with GitHub +# Integrate TiDB Cloud Serverless Branching (Beta) with GitHub > **Note:** > -> The integration is built upon [TiDB Serverless branching](/tidb-cloud/branch-overview.md). Make sure that you are familiar with TiDB Serverless branching before reading this document. +> The integration is built upon [TiDB Cloud Serverless branching](/tidb-cloud/branch-overview.md). Make sure that you are familiar with TiDB Cloud Serverless branching before reading this document. -If you use GitHub for application development, you can integrate TiDB Serverless branching into your GitHub CI/CD pipeline, which lets you automatically test your pull requests with branches without affecting the production database. +If you use GitHub for application development, you can integrate TiDB Cloud Serverless branching into your GitHub CI/CD pipeline, which lets you automatically test your pull requests with branches without affecting the production database. -In the integration process, you will be prompted to install the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) GitHub App. The app can automatically manage TiDB Serverless branches according to pull requests in your GitHub repository. For example, when you create a pull request, the app will create a corresponding branch for your TiDB Serverless cluster, in which you can work on new features or bug fixes in isolation without affecting the production database. +In the integration process, you will be prompted to install the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) GitHub App. The app can automatically manage TiDB Cloud Serverless branches according to pull requests in your GitHub repository. For example, when you create a pull request, the app will create a corresponding branch for your TiDB Cloud Serverless cluster, in which you can work on new features or bug fixes in isolation without affecting the production database. This document covers the following topics: -1. How to integrate TiDB Serverless branching with GitHub +1. How to integrate TiDB Cloud Serverless branching with GitHub 2. How does the TiDB Cloud Branching app work 3. How to build a branching-based CI workflow to test every pull request using branches rather than the production cluster @@ -25,13 +25,13 @@ Before the integration, make sure that you have the following: - A GitHub account - A GitHub repository for your application -- A [TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) +- A [TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) -## Integrate TiDB Serverless branching with your GitHub repository +## Integrate TiDB Cloud Serverless branching with your GitHub repository -To integrate TiDB Serverless branching with your GitHub repository, take the following steps: +To integrate TiDB Cloud Serverless branching with your GitHub repository, take the following steps: -1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Serverless cluster to go to its overview page. +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. 2. Click **Branches** in the left navigation pane. @@ -40,7 +40,7 @@ To integrate TiDB Serverless branching with your GitHub repository, take the fol - If you have not logged into GitHub, you will be asked to log into GitHub first. - If it is the first time you use the integration, you will be asked to authorize the **TiDB Cloud Branching** app. - + 4. In the **Connect to GitHub** dialog, select a GitHub account in the **GitHub Account** drop-down list. @@ -48,18 +48,18 @@ To integrate TiDB Serverless branching with your GitHub repository, take the fol 5. Select your target repository in the **GitHub Repository** drop-down list. If the list is long, you can search the repository by typing the name. -6. Click **Connect** to connect between your TiDB Serverless cluster and your GitHub repository. +6. Click **Connect** to connect between your TiDB Cloud Serverless cluster and your GitHub repository. - + ## TiDB Cloud Branching app behaviors -After you connect your TiDB Serverless cluster to your GitHub repository, for each pull request in this repository, the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) GitHub App can automatically manage its corresponding TiDB Serverless branch. The following lists the default behaviors for pull request changes: +After you connect your TiDB Cloud Serverless cluster to your GitHub repository, for each pull request in this repository, the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) GitHub App can automatically manage its corresponding TiDB Cloud Serverless branch. The following lists the default behaviors for pull request changes: | Pull request changes | TiDB Cloud Branching app behaviors | |------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| Create a pull request | When you create a pull request in the repository, the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) app creates a branch for your TiDB Serverless cluster. The branch name is in the `${github_branch_name}_${pr_id}_${commit_sha}` format. Note that the number of branches has a [limit](/tidb-cloud/branch-overview.md#limitations-and-quotas). | -| Push new commits to a pull request | Every time you push a new commit to a pull request in the repository, the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) app deletes the previous TiDB Serverless branch and creates a new branch for the latest commit. | +| Create a pull request | When you create a pull request in the repository, the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) app creates a branch for your TiDB Cloud Serverless cluster. When `branch.mode` is set to `reset`, the branch name follows the `${github_branch_name}_${pr_id}` format. When `branch.mode` is set to `reserve`, the branch name follows the `${github_branch_name}_${pr_id}_${commit_sha}` format. Note that the number of branches has a [limit](/tidb-cloud/branch-overview.md#limitations-and-quotas). | +| Push new commits to a pull request | When `branch.mode` is set to `reset`, every time you push a new commit to a pull request in the repository, the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) app resets the TiDB Cloud Serverless branch. When `branch.mode` is set to `reserve`, the app creates a new branch for the latest commit. | | Close or merge a pull request | When you close or merge a pull request, the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) app deletes the branch for this pull request. | | Reopen a pull request | When you reopen a pull request, the [TiDB Cloud Branching](https://github.com/apps/tidb-cloud-branching) app creates a branch for the lasted commit of the pull request. | @@ -94,23 +94,26 @@ github: - ".*_db" ``` -### branch.autoReserved +### branch.mode -**Type:** boolean. **Default:** `false`. +**Type:** string. **Default:** `reset`. -If it is set to `true`, the TiDB Cloud Branching app will not delete the TiDB Serverless branch that is created in the previous commit. +Specify how the TiDB Cloud Branching app handles branch updates: + +- If it is set to `reset`, the TiDB Cloud Branching app will update the existing branch with the latest data. +- If it is set to `reserve`, the TiDB Cloud Branching app will create a new branch for your latest commit. ```yaml github: branch: - autoReserved: false + mode: reset ``` ### branch.autoDestroy **Type:** boolean. **Default:** `true`. -If it is set to `false`, the TiDB Cloud Branching app will not delete the TiDB Serverless branch when a pull request is closed or merged. +If it is set to `false`, the TiDB Cloud Branching app will not delete the TiDB Cloud Serverless branch when a pull request is closed or merged. ```yaml github: @@ -120,21 +123,21 @@ github: ## Create a branching CI workflow -One of the best practices for using branches is to create a branching CI workflow. With the workflow, you can test your code using a TiDB Serverless branch instead of using the production cluster before merging the pull request. You can find a live demo [here](https://github.com/shiyuhang0/tidbcloud-branch-gorm-example). +One of the best practices for using branches is to create a branching CI workflow. With the workflow, you can test your code using a TiDB Cloud Serverless branch instead of using the production cluster before merging the pull request. You can find a live demo [here](https://github.com/shiyuhang0/tidbcloud-branch-gorm-example). Here are the main steps to create the workflow: -1. [Integrate TiDB Serverless branching with your GitHub repository](#integrate-tidb-serverless-branching-with-your-github-repository). +1. [Integrate TiDB Cloud Serverless branching with your GitHub repository](#integrate-tidb-cloud-serverless-branching-with-your-github-repository). 2. Get the branch connection information. - You can use the [wait-for-tidbcloud-branch](https://github.com/tidbcloud/wait-for-tidbcloud-branch) action to wait for the readiness of the TiDB Serverless branch and get the connection information of the branch. + You can use the [wait-for-tidbcloud-branch](https://github.com/tidbcloud/wait-for-tidbcloud-branch) action to wait for the readiness of the TiDB Cloud Serverless branch and get the connection information of the branch. Example usage: ```yaml steps: - - name: Wait for TiDB Serverless branch to be ready + - name: Wait for TiDB Cloud Serverless branch to be ready uses: tidbcloud/wait-for-tidbcloud-branch@v0 id: wait-for-branch with: @@ -142,7 +145,7 @@ Here are the main steps to create the workflow: public-key: ${{ secrets.TIDB_CLOUD_API_PUBLIC_KEY }} private-key: ${{ secrets.TIDB_CLOUD_API_PRIVATE_KEY }} - - name: Test with TiDB Serverless branch + - name: Test with TiDB Cloud Serverless branch run: | echo "The host is ${{ steps.wait-for-branch.outputs.host }}" echo "The user is ${{ steps.wait-for-branch.outputs.user }}" diff --git a/tidb-cloud/branch-manage.md b/tidb-cloud/branch-manage.md index 57b0b97ca7de3..da9f3b88a36a8 100644 --- a/tidb-cloud/branch-manage.md +++ b/tidb-cloud/branch-manage.md @@ -1,11 +1,11 @@ --- -title: Manage TiDB Serverless Branches -summary: Learn How to manage TiDB Serverless branches. +title: Manage TiDB Cloud Serverless Branches +summary: Learn How to manage TiDB Cloud Serverless branches. --- -# Manage TiDB Serverless Branches +# Manage TiDB Cloud Serverless Branches -This document describes how to manage TiDB Serverless branches using the [TiDB Cloud console](https://tidbcloud.com). To manage it using the TiDB Cloud CLI, see [`ticloud branch`](/tidb-cloud/ticloud-branch-create.md). +This document describes how to manage TiDB Cloud Serverless branches using the [TiDB Cloud console](https://tidbcloud.com). To manage it using the TiDB Cloud CLI, see [`ticloud branch`](/tidb-cloud/ticloud-branch-create.md). ## Required access @@ -18,14 +18,25 @@ For more information about permissions, see [User roles](/tidb-cloud/manage-user > **Note:** > -> You can only create branches for TiDB Serverless clusters that are created after July 5, 2023. See [Limitations and quotas](/tidb-cloud/branch-overview.md#limitations-and-quotas) for more limitations. +> You can only create branches for TiDB Cloud Serverless clusters that are created after July 5, 2023. See [Limitations and quotas](/tidb-cloud/branch-overview.md#limitations-and-quotas) for more limitations. To create a branch, perform the following steps: -1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Serverless cluster to go to its overview page. +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. 2. Click **Branches** in the left navigation pane. -3. Click **Create Branch** in the upper-right corner. -4. Enter the branch name, and then click **Create**. +3. In the upper-right corner of the **Branches** page, click **Create Branch**. A dialog is displayed. + + Alternatively, to create a branch from an existing parent branch, locate the row of your target parent branch, and then click **...** > **Create Branch** in the **Action** column. + +4. In the **Create Branch** dialog, configure the following options: + + - **Name**: enter a name for the branch. + - **Parent branch**: select the original cluster or an existing branch. `main` represents the current cluster. + - **Include data up to**: choose one of the following: + - **Current point in time**: create a branch from the current state. + - **Specific date and time**: create a branch from a specified time. + +5. Click **Create**. Depending on the data size in your cluster, the branch creation will be completed in a few minutes. @@ -33,7 +44,7 @@ Depending on the data size in your cluster, the branch creation will be complete To view branches for your cluster, perform the following steps: -1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Serverless cluster to go to its overview page. +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. 2. Click **Branches** in the left navigation pane. The branch list of the cluster is displayed in the right pane. @@ -42,7 +53,7 @@ To view branches for your cluster, perform the following steps: To connect to a branch, perform the following steps: -1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Serverless cluster to go to its overview page. +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. 2. Click **Branches** in the left navigation pane. 3. In the row of your target branch to be connected, click **...** in the **Action** column. 4. Click **Connect** in the drop-down list. The dialog for the connection information is displayed. @@ -51,7 +62,7 @@ To connect to a branch, perform the following steps: Alternatively, you can get the connection string from the cluster overview page: -1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Serverless cluster to go to its overview page. +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. 3. Select the branch you want to connect to in the `Branch` drop-down list. 4. Click **Generate Password** or **Reset Password** to create or reset the root password. @@ -61,12 +72,28 @@ Alternatively, you can get the connection string from the cluster overview page: To delete a branch, perform the following steps: -1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Serverless cluster to go to its overview page. +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. 2. Click **Branches** in the left navigation pane. 3. In the row of your target branch to be deleted, click **...** in the **Action** column. 4. Click **Delete** in the drop-down list. 5. Confirm the deletion. +## Reset a branch + +Resetting a branch synchronizes it with the latest data from its parent. + +> **Note:** +> +> This operation is irreversible. Before resetting a branch, make sure that you have backed up any important data. + +To reset a branch, perform the following steps: + +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. +2. Click **Branches** in the left navigation pane. +3. In the row of your target branch to be reset, click **...** in the **Action** column. +4. Click **Reset** in the drop-down list. +5. Confirm the reset. + ## What's next -- [Integrate TiDB Serverless branching into your GitHub CI/CD pipeline](/tidb-cloud/branch-github-integration.md) +- [Integrate TiDB Cloud Serverless branching into your GitHub CI/CD pipeline](/tidb-cloud/branch-github-integration.md) diff --git a/tidb-cloud/branch-overview.md b/tidb-cloud/branch-overview.md index aa75153ea68ff..6908ebf402984 100644 --- a/tidb-cloud/branch-overview.md +++ b/tidb-cloud/branch-overview.md @@ -1,19 +1,19 @@ --- -title: TiDB Serverless Branching (Beta) Overview -summary: Learn the concept of TiDB Serverless branches. +title: TiDB Cloud Serverless Branching (Beta) Overview +summary: Learn the concept of TiDB Cloud Serverless branches. --- -# TiDB Serverless Branching (Beta) Overview +# TiDB Cloud Serverless Branching (Beta) Overview -TiDB Cloud lets you create branches for TiDB Serverless clusters. A branch for a cluster is a separate instance that contains a diverged copy of data from the original cluster. It provides an isolated environment, allowing you to experiment freely without worrying about affecting the original cluster. +TiDB Cloud lets you create branches for TiDB Cloud Serverless clusters. A branch for a cluster is a separate instance that contains a diverged copy of data from the original cluster. It provides an isolated environment, allowing you to experiment freely without worrying about affecting the original cluster. -With TiDB Serverless branches, developers can work in parallel, iterate rapidly on new features, troubleshoot issues without affecting the production database, and easily revert changes if needed. This feature streamlines the development and deployment process while ensuring a high level of stability and reliability for the production database. +With TiDB Cloud Serverless branches, developers can work in parallel, iterate rapidly on new features, troubleshoot issues without affecting the production database, and easily revert changes if needed. This feature streamlines the development and deployment process while ensuring a high level of stability and reliability for the production database. ## Implementations -When a branch for a cluster is created, the data in the branch diverges from the original cluster. This means that subsequent changes made in either the original cluster or the branch will not be synchronized with each other. +When a branch for a cluster is created, the data in the branch diverges from the original cluster or its parent branch at a specific point in time. This means that subsequent changes made in either the parent or the branch will not be synchronized with each other. -To ensure fast and seamless branch creation, TiDB Serverless uses a copy-on-write technique for sharing data between the original cluster and its branches. This process usually completes within a few minutes and is imperceptible to users, ensuring that it does not affect the performance of your original cluster. +To ensure fast and seamless branch creation, TiDB Cloud Serverless uses a copy-on-write technique for sharing data between the original cluster and its branches. This process usually completes within a few minutes and is imperceptible to users, ensuring that it does not affect the performance of your original cluster. ## Scenarios @@ -33,13 +33,18 @@ You can create branches easily and quickly to get isolated data environments. Br ## Limitations and quotas -Currently, TiDB Serverless branches are in beta and free of charge. +Currently, TiDB Cloud Serverless branches are in beta and free of charge. -- You can only create branches for TiDB Serverless clusters that are created after July 5, 2023. +- You can only create branches for TiDB Cloud Serverless clusters that are created after July 5, 2023. -- For each organization in TiDB Cloud, you can create a maximum of five TiDB Serverless branches by default across all the clusters. The branches of a cluster will be created in the same region as the cluster, and you cannot create branches for a throttled cluster or a cluster larger than 100 GiB. +- For each organization in TiDB Cloud, you can create a maximum of five TiDB Cloud Serverless branches by default across all the clusters. The branches of a cluster will be created in the same region as the cluster, and you cannot create branches for a throttled cluster or a cluster larger than 100 GiB. -- For each branch, 5 GiB storage is allowed. Once the storage is reached, the read and write operations on this branch will be throttled until you reduce the storage. +- For each branch of a free cluster, 10 GiB storage is allowed. For each branch of a scalable cluster, 100 GiB storage is allowed. Once the storage is reached, the read and write operations on this branch will be throttled until you reduce the storage. + +- When [creating a branch](/tidb-cloud/branch-manage.md#create-a-branch) from a specific point in time: + + - For branches of a free cluster, you can select any time within the last 24 hours. + - For branches of a scalable cluster, you can select any time within the last 14 days. If you need more quotas, [contact TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). diff --git a/tidb-cloud/built-in-monitoring.md b/tidb-cloud/built-in-monitoring.md index fc18d7d7f1903..ff039fdc176cc 100644 --- a/tidb-cloud/built-in-monitoring.md +++ b/tidb-cloud/built-in-monitoring.md @@ -22,11 +22,11 @@ To view the metrics on the Metrics page, take the following steps: ## Metrics retention policy -For TiDB Dedicated clusters and TiDB Serverless clusters, the metrics data is kept for 7 days. +For TiDB Cloud Dedicated clusters and TiDB Cloud Serverless clusters, the metrics data is kept for 7 days. -## Metrics for TiDB Dedicated clusters +## Metrics for TiDB Cloud Dedicated clusters -The following sections illustrate the metrics on the Metrics page for TiDB Dedicated clusters. +The following sections illustrate the metrics on the Metrics page for TiDB Cloud Dedicated clusters. ### Overview @@ -67,22 +67,22 @@ The following sections illustrate the metrics on the Metrics page for TiDB Dedic | Metric name | Labels | Description | | :------------| :------| :-------------------------------------------- | | TiDB Uptime | node | The runtime of each TiDB node since last restart. | -| TiDB CPU Usage | node | The CPU usage statistics of each TiDB node. | -| TiDB Memory Usage | node | The memory usage statistics of each TiDB node. | +| TiDB CPU Usage | node, limit | The CPU usage statistics or upper limit of each TiDB node. | +| TiDB Memory Usage | node, limit | The memory usage statistics or upper limit of each TiDB node. | | TiKV Uptime | node | The runtime of each TiKV node since last restart. | -| TiKV CPU Usage | node | The CPU usage statistics of each TiKV node. | -| TiKV Memory Usage | node | The memory usage statistics of each TiKV node. | +| TiKV CPU Usage | node, limit | The CPU usage statistics or upper limit of each TiKV node. | +| TiKV Memory Usage | node, limit | The memory usage statistics or upper limit of each TiKV node. | | TiKV IO Bps | node-write, node-read | The total input/output bytes per second of read and write in each TiKV node. | -| TiKV Storage Usage | node | The storage usage statistics of each TiKV node. | +| TiKV Storage Usage | node, limit | The storage usage statistics or upper limit of each TiKV node. | | TiFlash Uptime | node | The runtime of each TiFlash node since last restart. | -| TiFlash CPU Usage | node | The CPU usage statistics of each TiFlash node. | -| TiFlash Memory | node | The memory usage statistics of each TiFlash node. | +| TiFlash CPU Usage | node, limit | The CPU usage statistics or upper limit of each TiFlash node. | +| TiFlash Memory Usage | node, limit | The memory usage statistics or upper limit of each TiFlash node. | | TiFlash IO MBps | node-write, node-read | The total bytes of read and write in each TiFlash node. | -| TiFlash Storage Usage | node | The storage usage statistics of each TiFlash node. | +| TiFlash Storage Usage | node, limit | The storage usage statistics or upper limit of each TiFlash node. | -## Metrics for TiDB Serverless clusters +## Metrics for TiDB Cloud Serverless clusters -The Metrics page provides two tabs for metrics of TiDB Serverless clusters: +The Metrics page provides two tabs for metrics of TiDB Cloud Serverless clusters: - Cluster Status: displays the cluster-level main metrics. - Database Status: displays the database-level main metrics. @@ -96,11 +96,11 @@ The following table illustrates the cluster-level main metrics under the **Clust | Request Units | RU per second | The Request Unit (RU) is a unit of measurement used to track the resource consumption of a query or transaction. In addition to queries that you run, Request Units can be consumed by background activities, so when the QPS is 0, the Request Units per second might not be zero. | | Used Storage Size | Row-based storage, Columnar storage | The size of the row store and the size of the column store. | | Query Per Second | All, {SQL type} | The number of SQL statements executed per second, which are collected by SQL types, such as `SELECT`, `INSERT`, and `UPDATE`. | -| Average Query Duration | All, {SQL type} | The duration from receiving a request from the client to the TiDB Serverless cluster until the cluster executes the request and returns the result to the client. | +| Average Query Duration | All, {SQL type} | The duration from receiving a request from the client to the TiDB Cloud Serverless cluster until the cluster executes the request and returns the result to the client. | | Failed Query | All | The number of SQL statement execution errors per second. | | Transaction Per Second | All | The number of transactions executed per second. | | Average Transaction Duration | All | The average execution duration of transactions. | -| Total Connection | All | The number of connections to the TiDB Serverless cluster. | +| Total Connection | All | The number of connections to the TiDB Cloud Serverless cluster. | ### Database Status diff --git a/tidb-cloud/changefeed-overview.md b/tidb-cloud/changefeed-overview.md index 979c0f1cf2194..0570e5a9659cf 100644 --- a/tidb-cloud/changefeed-overview.md +++ b/tidb-cloud/changefeed-overview.md @@ -11,7 +11,7 @@ TiDB Cloud changefeed helps you stream data from TiDB Cloud to other data servic > > - Currently, TiDB Cloud only allows up to 100 changefeeds per cluster. > - Currently, TiDB Cloud only allows up to 100 table filter rules per changefeed. -> - For [TiDB Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-serverless), the changefeed feature is unavailable. +> - For [TiDB Cloud Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless), the changefeed feature is unavailable. To access the changefeed feature, navigate to the cluster overview page of your TiDB cluster, and then click **Changefeed** in the left navigation pane. The changefeed page is displayed. diff --git a/tidb-cloud/changefeed-sink-to-apache-kafka.md b/tidb-cloud/changefeed-sink-to-apache-kafka.md index f7580d4d904ec..d55656267e310 100644 --- a/tidb-cloud/changefeed-sink-to-apache-kafka.md +++ b/tidb-cloud/changefeed-sink-to-apache-kafka.md @@ -1,6 +1,6 @@ --- title: Sink to Apache Kafka -Summary: Learn how to create a changefeed to stream data from TiDB Cloud to Apache Kafka. +summary: This document explains how to create a changefeed to stream data from TiDB Cloud to Apache Kafka. It includes restrictions, prerequisites, and steps to configure the changefeed for Apache Kafka. The process involves setting up network connections, adding permissions for Kafka ACL authorization, and configuring the changefeed specification. --- # Sink to Apache Kafka @@ -9,8 +9,8 @@ This document describes how to create a changefeed to stream data from TiDB Clou > **Note:** > -> - To use the changefeed feature, make sure that your TiDB Dedicated cluster version is v6.1.3 or later. -> - For [TiDB Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-serverless), the changefeed feature is unavailable. +> - To use the changefeed feature, make sure that your TiDB Cloud Dedicated cluster version is v6.1.3 or later. +> - For [TiDB Cloud Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless), the changefeed feature is unavailable. ## Restrictions @@ -18,6 +18,17 @@ This document describes how to create a changefeed to stream data from TiDB Clou - Currently, TiDB Cloud does not support uploading self-signed TLS certificates to connect to Kafka brokers. - Because TiDB Cloud uses TiCDC to establish changefeeds, it has the same [restrictions as TiCDC](https://docs.pingcap.com/tidb/stable/ticdc-overview#unsupported-scenarios). - If the table to be replicated does not have a primary key or a non-null unique index, the absence of a unique constraint during replication could result in duplicated data being inserted downstream in some retry scenarios. +- If you choose Private Link or Private Service Connect as the network connectivity method, ensure that your TiDB cluster version meets the following requirements: + + - For v6.5.x: version v6.5.9 or later + - For v7.1.x: version v7.1.4 or later + - For v7.5.x: version v7.5.1 or later + - For v8.1.x: all versions of v8.1.x and later are supported +- If you want to use Debezium as your data format, make sure the version of your TiDB cluster is v8.1.0 or later. +- For the partition distribution of Kafka messages, note the following: + + - If you want to distribute changelogs by primary key or index value to Kafka partition with a specified index name, make sure the version of your TiDB cluster is v7.5.0 or later. + - If you want to distribute changelogs by column value to Kafka partition, make sure the version of your TiDB cluster is v7.5.0 or later. ## Prerequisites @@ -28,7 +39,33 @@ Before creating a changefeed to stream data to Apache Kafka, you need to complet ### Network -Make sure that your TiDB cluster can connect to the Apache Kafka service. +Ensure that your TiDB cluster can connect to the Apache Kafka service. You can choose one of the following connection methods: + +- Private Connect (Beta): ideal for avoiding VPC CIDR conflicts and meeting security compliance, but incurs additional [Private Data Link Cost](/tidb-cloud/tidb-cloud-billing-ticdc-rcu.md#private-data-link-cost). +- VPC Peering: suitable as a cost-effective option, but requires managing potential VPC CIDR conflicts and security considerations. +- Public IP: suitable for a quick setup. + + +
+ +Private Connect leverages **Private Link** or **Private Service Connect** technologies from cloud providers to enable resources in your VPC to connect to services in other VPCs using private IP addresses, as if those services were hosted directly within your VPC. + +TiDB Cloud currently supports Private Connect only for self-hosted Kafka. It does not support direct integration with MSK, Confluent Kafka, or other Kafka SaaS services. To connect to these Kafka SaaS services via Private Connect, you can deploy a [kafka-proxy](https://github.com/grepplabs/kafka-proxy) as an intermediary, effectively exposing the Kafka service as self-hosted Kafka. For a detailed example, see [Set up self-hosted Kafka Private Service Connect by Kafka-proxy in Google Cloud](/tidb-cloud/setup-self-hosted-kafka-private-service-connect.md#set-up-self-hosted-kafka-private-service-connect-by-kafka-proxy). This setup is similar across all Kafka SaaS services. + +- If your Apache Kafka service is hosted in AWS, follow [Set Up Self-Hosted Kafka Private Link Service in AWS](/tidb-cloud/setup-self-hosted-kafka-private-link-service.md) to ensure that the network connection is properly configured. After setup, provide the following information in the TiDB Cloud console to create the changefeed: + + - The ID in Kafka Advertised Listener Pattern + - The Endpoint Service Name + - The Bootstrap Ports + +- If your Apache Kafka service is hosted in Google Cloud, follow [Set Up Self-Hosted Kafka Private Service Connect in Google Cloud](/tidb-cloud/setup-self-hosted-kafka-private-service-connect.md) to ensure that the network connection is properly configured. After setup, provide the following information in the TiDB Cloud console to create the changefeed: + + - The ID in Kafka Advertised Listener Pattern + - The Service Attachment + - The Bootstrap Ports + +
+
If your Apache Kafka service is in an AWS VPC that has no internet access, take the following steps: @@ -39,7 +76,7 @@ If your Apache Kafka service is in an AWS VPC that has no internet access, take 3. If the Apache Kafka URL contains hostnames, you need to allow TiDB Cloud to be able to resolve the DNS hostnames of the Apache Kafka brokers. - 1. Follow the steps in [Enable DNS resolution for a VPC peering connection](https://docs.aws.amazon.com/vpc/latest/peering/modify-peering-connections.html#vpc-peering-dns). + 1. Follow the steps in [Enable DNS resolution for a VPC peering connection](https://docs.aws.amazon.com/vpc/latest/peering/vpc-peering-dns.html). 2. Enable the **Accepter DNS resolution** option. If your Apache Kafka service is in a Google Cloud VPC that has no internet access, take the following steps: @@ -49,6 +86,16 @@ If your Apache Kafka service is in a Google Cloud VPC that has no internet acces You must add the CIDR of the region where your TiDB Cloud cluster is located to the ingress firewall rules. The CIDR can be found on the **VPC Peering** page. Doing so allows the traffic to flow from your TiDB cluster to the Kafka brokers. +
+
+ +If you want to provide Public IP access to your Apache Kafka service, assign Public IP addresses to all your Kafka brokers. + +It is **NOT** recommended to use Public IP in a production environment. + +
+ + ### Kafka ACL authorization To allow TiDB Cloud changefeeds to stream data to Apache Kafka and create Kafka topics automatically, ensure that the following permissions are added in Kafka: @@ -60,21 +107,71 @@ For example, if your Kafka cluster is in Confluent Cloud, you can see [Resources ## Step 1. Open the changefeed page for Apache Kafka -1. In the [TiDB Cloud console](https://tidbcloud.com), navigate to the cluster overview page of the target TiDB cluster, and then click **Changefeed** in the left navigation pane. -2. Click **Create Changefeed**, and select **Kafka** as **Target Type**. +1. Log in to the [TiDB Cloud console](https://tidbcloud.com). +2. Navigate to the cluster overview page of the target TiDB cluster, and then click **Changefeed** in the left navigation pane. +3. Click **Create Changefeed**, and select **Kafka** as **Target Type**. ## Step 2. Configure the changefeed target -1. Under **Brokers Configuration**, fill in your Kafka brokers endpoints. You can use commas `,` to separate multiple endpoints. -2. Select an authentication option according to your Kafka authentication configuration. +The steps vary depending on the connectivity method you select. + + +
+ +1. In **Connectivity Method**, select **VPC Peering** or **Public IP**, fill in your Kafka brokers endpoints. You can use commas `,` to separate multiple endpoints. +2. Select an **Authentication** option according to your Kafka authentication configuration. - If your Kafka does not require authentication, keep the default option **Disable**. - - If your Kafka requires authentication, select the corresponding authentication type, and then fill in the user name and password of your Kafka account for authentication. + - If your Kafka requires authentication, select the corresponding authentication type, and then fill in the **user name** and **password** of your Kafka account for authentication. -3. Select your Kafka version. If you do not know that, use Kafka V2. -4. Select a desired compression type for the data in this changefeed. +3. Select your **Kafka Version**. If you do not know which one to use, use **Kafka v2**. +4. Select a **Compression** type for the data in this changefeed. 5. Enable the **TLS Encryption** option if your Kafka has enabled TLS encryption and you want to use TLS encryption for the Kafka connection. -6. Click **Next** to check the configurations you set and go to the next page. +6. Click **Next** to test the network connection. If the test succeeds, you will be directed to the next page. + +
+
+ +1. In **Connectivity Method**, select **Private Link**. +2. Authorize the [AWS Principal](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_principal.html#principal-accounts) of TiDB Cloud to create an endpoint for your endpoint service. The AWS Principal is provided in the tip on the web page. +3. Make sure you select the same **Number of AZs** and **AZ IDs of Kafka Deployment**, and fill the same unique ID in **Kafka Advertised Listener Pattern** when you [Set Up Self-Hosted Kafka Private Link Service in AWS](/tidb-cloud/setup-self-hosted-kafka-private-link-service.md) in the **Network** section. +4. Fill in the **Endpoint Service Name** which is configured in [Set Up Self-Hosted Kafka Private Link Service in AWS](/tidb-cloud/setup-self-hosted-kafka-private-link-service.md). +5. Fill in the **Bootstrap Ports**. It is recommended that you set at least one port for one AZ. You can use commas `,` to separate multiple ports. +6. Select an **Authentication** option according to your Kafka authentication configuration. + + - If your Kafka does not require authentication, keep the default option **Disable**. + - If your Kafka requires authentication, select the corresponding authentication type, and then fill in the **user name** and **password** of your Kafka account for authentication. + +7. Select your **Kafka Version**. If you do not know which one to use, use **Kafka v2**. +8. Select a **Compression** type for the data in this changefeed. +9. Enable the **TLS Encryption** option if your Kafka has enabled TLS encryption and you want to use TLS encryption for the Kafka connection. +10. Click **Next** to test the network connection. If the test succeeds, you will be directed to the next page. +11. TiDB Cloud creates the endpoint for **Private Link**, which might take several minutes. +12. Once the endpoint is created, log in to your cloud provider console and accept the connection request. +13. Return to the [TiDB Cloud console](https://tidbcloud.com) to confirm that you have accepted the connection request. TiDB Cloud will test the connection and proceed to the next page if the test succeeds. + +
+
+ +1. In **Connectivity Method**, select **Private Service Connect**. +2. Ensure that you fill in the same unique ID in **Kafka Advertised Listener Pattern** when you [Set Up Self-Hosted Kafka Private Service Connect in Google Cloud](/tidb-cloud/setup-self-hosted-kafka-private-service-connect.md) in the **Network** section. +3. Fill in the **Service Attachment** that you have configured in [Setup Self Hosted Kafka Private Service Connect in Google Cloud](/tidb-cloud/setup-self-hosted-kafka-private-service-connect.md) +4. Fill in the **Bootstrap Ports**. It is recommended that you provide more than one port. You can use commas `,` to separate multiple ports. +5. Select an **Authentication** option according to your Kafka authentication configuration. + + - If your Kafka does not require authentication, keep the default option **Disable**. + - If your Kafka requires authentication, select the corresponding authentication type, and then fill in the **user name** and **password** of your Kafka account for authentication. + +6. Select your **Kafka Version**. If you do not know which one to use, use **Kafka v2**. +7. Select a **Compression** type for the data in this changefeed. +8. Enable the **TLS Encryption** option if your Kafka has enabled TLS encryption and you want to use TLS encryption for the Kafka connection. +9. Click **Next** to test the network connection. If the test succeeds, you will be directed to the next page. +10. TiDB Cloud creates the endpoint for **Private Service Connect**, which might take several minutes. +11. Once the endpoint is created, log in to your cloud provider console and accept the connection request. +12. Return to the [TiDB Cloud console](https://tidbcloud.com) to confirm that you have accepted the connection request. TiDB Cloud will test the connection and proceed to the next page if the test succeeds. + +
+ ## Step 3. Set the changefeed @@ -91,8 +188,10 @@ For example, if your Kafka cluster is in Confluent Cloud, you can see [Resources 3. In the **Data Format** area, select your desired format of Kafka messages. - - Avro is a compact, fast, and binary data format with rich data structures, which is widely used in various flow systems. For more information, see [Avro data format](https://docs.pingcap.com/tidb/stable/ticdc-avro-protocol). - - Canal-JSON is a plain JSON text format, which is easy to parse. For more information, see [Canal-JSON data format](https://docs.pingcap.com/tidb/stable/ticdc-canal-json). + - Avro is a compact, fast, and binary data format with rich data structures, which is widely used in various flow systems. For more information, see [Avro data format](https://docs.pingcap.com/tidb/stable/ticdc-avro-protocol). + - Canal-JSON is a plain JSON text format, which is easy to parse. For more information, see [Canal-JSON data format](https://docs.pingcap.com/tidb/stable/ticdc-canal-json). + - Open Protocol is a row-level data change notification protocol that provides data sources for monitoring, caching, full-text indexing, analysis engines, and primary-secondary replication between different databases. For more information, see [Open Protocol data format](https://docs.pingcap.com/tidb/stable/ticdc-open-protocol). + - Debezium is a tool for capturing database changes. It converts each captured database change into a message called an "event" and sends these events to Kafka. For more information, see [Debezium data format](https://docs.pingcap.com/tidb/stable/ticdc-debezium). 4. Enable the **TiDB Extension** option if you want to add TiDB-extension fields to the Kafka message body. @@ -109,36 +208,44 @@ For example, if your Kafka cluster is in Confluent Cloud, you can see [Resources The distribution mode controls how the changefeed creates Kafka topics, by table, by database, or creating one topic for all changelogs. - - **Distribute changelogs by table to Kafka Topics** + - **Distribute changelogs by table to Kafka Topics** If you want the changefeed to create a dedicated Kafka topic for each table, choose this mode. Then, all Kafka messages of a table are sent to a dedicated Kafka topic. You can customize topic names for tables by setting a topic prefix, a separator between a database name and table name, and a suffix. For example, if you set the separator as `_`, the topic names are in the format of `_`. For changelogs of non-row events, such as Create Schema Event, you can specify a topic name in the **Default Topic Name** field. The changefeed will create a topic accordingly to collect such changelogs. - - **Distribute changelogs by database to Kafka Topics** + - **Distribute changelogs by database to Kafka Topics** If you want the changefeed to create a dedicated Kafka topic for each database, choose this mode. Then, all Kafka messages of a database are sent to a dedicated Kafka topic. You can customize topic names of databases by setting a topic prefix and a suffix. For changelogs of non-row events, such as Resolved Ts Event, you can specify a topic name in the **Default Topic Name** field. The changefeed will create a topic accordingly to collect such changelogs. - - **Send all changelogs to one specified Kafka Topic** + - **Send all changelogs to one specified Kafka Topic** If you want the changefeed to create one Kafka topic for all changelogs, choose this mode. Then, all Kafka messages in the changefeed will be sent to one Kafka topic. You can define the topic name in the **Topic Name** field. -7. In the **Partition Distribution** area, you can decide which partition a Kafka message will be sent to: +7. In the **Partition Distribution** area, you can decide which partition a Kafka message will be sent to. You can define **a single partition dispatcher for all tables**, or **different partition dispatchers for different tables**. TiDB Cloud provides four types of dispatchers: - - **Distribute changelogs by index value to Kafka partition** + - **Distribute changelogs by primary key or index value to Kafka partition** - If you want the changefeed to send Kafka messages of a table to different partitions, choose this distribution method. The index value of a row changelog will determine which partition the changelog is sent to. This distribution method provides a better partition balance and ensures row-level orderliness. + If you want the changefeed to send Kafka messages of a table to different partitions, choose this distribution method. The primary key or index value of a row changelog will determine which partition the changelog is sent to. This distribution method provides a better partition balance and ensures row-level orderliness. - - **Distribute changelogs by table to Kafka partition** + - **Distribute changelogs by table to Kafka partition** If you want the changefeed to send Kafka messages of a table to one Kafka partition, choose this distribution method. The table name of a row changelog will determine which partition the changelog is sent to. This distribution method ensures table orderliness but might cause unbalanced partitions. + - **Distribute changelogs by timestamp to Kafka partition** + + If you want the changefeed to send Kafka messages to different Kafka partitions randomly, choose this distribution method. The commitTs of a row changelog will determine which partition the changelog is sent to. This distribution method provides a better partition balance and ensures orderliness in each partition. However, multiple changes of a data item might be sent to different partitions and the consumer progress of different consumers might be different, which might cause data inconsistency. Therefore, the consumer needs to sort the data from multiple partitions by commitTs before consuming. + + - **Distribute changelogs by column value to Kafka partition** + + If you want the changefeed to send Kafka messages of a table to different partitions, choose this distribution method. The specified column values of a row changelog will determine which partition the changelog is sent to. This distribution method ensures orderliness in each partition and guarantees that the changelog with the same column values is send to the same partition. + 8. In the **Topic Configuration** area, configure the following numbers. The changefeed will automatically create the Kafka topics according to the numbers. - - **Replication Factor**: controls how many Kafka servers each Kafka message is replicated to. - - **Partition Number**: controls how many partitions exist in a topic. + - **Replication Factor**: controls how many Kafka servers each Kafka message is replicated to. The valid value ranges from [`min.insync.replicas`](https://kafka.apache.org/33/documentation.html#brokerconfigs_min.insync.replicas) to the number of Kafka brokers. + - **Partition Number**: controls how many partitions exist in a topic. The valid value range is `[1, 10 * the number of Kafka brokers]`. 9. Click **Next**. diff --git a/tidb-cloud/changefeed-sink-to-cloud-storage.md b/tidb-cloud/changefeed-sink-to-cloud-storage.md index 391b53e14a602..596a2df4db24b 100644 --- a/tidb-cloud/changefeed-sink-to-cloud-storage.md +++ b/tidb-cloud/changefeed-sink-to-cloud-storage.md @@ -1,6 +1,6 @@ --- title: Sink to Cloud Storage -Summary: Learn how to create a changefeed to stream data from a TiDB Dedicated cluster to cloud storage, such as Amazon S3 and GCS. +summary: This document explains how to create a changefeed to stream data from TiDB Cloud to Amazon S3 or GCS. It includes restrictions, configuration steps for the destination, replication, and specification, as well as starting the replication process. --- # Sink to Cloud Storage @@ -9,8 +9,8 @@ This document describes how to create a changefeed to stream data from TiDB Clou > **Note:** > -> - To stream data to cloud storage, make sure that your TiDB cluster version is v7.1.1 or later. To upgrade your TiDB Dedicated cluster to v7.1.1 or later, [contact TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). -> - For [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters, the changefeed feature is unavailable. +> - To stream data to cloud storage, make sure that your TiDB cluster version is v7.1.1 or later. To upgrade your TiDB Cloud Dedicated cluster to v7.1.1 or later, [contact TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). +> - For [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters, the changefeed feature is unavailable. ## Restrictions @@ -45,8 +45,8 @@ For **GCS**, before filling **GCS Endpoint**, you need to first grant the GCS bu ![Create a role](/media/tidb-cloud/changefeed/sink-to-cloud-storage-gcs-create-role.png) - 3. Enter a name, description, ID, and role launch stage for the role. The role name cannot be changed after the role is created. - 4. Click **Add permissions**. Add the following read-only permissions to the role, and then click **Add**. + 3. Enter a name, description, ID, and role launch stage for the role. The role name cannot be changed after the role is created. + 4. Click **Add permissions**. Add the following permissions to the role, and then click **Add**. - storage.buckets.get - storage.objects.create @@ -87,7 +87,7 @@ For **GCS**, before filling **GCS Endpoint**, you need to first grant the GCS bu
-Click **Next** to establish the connection from the TiDB Dedicated cluster to Amazon S3 or GCS. TiDB Cloud will automatically test and verify if the connection is successful. +Click **Next** to establish the connection from the TiDB Cloud Dedicated cluster to Amazon S3 or GCS. TiDB Cloud will automatically test and verify if the connection is successful. - If yes, you are directed to the next step of configuration. - If not, a connectivity error is displayed, and you need to handle the error. After the error is resolved, click **Next** to retry the connection. diff --git a/tidb-cloud/changefeed-sink-to-mysql.md b/tidb-cloud/changefeed-sink-to-mysql.md index 97d3f7cc15787..f0c13cc51f99c 100644 --- a/tidb-cloud/changefeed-sink-to-mysql.md +++ b/tidb-cloud/changefeed-sink-to-mysql.md @@ -1,6 +1,6 @@ --- title: Sink to MySQL -Summary: Learn how to create a changefeed to stream data from TiDB Cloud to MySQL. +summary: This document explains how to stream data from TiDB Cloud to MySQL using the Sink to MySQL changefeed. It includes restrictions, prerequisites, and steps to create a MySQL sink for data replication. The process involves setting up network connections, loading existing data to MySQL, and creating target tables in MySQL. After completing the prerequisites, users can create a MySQL sink to replicate data to MySQL. --- # Sink to MySQL @@ -9,8 +9,8 @@ This document describes how to stream data from TiDB Cloud to MySQL using the ** > **Note:** > -> - To use the changefeed feature, make sure that your TiDB Dedicated cluster version is v6.1.3 or later. -> - For [TiDB Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-serverless), the changefeed feature is unavailable. +> - To use the changefeed feature, make sure that your TiDB Cloud Dedicated cluster version is v6.1.3 or later. +> - For [TiDB Cloud Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless), the changefeed feature is unavailable. ## Restrictions @@ -35,7 +35,7 @@ If your MySQL service is in an AWS VPC that has no public internet access, take 1. [Set up a VPC peering connection](/tidb-cloud/set-up-vpc-peering-connections.md) between the VPC of the MySQL service and your TiDB cluster. 2. Modify the inbound rules of the security group that the MySQL service is associated with. - You must add [the CIDR of the region where your TiDB Cloud cluster is located](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-project-cidr) to the inbound rules. Doing so allows the traffic to flow from your TiDB Cluster to the MySQL instance. + You must add [the CIDR of the region where your TiDB Cloud cluster is located](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-cidr-for-a-region) to the inbound rules. Doing so allows the traffic to flow from your TiDB Cluster to the MySQL instance. 3. If the MySQL URL contains a hostname, you need to allow TiDB Cloud to be able to resolve the DNS hostname of the MySQL service. @@ -48,7 +48,7 @@ If your MySQL service is in a Google Cloud VPC that has no public internet acces 2. [Set up a VPC peering connection](/tidb-cloud/set-up-vpc-peering-connections.md) between the VPC of the MySQL service and your TiDB cluster. 3. Modify the ingress firewall rules of the VPC where MySQL is located. - You must add [the CIDR of the region where your TiDB Cloud cluster is located](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-project-cidr) to the ingress firewall rules. Doing so allows the traffic to flow from your TiDB Cluster to the MySQL endpoint. + You must add [the CIDR of the region where your TiDB Cloud cluster is located](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-cidr-for-a-region) to the ingress firewall rules. Doing so allows the traffic to flow from your TiDB Cluster to the MySQL endpoint. ### Load existing data (optional) diff --git a/tidb-cloud/changefeed-sink-to-tidb-cloud.md b/tidb-cloud/changefeed-sink-to-tidb-cloud.md index 769b185cde21d..8366f1e84a208 100644 --- a/tidb-cloud/changefeed-sink-to-tidb-cloud.md +++ b/tidb-cloud/changefeed-sink-to-tidb-cloud.md @@ -1,37 +1,36 @@ --- title: Sink to TiDB Cloud -Summary: Learn how to create a changefeed to stream data from a TiDB Dedicated cluster to a TiDB Serverless cluster. +summary: This document explains how to stream data from a TiDB Cloud Dedicated cluster to a TiDB Cloud Serverless cluster. There are restrictions on the number of changefeeds and regions available for the feature. Prerequisites include extending tidb_gc_life_time, backing up data, and obtaining the start position of TiDB Cloud sink. To create a TiDB Cloud sink, navigate to the cluster overview page, establish the connection, customize table and event filters, fill in the start replication position, specify the changefeed specification, review the configuration, and create the sink. Finally, restore tidb_gc_life_time to its original value. --- # Sink to TiDB Cloud -This document describes how to stream data from a TiDB Dedicated cluster to a TiDB Serverless cluster. +This document describes how to stream data from a TiDB Cloud Dedicated cluster to a TiDB Cloud Serverless cluster. > **Note:** > -> To use the Changefeed feature, make sure that your TiDB Dedicated cluster version is v6.1.3 or later. +> To use the Changefeed feature, make sure that your TiDB Cloud Dedicated cluster version is v6.1.3 or later. ## Restrictions - For each TiDB Cloud cluster, you can create up to 100 changefeeds. - Because TiDB Cloud uses TiCDC to establish changefeeds, it has the same [restrictions as TiCDC](https://docs.pingcap.com/tidb/stable/ticdc-overview#unsupported-scenarios). - If the table to be replicated does not have a primary key or a non-null unique index, the absence of a unique constraint during replication could result in duplicated data being inserted downstream in some retry scenarios. -- The **Sink to TiDB Cloud** feature is only available to TiDB Dedicated clusters that are in the following AWS regions and created after November 9, 2022: +- The **Sink to TiDB Cloud** feature is only available to TiDB Cloud Dedicated clusters that are in the following AWS regions and created after November 9, 2022: - AWS Oregon (us-west-2) - AWS Frankfurt (eu-central-1) - AWS Singapore (ap-southeast-1) - AWS Tokyo (ap-northeast-1) - - AWS São Paulo (sa-east-1) -- The source TiDB Dedicated cluster and the destination TiDB Serverless cluster must be in the same project and the same region. -- The **Sink to TiDB Cloud** feature only supports network connection via private endpoints. When you create a changefeed to stream data from a TiDB Dedicated cluster to a TiDB Serverless cluster, TiDB Cloud will automatically set up the private endpoint connection between the two clusters. +- The source TiDB Cloud Dedicated cluster and the destination TiDB Cloud Serverless cluster must be in the same project and the same region. +- The **Sink to TiDB Cloud** feature only supports network connection via private endpoints. When you create a changefeed to stream data from a TiDB Cloud Dedicated cluster to a TiDB Cloud Serverless cluster, TiDB Cloud will automatically set up the private endpoint connection between the two clusters. ## Prerequisites -The **Sink to TiDB Cloud** connector can only sink incremental data from a TiDB Dedicated cluster to a TiDB Serverless cluster after a certain [TSO](https://docs.pingcap.com/tidb/stable/glossary#tso). +The **Sink to TiDB Cloud** connector can only sink incremental data from a TiDB Cloud Dedicated cluster to a TiDB Cloud Serverless cluster after a certain [TSO](https://docs.pingcap.com/tidb/stable/glossary#tso). -Before creating a changefeed, you need to export existing data from the source TiDB Dedicated cluster and load the data to the destination TiDB Serverless cluster. +Before creating a changefeed, you need to export existing data from the source TiDB Cloud Dedicated cluster and load the data to the destination TiDB Cloud Serverless cluster. 1. Extend the [tidb_gc_life_time](https://docs.pingcap.com/tidb/stable/system-variables#tidb_gc_life_time-new-in-v50) to be longer than the total time of the following two operations, so that historical data during the time is not garbage collected by TiDB. @@ -44,7 +43,7 @@ Before creating a changefeed, you need to export existing data from the source T SET GLOBAL tidb_gc_life_time = '720h'; ``` -2. [Back up data](/tidb-cloud/backup-and-restore.md#backup) from your TiDB Dedicated cluster, then use community tools such as [mydumper/myloader](https://centminmod.com/mydumper.html) to load data to the destination TiDB Serverless cluster. +2. Use [Dumpling](https://docs.pingcap.com/tidb/stable/dumpling-overview) to export data from your TiDB Cloud Dedicated cluster, then use [TiDB Cloud Serverless Import](/tidb-cloud/import-csv-files-serverless.md) to load data to the destination TiDB Cloud Serverless cluster. 3. From the [exported files of Dumpling](https://docs.pingcap.com/tidb/stable/dumpling-overview#format-of-exported-files), get the start position of TiDB Cloud sink from the metadata file: @@ -60,13 +59,13 @@ Before creating a changefeed, you need to export existing data from the source T ## Create a TiDB Cloud sink -After completing the prerequisites, you can sink your data to the destination TiDB Serverless cluster. +After completing the prerequisites, you can sink your data to the destination TiDB Cloud Serverless cluster. 1. Navigate to the cluster overview page of the target TiDB cluster, and then click **Changefeed** in the left navigation pane. 2. Click **Create Changefeed**, and select **TiDB Cloud** as the destination. -3. In the **TiDB Cloud Connection** area, select the destination TiDB Serverless cluster, and then fill in the user name and password of the destination cluster. +3. In the **TiDB Cloud Connection** area, select the destination TiDB Cloud Serverless cluster, and then fill in the user name and password of the destination cluster. 4. Click **Next** to establish the connection between the two TiDB clusters and test whether the changefeed can connect them successfully: diff --git a/tidb-cloud/cli-reference.md b/tidb-cloud/cli-reference.md index 44d96bdde1cda..63b52449cba66 100644 --- a/tidb-cloud/cli-reference.md +++ b/tidb-cloud/cli-reference.md @@ -3,7 +3,11 @@ title: TiDB Cloud CLI Reference summary: Provides an overview of TiDB Cloud CLI. --- -# TiDB Cloud CLI Reference +# TiDB Cloud CLI Reference (Beta) + +> **Note:** +> +> TiDB Cloud CLI is in beta. TiDB Cloud CLI is a command line interface, which allows you to operate TiDB Cloud from your terminal with a few lines of commands. In the TiDB Cloud CLI, you can easily manage your TiDB Cloud clusters, import data to your clusters, and perform more operations. @@ -17,17 +21,20 @@ The following table lists the commands available for the TiDB Cloud CLI. To use the `ticloud` CLI in your terminal, run `ticloud [command] [subcommand]`. If you are using [TiUP](https://docs.pingcap.com/tidb/stable/tiup-overview), use `tiup cloud [command] [subcommand]` instead. -| Command | Subcommand | Description | -|------------|------------------------------------------------------------|----------------------------------------------------------------------------------------------------------| -| cluster | create, delete, describe, list, connect-info | Manage clusters | -| branch | create, delete, describe, list, connect-info | Manage branches | -| completion | bash, fish, powershell, zsh | Generate completion script for specified shell | -| config | create, delete, describe, edit, list, set, use | Configure user profiles | -| connect | - | Connect to a TiDB cluster | -| help | cluster, completion, config, help, import, project, update | View help for any command | -| import | cancel, describe, list, start | Manage [import](/tidb-cloud/tidb-cloud-migration-overview.md#import-data-from-files-to-tidb-cloud) tasks | -| project | list | Manage projects | -| update | - | Update the CLI to the latest version | +| Command | Subcommand | Description | +|-----------------------|-----------------------------------------------------------------------|------------------------------------------------| +| auth | login, logout, whoami | Login and logout | +| serverless (alias: s) | create, delete, describe, list, update, spending-limit, region, shell | Manage TiDB Cloud Serverless clusters | +| serverless branch | create, delete, describe, list, shell | Manage TiDB Cloud Serverless branches | +| serverless import | cancel, describe, list, start | Manage TiDB Cloud Serverless import tasks | +| serverless export | create, describe, list, cancel, download | Manage TiDB Cloud Serverless export tasks | +| serverless sql-user | create, list, delete, update | Manage TiDB Cloud Serverless SQL users | +| ai | - | Chat with TiDB Bot | +| completion | bash, fish, powershell, zsh | Generate completion script for specified shell | +| config | create, delete, describe, edit, list, set, use | Configure user profiles | +| project | list | Manage projects | +| upgrade | - | Update the CLI to the latest version | +| help | auth, config, serverless, ai, project, upgrade, help, completion | View help for any command | ## Command modes @@ -43,12 +50,20 @@ The TiDB Cloud CLI provides two modes for some commands for easy use: ## User profile -For the TiDB Cloud CLI, a user profile is a collection of properties associated with a user, including the profile name, public key, and private key. To use TiDB Cloud CLI, you must create a user profile first. +For the TiDB Cloud CLI, a user profile is a collection of properties associated with a user, including the profile name, public key, private key, and OAuth token. To use TiDB Cloud CLI, you must have a user profile. -### Create a user profile +### Create a user profile with TiDB Cloud API key Use [`ticloud config create`](/tidb-cloud/ticloud-config-create.md) to create a user profile. +### Create a user profile with OAuth token + +Use [`ticloud auth login`](/tidb-cloud/ticloud-auth-login.md) to assign OAuth token to the current profile. If no profiles exist, a profile named `default` will be created automatically. + +> **Note:** +> +> In the preceding two methods, the TiDB Cloud API key takes precedence over the OAuth token. If both are available in the current profile, the API key will be used. + ### List all user profiles Use [`ticloud config list`](/tidb-cloud/ticloud-config-list.md) to list all user profiles. @@ -103,10 +118,11 @@ Use [`ticloud config delete`](/tidb-cloud/ticloud-config-delete.md) to delete a The following table lists the global flags for the TiDB Cloud CLI. -| Flag | Description | Required | Note | -|----------------------|-----------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active user profile used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|---------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active user profile used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enable debug mode | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/config-s3-and-gcs-access.md b/tidb-cloud/config-s3-and-gcs-access.md index 784df3f8ce80d..14f399e309045 100644 --- a/tidb-cloud/config-s3-and-gcs-access.md +++ b/tidb-cloud/config-s3-and-gcs-access.md @@ -1,25 +1,26 @@ --- -title: Configure Amazon S3 Access and GCS Access +title: Configure External Storage Access for TiDB Cloud Dedicated summary: Learn how to configure Amazon Simple Storage Service (Amazon S3) access and Google Cloud Storage (GCS) access. --- -# Configure Amazon S3 Access and GCS Access +# Configure External Storage Access for TiDB Cloud Dedicated -If your source data is stored in Amazon S3 or Google Cloud Storage (GCS) buckets, before importing or migrating the data to TiDB Cloud, you need to configure cross-account access to the buckets. This document describes how to do this. +If your source data is stored in Amazon S3 or Google Cloud Storage (GCS) buckets, before importing or migrating the data to TiDB Cloud, you need to configure cross-account access to the buckets. This document describes how to do this for TiDB Cloud Dedicated clusters. + +If you need to configure these external storages for TiDB Cloud Serverless clusters, see [Configure External Storage Access for TiDB Cloud Serverless](/tidb-cloud/serverless-external-storage.md). ## Configure Amazon S3 access -To allow TiDB Cloud to access the source data in your Amazon S3 bucket, you need to configure the bucket access for TiDB Cloud. You can use either of the following methods to configure the bucket access: +To allow a TiDB Cloud Dedicated cluster to access the source data in your Amazon S3 bucket, configure the bucket access for the cluster using either of the following methods: -- Use an AWS access key: use the access key of an IAM user to access your Amazon S3 bucket. -- Use a Role ARN: use a Role ARN to access your Amazon S3 bucket. +- [Use a Role ARN](#configure-amazon-s3-access-using-a-role-arn): use a Role ARN to access your Amazon S3 bucket. +- [Use an AWS access key](#configure-amazon-s3-access-using-an-aws-access-key): use the access key of an IAM user to access your Amazon S3 bucket. - -
+### Configure Amazon S3 access using a Role ARN Configure the bucket access for TiDB Cloud and get the Role ARN as follows: -1. In the [TiDB Cloud console](https://tidbcloud.com/), get the TiDB Cloud account ID and external ID of the target TiDB cluster. +1. In the [TiDB Cloud console](https://tidbcloud.com/), get the corresponding TiDB Cloud account ID and external ID of the target TiDB cluster. 1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. @@ -29,9 +30,13 @@ Configure the bucket access for TiDB Cloud and get the Role ARN as follows: 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. - 3. On the **Import** page, click **Import Data** in the upper-right corner and select **From S3**. + 3. On the **Import** page, select **Import data from S3**. + + If this is your first time importing data into this cluster, select **Import From Amazon S3**. - 4. On the **Import from S3** page, click **Guide for getting the required Role ARN** to get the TiDB Cloud Account ID and TiDB Cloud External ID. Take a note of these IDs for later use. + 4. On the **Import Data from Amazon S3** page, click the link under **Role ARN**. The **Add New Role ARN** dialog is displayed. + + 5. Expand **Create Role ARN manually** to get the TiDB Cloud Account ID and TiDB Cloud External ID. Take a note of these IDs for later use. 2. In the AWS Management Console, create a managed policy for your Amazon S3 bucket. @@ -98,9 +103,8 @@ Configure the bucket access for TiDB Cloud and get the Role ARN as follows: If the objects in your bucket have been copied from another encrypted bucket, the KMS key value needs to include the keys of both buckets. For example, `"Resource": ["arn:aws:kms:ap-northeast-1:105880447796:key/c3046e91-fdfc-4f3a-acff-00597dd3801f","arn:aws:kms:ap-northeast-1:495580073302:key/0d7926a7-6ecc-4bf7-a9c1-a38f0faec0cd"]`. - 6. Click **Next: Tags**, add a tag of the policy (optional), and then click **Next:Review**. - - 7. Set a policy name, and then click **Create policy**. + 6. Click **Next**. + 7. Set a policy name, add a tag of the policy (optional), and then click **Create policy**. 3. In the AWS Management Console, create an access role for TiDB Cloud and get the role ARN. @@ -112,7 +116,7 @@ Configure the bucket access for TiDB Cloud and get the Role ARN as follows: - Under **Trusted entity type**, select **AWS account**. - Under **An AWS account**, select **Another AWS account**, and then paste the TiDB Cloud account ID to the **Account ID** field. - - Under **Options**, click **Require external ID (Best practice when a third party will assume this role)**, and then paste the TiDB Cloud External ID to the **External ID** field. If the role is created without "Require external ID", once the configuration is done for one TiDB cluster in a project, all TiDB clusters in that project can use the same Role ARN to access your Amazon S3 bucket. If the role is created with the account ID and external ID, only the corresponding TiDB cluster can access the bucket. + - Under **Options**, click **Require external ID** to avoid the [confused deputy problem](https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html), and then paste the TiDB Cloud External ID to the **External ID** field. If the role is created without "Require external ID", anyone with your S3 bucket URI and IAM role ARN might be able to access your Amazon S3 bucket. If the role is created with both the account ID and external ID, only TiDB clusters running in the same project and the same region can access the bucket. 3. Click **Next** to open the policy list, choose the policy you just created, and then click **Next**. 4. Under **Role details**, set a name for the role, and then click **Create role** in the lower-right corner. After the role is created, the list of roles is displayed. @@ -122,8 +126,7 @@ Configure the bucket access for TiDB Cloud and get the Role ARN as follows: 4. In the TiDB Cloud console, go to the **Data Import** page where you get the TiDB Cloud account ID and external ID, and then paste the role ARN to the **Role ARN** field. -
-
+### Configure Amazon S3 access using an AWS access key It is recommended that you use an IAM user (instead of the AWS account root user) to create an access key. @@ -146,9 +149,6 @@ Take the following steps to configure an access key: > > TiDB Cloud does not store your access keys. It is recommended that you [delete the access key](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey) after the import is complete. -
- - ## Configure GCS access To allow TiDB Cloud to access the source data in your GCS bucket, you need to configure the GCS access for the bucket. Once the configuration is done for one TiDB cluster in a project, all TiDB clusters in that project can access the GCS bucket. @@ -163,7 +163,11 @@ To allow TiDB Cloud to access the source data in your GCS bucket, you need to co 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. - 3. Click **Import Data** in the upper-right corner, click **Show Google Cloud Service Account ID**, and then copy the Service Account ID for later use. + 3. Click **Import Data** in the upper-right corner. + + If this is your first time importing data into this cluster, select **Import From GCS**. + + 4. Click **Show Google Cloud Server Account ID**, and then copy the Service Account ID for later use. 2. In the Google Cloud console, create an IAM role for your GCS bucket. diff --git a/tidb-cloud/configure-ip-access-list.md b/tidb-cloud/configure-ip-access-list.md index 1b035c7bc1101..d780fe742dcda 100644 --- a/tidb-cloud/configure-ip-access-list.md +++ b/tidb-cloud/configure-ip-access-list.md @@ -1,58 +1,24 @@ --- title: Configure an IP Access List -summary: Learn how to configure IP addresses that are allowed to access your TiDB Dedicated cluster. +summary: Learn how to configure IP addresses that are allowed to access your TiDB Cloud Dedicated cluster. --- # Configure an IP Access List -For each TiDB Dedicated cluster in TiDB Cloud, you can configure an IP access list to filter internet traffic trying to access the cluster, which works similarly to a firewall access control list. After the configuration, only the clients and applications whose IP addresses are in the IP access list can connect to your TiDB Dedicated cluster. +For each TiDB Cloud Dedicated cluster in TiDB Cloud, you can configure an IP access list to filter internet traffic trying to access the cluster, which works similarly to a firewall access control list. After the configuration, only the clients and applications whose IP addresses are in the IP access list can connect to your TiDB Cloud Dedicated cluster. > **Note:** > -> Configuring the IP access list is only available for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. +> This document applies to [**TiDB Cloud Dedicated**](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). For instructions on configuring an IP access list for **TiDB Cloud Serverless**, see [Configure TiDB Cloud Serverless Firewall Rules for Public Endpoints](/tidb-cloud/configure-serverless-firewall-rules-for-public-endpoints.md). -For a TiDB Dedicated cluster, you can configure its IP access list in either of the following ways: +To configure an IP access list, take the following steps in the [TiDB Cloud console](https://tidbcloud.com/console/clusters): -- [Configure an IP access list in standard connection](#configure-an-ip-access-list-in-standard-connection) +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. +2. In the left navigation pane, click **Networking**, and then click **Add IP Address**. +3. In the dialog, choose one of the following options: -- [Configure an IP access list in security settings](#configure-an-ip-access-list-in-security-settings) + - **Allow access from anywhere**: allows all IP addresses to access TiDB Cloud. This option exposes your cluster to the internet completely and is highly risky. + - **Use IP addresses** (recommended): you can add a list of IPs and CIDR addresses that are allowed to access TiDB Cloud via a SQL client. -## Configure an IP access list in standard connection - -To configure an IP access list for your TiDB Dedicated cluster in standard connection, take the following steps: - -1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. -2. In the row of your TiDB Dedicated cluster, click **...** and select **Connect**. A dialog is displayed. -3. In the dialog, locate **Step 1: Create traffic filter** on the **Standard Connection** tab and configure the IP access list. - - - If the IP access list of your cluster has not been set, you can click **Add My Current IP Address** to add your current IP address to the IP access list, and then click **Add Item** to add more IP addresses if necessary. Next, click **Update Filter** to save the configuration. - - > **Note:** - > - > For each TiDB Dedicated cluster, you can add up to 100 IP addresses to the IP access list. To apply for a quota to add more IP addresses, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). - - - If the IP access list of your cluster has been set, click **Edit** to add, edit, or remove IP addresses, and then click **Update Filter** to save the configuration. - - - To allow any IP address to access your cluster (not recommended), click **Allow Access From Anywhere**, and then click **Update Filter**. According to security best practices, it is NOT recommended that you allow any IP address to access your cluster, as this would expose your cluster to the internet completely, which is highly risky. - -## Configure an IP access list in security settings - -To configure an IP access list for your TiDB Dedicated cluster in security settings, take the following steps: - -1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. -2. In the row of your TiDB Dedicated cluster, click **...** and select **Security Settings**. A security setting dialog is displayed. -3. In the dialog, configure the IP access list as follows: - - - To add your current IP address to the IP access list, click **Add My Current IP Address**. - - - To add an IP address to the IP access list, enter the IP address and description, and click **Add to IP List**. - - > **Note:** - > - > For each TiDB Dedicated cluster, you can add up to 100 IP addresses to the IP access list. To apply for a quota to add more IP addresses, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). - - - To allow any IP address to access your cluster (not recommended), click **Allow Access From Anywhere**. According to security best practices, it is NOT recommended that you allow any IP address to access your cluster, as this would expose your cluster to the internet completely, which is highly risky. - - - To remove an IP address from the access list, click **Remove** in the line of the IP address. - -4. Click **Apply** to save the configuration. \ No newline at end of file +4. If you choose **Use IP addresses**, add IP addresses or CIDR range with an optional description. For each TiDB Cloud Dedicated cluster, you can add up to 100 IP addresses. +5. Click **Confirm** to save your changes. diff --git a/tidb-cloud/configure-maintenance-window.md b/tidb-cloud/configure-maintenance-window.md index 21e36c8b5c86b..42cfc8eed937a 100644 --- a/tidb-cloud/configure-maintenance-window.md +++ b/tidb-cloud/configure-maintenance-window.md @@ -7,13 +7,13 @@ summary: Learn how to configure maintenance window for your cluster. A maintenance window is a designated timeframe during which planned maintenance tasks, such as operating system updates, security patches, and infrastructure upgrades, are performed automatically to ensure the reliability, security, and performance of the TiDB Cloud service. -During a maintenance window, the maintenance is executed on TiDB Dedicated clusters one by one so the overall impact is minimal. Although there might be temporary connection disruptions or QPS fluctuations, the clusters remain available, and the existing data import, backup, restore, migration, and replication tasks can still run normally. +During a maintenance window, the maintenance is executed on TiDB Cloud Dedicated clusters one by one so the overall impact is minimal. Although there might be temporary connection disruptions or QPS fluctuations, the clusters remain available, and the existing data import, backup, restore, migration, and replication tasks can still run normally. By configuring the maintenance window, you can easily schedule and manage maintenance tasks to minimize the maintenance impact. For example, you can set the start time of the maintenance window to avoid peak hours of your application workloads. > **Note:** > -> The maintenance window feature is only available for [TiDB Dedicated clusters](/tidb-cloud/select-cluster-tier.md#tidb-dedicated). +> The maintenance window feature is only available for [TiDB Cloud Dedicated clusters](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). ## Allowed and disallowed operations during a maintenance window @@ -51,9 +51,12 @@ For every maintenance window, TiDB Cloud sends four email notifications to all p Regular maintenance ensures that essential updates are performed to safeguard TiDB Cloud from security threats, performance issues, and unreliability. Therefore, the maintenance window is enabled by default and cannot be disabled. -If a maintenance window is planned, the default start time of the window is 03:00 Wednesday (based on the time zone of your TiDB Cloud organization) of the target week. +> **Note:** +> +> - For the default project automatically created when you first sign up for TiDB Cloud, the maintenance window starts at 03:00 AM every Wednesday (based on the time zone of your TiDB Cloud organization). +> - For new projects that you create, you can set a custom start time for the maintenance window during project setup. -You can modify the start time to your preferred time or defer maintenance tasks until the deadline as follows: +You can modify the start time to your preferred time or reschedule maintenance tasks until the deadline as follows: 1. Log in to the [TiDB Cloud console](https://tidbcloud.com). 2. Click in the lower-left corner, switch to the target project if you have multiple projects, and then click **Project Settings**. @@ -63,11 +66,11 @@ You can modify the start time to your preferred time or defer maintenance tasks - If there is no maintenance data, it means no maintenance task is scheduled recently. -4. (Optional) Click to modify the start time of the maintenance window. Note that the maintenance will be performed at the specified start time only if there is a maintenance window planned for that week. +4. (Optional) Click **Maintenance Window Setting** to modify the start time of the maintenance window. Note that the maintenance will be performed at the specified start time only if there is a maintenance window planned for that week. -5. To defer the start time of a scheduled maintenance task, click **Defer** in the **Action** column and change it to the next feasible maintenance window before the deadline. +5. To reschedule a specific maintenance task, click **...** > **Reschedule** in the **Action** column, and choose a new time before the deadline. - If you need to defer the maintenance task beyond the deadline, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md#tidb-cloud-support) for assistance. + If you need to reschedule the maintenance task beyond the deadline, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md#tidb-cloud-support) for assistance. ## FAQs @@ -77,7 +80,7 @@ You can modify the start time to your preferred time or defer maintenance tasks - Can I disable a maintenance window? - No. The maintenance window is enabled by default and cannot be disabled. You can modify the start time of the maintenance window or defer a maintenance task to 2 to 4 weeks until the deadline. For more information, see [View and configure maintenance windows](#view-and-configure-maintenance-windows). + No. The maintenance window is enabled by default and cannot be disabled. You can modify the start time of the maintenance window or reschedule a maintenance task until the deadline. For more information, see [View and configure maintenance windows](#view-and-configure-maintenance-windows). - How long does a maintenance window last? diff --git a/tidb-cloud/configure-security-settings.md b/tidb-cloud/configure-security-settings.md index c45b5705bbd4d..c60c48dfdb050 100644 --- a/tidb-cloud/configure-security-settings.md +++ b/tidb-cloud/configure-security-settings.md @@ -1,15 +1,15 @@ --- -title: Configure Cluster Security Settings -summary: Learn how to configure the root password and allowed IP addresses to connect to your cluster. +title: Configure Cluster Password Settings +summary: Learn how to configure the root password to connect to your cluster. --- -# Configure Cluster Security Settings +# Configure Cluster Password Settings -For TiDB Dedicated clusters, you can configure the root password and allowed IP addresses to connect to your cluster. +For TiDB Cloud Dedicated clusters, you can configure the root password and allowed IP addresses to connect to your cluster. > **Note:** > -> For TiDB Serverless clusters, this document is inapplicable and you can refer to [TLS Connection to TiDB Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md) instead. +> For TiDB Cloud Serverless clusters, this document is inapplicable and you can refer to [TLS Connection to TiDB Cloud Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md) instead. 1. In the TiDB Cloud console, navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. @@ -17,13 +17,11 @@ For TiDB Dedicated clusters, you can configure the root password and allowed IP > > If you have multiple projects, you can click in the lower-left corner and switch to another project. -2. In the row of your target cluster, click **...** and select **Security Settings**. -3. In the **Security Settings** dialog, configure the root password and allowed IP addresses. +2. In the row of your target cluster, click **...** and select **Password Settings**. +3. Set the root password to connect to your cluster, and then click **Save**. - To allow your cluster to be accessible by any IP addresses, click **Allow Access from Anywhere**. - -4. Click **Apply**. + You can click **Auto-generate Password** to generate a random password. The generated password will not show again, so save your password in a secure location. > **Tip:** > -> If you are viewing the overview page of your cluster, you can click the **...** in the upper-right corner of the page, select **Security Settings**, and configure these settings, too. +> If you are viewing the overview page of your cluster, you can click the **...** in the upper-right corner of the page, select **Password Settings**, and configure these settings, too. diff --git a/tidb-cloud/configure-serverless-firewall-rules-for-public-endpoints.md b/tidb-cloud/configure-serverless-firewall-rules-for-public-endpoints.md new file mode 100644 index 0000000000000..316588d3bbafc --- /dev/null +++ b/tidb-cloud/configure-serverless-firewall-rules-for-public-endpoints.md @@ -0,0 +1,56 @@ +--- +title: Configure TiDB Cloud Serverless Firewall Rules for Public Endpoints +summary: Learn how to configure and manage firewall rules with public access to your TiDB Cloud Serverless cluster securely. +--- + +# Configure TiDB Cloud Serverless Firewall Rules for Public Endpoints + +This document describes the public connectivity option for TiDB Cloud Serverless. You will learn key concepts for securely managing a TiDB Cloud Serverless cluster accessible via the internet. + +> **Note:** +> +> This document applies to **TiDB Cloud Serverless**. For instructions on configuring an IP access list for **TiDB Cloud Dedicated**, see [Configure an IP Access List for TiDB Cloud Dedicated](/tidb-cloud/configure-ip-access-list.md). + +## Public endpoints + +Configuring public access on your TiDB Cloud Serverless cluster allows the cluster access through a public endpoint. That is, the cluster is accessible through the internet. The public endpoint is a publicly resolvable DNS address. The term "authorized network" refers to a range of IP addresses you choose to permit access to your cluster. These permissions are enforced through **firewall rules**. + +### Characteristics of public access + +- Only specified IP addresses can access TiDB Cloud Serverless. + - By default, all IP addresses (`0.0.0.0 - 255.255.255.255`) are allowed. + - You can update allowed IP addresses after cluster creation. +- Your cluster has a publicly resolvable DNS name. +- Network traffic to and from your cluster is routed over the **public internet** rather than a private network. + +### Firewall rules + +Granting access to an IP address is done via **firewall rules**. If a connection attempt originates from an unapproved IP address, the client will receive an error. + +You can create a maximum of 200 IP firewall rules. + +### Allow AWS access + +You can enable access from **all AWS IP addresses** by referring to the official [AWS IP address list](https://docs.aws.amazon.com/vpc/latest/userguide/aws-ip-ranges.html). + +TiDB Cloud regularly updates this list and uses the reserved IP address **169.254.65.87** to represent all AWS IP addresses. + +## Create and manage a firewall rule + +This section describes how to manage firewall rules for a TiDB Cloud Serverless cluster. With a public endpoint, the connections to the TiDB Cloud Serverless cluster are restricted to the IP addresses specified in the firewall rules. + +1. In the [TiDB Cloud console](https://tidbcloud.com/), select the TiDB Cloud Serverless cluster on which you want to add firewall rules. + +2. In the left navigation pane, click **Networking** to open the Networking page for the TiDB Cloud Serverless cluster. + +3. **Enable** Public Endpoint if it is disabled. In **Firewall Rules**, click **+ Add Current IP**. This automatically creates a firewall rule with the public IP address of your computer, as perceived by TiDB Cloud. + + > **Note:** + > + > In some situations, the IP address observed by the TiDB Cloud console differs from the IP address used when accessing the internet. Therefore, you might need to change the start and end IP addresses to make the rule function as expected. You can use a search engine or other online tool to check your own IP address. For example, search for "what is my IP." + +4. Click **Add rule** to add more address ranges. In the **Add Firewall Rule** window, you can specify a single IP address or a range of IP addresses. If you want to limit the rule to a single IP address, type the same IP address in the **Start IP Address** and **End IP Address** fields. Opening the firewall enables administrators, users, and applications to access any database on your TiDB Cloud Serverless cluster to which they have valid credentials. Click **Submit** to add the firewall rule. + +## What's next + +- [Connect to TiDB Cloud Serverless via Public Endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md) \ No newline at end of file diff --git a/tidb-cloud/connect-to-tidb-cluster-serverless.md b/tidb-cloud/connect-to-tidb-cluster-serverless.md index 8af2cd509393f..604786f4776c1 100644 --- a/tidb-cloud/connect-to-tidb-cluster-serverless.md +++ b/tidb-cloud/connect-to-tidb-cluster-serverless.md @@ -1,31 +1,31 @@ --- -title: Connect to Your TiDB Serverless Cluster -summary: Learn how to connect to your TiDB Serverless cluster via different methods. +title: Connect to Your TiDB Cloud Serverless Cluster +summary: Learn how to connect to your TiDB Cloud Serverless cluster via different methods. --- -# Connect to Your TiDB Serverless Cluster +# Connect to Your TiDB Cloud Serverless Cluster -This document describes how to connect to your TiDB Serverless cluster. +This document describes how to connect to your TiDB Cloud Serverless cluster. > **Tip:** > -> To learn how to connect to a TiDB Dedicated cluster, see [Connect to Your TiDB Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md). +> To learn how to connect to a TiDB Cloud Dedicated cluster, see [Connect to Your TiDB Cloud Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md). ## Connection methods -After your TiDB Serverless cluster is created on TiDB Cloud, you can connect to it via one of the following methods: +After your TiDB Cloud Serverless cluster is created on TiDB Cloud, you can connect to it via one of the following methods: - Direct connections - Direct connections mean the MySQL native connection system over TCP. You can connect to your TiDB Serverless cluster using any tool that supports MySQL connection, such as [MySQL client](https://dev.mysql.com/doc/refman/8.0/en/mysql.html). + Direct connections mean the MySQL native connection system over TCP. You can connect to your TiDB Cloud Serverless cluster using any tool that supports MySQL connection, such as [MySQL client](https://dev.mysql.com/doc/refman/8.0/en/mysql.html). - [Data Service (beta)](/tidb-cloud/data-service-overview.md) - TiDB Cloud provides a Data Service feature that enables you to connect to your TiDB Serverless cluster via an HTTPS request using a custom API endpoint. Unlike direct connections, Data Service accesses TiDB Serverless data via a RESTful API rather than raw SQL. + TiDB Cloud provides a Data Service feature that enables you to connect to your TiDB Cloud Serverless cluster via an HTTPS request using a custom API endpoint. Unlike direct connections, Data Service accesses TiDB Cloud Serverless data via a RESTful API rather than raw SQL. - [Serverless Driver (beta)](/tidb-cloud/serverless-driver.md) - TiDB Cloud provides a serverless driver for JavaScript, which allows you to connect to your TiDB Serverless cluster in edge environments with the same experience as direct connections. + TiDB Cloud provides a serverless driver for JavaScript, which allows you to connect to your TiDB Cloud Serverless cluster in edge environments with the same experience as direct connections. In the preceding connection methods, you can choose your desired one based on your needs: @@ -37,7 +37,7 @@ In the preceding connection methods, you can choose your desired one based on yo ## Network -There are two network connection types for TiDB Serverless: +There are two network connection types for TiDB Cloud Serverless: - [Private endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md) (recommended) @@ -47,14 +47,14 @@ There are two network connection types for TiDB Serverless: The standard connection exposes a public endpoint, so you can connect to your TiDB cluster via a SQL client from your laptop. - TiDB Serverless requires [TLS connections](/tidb-cloud/secure-connections-to-serverless-clusters.md), which ensures the security of data transmission from your applications to TiDB clusters. + TiDB Cloud Serverless requires [TLS connections](/tidb-cloud/secure-connections-to-serverless-clusters.md), which ensures the security of data transmission from your applications to TiDB clusters. The following table shows the network you can use in different connection methods: | Connection method | Network | Description | |----------------------------|------------------------------|-------------------------------------------------------------------------------------------------------------------| | Direct connections | Public or private endpoint | Direct connections can be made via both public and private endpoints. | -| Data Service (beta) | / | Accessing TiDB Serverless via Data Service (beta) does not need to specify the network type. | +| Data Service (beta) | / | Accessing TiDB Cloud Serverless via Data Service (beta) does not need to specify the network type. | | Serverless Driver (beta) | Public endpoint | Serverless Driver only supports connections via public endpoint. | ## What's next diff --git a/tidb-cloud/connect-to-tidb-cluster.md b/tidb-cloud/connect-to-tidb-cluster.md index 2c36e21103fdf..7fb8a90c76df9 100644 --- a/tidb-cloud/connect-to-tidb-cluster.md +++ b/tidb-cloud/connect-to-tidb-cluster.md @@ -1,45 +1,48 @@ --- -title: Connect to Your TiDB Dedicated Cluster -summary: Learn how to connect to your TiDB Dedicated cluster via different methods. +title: Connect to Your TiDB Cloud Dedicated Cluster +summary: Learn how to connect to your TiDB Cloud Dedicated cluster via different methods. --- -# Connect to Your TiDB Dedicated Cluster +# Connect to Your TiDB Cloud Dedicated Cluster -This document introduces the methods to connect to your TiDB Dedicated cluster. +This document introduces the methods to connect to your TiDB Cloud Dedicated cluster. > **Tip:** > -> To learn how to connect to a TiDB Serverless cluster, see [Connect to Your TiDB Serverless Cluster](/tidb-cloud/connect-to-tidb-cluster-serverless.md). +> To learn how to connect to a TiDB Cloud Serverless cluster, see [Connect to Your TiDB Cloud Serverless Cluster](/tidb-cloud/connect-to-tidb-cluster-serverless.md). -After your TiDB Dedicated cluster is created on TiDB Cloud, you can connect to it via one of the following methods: +After your TiDB Cloud Dedicated cluster is created on TiDB Cloud, you can connect to it via one of the following methods: -- [Connect via standard connection](/tidb-cloud/connect-via-standard-connection.md) +- Direct connections - The standard connection exposes a public endpoint with traffic filters, so you can connect to your TiDB cluster via a SQL client from your laptop. You can connect to your TiDB clusters using TLS, which ensures the security of data transmission from your applications to TiDB clusters. + Direct connections use the MySQL native connection system over TCP. You can connect to your TiDB Cloud Dedicated cluster using any tool that supports MySQL connections, such as the [MySQL Command-Line Client](https://dev.mysql.com/doc/refman/8.0/en/mysql.html). TiDB Cloud also provides [SQL Shell](/tidb-cloud/connect-via-sql-shell.md), which enables you to try TiDB SQL, test out TiDB's compatibility with MySQL quickly, and manage user privileges. -- [Connect via private endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md) (recommended) + TiDB Cloud Dedicated provides three network connection types: - For TiDB Dedicated clusters hosted on AWS, private endpoint connection provides a private endpoint to allow SQL clients in your VPC to securely access services over AWS PrivateLink, which provides highly secure and one-way access to database services with simplified network management. + - [Public connection](/tidb-cloud/connect-via-standard-connection.md) -- [Connect via private endpoint with Google Cloud](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md) (recommended) + The public connection exposes a public endpoint with traffic filters, so you can connect to your TiDB cluster via a SQL client from your laptop. You can connect to your TiDB clusters using TLS, which ensures the security of data transmission from your applications to TiDB clusters. For more information, see [Connect to TiDB Cloud Dedicated via Public Connection](/tidb-cloud/connect-via-standard-connection.md). - For TiDB Dedicated clusters hosted on Google Cloud, private endpoint connection provides a private endpoint to allow SQL clients in your VPC to securely access services over Google Cloud Private Service Connect, which provides highly secure and one-way access to database services with simplified network management. + - Private endpoint (recommended) -- [Connect via VPC peering](/tidb-cloud/set-up-vpc-peering-connections.md) + Private endpoint connection provides a private endpoint to allow SQL clients in your VPC to securely access TiDB Cloud Dedicated clusters. This uses the private link service provided by different cloud providers, which provides highly secure and one-way access to database services with simplified network management. - If you want lower latency and more security, set up VPC peering and connect via a private endpoint using a VM instance on the corresponding cloud provider in your cloud account. + - For TiDB Cloud Dedicated clusters hosted on AWS, the private endpoint connection uses AWS PrivateLink. For more information, see [Connect to a TiDB Cloud Dedicated Cluster via Private Endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md). + - For TiDB Cloud Dedicated clusters hosted on Google Cloud, the private endpoint connection uses Google Cloud Private Service Connect. For more information, see [Connect to a TiDB Cloud Dedicated Cluster via Google Cloud Private Service Connect](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md). -- [Connect via Chat2Query (beta)](/tidb-cloud/explore-data-with-chat2query.md) + - [VPC peering](/tidb-cloud/set-up-vpc-peering-connections.md) + + If you want lower latency and more security, set up VPC peering and connect via a private endpoint using a VM instance on the corresponding cloud provider in your cloud account. For more information, see [Connect to TiDB Cloud Dedicated via VPC Peering](/tidb-cloud/set-up-vpc-peering-connections.md). + +- [Built-in SQL Editor](/tidb-cloud/explore-data-with-chat2query.md) > **Note:** > - > To use Chat2Query on [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md). - - TiDB Cloud is powered by artificial intelligence (AI). If your cluster is hosted on AWS and the TiDB version of the cluster is v6.5.0 or later, you can use Chat2Query (beta), an AI-powered SQL editor in the [TiDB Cloud console](https://tidbcloud.com/), to maximize your data value. + > To use SQL Editor on [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md). - In Chat2Query, you can either simply type `--` followed by your instructions to let AI generate SQL queries automatically or write SQL queries manually, and then run SQL queries against databases without a terminal. You can find the query results in tables intuitively and check the query logs easily. + If your cluster is hosted on AWS and the TiDB version of the cluster is v6.5.0 or later, you can use the AI-assisted SQL Editor in the [TiDB Cloud console](https://tidbcloud.com/) to maximize your data value. -- [Connect via SQL Shell](/tidb-cloud/connect-via-sql-shell.md): to try TiDB SQL and test out TiDB's compatibility with MySQL quickly, or administer user privileges. + In SQL Editor, you can either write SQL queries manually or simply press ⌘ + I on macOS (or Control + I on Windows or Linux) to instruct [Chat2Query (beta)](/tidb-cloud/tidb-cloud-glossary.md#chat2query) to generate SQL queries automatically. This enables you to run SQL queries against databases without a local SQL client. You can intuitively view the query results in tables or charts and easily check the query logs. ## What's next diff --git a/tidb-cloud/connect-via-sql-shell.md b/tidb-cloud/connect-via-sql-shell.md index a6d3c26e1b6fe..5fa72f648bb95 100644 --- a/tidb-cloud/connect-via-sql-shell.md +++ b/tidb-cloud/connect-via-sql-shell.md @@ -9,7 +9,7 @@ In TiDB Cloud SQL Shell, you can try TiDB SQL, test out TiDB's compatibility wit > **Note:** > -> You cannot connect to [TiDB Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-serverless) using SQL Shell. To connect to your TiDB Serverless cluster, see [Connect to TiDB Serverless clusters](/tidb-cloud/connect-to-tidb-cluster-serverless.md). +> You cannot connect to [TiDB Cloud Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) using SQL Shell. To connect to your TiDB Cloud Serverless cluster, see [Connect to TiDB Cloud Serverless clusters](/tidb-cloud/connect-to-tidb-cluster-serverless.md). To connect to your TiDB cluster using SQL shell, perform the following steps: @@ -19,8 +19,6 @@ To connect to your TiDB cluster using SQL shell, perform the following steps: > > If you have multiple projects, you can click in the lower-left corner and switch to another project. -2. Click the name of your target cluster to go to its cluster overview page, and then click **Connect** in the upper-right corner. A connection dialog is displayed. - -3. In the dialog, select the **Web SQL Shell** tab, and then click **Open SQL Shell**. - +2. Click the name of your target cluster to go to its cluster overview page, and then click **Networking** in the left navigation pane. +3. On the **Networking** page, click **Web SQL Shell** in the upper-right corner. 4. On the prompted **Enter password** line, enter the root password of the current cluster. Then your application is connected to the TiDB cluster. \ No newline at end of file diff --git a/tidb-cloud/connect-via-standard-connection-serverless.md b/tidb-cloud/connect-via-standard-connection-serverless.md index f1d0fe58281a0..d3737ac6782de 100644 --- a/tidb-cloud/connect-via-standard-connection-serverless.md +++ b/tidb-cloud/connect-via-standard-connection-serverless.md @@ -1,30 +1,32 @@ --- -title: Connect to TiDB Serverless via Public Endpoint -summary: Learn how to connect to your TiDB Serverless cluster via public endpoint. +title: Connect to TiDB Cloud Serverless via Public Endpoint +summary: Learn how to connect to your TiDB Cloud Serverless cluster via public endpoint. --- -# Connect to TiDB Serverless via Public Endpoint +# Connect to TiDB Cloud Serverless via Public Endpoint -This document describes how to connect to your TiDB Serverless cluster via public endpoint. With the public endpoint, you can connect to your TiDB Serverless cluster via a SQL client from your laptop. +This document describes how to connect to your TiDB Cloud Serverless cluster via a public endpoint, using a SQL client from your computer, as well as how to disable a public endpoint. + +## Connect via a public endpoint > **Tip:** > -> To learn how to connect to a TiDB Dedicated cluster via public endpoint, see [Connect to TiDB Dedicated via Standard Connection](/tidb-cloud/connect-via-standard-connection.md). +> To learn how to connect to a TiDB Cloud Dedicated cluster via public endpoint, see [Connect to TiDB Cloud Dedicated via Public Connection](/tidb-cloud/connect-via-standard-connection.md). -To connect to a TiDB Serverless cluster via public endpoint, take the following steps: +To connect to a TiDB Cloud Serverless cluster via public endpoint, take the following steps: 1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. In the dialog, keep the default setting of the endpoint type as `Public`, and select your preferred connection method and operating system to get the corresponding connection string. +3. In the dialog, keep the default setting of the connection type as `Public`, and select your preferred connection method and operating system to get the corresponding connection string. > **Note:** > - > - Keeping the endpoint type as `Public` means the connection is via standard TLS connection. For more information, see [TLS Connection to TiDB Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md). - > - If you choose **Private** in the **Endpoint Type** drop-down list, it means that the connection is via private endpoint. For more information, see [Connect to TiDB Serverless via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md). + > - Keeping the connection type as `Public` means the connection is via standard TLS connection. For more information, see [TLS Connection to TiDB Cloud Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md). + > - If you choose **Private Endpoint** in the **Connection Type** drop-down list, it means that the connection is via private endpoint. For more information, see [Connect to TiDB Cloud Serverless via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md). -4. TiDB Serverless lets you create [branches](/tidb-cloud/branch-overview.md) for your cluster. After a branch is created, you can choose to connect to the branch via the **Branch** drop-down list. `main` represents the cluster itself. +4. TiDB Cloud Serverless lets you create [branches](/tidb-cloud/branch-overview.md) for your cluster. After a branch is created, you can choose to connect to the branch via the **Branch** drop-down list. `main` represents the cluster itself. 5. If you have not set a password yet, click **Generate Password** to generate a random password. The generated password will not show again, so save your password in a secure location. @@ -32,7 +34,30 @@ To connect to a TiDB Serverless cluster via public endpoint, take the following > **Note:** > - > When you connect to a TiDB Serverless cluster, you must include the prefix for your cluster in the user name and wrap the name with quotation marks. For more information, see [User name prefix](/tidb-cloud/select-cluster-tier.md#user-name-prefix). + > When you connect to a TiDB Cloud Serverless cluster, you must include the prefix for your cluster in the user name and wrap the name with quotation marks. For more information, see [User name prefix](/tidb-cloud/select-cluster-tier.md#user-name-prefix). + > Your client IP must be in the allowed IP rules of the public endpoint of your cluster. For more information, see [Configure TiDB Cloud Serverless Firewall Rules for Public Endpoints](/tidb-cloud/configure-serverless-firewall-rules-for-public-endpoints.md). + +## Disable a public endpoint + +If you do not need to use a public endpoint of a TiDB Cloud Serverless cluster, you can disable it to prevent connections from the internet: + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Networking** in the left navigation pane and click **Disable** in the right pane. A confirmation dialog is displayed. + +3. Click **Disable** in the confirmation dialog. + +After disabling the public endpoint, the `Public` entry in the **Connection Type** drop-down list of the connect dialog is disabled. If users are still trying to access the cluster from the public endpoint, they will get an error. + +> **Note:** +> +> Disabling the public endpoint does not affect existing connections. It only prevents new connections from the internet. + +You can re-enable the public endpoint after disabling it: + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Networking** in the left navigation pane and click **Enable** in the right pane. ## What's next diff --git a/tidb-cloud/connect-via-standard-connection.md b/tidb-cloud/connect-via-standard-connection.md index dfe68f465736f..9b7cecd457c6b 100644 --- a/tidb-cloud/connect-via-standard-connection.md +++ b/tidb-cloud/connect-via-standard-connection.md @@ -1,17 +1,23 @@ --- -title: Connect to TiDB Dedicated via Standard Connection -summary: Learn how to connect to your TiDB Cloud cluster via standard connection. +title: Connect to TiDB Cloud Dedicated via Public Connection +summary: Learn how to connect to your TiDB Cloud cluster via public connection. --- -# Connect to TiDB Dedicated via Standard Connection +# Connect to TiDB Cloud Dedicated via Public Connection -This document describes how to connect to your TiDB Dedicated cluster via standard connection. The standard connection exposes a public endpoint with traffic filters, so you can connect to your TiDB Dedicated cluster via a SQL client from your laptop. +This document describes how to connect to your TiDB Cloud Dedicated cluster via public connection. The public connection exposes a public endpoint with traffic filters, so you can connect to your TiDB Cloud Dedicated cluster via a SQL client from your laptop. > **Tip:** > -> To learn how to connect to a TiDB Serverless cluster via standard connection, see [Connect to TiDB Serverless via Public Endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md). +> To learn how to connect to a TiDB Cloud Serverless cluster via public connection, see [Connect to TiDB Cloud Serverless via Public Endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md). -To connect to a TiDB Dedicated cluster via standard connection, take the following steps: +## Prerequisite: Configure IP access list + +For public connections, TiDB Cloud Dedicated only allows client connections from addresses in the IP access list. If you have not configured the IP access list, follow the steps in [Configure an IP Access List](/tidb-cloud/configure-ip-access-list.md) to configure it before your first connection. + +## Connect to the cluster + +To connect to a TiDB Cloud Dedicated cluster via public connection, take the following steps: 1. Open the overview page of the target cluster. @@ -25,29 +31,13 @@ To connect to a TiDB Dedicated cluster via standard connection, take the followi 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. Create a traffic filter for the cluster. Traffic filter is a list of IPs and CIDR addresses that are allowed to access TiDB Cloud via a SQL client. - - If the traffic filter is already set, skip the following sub-steps. If the traffic filter is empty, take the following sub-steps to add one. - - 1. Click one of the buttons to add some rules quickly. - - - **Add My Current IP Address** - - **Allow Access from Anywhere** - - 2. Provide an optional description for the newly added IP address or CIDR range. - - 3. Click **Create Filter** to confirm the changes. - -4. Under **Step 2: Download CA cert** in the dialog, click **Download CA cert** for TLS connection to TiDB clusters. The CA cert supports TLS 1.2 version by default. +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list. - > **Note:** - > - > - The CA cert is only available for TiDB Dedicated clusters. - > - Currently, TiDB Cloud only provides the connection strings and sample code for these connection methods: MySQL, MyCLI, JDBC, Python, Go, and Node.js. + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](/tidb-cloud/configure-ip-access-list.md) to configure it before your first connection. -5. Under **Step 3: Connect with a SQL client** in the dialog, click the tab of your preferred connection method, and then refer to the connection string and sample code on the tab to connect to your cluster. +4. Click **CA cert** to download CA cert for TLS connection to TiDB clusters. The CA cert supports TLS 1.2 version by default. - Note that you need to use the path of the downloaded CA file as the argument of the `--ssl-ca` option in the connection string. +5. Choose your preferred connection method, and then refer to the connection string and sample code on the tab to connect to your cluster. ## What's next diff --git a/tidb-cloud/connected-ai-chat-in-im.md b/tidb-cloud/connected-ai-chat-in-im.md new file mode 100644 index 0000000000000..68f189af539e0 --- /dev/null +++ b/tidb-cloud/connected-ai-chat-in-im.md @@ -0,0 +1,16 @@ +--- +title: "Connected: AI chat in IM" +summary: Introduces detailed information about the AI chat in Instant Message (IM). +--- + +# Connected: AI chat in IM + +The AI chat in Instant Message (IM) powered by PingCAP, is an offering to invite TiDB AI assistant chatbot to specific IM channels for preliminary technical support and consultation. This service is based on Graph RAG (Retrieval-Augmented Generation) built on top of [TiDB Vector Search](/tidb-cloud/vector-search-overview.md). + +## Limitation + +Currently, the ticket support feature for [Slack](https://slack.com/) and [Lark](https://www.larksuite.com/) is available upon request. If you are interested in trying this feature, contact TiDB Cloud support at support@pingcap.com or reach out to your Technical Account Manager (TAM). + +## Example + +![ai-chat-example](/media/tidb-cloud/connected-ai-chat-example.png) diff --git a/tidb-cloud/connected-care-detail.md b/tidb-cloud/connected-care-detail.md new file mode 100644 index 0000000000000..e037efa18774a --- /dev/null +++ b/tidb-cloud/connected-care-detail.md @@ -0,0 +1,144 @@ +--- +title: Connected Care Details +summary: Introduces detailed information about the Connected Care support services for TiDB Cloud. +--- + +# Connected Care Details + +This document provides detailed information about the [Connected Care](/tidb-cloud/connected-care-overview.md) support services. + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Support planBasicDeveloperEnterprisePremium
Features and ServicesGreat for personal starter projects, connecting you with an active community and AI-powered assistance for essential support needs.Ideal for development and testing phases, offering direct connections and unlimited access to tech support, to kickstart your cloud journey.Designed for production workloads, connecting your operations with advanced monitoring, prompt alerts, and fast 7x24 response times for a seamless experience.Built for enterprises with mission-critical workloads, offering the fastest response times, proactive support, and dedicated connections through named Technical Account Managers.
Initial Response Time-S1: 4 business hoursS1: 1 hourS1: 30 minutes
-S2: 8 business hoursS2: 2 hoursS2: 1 hour
-S3: 12 business hoursS3: 8 business hoursS3: 4 business hours
-S4: 24 business hoursS4: 24 business hoursS4: 24 business hours
ServicesBilling and account supportBilling and account supportBilling and account supportBilling and account support
-Ticket-based technical supportTicket-based technical supportTicket-based technical support
-End-to-end database guidanceEnd-to-end database guidanceEnd-to-end database guidance
--Connected: Clinic service for advanced monitoringConnected: Clinic service for advanced monitoring
--Connected: AI chat in IMConnected: AI chat in IM
--Connected: IM subscription for TiDB Cloud alerts (via Slack or Zoom)Connected: IM subscription for TiDB Cloud alerts (via Slack or Zoom)
--Connected: IM ticket creation and update subscription (via Slack or Lark)Connected: IM ticket creation and update subscription (via Slack or Lark)
---Connected: IM interaction for support tickets (via Slack or Lark)
---Technical Account Manager
Customer ResourcesDocumentationsDocumentationsDocumentationsDocumentations
FAQFAQFAQFAQ
Community WorkspaceCommunity WorkspaceCommunity WorkspaceCommunity Workspace
TiDB.AITiDB.AITiDB.AITiDB.AI
-Knowledge BaseKnowledge BaseKnowledge Base
+ +- Business hours: 9 AM - 6 PM local time, Monday to Friday +- IM: Instant Message. Currently, [Slack](https://slack.com/) and [Lark](https://www.larksuite.com/) are supported. We are planning to support [Microsoft Teams](https://www.microsoft.com/en-us/microsoft-teams/group-chat-software) in the future. +- [Clinic](https://clinic.pingcap.com/): a modern monitoring service that provides advanced monitoring and diagnostic capabilities for your TiDB Cloud clusters. diff --git a/tidb-cloud/connected-care-overview.md b/tidb-cloud/connected-care-overview.md new file mode 100644 index 0000000000000..380b0686185e1 --- /dev/null +++ b/tidb-cloud/connected-care-overview.md @@ -0,0 +1,102 @@ +--- +title: Connected Care Overview +summary: Introduces the new generation of TiDB Cloud support services - Connected Care. +aliases: ['/tidbcloud/connected-care-announcement'] +--- + +# Connected Care Overview + +As customers of all sizes continue to scale use cases and operations on TiDB Cloud, TiDB Cloud is committed to re-imagining its support services to accommodate their evolving needs. To deliver even greater value and seamless experiences, TiDB Cloud is excited to announce the launch of the new support services, **Connected Care**, on **Feb 17, 2025**. + +As part of this transition, the current support plan offerings are no longer available for purchase and are classified as legacy support plans starting from **Feb 17, 2025**. However, TiDB Cloud continues to provide full support to customers subscribing to legacy plans until their respective [retirement dates](#transition-to-connected-care). + +To ensure a smooth transition and access to the latest features, TiDB Cloud encourages customers to transition and adopt the Connected Care services. + +## Connected Care + +The Connected Care services are designed to strengthen your connection with TiDB Cloud through modern communication tools, proactive support, and advanced AI capabilities, delivering a seamless and customer-centric experience. + +In the Connected Care services, there are four support plans: **Basic**, **Developer** (corresponding to the legacy **Standard** plan), **Enterprise**, and **Premium**. + +> **Note** +> +> Although **Basic**, **Enterprise**, and **Premium** support plans use the same plan names as those in legacy plans, they refer to different plans with different service commitments. + +The following table provides an overview for each support plan in the Connected Care services. For more information, see [Connected Care Details](/tidb-cloud/connected-care-detail.md). + +| Support plan | Basic | Developer | Enterprise | Premium | +|:-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|:-----------------------------|:---------------------------------------------|:-----------------------------------------------|:------------------------------------------| +| Recommended workloads | Personal or starter projects | Workloads under development | Workloads under production | Business-critical workloads in production | +| Billing and account support | ✔ | ✔ | ✔ | ✔ | +| Technical support | - | ✔ | ✔ | ✔ | +| Initial response time | - | Business hours | 7x24 | 7x24 | +| [Connected: Clinic service](/tidb-cloud/tidb-cloud-clinic.md) | - | - | ✔ | ✔ | +| [Connected: AI chat in IM](/tidb-cloud/connected-ai-chat-in-im.md) | - | - | ✔ | ✔ | +| Connected: IM subscription for TiDB Cloud alerts ([Slack](/tidb-cloud/monitor-alert-slack.md), [Zoom](/tidb-cloud/monitor-alert-zoom.md)) | - | - | ✔ | ✔ | +| Connected: IM ticket creation and update subscription ([Slack](/tidb-cloud/connected-slack-ticket-creation.md), [Lark](/tidb-cloud/connected-lark-ticket-creation.md)) | - | - | ✔ | ✔ | +| Connected: IM interaction for support tickets ([Slack](/tidb-cloud/connected-slack-ticket-interaction.md), [Lark](/tidb-cloud/connected-lark-ticket-interaction.md)) | - | - | - | ✔ | +| Technical Account Manager | - | - | - | ✔ | + +> **Note** +> +> Customers from all four support plans have access to the [PingCAP support portal](https://tidb.support.pingcap.com/) for service requests. + +## Differences between legacy support services and Connected Care support services + +The support plans in the Connected Care services introduce a completely new set of features as follows: + +- Connected: Clinic service + + This feature offers an advanced monitoring and diagnostic service, Clinic, designed to help you quickly identify performance issues, optimize your database, and enhance overall performance with detailed analysis and actionable insights. For more information, see [Connected: Clinic Service](/tidb-cloud/tidb-cloud-clinic.md). + +- Connected: AI chat in IM + + This feature enables you to chat with an AI assistant through an instant message (IM) tool to receive immediate answers to your questions. For more information, see [Connected: AI chat in IM](/tidb-cloud/connected-ai-chat-in-im.md). + +- Connected: IM subscription for TiDB Cloud alerts + + This feature provides you with an easy way to subscribe to alert notifications via an IM tool, keeping you informed about critical updates. For more information, see [Subscribe via Slack](/tidb-cloud/monitor-alert-slack.md) and [Subscribe via Zoom](/tidb-cloud/monitor-alert-zoom.md). + +- Connected: IM ticket creation and update subscription + + This feature enables you to create a support ticket and subscribe to your support ticket updates through an IM tool. For more information, see [Create Tickets and Subscribe to Ticket Updates via Slack](/tidb-cloud/connected-slack-ticket-creation.md) and [Create Tickets and Subscribe to Ticket Updates via Lark](/tidb-cloud/connected-lark-ticket-creation.md). + +- Connected: IM interaction for support tickets + + This feature enables you to quickly create and interact with support tickets through an IM tool for streamlined communication. For more information, see [Interact with Support Tickets via Slack](/tidb-cloud/connected-slack-ticket-interaction.md) and [Interact with Support Tickets via Lark](/tidb-cloud/connected-lark-ticket-interaction.md). + +With these new features, the Connected Care services offer you better connectivity, more personalized support, and cost-effective solutions for different customer needs. + +- New **Enterprise** and **Premium** plans: connect customers with modern communication tools and advanced AI capabilities through advanced monitoring service in Clinic, IM subscription for TiDB Cloud alerts, IM subscription for ticket updates, AI chat in IM, and IM interaction for support tickets. + +- New **Developer** plan: customers benefit from access to the same community and [TiDB.AI](https://tidb.ai/) assistance as the **Basic** plan, while also enjoying direct connections with unlimited access to technical support. + +- New **Basic** plan: customers will be guided to join the active community channels, where they can engage with other community members and interact with [TiDB.AI](https://tidb.ai/) for technical assistance. + +## Transition to Connected Care + +The following table lists the shutdown schedules for the legacy support plans: + +| Support plans | Shutdown date | +|:----------------------------------------|:--------------| +| Legacy **Basic** plan | Feb 17, 2025 | +| Legacy **Standard** plan | Feb 17, 2025 | +| Legacy **Enterprise** and **Premium** plans | Jan 15, 2026 | + +Once a legacy support plan shuts down, TiDB Cloud will no longer support it. If you do not transition to any support plan in Connected Care by the relevant shutdown date, you will be automatically transitioned to the **Basic** support plan in Connected Care. + +## FAQs + +### How do I check or make changes to my current support plan? + +Visit the [Support](https://tidbcloud.com/console/org-settings/support) page in the TiDB Cloud console. Your current support plan is displayed in the upper-left area of the page. + +Starting from Feb 17, 2025, you can transition to the new support plans on the [Support](https://tidbcloud.com/console/org-settings/support) page, except for the **Premium** support plan. To upgrade to the **Premium** plan, please [contact sales](https://www.pingcap.com/contact-us). + +### Do I need to pay more for similar services? + +While the new Connected Care services offer a more comprehensive and feature-rich support experience, the pricing remains closely aligned to the previous offerings. TiDB Cloud remains dedicated to delivering added value to better support your journey. + +### How can I get technical support after the legacy **Basic** plan shuts down? + +You can still access [billing and account support](/tidb-cloud/tidb-cloud-support.md#create-an-account-or-billing-support-ticket). For technical support, consider purchasing a support plan in the Connected Care services. It is recommended that you start with the **Developer** plan, which includes a one-month free trial. diff --git a/tidb-cloud/connected-lark-ticket-creation.md b/tidb-cloud/connected-lark-ticket-creation.md new file mode 100644 index 0000000000000..1eeb01afa884e --- /dev/null +++ b/tidb-cloud/connected-lark-ticket-creation.md @@ -0,0 +1,42 @@ +--- +title: Create Tickets and Subscribe to Ticket Updates via Lark +summary: Introduces detailed information about the Lark ticket creation and update subscription. +--- + +# Create Tickets and Subscribe to Ticket Updates via Lark + +For customers subscribed to the **Enterprise** [support plan](/tidb-cloud/connected-care-detail.md), TiDB Cloud provides a ticket bot called **PingCAP Support Bot** in [Lark](https://www.larksuite.com/), to manage ticket creation and updates from the support ticket system. + +> **Note:** +> +> The ticket support feature for Lark is available upon request. If you are interested in trying this feature, contact TiDB Cloud support at support@pingcap.com or reach out to your Technical Account Manager (TAM). + +## Create a support ticket + +In the **Customer Support Group** Lark group, type `@PingCAP Support Bot create ticket` in a message. Then, **PingCAP Support Bot** will reply with a Lark message card for ticket creation. + +![lark-ticket-creation-1](/media/tidb-cloud/connected-lark-ticket-creation-1.png) + +Fill in the required fields and click **Submit**. Once submitted, the card will be updated to **Ticket Submitted**, indicating that your request is being processed. + +![lark-ticket-creation-2](/media/tidb-cloud/connected-lark-ticket-creation-2.png) + +After the support ticket is created, the **Ticket Submitted** card will be updated to the **Support Ticket Created** card, providing the ticket name and a link to view the ticket. + +![lark-ticket-creation-3](/media/tidb-cloud/connected-lark-ticket-creation-3.png) + +## Subscribe to ticket updates + +Whenever a PingCAP support engineer comments on the ticket, **PingCAP Support Bot** will send a **New Comment on Your Ticket** card to the Lark group. + +![connected-lark-ticket-creation-4](/media/tidb-cloud/connected-lark-ticket-creation-4.png) + +## FAQs + +- How can I check the status of my ticket? + + Log in to the [PingCAP Help Center](https://tidb.support.pingcap.com/servicedesk/customer/user/requests) with the email address used to create the ticket. You can view all historical tickets and their status for the current account. + +## Contact support + +For help or questions, please contact our support team at support@pingcap.com. diff --git a/tidb-cloud/connected-lark-ticket-interaction.md b/tidb-cloud/connected-lark-ticket-interaction.md new file mode 100644 index 0000000000000..c39b8f0c2d81b --- /dev/null +++ b/tidb-cloud/connected-lark-ticket-interaction.md @@ -0,0 +1,32 @@ +--- +title: Interact with Support Tickets via Lark +summary: Introduces detailed information about the Lark interaction for support tickets +--- + +# Interact with Support Tickets via Lark + +For customers subscribed to the **Premium** [support plan](/tidb-cloud/connected-care-detail.md), TiDB Cloud provides a ticket bot called **PingCAP Support Bot** in [Lark](https://www.larksuite.com/), to support more comprehensive interaction and management of support tickets. + +> **Note:** +> +> The ticket support feature for Lark is available upon request. If you are interested in trying this feature, contact TiDB Cloud support at support@pingcap.com or reach out to your Technical Account Manager (TAM). + +## Interact with support tickets + +If Interaction for Support Tickets is enabled, **PingCAP Support Bot** will convert the **Support Ticket Created** message card to a thread to manage ticket interactions. + +![lark-ticket-interaction-1](/media/tidb-cloud/connected-lark-ticket-interaction-1.png) + +When a PingCAP support engineer comments on the ticket, the comment will be automatically synchronized to the thread. + +When anyone comments in the thread, the comment will be synchronized back to the corresponding support ticket. + +## FAQs + +- How can I check the status of my ticket? + + Log in to the [PingCAP Help Center](https://tidb.support.pingcap.com/servicedesk/customer/user/requests) with the email address used to create the ticket. You can view all historical tickets and their status for the current account. + +## Contact support + +For help or questions, please contact our support team at support@pingcap.com. diff --git a/tidb-cloud/connected-slack-ticket-creation.md b/tidb-cloud/connected-slack-ticket-creation.md new file mode 100644 index 0000000000000..c42f1d14295d3 --- /dev/null +++ b/tidb-cloud/connected-slack-ticket-creation.md @@ -0,0 +1,44 @@ +--- +title: Create Tickets and Subscribe to Ticket Updates in Slack +summary: Introduces detailed information about the Slack ticket creation and update subscription. +--- + +# Create Tickets and Subscribe to Ticket Updates in Slack + +For customers subscribed to the **Enterprise** [support plan](/tidb-cloud/connected-care-detail.md), TiDB Cloud provides a ticket bot called **PingCAP Support Bot** in [Slack](https://slack.com/), to support quick ticket creation. + +> **Note:** +> +> The ticket support feature for Slack is available upon request. If you are interested in trying this feature, contact TiDB Cloud support at support@pingcap.com or reach out to your Technical Account Manager (TAM). + +## Create a support ticket + +In the Slack support channel, you only need to mention **PingCAP Support Bot** and describe the problem in a message. Then, the ticket bot will send you a message with a **Raise request** button. + +![slack-ticket-creation-1](/media/tidb-cloud/connected-slack-ticket-creation-1.png) + +Click **Raise request** to open a form, fill it out according to the problem, and then click **Create** to submit the ticket. + +![slack-ticket-creation-2](/media/tidb-cloud/connected-slack-ticket-creation-2.png) + +After successfully creating the ticket, the bot will reply with the ticket link. You can click the link to view the ticket details in the [PingCAP Help Center](https://tidb.support.pingcap.com/servicedesk/customer/portals). + +![slack-ticket-creation-3](/media/tidb-cloud/connected-slack-ticket-creation-3.png) + +![slack-ticket-creation-4](/media/tidb-cloud/connected-slack-ticket-creation-4.png) + +## Subscribe to ticket updates + +The **Enterprise** support plan supports subscriptions to ticket updates in Slack. When a support engineer responds to a ticket, a ticket update message will appear in the Slack channel. The message includes information such as the ticket title, ticket link, and last comment content. + +![slack-ticket-creation-5](/media/tidb-cloud/connected-slack-ticket-creation-5.png) + +## FAQs + +- How can I check the status of my ticket? + + Log in to the [PingCAP Help Center](https://tidb.support.pingcap.com/servicedesk/customer/user/requests) with the email address used to create the ticket. You can view all historical tickets and their status for the current account. + +## Contact support + +For help or questions, please contact our support team at support@pingcap.com. diff --git a/tidb-cloud/connected-slack-ticket-interaction.md b/tidb-cloud/connected-slack-ticket-interaction.md new file mode 100644 index 0000000000000..673e9389d51fa --- /dev/null +++ b/tidb-cloud/connected-slack-ticket-interaction.md @@ -0,0 +1,56 @@ +--- +title: Interact with Support Tickets via Slack +summary: Introduces detailed information about the Slack interaction for support tickets. +--- + +# Interact with Support Tickets via Slack + +For customers subscribed to the **Premium** [support plan](/tidb-cloud/connected-care-detail.md), TiDB Cloud provides a ticket bot called **Assist** in [Slack](https://slack.com/), to support more comprehensive interaction and management of support tickets. + +> **Note:** +> +> The ticket support feature for Slack is available upon request. If you are interested in trying this feature, contact TiDB Cloud support at support@pingcap.com or reach out to your Technical Account Manager (TAM). + +## Interact with support tickets + +If an **Assist** app is added to the channel, you can use **Assist** to submit the ticket. There are two ways to submit the request. + +- **Method 1: Add a 🎫 emoji to the message for ticket creation** + + Click the emoji icon next to the message that you need to create a ticket. Type `ticket` in the search box to quickly find the 🎫 emoji, and then click 🎫. + + ![slack-ticket-interaction-1](/media/tidb-cloud/connected-slack-ticket-interaction-1.png) + + ![slack-ticket-interaction-2](/media/tidb-cloud/connected-slack-ticket-interaction-2.png) + + The **Assist** app will send you a message with a **Raise a request** button. Then, you can click the button, fill out the form, and submit your request. + + ![slack-ticket-interaction-3](/media/tidb-cloud/connected-slack-ticket-interaction-3.png) + + ![slack-ticket-interaction-4](/media/tidb-cloud/connected-slack-ticket-interaction-4.png) + +- **Method 2: Type `/assist` or `/assist` along with the problem description for ticket creation** + + Another faster way is to type `/assist` or `/assist [problem description]` in the message box and press **Enter**. A request submission form will appear for you to complete and submit. + + ![slack-ticket-interaction-5](/media/tidb-cloud/connected-slack-ticket-interaction-5.png) + +After submission, the Assist app will send a confirmation message in the thread, which includes the ticket link and ticket status. + +For customers subscribed to the **Premium** [support plan](/tidb-cloud/connected-care-detail.md), two-way information synchronization is supported between Slack and the ticket system. + +The support engineer's comments on the ticket will be synchronized to the Slack message thread, and users do not need to jump to the support portal to view them. Users can reply directly in this message thread, and these replies will be synchronized to the ticket system. + +In this way, customers subscribed to the **Premium** support plan can quickly create, respond, and manage tickets without leaving Slack. + +![slack-ticket-interaction-6](/media/tidb-cloud/connected-slack-ticket-interaction-6.png) + +## FAQs + +- How can I check the status of my ticket? + + Log in to the [PingCAP Help Center](https://tidb.support.pingcap.com/servicedesk/customer/user/requests) with the email address used to create the ticket. You can view all historical tickets and their status for the current account. + +## Contact support + +For help or questions, please contact our support team at support@pingcap.com. diff --git a/tidb-cloud/cppo-customer.md b/tidb-cloud/cppo-customer.md new file mode 100644 index 0000000000000..80a4ebdfa417e --- /dev/null +++ b/tidb-cloud/cppo-customer.md @@ -0,0 +1,32 @@ +--- +title: Reseller's Customer +summary: Learn how to become a reseller's customer. +--- + +# Reseller's Customer + +A reseller's customer subscribes to TiDB Cloud services through a reseller in the AWS Marketplace. + +Unlike direct TiDB Cloud customers, a reseller's customers pay invoices via the AWS Marketplace rather than directly to PingCAP. However, all other daily operations in the TiDB Cloud console remain the same for both AWS Marketplace Channel Partner Private Offer (CPPO) and direct TiDB Cloud customers. + +This document outlines how to check both future and historical bills. + +## Migrate from a direct TiDB Cloud account to a reseller's customer account + +> **Tip:** +> +> Direct customers are the end customers who directly purchase TiDB Cloud from and pay invoices to PingCAP. + +If you are currently a direct customer with a TiDB Cloud account, you can ask your reseller to migrate your account to a reseller's customer account. + +The migration will take effect on the first day of a future month. Discuss with your reseller to determine the specific effective date. + +On the effective day of migration, you will receive an email notification. + +## View your future bills + +As a reseller's customer, you will have your invoices processed and paid through the AWS Marketplace. + +## View your historical bills + +If you have migrated from a direct TiDB Cloud account to a reseller's customer account, you can still access your billing history before the migration. To do so, go to **Billing** > **Bills** > **History** in the TiDB Cloud console. diff --git a/tidb-cloud/create-tidb-cluster-serverless.md b/tidb-cloud/create-tidb-cluster-serverless.md index 474ae6408df88..4bab3ee13498d 100644 --- a/tidb-cloud/create-tidb-cluster-serverless.md +++ b/tidb-cloud/create-tidb-cluster-serverless.md @@ -1,15 +1,15 @@ --- -title: Create a TiDB Serverless Cluster -summary: Learn how to create your TiDB Serverless cluster. +title: Create a TiDB Cloud Serverless Cluster +summary: Learn how to create your TiDB Cloud Serverless cluster. --- -# Create a TiDB Serverless Cluster +# Create a TiDB Cloud Serverless Cluster -This document describes how to create a TiDB Serverless cluster in the [TiDB Cloud console](https://tidbcloud.com/). +This document describes how to create a TiDB Cloud Serverless cluster in the [TiDB Cloud console](https://tidbcloud.com/). > **Tip:** > -> To learn how to create a TiDB Dedicated cluster, see [Create a TiDB Dedicated Cluster](/tidb-cloud/create-tidb-cluster.md). +> To learn how to create a TiDB Cloud Dedicated cluster, see [Create a TiDB Cloud Dedicated Cluster](/tidb-cloud/create-tidb-cluster.md). ## Before you begin @@ -21,7 +21,7 @@ If you do not have a TiDB Cloud account, click [here](https://tidbcloud.com/sign ## Steps -If you are in the `Organization Owner` or the `Project Owner` role, you can create a TiDB Serverless cluster as follows: +If you are in the `Organization Owner` or the `Project Owner` role, you can create a TiDB Cloud Serverless cluster as follows: 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/), and then navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page. @@ -29,22 +29,24 @@ If you are in the `Organization Owner` or the `Project Owner` role, you can crea 3. On the **Create Cluster** page, **Serverless** is selected by default. -4. The cloud provider of TiDB Serverless is AWS. You can select an AWS region where you want to host your cluster. +4. The cloud provider of TiDB Cloud Serverless is AWS. You can select an AWS region where you want to host your cluster. -5. (Optional) Change the spending limit if you plan to use more storage and compute resources than the [free quota](/tidb-cloud/select-cluster-tier.md#usage-quota). If you have not added a payment method, you need to add a credit card after editing the limit. +5. Update the default cluster name if necessary. + +6. Select a cluster plan. TiDB Cloud Serverless provides two [cluster plans](/tidb-cloud/select-cluster-tier.md#cluster-plans): **Free Cluster** and **Scalable Cluster**. You can start with a free cluster and later upgrade to a scalable cluster as your needs grow. To create a scalable cluster, you need to specify a **Monthly Spending Limit** and add a credit card. > **Note:** > - > For each organization in TiDB Cloud, you can create a maximum of five TiDB Serverless clusters by default. To create more TiDB Serverless clusters, you need to add a credit card and set a [spending limit](/tidb-cloud/tidb-cloud-glossary.md#spending-limit) for the usage. + > For each organization in TiDB Cloud, you can create a maximum of five [free clusters](/tidb-cloud/select-cluster-tier.md#free-cluster-plan) by default. To create more TiDB Cloud Serverless clusters, you need to add a credit card and create [scalable clusters](/tidb-cloud/select-cluster-tier.md#scalable-cluster-plan) for the usage. -6. Update the default cluster name if necessary, and then click **Create**. +7. Click **Create**. The cluster creation process starts and your TiDB Cloud cluster will be created in approximately 30 seconds. ## What's next -After your cluster is created, follow the instructions in [Connect to TiDB Serverless via Public Endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md) to create a password for your cluster. +After your cluster is created, follow the instructions in [Connect to TiDB Cloud Serverless via Public Endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md) to create a password for your cluster. > **Note:** > -> If you do not set a password, you cannot connect to the cluster. \ No newline at end of file +> If you do not set a password, you cannot connect to the cluster. diff --git a/tidb-cloud/create-tidb-cluster.md b/tidb-cloud/create-tidb-cluster.md index 520ba710d4509..274de0ba9bae1 100644 --- a/tidb-cloud/create-tidb-cluster.md +++ b/tidb-cloud/create-tidb-cluster.md @@ -1,15 +1,15 @@ --- -title: Create a TiDB Dedicated Cluster -summary: Learn how to create your TiDB Dedicated cluster. +title: Create a TiDB Cloud Dedicated Cluster +summary: Learn how to create your TiDB Cloud Dedicated cluster. --- -# Create a TiDB Dedicated Cluster +# Create a TiDB Cloud Dedicated Cluster -This tutorial guides you through signing up and creating a TiDB Dedicated cluster. +This tutorial guides you through signing up and creating a TiDB Cloud Dedicated cluster. > **Tip:** > -> To learn how to create a TiDB Serverless cluster, see [Create a TiDB Serverless Cluster](/tidb-cloud/create-tidb-cluster-serverless.md). +> To learn how to create a TiDB Cloud Serverless cluster, see [Create a TiDB Cloud Serverless Cluster](/tidb-cloud/create-tidb-cluster-serverless.md). ## Before you begin @@ -27,9 +27,7 @@ If you are an organization owner, you can rename the default project or create a 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/), and then click in the lower-left corner. -2. Click **Organization Settings**. - - The **Projects** tab is displayed by default. +2. Click **Organization Settings**, and click the **Projects** tab in the left navigation pane. The **Projects** tab is displayed. 3. Do one of the following: @@ -38,9 +36,9 @@ If you are an organization owner, you can rename the default project or create a 4. To return to the cluster page, click the TiDB Cloud logo in the upper-left corner of the window. -## Step 2. Create a TiDB Dedicated cluster +## Step 2. Create a TiDB Cloud Dedicated cluster -If you are in the `Organization Owner` or the `Project Owner` role, you can create a TiDB Dedicated cluster as follows: +If you are in the `Organization Owner` or the `Project Owner` role, you can create a TiDB Cloud Dedicated cluster as follows: 1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. @@ -61,11 +59,12 @@ If you are in the `Organization Owner` or the `Project Owner` role, you can crea 2. Configure the [cluster size](/tidb-cloud/size-your-cluster.md) for TiDB, TiKV, and TiFlash (optional) respectively. 3. Update the default cluster name and port number if necessary. - 4. If this is the first cluster of your current project and CIDR has not been configured for this project, you need to set the project CIDR. If you do not see the **Project CIDR** field, it means that CIDR has already been configured for this project. + 4. If CIDR has not been configured for this region, you need to set the CIDR. If you do not see the **Project CIDR** field, it means that CIDR has already been configured for this region. > **Note:** > - > When setting the project CIDR, avoid any conflicts with the CIDR of the VPC where your application is located. You cannot modify your project CIDR once it is set. + > - TiDB Cloud will create a VPC with this CIDR when the first cluster in this region is created. All the subsequent clusters of the same project in this region will use this VPC. + > - When setting the CIDR, avoid any conflicts with the CIDR of the VPC where your application is located. You cannot modify your CIDR once the VPC is created. 4. Confirm the cluster and billing information on the right side. @@ -79,14 +78,16 @@ If you are in the `Organization Owner` or the `Project Owner` role, you can crea Your TiDB Cloud cluster will be created in approximately 20 to 30 minutes. -## Step 3. Configure secure settings +## Step 3. Set the root password + +After your cluster is created, take the following steps to set the root password: -After your cluster is created, take the following steps to configure the security settings: +1. In the upper-right corner of your cluster overview page, click **...** and select **Password Settings**. -1. In the upper-right corner of your cluster overview page, click **...** and select **Security Settings**. +2. Set the root password to connect to your cluster, and then click **Save**. -2. Set the root password and allowed IP addresses to connect to your cluster, and then click **Apply**. + You can click **Auto-generate Password** to generate a random password. The generated password will not show again, so save your password in a secure location. ## What's next -After your cluster is created on TiDB Cloud, you can connect to it via the methods provided in [Connect to Your TiDB Dedicated Cluster](/tidb-cloud/connect-via-standard-connection-serverless.md). \ No newline at end of file +After your cluster is created on TiDB Cloud, you can connect to it via the methods provided in [Connect to Your TiDB Cloud Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md). diff --git a/tidb-cloud/csv-config-for-import-data.md b/tidb-cloud/csv-config-for-import-data.md index 2e0419a47fc04..397374c18886e 100644 --- a/tidb-cloud/csv-config-for-import-data.md +++ b/tidb-cloud/csv-config-for-import-data.md @@ -33,12 +33,6 @@ The following is the CSV Configuration window when you use the Import Data servi - Default: `"` -## With header - -- Definition: whether *all* CSV files contain a header row. If **With header** is `True`, the first row is used as the column names. If **With header** is `False`, the first row is treated as an ordinary data row. - -- Default: `True` - ## Backslash escape - Definition: whether to parse backslash inside fields as escape characters. If **Backslash escape** is `True`, the following sequences are recognized and converted: @@ -72,3 +66,11 @@ The following is the CSV Configuration window when you use the Import Data servi `"{\"key1\": \"val1\", \"key2\":\"val2\" }"` - Default: `True` + +## NULL value + +- Definition: defines the string that represents a `NULL` value in the CSV file. + +- Default: `\N` + +- Custom null values are not supported in the console. You can use the [TiDB Cloud CLI](/tidb-cloud/get-started-with-cli.md) instead. For more information, see [`ticloud serverless import start`](/tidb-cloud/ticloud-import-start.md). diff --git a/tidb-cloud/data-service-api-key.md b/tidb-cloud/data-service-api-key.md index c01b55c0a471b..199c3e9c67696 100644 --- a/tidb-cloud/data-service-api-key.md +++ b/tidb-cloud/data-service-api-key.md @@ -26,7 +26,7 @@ Request quotas are subject to rate limits as follows: - TiDB Cloud Data Service allows up to 100 requests per minute (rpm) per API key by default. - You can edit the rate limit of an API key when you [create](#create-an-api-key) or [edit](#edit-an-api-key) the key. The supported value range is from `1` to `1000`. If your requests per minute exceed the rate limit, the API returns a `429` error. To get a quota of more than 1000 rpm per API key, you can [submit a request](https://support.pingcap.com/hc/en-us/requests/new?ticket_form_id=7800003722519) to our support team. + You can edit the rate limit of an API key when you [create](#create-an-api-key) or [edit](#edit-an-api-key) the key. The supported value range is from `1` to `1000`. If your requests per minute exceed the rate limit, the API returns a `429` error. To get a quota of more than 1000 rpm per API key, you can [submit a request](https://tidb.support.pingcap.com/) to our support team. Each API request returns the following headers about the limit. @@ -48,14 +48,38 @@ Request quotas are subject to rate limits as follows: x-kong-response-latency: 1 server: kong/2.8.1 - {"type":"","data":{"columns":[],"rows":[],"result":{"latency":"","row_affect":0,"code":49900007,"row_count":0,"end_ms":0,"limit":0,"message":"API key rate limit exceeded. The limit can be increased up to 1000 requests per minute per API key in TiDB Cloud console. For an increase in quota beyond 1000 rpm, please contact us: https://support.pingcap.com/hc/en-us/requests/new?ticket_form_id=7800003722519","start_ms":0}}} + {"type":"","data":{"columns":[],"rows":[],"result":{"latency":"","row_affect":0,"code":49900007,"row_count":0,"end_ms":0,"limit":0,"message":"API key rate limit exceeded. The limit can be increased up to 1000 requests per minute per API key in TiDB Cloud console. For an increase in quota beyond 1000 rpm, please contact us: https://tidb.support.pingcap.com/","start_ms":0}}} ``` - TiDB Cloud Data Service allows up to 100 requests per day for each Chat2Query Data App. +## API key expiration + +By default, API keys never expire. However, for security considerations, you can specify an expiration time for your API key when you [create](#create-an-api-key) or [edit](#edit-an-api-key) the key. + +- An API key is valid only before its expiration time. Once expired, all requests using that key will fail with a `401` error, and the response is similar as follows: + + ```bash + HTTP/2 401 + date: Mon, 05 Sep 2023 02:50:52 GMT + content-type: application/json + content-length: 420 + x-debug-trace-id: 202309040250529dcdf2055e7b2ae5e9 + x-kong-response-latency: 1 + server: kong/2.8.1 + + {"data":{"result":{"start_ms":0,"end_ms":0,"latency":"","row_affect":0,"limit":0,"code":49900002,"message":"API Key is no longer valid","row_count":0},"columns":[],"rows":[]},"type":""} + ``` + +- You can also expire API keys manually. For detailed steps, see [Expire an API key](#expire-an-api-key) and [Expire all API keys](#expire-all-api-keys). Once you manually expire an API key, the expiration takes effect immediately. + +- You can check the status and expiration time of your API keys in the **Authentication** area of your target Data App. + +- Once expired, an API key cannot be activated or edited again. + ## Manage API keys -The following sections describe how to create, edit, and delete an API key for a Data App. +The following sections describe how to create, edit, delete, and expire API keys for a Data App. ### Create an API key @@ -76,7 +100,11 @@ To create an API key for a Data App, perform the following steps: 3. (Optional) Set a desired rate limit for your API key. - If your requests per minute exceed the rate limit, the API returns a `429` error. To get a quota of more than 1000 requests per minute (rpm) per API key, you can [submit a request](https://support.pingcap.com/hc/en-us/requests/new?ticket_form_id=7800003722519) to our support team. + If your requests per minute exceed the rate limit, the API returns a `429` error. To get a quota of more than 1000 requests per minute (rpm) per API key, you can [submit a request](https://tidb.support.pingcap.com/) to our support team. + + 4. (Optional) Set a desired expiration time for your API key. + + By default, an API key never expires. If you prefer to specify an expiration time for the API key, click **Expires in**, select a time unit (`Minutes`, `Days`, or `Months`), and then fill in a desired number for the time unit. 5. Click **Next**. The public key and private key are displayed. @@ -86,12 +114,16 @@ To create an API key for a Data App, perform the following steps: ### Edit an API key +> **Note**: +> +> You cannot edit an expired key. + To edit the description or rate limit of an API key, perform the following steps: 1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. 2. In the left pane, click the name of your target Data App to view its details. -3. In the **API Key** area, locate the **Action** column, and then click **...** > **Edit** in the API key row that you want to change. -4. Update the description, role, or rate limit of the API key. +3. In the **Authentication** area, locate the **Action** column, and then click **...** > **Edit** in the API key row that you want to change. +4. Update the description, role, rate limit, or expiration time of the API key. 5. Click **Update**. ### Delete an API key @@ -106,3 +138,25 @@ To delete an API key for a Data App, perform the following steps: 2. In the left pane, click the name of your target Data App to view its details. 3. In the **API Key** area, locate the **Action** column, and then click **...** > **Delete** in the API key row that you want to delete. 4. In the displayed dialog box, confirm the deletion. + +### Expire an API key + +> **Note**: +> +> You cannot expire an expired key. + +To expire an API key for a Data App, perform the following steps: + +1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. +2. In the left pane, click the name of your target Data App to view its details. +3. In the **Authentication** area, locate the **Action** column, and then click **...** > **Expire Now** in the API key row that you want to expire. +4. In the displayed dialog box, confirm the expiration. + +### Expire all API keys + +To expire all API keys for a Data App, perform the following steps: + +1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. +2. In the left pane, click the name of your target Data App to view its details. +3. In the **Authentication** area, click **Expire All**. +4. In the displayed dialog box, confirm the expiration. \ No newline at end of file diff --git a/tidb-cloud/data-service-app-config-files.md b/tidb-cloud/data-service-app-config-files.md index 42e62c9451a98..ff45d0a5a5997 100644 --- a/tidb-cloud/data-service-app-config-files.md +++ b/tidb-cloud/data-service-app-config-files.md @@ -142,7 +142,8 @@ The following is an example configuration of `config.json`. In this example, the "type": "", "required": <0 | 1>, "default": "", - "description": "" + "description": "", + "is_path_parameter": } ], "settings": { @@ -177,6 +178,7 @@ The description of each field is as follows: | `params.enum` | String | (Optional) Specifies the value options of the parameter. This field is only valid when `params.type` is set to `string`, `number`, or `integer`. To specify multiple values, you can separate them with a comma (`,`). | | `params.default` | String | The default value of the parameter. Make sure that the value matches the type of parameter you specified. Otherwise, the endpoint returns an error. The default value of an `ARRAY` type parameter is a string and you can use a comma (`,`) to separate multiple values. | | `params.description` | String | The description of the parameter. | +| `params.is_path_parameter` | Boolean | Specifies whether the parameter is a path parameter. If it is set to `true`, ensure that the `endpoint` field contains the corresponding parameter placeholders; otherwise, it will cause deployment failures. Conversely, if the `endpoint` field contains the corresponding parameter placeholders but this field is set to `false`, it will also cause deployment failures. | | `settings.timeout` | Integer | The timeout for the endpoint in milliseconds, which is `30000` by default. You can set it to an integer from `1` to `60000`. | | `settings.row_limit` | Integer | The maximum number of rows that the endpoint can operate or return, which is `1000` by default. When `batch_operation` is set to `0`, you can set it to an integer from `1` to `2000`. When `batch_operation` is set to `1`, you can set it to an integer from `1` to `100`. | | `settings.enable_pagination` | Integer | Controls whether to enable the pagination for the results returned by the request. Supported values are `0` (disabled) and `1` (enabled). The default value is `0`. | @@ -185,7 +187,7 @@ The description of each field is as follows: | `tag` | String | The tag for the endpoint. The default value is `"Default"`. | | `batch_operation` | Integer | Controls whether to enable the endpoint to operate in batch mode. Supported values are `0` (disabled) and `1` (enabled). When it is set to `1`, you can operate on multiple rows in a single request. To enable this option, make sure that the request method is `POST` or `PUT`. | | `sql_file` | String | The SQL file directory for the endpoint. For example, `"sql/GET-v1.sql"`. | -| `type` | String | The type of the endpoint, which can only be `"sql_endpoint"`. | +| `type` | String | The type of the endpoint. The value is `"system-data"` for predefined system endpoints and `"sql_endpoint"` for other endpoints. | | `return_type` | String | The response format of the endpoint, which can only be `"json"`. | ### SQL file configuration diff --git a/tidb-cloud/data-service-concepts.md b/tidb-cloud/data-service-concepts.md new file mode 100644 index 0000000000000..264fde8e54827 --- /dev/null +++ b/tidb-cloud/data-service-concepts.md @@ -0,0 +1,48 @@ +--- +title: Data Service (Beta) +summary: Learn about Data Service concepts for TiDB Cloud. +--- + +# Data Service (Beta) + +TiDB Cloud [Data Service (beta)](https://tidbcloud.com/console/data-service) is a fully managed low-code backend-as-a-service solution that simplifies backend application development, empowering developers to rapidly build highly scalable, secure, data-driven applications. + +Data Service enables you to access TiDB Cloud data via an HTTPS request using a custom API endpoint. This feature uses a serverless architecture to handle computing resources and elastic scaling, so you can focus on the query logic in endpoints without worrying about infrastructure or maintenance costs. + +For more information, see [TiDB Cloud Data Service (Beta) Overview](/tidb-cloud/data-service-overview.md). + +## Data App + +A Data App in [Data Service (beta)](https://tidbcloud.com/console/data-service) is a collection of endpoints that you can use to access data for a specific application. By creating a Data App, you can group your endpoints and configure authorization settings using API keys to restrict access to endpoints. In this way, you can ensure that only authorized users can access and manipulate your data, making your application more secure. + +For more information, see [Manage a Data App](/tidb-cloud/data-service-manage-data-app.md). + +## Data App endpoints + +An endpoint in [Data Service (beta)](https://tidbcloud.com/console/data-service) is a web API that you can customize to execute SQL statements. You can specify parameters for your SQL statements, such as the value used in the `WHERE` clause. When a client calls an endpoint and provides values for the parameters in a request URL, the endpoint executes the corresponding SQL statement with the provided parameters and returns the results as part of the HTTP response. + +For more information, see [Manage an Endpoint](/tidb-cloud/data-service-manage-endpoint.md). + +## Chat2Query API + +In TiDB Cloud, Chat2Query API is a RESTful interface that enables you to generate and execute SQL statements using AI by providing instructions. Then, the API returns the query results for you. + +For more information, see [Get Started with Chat2Query API](/tidb-cloud/use-chat2query-api.md). + +## AI integrations + +Integrating third-party tools with your Data App enhances your applications with advanced natural language processing and artificial intelligence (AI) capabilities provided by third-party tools. This integration enables your applications to perform more complex tasks and deliver intelligent solutions. + +Currently, you can integrate third-party tools, such as GPTs and Dify, in the TiDB Cloud console. + +For more information, see [Integrate a Data App with Third-Party Tools](/tidb-cloud/data-service-integrations.md). + +## Configuration as Code + +TiDB Cloud provides a Configuration as Code (CaC) approach to represent your entire Data App configurations as code using the JSON syntax. + +By connecting your Data App to GitHub, TiDB Cloud can use the CaC approach and push your Data App configurations as [configuration files](/tidb-cloud/data-service-app-config-files.md) to your preferred GitHub repository and branch. + +If Auto Sync & Deployment is enabled for your GitHub connection, you can also modify your Data App by updating its configuration files on GitHub. After you push the configuration file changes to GitHub, the new configurations will be deployed in TiDB Cloud automatically. + +For more information, see [Deploy Data App Automatically with GitHub](/tidb-cloud/data-service-manage-github-connection.md). \ No newline at end of file diff --git a/tidb-cloud/data-service-custom-domain.md b/tidb-cloud/data-service-custom-domain.md new file mode 100644 index 0000000000000..dc60c8030f938 --- /dev/null +++ b/tidb-cloud/data-service-custom-domain.md @@ -0,0 +1,72 @@ +--- +title: Custom Domain in Data Service +summary: Learn how to use a custom domain to access your Data App in TiDB Cloud Data Service. +--- + +# Custom Domain in Data Service + +TiDB Cloud Data Service provides a default domain `.data.tidbcloud.com` to access each Data App's endpoints. For enhanced personalization and flexibility, you can configure a custom domain for your Data App instead of using the default domain. + +This document describes how to manage custom domains in your Data App. + +## Before you begin + +Before configuring a custom domain for your Data App, note the following: + +- Custom domain requests exclusively support HTTPS for security. Once you successfully configure a custom domain, a "Let's Encrypt" certificate is automatically applied. +- Your custom domain must be unique within the TiDB Cloud Data Service. +- You can configure only one custom domain for each default domain, which is determined by the region of your cluster. + +## Manage custom domains + +The following sections describe how to create, edit, and remove a custom domain for a Data App. + +### Create a custom domain + +To create a custom domain for a Data App, perform the following steps: + +1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. +2. In the left pane, click the name of your target Data App to view its details. +3. In the **Manage Custom Domain** area, click **Add Custom Domain**. +4. In the **Add Custom Domain** dialog box, do the following: + 1. Select the default domain you want to replace. + 2. Enter your desired custom domain name. + 3. Optional: configure a custom path as the prefix for your endpoints. If **Custom Path** is left empty, the default path is used. +5. Preview your **Base URL** to ensure it meets your expectations. If it looks correct, click **Save**. +6. Follow the instructions in the **DNS Settings** dialog to add a `CNAME` record for the default domain in your DNS provider. + +The custom domain is in a **Pending** status initially while the system validates your DNS settings. Once the DNS validation is successful, the status of your custom domain will update to **Success**. + +> **Note:** +> +> Depending on your DNS provider, it might take up to 24 hours for the DNS record to be validated. If a custom domain remains unvalidated for over 24 hours, it will be in an **Expired** status. In this case, you can only remove the custom domain and try again. + +After your custom domain status is set to **Success**, you can use it to access your endpoint. The code example provided by TiDB Cloud Data Service is automatically updated to your custom domain and path. For more information, see [Call an endpoint](/tidb-cloud/data-service-manage-endpoint.md#call-an-endpoint). + +### Edit a custom domain + +> **Note:** +> +> After you complete the following changes, the previous custom domain and custom path will become invalid immediately. If you modify the custom domain, you need to wait for the new DNS record to be validated. + +To edit a custom domain for a Data App, perform the following steps: + +1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. +2. In the left pane, click the name of your target Data App to view its details. +3. In the **Manage Custom Domain** area, locate the **Action** column, and then click **Edit** in the custom domain row that you want to edit. +4. In the displayed dialog box, update the custom domain or custom path. +5. Preview your **Base URL** to ensure it meets your expectations. If it looks correct, click **Save**. +6. If you have changed the custom domain, follow the instructions in the **DNS Settings** dialog to add a `CNAME` record for the default domain in your DNS provider. + +### Remove a custom domain + +> **Note:** +> +> Before you delete a custom domain, make sure that the custom domain is not used anymore. + +To remove a custom domain for a Data App, perform the following steps: + +1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. +2. In the left pane, click the name of your target Data App to view its details. +3. In the **Manage Custom Domain** area, locate the **Action** column, and then click **Delete** in the custom domain row that you want to delete. +4. In the displayed dialog box, confirm the deletion. diff --git a/tidb-cloud/data-service-get-started.md b/tidb-cloud/data-service-get-started.md index bc1c22137503e..324d9ce3dc74e 100644 --- a/tidb-cloud/data-service-get-started.md +++ b/tidb-cloud/data-service-get-started.md @@ -17,7 +17,7 @@ This document introduces how to quickly get started with TiDB Cloud Data Service ## Before you begin -Before creating a Data App, make sure that you have created a [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster. If you do not have one, follow the steps in [Create a TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create one. +Before creating a Data App, make sure that you have created a [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) cluster. If you do not have one, follow the steps in [Create a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create one. ## Get started with a sample Data App @@ -64,7 +64,7 @@ To get started with Data Service, you can also create your own Data App, and the 4. (Optional) To automatically deploy endpoints of the Data App to your preferred GitHub repository and branch, enable **Connect to GitHub**, and then do the following: 1. Click **Install on GitHub**, and then follow the on-screen instructions to install **TiDB Cloud Data Service** as an application on your target repository. - 2. Click **Authorize** to authorize access to the application on GitHub. + 2. Go back to the TiDB Cloud console, and then click **Authorize** to authorize access to the application on GitHub. 3. Specify the target repository, branch, and directory where you want to save the configuration files of your Data App. > **Note:** @@ -82,9 +82,7 @@ To get started with Data Service, you can also create your own Data App, and the An endpoint is a web API that you can customize to execute SQL statements. -After creating a Data App, a default `untitled endpoint` is created for you automatically. You can use the default endpoint to access your TiDB Cloud cluster. - -If you want to create a new endpoint, locate the newly created Data App and click **+** **Create Endpoint** to the right of the App name. +To create a new endpoint, locate the newly created Data App and click **+** **Create Endpoint** to the right of the App name. #### Configure properties @@ -116,6 +114,10 @@ You can customize SQL statements for the endpoint in the SQL editor, which is th In the SQL editor, you can write statements such as table join queries, complex queries, and aggregate functions. You can also simply type `--` followed by your instructions to let AI generate SQL statements automatically. + > **Note:** + > + > To try the AI capacity of TiDB Cloud, you need to allow PingCAP and Amazon Bedrock to use your code snippets for research and service improvement. For more information, see [Enable or disable AI to generate SQL queries](/tidb-cloud/explore-data-with-chat2query.md#enable-or-disable-ai-to-generate-sql-queries). + To define a parameter, you can insert it as a variable placeholder like `${ID}` in the SQL statement. For example, `SELECT * FROM table_name WHERE id = ${ID}`. Then, you can click the **Params** tab on the right pane to change the parameter definition and test values. > **Note:** @@ -131,9 +133,31 @@ You can customize SQL statements for the endpoint in the SQL editor, which is th If you have inserted parameters in the SQL statements, make sure that you have set test values or default values for the parameters in the **Params** tab on the right pane. Otherwise, an error is returned. - To run a SQL statement, select the line of the SQL with your cursor and click **Run** > **Run at cursor**. + +
+ + For macOS: + + - If you have only one statement in the editor, to run it, press **⌘ + Enter** or click **Run**. + + - If you have multiple statements in the editor, to run one or several of them sequentially, place your cursor on your target statement or select the lines of the target statements with your cursor, and then press **⌘ + Enter** or click **Run**. + + - To run all statements in the editor sequentially, press **⇧ + ⌘ + Enter**, or select the lines of all statements with your cursor and click **Run**. - To run all SQL statements in the SQL editor, click **Run**. In this case, only the last SQL results are returned. +
+ +
+ + For Windows or Linux: + + - If you have only one statement in the editor, to run it, press **Ctrl + Enter** or click **Run**. + + - If you have multiple statements in the editor, to run one or several of them sequentially, place your cursor on your target statement or select the lines of the target statements with your cursor, and then press **Ctrl + Enter** or click **Run**. + + - To run all statements in the editor sequentially, press **Shift + Ctrl + Enter**, or select the lines of all statements with your cursor and click **Run**. + +
+ After running the statements, you can see the query results immediately in the **Result** tab at the bottom of the page. @@ -153,7 +177,7 @@ To deploy the endpoint, perform the following steps: 2. Click **Deploy** to confirm the deployment. You will get the **Endpoint has been deployed** prompt if the endpoint is successfully deployed. - On the right pane of the endpoint details page, you can click the **Deployments** tab to view the deployed history. + To view the deployment history, you can click the name of your Data App in the left pane, and then click the **Deployments** tab in the right pane. ### Step 5. Call the endpoint diff --git a/tidb-cloud/data-service-integrations.md b/tidb-cloud/data-service-integrations.md new file mode 100644 index 0000000000000..32b6ddc0f72a7 --- /dev/null +++ b/tidb-cloud/data-service-integrations.md @@ -0,0 +1,40 @@ +--- +title: Integrate a Data App with Third-Party Tools +summary: Learn how to integrate a TiDB Cloud Data App with third-party tools, such as GPTs and Dify, in the TiDB Cloud console. +--- + +# Integrate a Data App with Third-Party Tools + +Integrating third-party tools with your Data App enhances your applications with advanced natural language processing and artificial intelligence (AI) capabilities provided by third-party tools. This integration enables your applications to perform more complex tasks and deliver intelligent solutions. + +This document describes how to integrate a Data App with third-party tools, such as GPTs and Dify, in the TiDB Cloud console. + +## Integrate your Data App with GPTs + +You can integrate your Data App with [GPTs](https://openai.com/blog/introducing-gpts) to enhance your applications with intelligent capabilities. + +To integrate your Data App with GPTs, perform the following steps: + +1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. +2. In the left pane, locate your target Data App, click the name of your target Data App, and then click the **Integrations** tab. +3. In the **Integrate with GPTs** area, click **Get Configuration**. + + ![Get Configuration](/media/tidb-cloud/data-service/GPTs1.png) + +4. In the displayed dialog box, you can see the following fields: + + a. **API Specification URL**: copy the URL of the OpenAPI Specification of your Data App. For more information, see [Use the OpenAPI Specification](/tidb-cloud/data-service-manage-data-app.md#use-the-openapi-specification). + + b. **API Key**: enter the API key of your Data App. If you do not have an API key yet, click **Create API Key** to create one. For more information, see [Create an API key](/tidb-cloud/data-service-api-key.md#create-an-api-key). + + c. **API Key Encoded**: copy the base64 encoded string equivalent to the API key you have provided. + + ![GPTs Dialog Box](/media/tidb-cloud/data-service/GPTs2.png) + +5. Use the copied API Specification URL and the encoded API key in your GPT configuration. + +## Integrate your Data App with Dify + +You can integrate your Data App with [Dify](https://docs.dify.ai/guides/tools) to enhance your applications with intelligent capabilities, such as vector distance calculations, advanced similarity searches, and vector analysis. + +To integrate your Data App with Dify, follow the same steps as for [GPTs integration](#integrate-your-data-app-with-gpts). The only difference is that on the **Integrations** tab, you need to click **Get Configuration** in the **Integrate with Dify** area. diff --git a/tidb-cloud/data-service-manage-data-app.md b/tidb-cloud/data-service-manage-data-app.md index ef8525188009a..962dc11193490 100644 --- a/tidb-cloud/data-service-manage-data-app.md +++ b/tidb-cloud/data-service-manage-data-app.md @@ -61,30 +61,6 @@ You can edit the name, version, and description of a Data App. To edit Data App For more information, see [Deploy automatically with GitHub](/tidb-cloud/data-service-manage-github-connection.md). -### Integrate your Data App with GPTs - -You can integrate your Data App with [GPTs](https://openai.com/blog/introducing-gpts) to enhance your applications with intelligent capabilities. - -To integrate your Data App with GPTs, perform the following steps: - -1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. -2. In the left pane, locate your target Data App and click the name of your target Data App to view its details. -3. In the **Integration with GPTs** area, click **Get Configuration**. - - ![Get Configuration](/media/tidb-cloud/data-service/GPTs1.png) - -4. In the displayed dialog box, you can see the following fields: - - a. **API Specification URL**: copy the URL of the OpenAPI Specification of your Data App. For more information, see [Use the OpenAPI Specification](#use-the-openapi-specification). - - b. **API Key**: enter the API key of your Data App. If you do not have an API key yet, click **Create API Key** to create one. For more information, see [Create an API key](/tidb-cloud/data-service-api-key.md#create-an-api-key). - - c. **API Key Encoded**: copy the base64 encoded string equivalent to the API key you have provided. - - ![GPTs Dialog Box](/media/tidb-cloud/data-service/GPTs2.png) - -5. Use the copied API Specification URL and the encoded API key in your GPT configuration. - ### Manage linked data sources You can add or remove linked clusters for a Data App. @@ -113,6 +89,10 @@ For more information, see [Manage an API key](/tidb-cloud/data-service-api-key.m For more information, see [Manage an endpoint](/tidb-cloud/data-service-manage-endpoint.md). +### Manage a custom domain + +For more information, see [Manage a custom domain](/tidb-cloud/data-service-custom-domain.md). + ### Manage deployments To manage deployments, perform the following steps: diff --git a/tidb-cloud/data-service-manage-endpoint.md b/tidb-cloud/data-service-manage-endpoint.md index e0d885498da9e..b0e154c8e38ea 100644 --- a/tidb-cloud/data-service-manage-endpoint.md +++ b/tidb-cloud/data-service-manage-endpoint.md @@ -20,11 +20,11 @@ This document describes how to manage your endpoints in a Data App in the TiDB C ## Create an endpoint -In Data Service, you can either generate an endpoint automatically or create an endpoint manually. +In Data Service, you can automatically generate endpoints, manually create endpoints, or add predefined system endpoints. > **Tip:** > -> You can also create an endpoint from a SQL file in Chat2Query (beta). For more information, see [Generate an endpoint from a SQL file](/tidb-cloud/explore-data-with-chat2query.md#generate-an-endpoint-from-a-sql-file). +> You can also create an endpoint from a SQL file in SQL Editor. For more information, see [Generate an endpoint from a SQL file](/tidb-cloud/explore-data-with-chat2query.md#generate-an-endpoint-from-a-sql-file). ### Generate an endpoint automatically @@ -42,8 +42,15 @@ In TiDB Cloud Data Service, you can generate one or multiple endpoints automatic 2. Select at least one HTTP operation (such as `GET (Retrieve)`, `POST (Create)`, and `PUT (Update)`) for the endpoint to be generated. - For each operation you selected, TiDB Cloud Data Service will generate a corresponding endpoint. If you have selected a batch operation (such as `POST (Batch Create)`), the generated endpoint lets you operate on multiple rows in a single request. + For each operation you select, TiDB Cloud Data Service will generate a corresponding endpoint. If you select a batch operation (such as `POST (Batch Create)`), the generated endpoint lets you operate on multiple rows in a single request. + If the table you selected contains [vector data types](/tidb-cloud/vector-search-data-types.md), you can enable the **Vector Search Operations** option and select a vector distance function to generate a vector search endpoint that automatically calculates vector distances based on your selected distance function. The supported [vector distance functions](/tidb-cloud/vector-search-functions-and-operators.md) include the following: + + - `VEC_L2_DISTANCE` (default): calculates the L2 distance (Euclidean distance) between two vectors. + - `VEC_COSINE_DISTANCE`: calculates the cosine distance between two vectors. + - `VEC_NEGATIVE_INNER_PRODUCT`: calculates the distance by using the negative of the inner product between two vectors. + - `VEC_L1_DISTANCE`: calculates the L1 distance (Manhattan distance) between two vectors. + 3. (Optional) Configure a timeout and tag for the operations. All the generated endpoints will automatically inherit the configured properties, which can be modified later as needed. 4. (Optional) The **Auto-Deploy Endpoint** option (disabled by default) controls whether to enable the direct deployment of the generated endpoints. When it is enabled, the draft review process is skipped, and the generated endpoints are deployed immediately without further manual review or approval. @@ -56,6 +63,7 @@ In TiDB Cloud Data Service, you can generate one or multiple endpoints automatic - Endpoint name: the generated endpoint name is in the `/` format, and the request method (such as `GET`, `POST`, and `PUT`) is displayed before the endpoint name. For example, if the selected table name is `sample_table` and the selected operation is `POST (Create)`, the generated endpoint is displayed as `POST /sample_table`. - If a batch operation is selected, TiDB Cloud Data Service appends `/bulk` to the name of the generated endpoint. For example, if the selected table name is `/sample_table` and the selected operation is `POST (Batch Create)`, the generated endpoint is displayed as `POST /sample_table/bulk`. + - If `POST (Vector Similarity Search)` is selected, TiDB Cloud Data Service appends `/vector_search` to the name of the generated endpoint. For example, if the selected table name is `/sample_table` and the selected operation is `POST (Vector Similarity Search)`, the generated endpoint is displayed as `POST /sample_table/vector_search`. - If there has been already an endpoint with the same request method and endpoint name, TiDB Cloud Data Service appends `_dump_` to the name of the generated endpoint. For example, `/sample_table_dump_EUKRfl`. - SQL statements: TiDB Cloud Data Service automatically writes SQL statements for the generated endpoints according to the table column specifications and the selected endpoint operations. You can click the endpoint name to view its SQL statements in the middle section of the page. @@ -73,6 +81,44 @@ To create an endpoint manually, perform the following steps: 3. Update the default name if necessary. The newly created endpoint is added to the top of the endpoint list. 4. Configure the new endpoint according to the instructions in [Develop an endpoint](#develop-an-endpoint). +### Add a predefined system endpoint + +Data Service provides an endpoint library with predefined system endpoints that you can directly add to your Data App, reducing the effort in your endpoint development. Currently, the library only includes the `/system/query` endpoint, which enables you to execute any SQL statement by simply passing the statement in the predefined `sql` parameter. + +To add a predefined system endpoint to your Data App, perform the following steps: + +1. Navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page of your project. + +2. In the left pane, locate your target Data App, click **+** to the right of the App name, and then click **Manage Endpoint Library**. + + A dialog for endpoint library management is displayed. Currently, only **Execute Query** (that is, the `/system/query` endpoint) is provided in the dialog. + +3. To add the `/system/query` endpoint to your Data App, toggle the **Execute Query** switch to **Added**. + + > **Tip:** + > + > To remove an added predefined endpoint from your Data App, toggle the **Execute Query** switch to **Removed**. + +4. Click **Save**. + + > **Note:** + > + > - After you click **Save**, the added or removed endpoint is deployed to production immediately, which makes the added endpoint accessible and the removed endpoint inaccessible immediately. + > - If a non-predefined endpoint with the same path and method already exists in the current App, the creation of the system endpoint will fail. + + The added system-provided endpoint is displayed at the top of the endpoint list. + +5. Check the endpoint name, SQL statements, properties, and parameters of the new endpoint. + + > **Note:** + > + > The `/system/query` endpoint is powerful and versatile but can be potentially destructive. Use it with discretion and ensure the queries are secure and well-considered to prevent unintended consequences. + + - Endpoint name: the endpoint name and path is `/system/query`, and the request method `POST`. + - SQL statements: the `/system/query` endpoint does not come with any SQL statement. You can find the SQL editor in the middle section of the page and write your desired SQL statements in the SQL editor. Note that the SQL statements written in the SQL editor for the `/system/query` endpoint will be saved in the SQL editor so you can further develop and test them next time but they will not be saved in the endpoint configuration. + - Endpoint properties: in the right pane of the page, you can find the endpoint properties on the **Properties** tab. Unlike other custom endpoints, only the `timeout` and `max rows` properties can be customized for system endpoints. + - Endpoint parameters: in the right pane of the page, you can find the endpoint parameters on the **Params** tab. The parameters of the `/system/query` endpoint are configured automatically and cannot be modified. + ## Develop an endpoint For each endpoint, you can write SQL statements to execute on a TiDB cluster, define parameters for the SQL statements, or manage the name and version. @@ -89,11 +135,38 @@ On the right pane of the endpoint details page, you can click the **Properties** - **Path**: the path that users use to access the endpoint. - - The combination of the request method and the path must be unique within a Data App. - - Only letters, numbers, underscores (`_`), and slashes (`/`) are allowed in a path. A path must start with a slash (`/`) and end with a letter, number, or underscore (`_`). For example, `/my_endpoint/get_id`. - The length of the path must be less than 64 characters. + - The combination of the request method and the path must be unique within a Data App. + - Only letters, numbers, underscores (`_`), slashes (`/`), and parameters enclosed in curly braces (such as `{var}`) are allowed in a path. Each path must start with a slash (`/`) and end with a letter, number, or underscore (`_`). For example, `/my_endpoint/get_id`. + - For parameters enclosed in `{ }`, only letters, numbers, and underscores (`_`) are allowed. Each parameter enclosed in `{ }` must start with a letter or underscore (`_`). + + > **Note:** + > + > - In a path, each parameter must be at a separate level and does not support prefixes or suffixes. + > + > Valid path: ```/var/{var}``` and ```/{var}``` + > + > Invalid path: ```/var{var}``` and ```/{var}var``` + > + > - Paths with the same method and prefix might conflict, as in the following example: + > + > ```GET /var/{var1}``` + > + > ```GET /var/{var2}``` + > + > These two paths will conflict with each other because `GET /var/123` matches both. + > + > - Paths with parameters have lower priority than paths without parameters. For example: + > + > ```GET /var/{var1}``` + > + > ```GET /var/123``` + > + > These two paths will not conflict because `GET /var/123` takes precedence. + > + > - Path parameters can be used directly in SQL. For more information, see [Configure parameters](#configure-parameters). -- **Endpoint URL**: (read-only) the URL is automatically generated based on the region where the corresponding cluster is located, the service URL of the Data App, and the path of the endpoint. For example, if the path of the endpoint is `/my_endpoint/get_id`, the endpoint URL is `https://.data.tidbcloud.com/api/v1beta/app//endpoint/my_endpoint/get_id`. +- **Endpoint URL**: (read-only) the default URL is automatically generated based on the region where the corresponding cluster is located, the service URL of the Data App, and the path of the endpoint. For example, if the path of the endpoint is `/my_endpoint/get_id`, the endpoint URL is `https://.data.tidbcloud.com/api/v1beta/app//endpoint/my_endpoint/get_id`. To configure a custom domain for the Data App, see [Custom Domain in Data Service](/tidb-cloud/data-service-custom-domain.md). - **Request Method**: the HTTP method of the endpoint. The following methods are supported: @@ -136,9 +209,12 @@ On the SQL editor of the endpoint details page, you can write and run the SQL st On the upper part of the SQL editor, select a cluster on which you want to execute SQL statements from the drop-down list. Then, you can view all databases of this cluster in the **Schema** tab on the right pane. -2. Write SQL statements. +2. Depending on your endpoint type, do one of the following to select a database: - Before querying or modifying data, you need to first specify the database in the SQL statements. For example, `USE database_name;`. + - Predefined system endpoints: on the upper part of the SQL editor, select the target database from the drop-down list. + - Other endpoints: write a SQL statement to specify the target database in the SQL editor. For example, `USE database_name;`. + +3. Write SQL statements. In the SQL editor, you can write statements such as table join queries, complex queries, and aggregate functions. You can also simply type `--` followed by your instructions to let AI generate SQL statements automatically. @@ -151,7 +227,7 @@ On the SQL editor of the endpoint details page, you can write and run the SQL st > - The parameter name is case-sensitive. > - The parameter cannot be used as a table name or column name. -3. Run SQL statements. +4. Run SQL statements. If you have inserted parameters in the SQL statements, make sure that you have set test values or default values for the parameters in the **Params** tab on the right pane. Otherwise, an error is returned. @@ -183,6 +259,10 @@ On the SQL editor of the endpoint details page, you can write and run the SQL st After running the statements, you can see the query results immediately in the **Result** tab at the bottom of the page. + > **Note:** + > + > The returned result has a size limit of 8 MiB. + ### Configure parameters On the right pane of the endpoint details page, you can click the **Params** tab to view and manage the parameters used in the endpoint. @@ -190,8 +270,11 @@ On the right pane of the endpoint details page, you can click the **Params** tab In the **Definition** section, you can view and manage the following properties for a parameter: - The parameter name: the name can only include letters, digits, and underscores (`_`) and must start with a letter or an underscore (`_`). **DO NOT** use `page` and `page_size` as parameter names, which are reserved for pagination of request results. -- **Required**: specifies whether the parameter is required in the request. The default configuration is set to not required. -- **Type**: specifies the data type of the parameter. Supported values are `STRING`, `NUMBER`, `INTEGER`, `BOOLEAN`, and `ARRAY`. When using a `STRING` type parameter, you do not need to add quotation marks (`'` or `"`). For example, `foo` is valid for the `STRING` type and is processed as `"foo"`, whereas `"foo"` is processed as `"\"foo\""`. +- **Required**: specifies whether the parameter is required in the request. For path parameters, the configuration is required and cannot be modified. For other parameters, the default configuration is not required. +- **Type**: specifies the data type of the parameter. For path parameters, only `STRING` and `INTEGER` are supported. For other parameters, `STRING`, `NUMBER`, `INTEGER`, `BOOLEAN`, and `ARRAY` are supported. + + When using a `STRING` type parameter, you do not need to add quotation marks (`'` or `"`). For example, `foo` is valid for the `STRING` type and is processed as `"foo"`, whereas `"foo"` is processed as `"\"foo\""`. + - **Enum Value**: (optional) specifies the valid values for the parameter and is available only when the parameter type is `STRING`, `INTEGER`, or `NUMBER`. - If you leave this field empty, the parameter can be any value of the specified type. @@ -203,15 +286,12 @@ In the **Definition** section, you can view and manage the following properties - For `ARRAY` type, you need to separate multiple values with a comma (`,`). - Make sure that the value can be converted to the type of parameter. Otherwise, the endpoint returns an error. - If you do not set a test value for a parameter, the default value is used when testing the endpoint. +- **Location**: indicates the location of the parameter. This property cannot be modified. + - For path parameters, this property is `Path`. + - For other parameters, if the request method is `GET` or `DELETE`, this property is `Query`. If the request method is `POST` or `PUT`, this property is `Body`. In the **Test Values** section, you can view and set test parameters. These values are used as the parameter values when you test the endpoint. Make sure that the value can be converted to the type of parameter. Otherwise, the endpoint returns an error. -### Manage versions - -On the right pane of the endpoint details page, you can click the **Deployments** tab to view and manage the deployed versions of the endpoint. - -In the **Deployments** tab, you can deploy a draft version and undeploy the online version. - ### Rename To rename an endpoint, perform the following steps: @@ -220,6 +300,10 @@ To rename an endpoint, perform the following steps: 2. In the left pane, click the name of your target Data App to view its endpoints. 3. Locate the endpoint you want to rename, click **...** > **Rename**., and enter a new name for the endpoint. +> **Note:** +> +> Predefined system endpoints do not support renaming. + ## Test an endpoint To test an endpoint, perform the following steps: diff --git a/tidb-cloud/data-service-oas-with-nextjs.md b/tidb-cloud/data-service-oas-with-nextjs.md index d2a9c5a741a3c..022099ef86ccc 100644 --- a/tidb-cloud/data-service-oas-with-nextjs.md +++ b/tidb-cloud/data-service-oas-with-nextjs.md @@ -11,18 +11,18 @@ This document introduces how to use the OpenAPI Specification of a [Data App](/t Before using OpenAPI Specification with Next.js, make sure that you have the following: -- A TiDB cluster. For more information, see [Create a TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) or [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). +- A TiDB cluster. For more information, see [Create a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) or [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). - [Node.js](https://nodejs.org/en/download) - [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) - [yarn](https://yarnpkg.com/getting-started/install) -This document uses a TiDB Serverless cluster as an example. +This document uses a TiDB Cloud Serverless cluster as an example. ## Step 1. Prepare data To begin with, create a table `test.repository` in your TiDB cluster and insert some sample data into it. The following example inserts some open source projects developed by PingCAP as data for demonstration purposes. -To execute the SQL statements, you can use [Chat2Query](/tidb-cloud/explore-data-with-chat2query.md) in the [TiDB Cloud console](https://tidbcloud.com). +To execute the SQL statements, you can use [SQL Editor](/tidb-cloud/explore-data-with-chat2query.md) in the [TiDB Cloud console](https://tidbcloud.com). ```sql -- Select the database diff --git a/tidb-cloud/data-service-overview.md b/tidb-cloud/data-service-overview.md index 7f05ebddbf86d..87eb3dec02312 100644 --- a/tidb-cloud/data-service-overview.md +++ b/tidb-cloud/data-service-overview.md @@ -11,7 +11,7 @@ Data Service enables you to access TiDB Cloud data via an HTTPS request using a > **Note:** > -> Data Service is available for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. To use Data Service in TiDB Dedicated clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). +> Data Service is available for [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters. To use Data Service in TiDB Cloud Dedicated clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). An endpoint in Data Service is a web API that you can customize to execute SQL statements. You can specify parameters for your SQL statements, such as the value used in the `WHERE` clause. When a client calls an endpoint and provides values for the parameters in a request URL, the endpoint executes the corresponding SQL statement with the provided parameters and returns the results as part of the HTTP response. diff --git a/tidb-cloud/data-service-response-and-status-code.md b/tidb-cloud/data-service-response-and-status-code.md index 08b8a0e1ef435..e36f5b812c3ff 100644 --- a/tidb-cloud/data-service-response-and-status-code.md +++ b/tidb-cloud/data-service-response-and-status-code.md @@ -275,7 +275,7 @@ An example response is as follows: ### 429 -This status code indicates that the request exceeds the rate limit of the API key. For more quota, you can [submit a request](https://support.pingcap.com/hc/en-us/requests/new?ticket_form_id=7800003722519) to our support team. +This status code indicates that the request exceeds the rate limit of the API key. For more quota, you can [submit a request](https://tidb.support.pingcap.com/) to our support team. An example response is as follows: @@ -290,7 +290,7 @@ An example response is as follows: "rows": [], "result": { "code": 49900007, - "message": "The request exceeded the limit of 100 times per apikey per minute. For more quota, please contact us: https://support.pingcap.com/hc/en-us/requests/new?ticket_form_id=7800003722519", + "message": "The request exceeded the limit of 100 times per apikey per minute. For more quota, please contact us: https://tidb.support.pingcap.com/", "start_ms": "", "end_ms": "", "latency": "", @@ -314,7 +314,7 @@ An example response is as follows: "rows": [], "result": { "code": 429, - "message": "The AI request exceeded the limit of 100 times per day. For more quota, please contact us: https://support.pingcap.com/hc/en-us/requests/new?ticket_form_id=7800003722519", + "message": "The AI request exceeded the limit of 100 times per day. For more quota, please contact us: https://tidb.support.pingcap.com/", "start_ms": "", "end_ms": "", "latency": "", diff --git a/tidb-cloud/data-streaming-concepts.md b/tidb-cloud/data-streaming-concepts.md new file mode 100644 index 0000000000000..4a9d53326e0ca --- /dev/null +++ b/tidb-cloud/data-streaming-concepts.md @@ -0,0 +1,22 @@ +--- +title: Data Streaming +summary: Learn about data streaming concepts for TiDB Cloud. +--- + +# Data Streaming + +TiDB Cloud lets you stream data changes from your TiDB Cluster to other systems like Kafka, MySQL, and object storage. + +Currently, TiDB Cloud supports streaming data to Apache Kafka, MySQL, TiDB Cloud, and cloud storage. + +## Changefeed + +TiDB Cloud changefeed is a continuous data stream that helps you replicate data changes from TiDB Cloud to other data services. + +On the changefeed page in the TiDB Cloud console, you can create a changefeed, view a list of existing changefeeds, and operate the existing changefeeds (such as scaling, pausing, resuming, editing, and deleting a changefeed). + +Replication includes only incremental data changes by default. If existing data must be replicated, it must be exported and loaded into the target system manually before starting the changefeed. + +In TiDB Cloud, replication can be tailored by defining table filters (to specify which tables to replicate) and event filters (to include or exclude specific types of events like INSERT or DELETE). + +For more information, see [Changefeed](/tidb-cloud/changefeed-overview.md). \ No newline at end of file diff --git a/tidb-cloud/database-schema-concepts.md b/tidb-cloud/database-schema-concepts.md new file mode 100644 index 0000000000000..fbf3bccdd6bc2 --- /dev/null +++ b/tidb-cloud/database-schema-concepts.md @@ -0,0 +1,176 @@ +--- +title: Database Schema +summary: Learn about database schema concepts for TiDB Cloud. +--- + +# Database Schema + +A database schema defines the structure and organization of data within databases, tables, columns, indexes, and other objects. + +This document introduces the key concepts of database schemas, such as databases, tables, columns, data types, constraints, and indexes. It also introduces advanced features such as temporary tables for managing intermediate data seamlessly, vector indexes for efficient approximate nearest neighbor (ANN) searches, and cached tables to improve read performance. + +## Databases + +A database in TiDB is a collection of objects such as tables and indexes. + +### System databases + +System databases are default databases created by TiDB to store system tables. TiDB provides the following system databases: + +- [`INFORMATION_SCHEMA`](/information-schema/information-schema.md) + +- [`mysql`](/mysql-schema/mysql-schema.md) + +- [`performance_schema`](/performance-schema/performance-schema.md) + +- [`sys`](/sys-schema/sys-schema.md) + +### `test` database + +TiDB comes with a default database named `test`. However, it is recommended that you create your own database instead of using the `test` database. + +## Tables + +A table is a collection of related data in a [database](/develop/dev-guide-schema-design-overview.md#database). + +Each table consists of rows and columns. Each value in a row belongs to a specific column. Each column allows only a single data type. To further qualify columns, you can add some [constraints](/constraints.md). To accelerate calculations, you can add [generated columns](/generated-columns.md). + +### System table + +- The `mysql` schema contains TiDB system tables. The design is similar to the `mysql` schema in MySQL, where tables such as `mysql.user` can be edited directly. It also contains a number of tables that are extensions to MySQL. + +- Information Schema provides an ANSI-standard way of viewing system metadata. TiDB also provides a number of custom `INFORMATION_SCHEMA` tables, in addition to the tables included for MySQL compatibility. Many `INFORMATION_SCHEMA` tables have a corresponding `SHOW` command. The benefit of querying `INFORMATION_SCHEMA` is that it is possible to join between tables. + +- Performance Schema. TiDB implements performance schema tables for MySQL compatibility. + +### Cached table + +TiDB introduces the [cached table](/cached-tables.md) feature for frequently accessed but rarely updated small hotspot tables. When this feature is used, the data of an entire table is loaded into the memory of the TiDB server, and TiDB directly gets the table data from the memory without accessing TiKV, which improves the read performance. + +### Temporary table + +The temporary tables feature solves the issue of temporarily storing the intermediate results of an application, which frees you from frequently creating and dropping tables. You can store the intermediate calculation data in temporary tables. When the intermediate data is no longer needed, TiDB automatically cleans up and recycles the temporary tables. This avoids user applications being too complicated, reduces table management overhead, and improves performance. + +### Partitioned table + +In TiDB, [partitioning](/partitioned-table.md) enables you to divide a large table into one or more manageable pieces called partitions. Each partition is independent and can be managed individually. + +## Columns + +A column is subordinate to a table. Each table has at least one column. Columns provide a structure to a table by dividing the values in each row into small cells of a single data type. + +For more information, see [Define columns](/develop/dev-guide-create-table.md#define-columns). + +## Generated columns + +TiDB lets you extract data from the JSON data type as a [generated column](/generated-columns.md). + +Unlike general columns, the value of the generated column is calculated by the expression in the column definition. When inserting or updating a generated column, you cannot assign a value, but only use `DEFAULT`. + +There are two kinds of generated columns: virtual and stored. A virtual generated column occupies no storage and is computed when it is read. A stored generated column is computed when it is written (inserted or updated) and occupies storage. Compared with the virtual generated columns, the stored generated columns have better read performance, but take up more disk space. + +## Data types + +TiDB supports all the data types in MySQL except the `SPATIAL` type. This includes all the [numeric types](/data-type-numeric.md), [string types](/data-type-string.md), [date & time types](/data-type-date-and-time.md), and [the JSON type](/data-type-json.md). + +## Indexes + +An index is a copy of selected columns in a table. You can create an index using one or more columns of a [table](/develop/dev-guide-schema-design-overview.md#table). With indexes, TiDB can quickly locate data without having to search every row in a table every time, which greatly improves your query performance. + +There are two common types of indexes: + +- Primary Key: indexes on the primary key column. + +- Secondary Index: indexes on non-primary key column + +### Unique indexes + +A unique index in TiDB enforces uniqueness on one or more columns, ensuring that no two rows in a table can have the same values in the indexed column(s). This constraint provides a way to maintain data integrity by preventing duplicate values, making unique indexes ideal for fields that should naturally be unique, like email addresses, usernames, or product codes. + +### Primary key index + +A primary key index is a unique index on one or more columns in a table, which serves as the primary identifier for each row. In TiDB, every table must have a primary key, and it can be defined explicitly by the user or implicitly by TiDB if no primary key is specified. + +### Composite index + +A composite index is an index built on two or more columns of a table, which is particularly useful for queries that filter or sort data by multiple fields. For example, creating a composite index on `last_name` and `first_name` in a person table allows TiDB to quickly locate records based on both names. + +### Invisible indexes + +Invisible indexes are indexes that exist in the database but are hidden from the query optimizer, meaning they are ignored in query plans. In TiDB, invisible indexes are useful for testing and debugging, allowing you to assess the impact of an index on performance without fully dropping it. + +Starting from TiDB v8.0.0, you can make the optimizer select invisible indexes by modifying the [`tidb_opt_use_invisible_indexes`](/system-variables.md#tidb_opt_use_invisible_indexes-new-in-v800) system variable. + +### Clustered indexes + +In clustered indexes, the term clustered refers to the organization of how data is stored and not a group of database servers working together. Some database management systems refer to clustered indexes as index-organized tables (IOT). + +This feature controls how data is stored in tables containing primary keys. It provides TiDB with the ability to organize tables in a way that can improve the performance of certain queries. + +For more information, see [Clustered Indexes](/clustered-indexes.md). + +### Secondary index + +A secondary index is a logical object in a TiDB cluster. You can simply regard it as a sorting type of data that TiDB uses to improve the query performance. In TiDB, creating a secondary index is an online operation, which does not block any data read and write operations on a table. For each index, TiDB creates references for each row in a table and sorts the references by selected columns instead of by data directly. + +For more information about secondary indexes, see [Secondary Indexes](https://docs.pingcap.com/tidb/stable/tidb-best-practices#secondary-index). + +In TiDB, you can either [add a secondary index to an existing table](/develop/dev-guide-create-secondary-indexes.md#add-a-secondary-index-to-an-existing-table) or [create a secondary index when creating a new table](/develop/dev-guide-create-secondary-indexes.md#create-a-secondary-index-when-creating-a-new-table). + +### Vector index + +For the following TiDB deployment options, TiDB supports vector data types and vector search indexes. + +- TiDB Cloud Serverless + +- TiDB Self-Managed v8.4.0 or later versions + +In TiDB, a vector index is a specialized index designed for efficient approximate nearest neighbor (ANN) searches over columns containing vector data. Vector indexes, particularly the HNSW (Hierarchical Navigable Small World) algorithm, allow K-nearest neighbors (KNN) searches to identify the closest data points in a vector space quickly. This significantly speeds up query performance, enabling results in milliseconds compared to brute-force methods. + +Vector indexes rely on TiFlash replicas for data storage and search functionality. Before creating and using vector indexes, make sure that TiFlash nodes are available in your cluster. + +## Constraints + +TiDB supports almost the same constraints as MySQL. + +### NOT NULL constraints + +A `NOT NULL` constraint ensures that a column cannot contain `NULL` values. + +When a column is defined with the `NOT NULL` constraint, TiDB ensures that any attempt to insert or update a row with a `NULL` value in that column will result in an error. This behavior is consistent with MySQL's implementation of `NOT NULL` constraints. + +### CHECK constraints + +A `CHECK` constraint restricts the values of a column in a table to meet your specified conditions. When the `CHECK` constraint is added to a table, TiDB checks whether the constraint is satisfied during the insertion or updates of data into the table. If the constraint is not met, an error is returned. + +### Primary key constraints + +Like MySQL, primary key constraints in TiDB contain unique constraints, that is, creating a primary key constraint is equivalent to having a unique constraint. In addition, other primary key constraints of TiDB are also similar to those of MySQL. + +### Unique key constraints + +Unique constraints mean that all non-null values in a unique index and a primary key column are unique. + +### FOREIGN KEY constraints + +A FOREIGN KEY is a database constraint that enforces referential integrity between two tables by linking a column in one table (the child table) to a column in another table (the parent table). This ensures that the values in the foreign key column of the child table match values in the primary or unique key column of the parent table. For example, a record in an `orders` table might have a foreign key linking to a customer in a `customers` table, which ensures that each order is associated with a valid customer. + +Starting from v6.6.0, TiDB supports foreign key constraints as an experimental feature. This feature allows cross-table referencing of related data and helps maintain data consistency by enforcing referential integrity. However, it is important to note that this feature is experimental and not recommended for production environments due to potential performance issues, especially with large data volumes. + +For more information, see [FOREIGN KEY constraints](/foreign-key.md). + +## Views + +A view acts as a virtual table, whose schema is defined by the `SELECT` statement that creates the view. Using views has the following benefits: + +- Exposing only safe fields and data to users to ensure the security of sensitive fields and data stored in the underlying table. + +- Defining complex queries that frequently appear as views to make complex queries easier and more convenient. + +For more information, see [Views](/views.md). + +## Sequence + +A sequence is a database object designed to generate a sequence of numbers according to a specified set of rules. This feature is especially useful in scenarios where unique identifiers are required, such as in the creation of primary keys for database tables. + +For more information, see [sequence](/sql-statements/sql-statement-create-sequence.md). \ No newline at end of file diff --git a/tidb-cloud/delete-tidb-cluster.md b/tidb-cloud/delete-tidb-cluster.md index d8d042d9bbec0..930faf7a42f7e 100644 --- a/tidb-cloud/delete-tidb-cluster.md +++ b/tidb-cloud/delete-tidb-cluster.md @@ -17,19 +17,26 @@ You can delete a cluster at any time by performing the following steps: > Alternatively, you can also click the name of the target cluster to go to its overview page, and then click **...** in the upper-right corner. 3. Click **Delete** in the drop-down menu. -4. In the cluster deleting window, enter your `//`. +4. In the cluster deleting window, confirm the deletion: - If you want to restore the cluster sometime in the future, make sure that you have a backup of the cluster. Otherwise, you cannot restore it anymore. For more information about how to back up TiDB Dedicated clusters, see [Back Up and Restore TiDB Dedicated Data](/tidb-cloud/backup-and-restore.md). + - If you have at least one manual or automatic backup, you can see the number of backups and the charging policy for backups. Click **Continue** and enter `//`. + - If you do not have any backups, just enter `//`. + + If you want to restore the cluster sometime in the future, make sure that you have a backup of the cluster. Otherwise, you cannot restore it anymore. For more information about how to back up TiDB Cloud Dedicated clusters, see [Back Up and Restore TiDB Cloud Dedicated Data](/tidb-cloud/backup-and-restore.md). > **Note:** > - > [TiDB Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-serverless) only support [in-place restoring from backups](/tidb-cloud/backup-and-restore-serverless.md#restore) and do not support restoring data after the deletion. If you want to delete a TiDB Serverless cluster and restore its data in the future, you can use [Dumpling](https://docs.pingcap.com/tidb/stable/dumpling-overview) to export your data as a backup. + > [TiDB Cloud Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) do not support restoring data after the deletion. If you want to delete a TiDB Cloud Serverless cluster and restore its data in the future, see [Export Data from TiDB Cloud Serverless](/tidb-cloud/serverless-export.md) to export your data as a backup. + +5. Click **I understand, delete it**. -5. Click **I understand the consequences. Delete this cluster**. + Once a backed up TiDB Cloud Dedicated cluster is deleted, the existing backup files of the cluster are moved to the recycle bin. - Once a backed up TiDB Dedicated cluster is deleted, the existing backup files of the cluster are moved to the recycle bin. + - Automatic backups will expire and be automatically deleted once the retention period ends. The default retention period is 7 days if you don't modify it. + - Manual backups will be kept in the Recycle Bin until manually deleted. -- For backup files from an automatic backup, the recycle bin can retain them for 7 days. -- For backup files from a manual backup, there is no expiration date. + > **Note:** + > + > Please be aware that backups will continue to incur charges until deleted. - If you want to restore a TiDB Dedicated cluster from recycle bin, see [Restore a deleted cluster](/tidb-cloud/backup-and-restore.md#restore-a-deleted-cluster). + If you want to restore a TiDB Cloud Dedicated cluster from recycle bin, see [Restore a deleted cluster](/tidb-cloud/backup-and-restore.md#restore-a-deleted-cluster). diff --git a/tidb-cloud/dev-guide-bi-looker-studio.md b/tidb-cloud/dev-guide-bi-looker-studio.md index 29a446a68f752..e1b3c8b357b1a 100644 --- a/tidb-cloud/dev-guide-bi-looker-studio.md +++ b/tidb-cloud/dev-guide-bi-looker-studio.md @@ -1,47 +1,47 @@ --- -title: Connect to TiDB Serverless with Looker Studio -summary: Learn how to connect to TiDB Serverless using Looker Studio. +title: Connect to TiDB Cloud Serverless with Looker Studio +summary: Learn how to connect to TiDB Cloud Serverless using Looker Studio. --- -# Connect to TiDB Serverless with Looker Studio +# Connect to TiDB Cloud Serverless with Looker Studio -TiDB is a MySQL-compatible database, TiDB Serverless is a fully managed TiDB offering, and [Looker Studio](https://lookerstudio.google.com/) is a free web-based BI tool that can visualize data from various sources. +TiDB is a MySQL-compatible database, TiDB Cloud Serverless is a fully managed TiDB offering, and [Looker Studio](https://lookerstudio.google.com/) is a free web-based BI tool that can visualize data from various sources. -In this tutorial, you can learn how to connect to your TiDB Serverless cluster with Looker Studio. +In this tutorial, you can learn how to connect to your TiDB Cloud Serverless cluster with Looker Studio. > **Note:** > -> Most steps in this tutorial work with TiDB Dedicated as well. However, for TiDB Dedicated, you need to note the following: +> Most steps in this tutorial work with TiDB Cloud Dedicated as well. However, for TiDB Cloud Dedicated, you need to note the following: > > - Import your dataset following [Import data from files to TiDB Cloud](/tidb-cloud/tidb-cloud-migration-overview.md#import-data-from-files-to-tidb-cloud). -> - Get the connection information for your cluster following [Connect to TiDB Dedicated](/tidb-cloud/connect-via-standard-connection.md). When connecting to TiDB Dedicated, you need to allow access from `142.251.74.0/23`. For more information about connections from Looker Studio, see [Looker Studio documentation](https://support.google.com/looker-studio/answer/7088031#zippy=%2Cin-this-article). +> - Get the connection information for your cluster following [Connect to TiDB Cloud Dedicated](/tidb-cloud/connect-via-standard-connection.md). When connecting to TiDB Cloud Dedicated, you need to allow access from `142.251.74.0/23`. For more information about connections from Looker Studio, see [Looker Studio documentation](https://support.google.com/looker-studio/answer/7088031#zippy=%2Cin-this-article). ## Prerequisites To complete this tutorial, you need: - A Google account -- A TiDB Serverless cluster +- A TiDB Cloud Serverless cluster -**If you don't have a TiDB Serverless cluster, you can create one as follows:** +**If you don't have a TiDB Cloud Serverless cluster, you can create one as follows:** -- [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster) +- [Create a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-cloud-serverless-cluster) ## Step 1. Import a dataset -You can import the S&P 500 dataset provided in the interactive tutorial of TiDB Serverless. +You can import the S&P 500 dataset provided in the interactive tutorial of TiDB Cloud Serverless. 1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and click **?** in the lower-right corner. A **Help** dialog is displayed. 2. In the dialog, click **Interactive Tutorials**, and then click **S&P 500 Analysis**. -3. Select your TiDB Serverless cluster, and then click **Import Dataset** to import the S&P 500 dataset to your cluster. +3. Select your TiDB Cloud Serverless cluster, and then click **Import Dataset** to import the S&P 500 dataset to your cluster. 4. After the import status changes to **IMPORTED**, click **Exit Tutorial** to close this dialog. If you encounter any issues during import, you can cancel this import task as follows: -1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click the name of your TiDB Serverless cluster to go to its overview page. +1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click the name of your TiDB Cloud Serverless cluster to go to its overview page. 2. In the left navigation pane, click **Import**. 3. Find the import task named **sp500-insight**, click **...** in the **Action** column, and then click **Cancel**. @@ -61,7 +61,7 @@ If you encounter any issues during import, you can cancel this import task as fo > **Tip:** > - > TiDB Serverless requires a secure TLS connection between the client and the cluster, so you need this CA cert for connection settings in Looker Studio. + > TiDB Cloud Serverless requires a secure TLS connection between the client and the cluster, so you need this CA cert for connection settings in Looker Studio. ## Step 3. Connect to your TiDB cluster with Looker Studio @@ -71,14 +71,14 @@ If you encounter any issues during import, you can cancel this import task as fo 3. In the **BASIC** setting pane, configure the connection parameters. - - **Host Name or IP**: enter the `HOST` parameter from the TiDB Serverless connection dialog. - - **Port(Optional)**: enter the `PORT` parameter from the TiDB Serverless connection dialog. + - **Host Name or IP**: enter the `HOST` parameter from the TiDB Cloud Serverless connection dialog. + - **Port(Optional)**: enter the `PORT` parameter from the TiDB Cloud Serverless connection dialog. - **Database**: enter the database you want to connect to. For this tutorial, enter `sp500insight`. - - **Username**: enter the `USERNAME` parameter from the TiDB Serverless connection dialog. - - **Password**: enter the `PASSWORD` parameter from the TiDB Serverless connection dialog. + - **Username**: enter the `USERNAME` parameter from the TiDB Cloud Serverless connection dialog. + - **Password**: enter the `PASSWORD` parameter from the TiDB Cloud Serverless connection dialog. - **Enable SSL**: select this option, and then click the upload icon to the right of **MySQL SSL Client Configuration Files** to upload the CA file downloaded from [Step 2](#step-2-get-the-connection-information-for-your-cluster). - ![Looker Studio: configure connection settings for TiDB Serverless](/media/tidb-cloud/looker-studio-configure-connection.png) + ![Looker Studio: configure connection settings for TiDB Cloud Serverless](/media/tidb-cloud/looker-studio-configure-connection.png) 4. Click **AUTHENTICATE**. @@ -134,4 +134,4 @@ Then, you can see a combo chart similar as follows: ## Need help? -Ask questions on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [create a support ticket](https://support.pingcap.com/). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/tidb-cloud/dev-guide-wordpress.md b/tidb-cloud/dev-guide-wordpress.md index 4fb3ec59b81f0..c699e6215a2d7 100644 --- a/tidb-cloud/dev-guide-wordpress.md +++ b/tidb-cloud/dev-guide-wordpress.md @@ -1,27 +1,27 @@ --- -title: Connect to TiDB Serverless with WordPress -summary: Learn how to use TiDB Serverless to run WordPress. This tutorial gives step-by-step guidance to run WordPress + TiDB Serverless in a few minutes. +title: Connect to TiDB Cloud Serverless with WordPress +summary: Learn how to use TiDB Cloud Serverless to run WordPress. This tutorial gives step-by-step guidance to run WordPress + TiDB Cloud Serverless in a few minutes. --- -# Connect to TiDB Serverless with WordPress +# Connect to TiDB Cloud Serverless with WordPress -TiDB is a MySQL-compatible database, TiDB Serverless is a fully managed TiDB offering, and [WordPress](https://github.com/WordPress) is a free, open-source content management system (CMS) that lets users create and manage websites. WordPress is written in PHP and uses a MySQL database. +TiDB is a MySQL-compatible database, TiDB Cloud Serverless is a fully managed TiDB offering, and [WordPress](https://github.com/WordPress) is a free, open-source content management system (CMS) that lets users create and manage websites. WordPress is written in PHP and uses a MySQL database. -In this tutorial, you can learn how to use TiDB Serverless to run WordPress for free. +In this tutorial, you can learn how to use TiDB Cloud Serverless to run WordPress for free. > **Note:** > -> In addition to TiDB Serverless, this tutorial works with TiDB Dedicated and TiDB Self-Hosted clusters as well. However, it is highly recommended to run WordPress with TiDB Serverless for cost efficiency. +> In addition to TiDB Cloud Serverless, this tutorial works with TiDB Cloud Dedicated and TiDB Self-Managed clusters as well. However, it is highly recommended to run WordPress with TiDB Cloud Serverless for cost efficiency. ## Prerequisites To complete this tutorial, you need: -- A TiDB Serverless cluster. Follow [creating a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster if you don't have one. +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create your own TiDB Cloud cluster if you don't have one. -## Run WordPress with TiDB Serverless +## Run WordPress with TiDB Cloud Serverless -This section demonstrates how to run WordPress with TiDB Serverless. +This section demonstrates how to run WordPress with TiDB Cloud Serverless. ### Step 1: Clone the WordPress sample repository @@ -34,7 +34,7 @@ cd wordpress-tidb-docker ### Step 2: Install dependencies -1. The sample repository requires [Docker](https://www.docker.com/) and [Docker Compose](https://docs.docker.com/compose/) to start WordPress. If you have them installed, you can skip this step. It is highly recommended to run your WordPress in a Linux environment (such as Ubuntu). Run the following command to install them: +1. The sample repository requires [Docker](https://www.docker.com/) and [Docker Compose](https://docs.docker.com/compose/) to start WordPress. If you have them installed, you can skip this step. It is highly recommended to run your WordPress in a Linux environment (such as Ubuntu). Run the following command to install Docker and Docker Compose: ```shell sudo sh install.sh @@ -48,7 +48,7 @@ cd wordpress-tidb-docker ### Step 3: Configure connection information -Configure the WordPress database connection to TiDB Serverless. +Configure the WordPress database connection to TiDB Cloud Serverless. 1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. @@ -56,9 +56,10 @@ Configure the WordPress database connection to TiDB Serverless. 3. Ensure the configurations in the connection dialog match your operating environment. - - **Endpoint Type** is set to `Public`. + - **Connection Type** is set to `Public`. - **Connect With** is set to `WordPress`. - **Operating System** is set to `Debian/Ubuntu/Arch`. + - **Database** is set to the database you want to use—for example, `test`. 4. Click **Generate Password** to create a random password. @@ -82,11 +83,11 @@ Configure the WordPress database connection to TiDB Serverless. TIDB_DB_NAME='test' ``` - Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. By default, your TiDB Serverless comes with a `test` database. If you have already created another database in your TiDB Serverless cluster, you can replace `test` with your database name. + Be sure to replace the placeholders `{}` with the connection parameters obtained from the connection dialog. By default, your TiDB Cloud Serverless comes with a `test` database. If you have already created another database in your TiDB Cloud Serverless cluster, you can replace `test` with your database name. 7. Save the `.env` file. -### Step 4: Start WordPress with TiDB Serverless +### Step 4: Start WordPress with TiDB Cloud Serverless 1. Execute the following command to run WordPress as a Docker container: @@ -96,6 +97,12 @@ Configure the WordPress database connection to TiDB Serverless. 2. Set up your WordPress site by visiting [localhost](http://localhost/) if you start the container on your local machine or `http://` if the WordPress is running on a remote machine. +### Step 5: Confirm the database connection + +1. Close the connection dialog for your cluster on the TiDB Cloud console, and open the **SQL Editor** page. +2. Under the **Schemas** tab on the left, click the database you connected to Wordpress. +3. Confirm that you now see the Wordpress tables (such as `wp_posts` and `wp_comments`) in the list of tables for that database. + ## Need help? -Ask questions on the [Discord](https://discord.gg/vYU9h56kAX), or [create a support ticket](/tidb-cloud/tidb-cloud-support.md). +Ask the community on [Discord](https://discord.gg/DQZ2dy3cuc?utm_source=doc) or [Slack](https://slack.tidb.io/invite?team=tidb-community&channel=everyone&ref=pingcap-docs), or [submit a support ticket](https://tidb.support.pingcap.com/). diff --git a/tidb-cloud/explore-data-with-chat2query.md b/tidb-cloud/explore-data-with-chat2query.md index 27d2eb2d3d64a..2caf5475d50eb 100644 --- a/tidb-cloud/explore-data-with-chat2query.md +++ b/tidb-cloud/explore-data-with-chat2query.md @@ -1,35 +1,29 @@ --- -title: Explore Your Data with AI-Powered Chat2Query (beta) -summary: Learn how to use Chat2Query, an AI-powered SQL editor in the TiDB Cloud console, to maximize your data value. +title: Explore Your Data with AI-Assisted SQL Editor +summary: Learn how to use AI-assisted SQL Editor in the TiDB Cloud console, to maximize your data value. --- -# Explore Your Data with AI-Powered Chat2Query (beta) +# Explore Your Data with AI-Assisted SQL Editor -TiDB Cloud is powered by AI. You can use Chat2Query (beta), an AI-powered SQL editor in the [TiDB Cloud console](https://tidbcloud.com/), to maximize your data value. +You can use the built-in AI-assisted SQL Editor in the [TiDB Cloud console](https://tidbcloud.com/) to maximize your data value. -In Chat2Query, you can either simply type `--` followed by your instructions to let AI generate SQL queries automatically or write SQL queries manually, and then run SQL queries against databases without a terminal. You can find the query results in tables intuitively and check the query logs easily. - -> **Note:** -> -> Chat2Query is supported for TiDB clusters that are v6.5.0 or later and are hosted on AWS. -> -> - For [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters, Chat2Query is available by default. -> - For [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters, Chat2Query is only available upon request. To use Chat2Query on TiDB Dedicated clusters, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md). +In SQL Editor, you can either write SQL queries manually or simply press ⌘ + I on macOS (or Control + I on Windows or Linux) to instruct [Chat2Query (beta)](/tidb-cloud/tidb-cloud-glossary.md#chat2query) to generate SQL queries automatically. This enables you to run SQL queries against databases without a local SQL client. You can intuitively view the query results in tables or charts and easily check the query logs. ## Use cases -The recommended use cases of Chat2Query are as follows: +The recommended use cases of SQL Editor are as follows: -- Use the AI capacity of Chat2Query to help you generate complex SQL queries instantly. -- Test out the MySQL compatibility of TiDB quickly. -- Explore TiDB SQL features easily. +- Utilize the AI capabilities of Chat2Query to generate, debug, or rewrite complex SQL queries instantly. +- Quickly test the MySQL compatibility of TiDB. +- Easily explore the SQL features in TiDB using your own datasets. -## Limitation +## Limitations -- SQL queries generated by the AI are not 100% accurate and might still need your further tweak. -- The [Chat2Query API](/tidb-cloud/use-chat2query-api.md) is available for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. To use the Chat2Query API on [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md). +- SQL queries generated by the AI might not be 100% accurate, and you might need to refine them. +- SQL Editor is only supported for TiDB clusters that are v6.5.0 or later and hosted on AWS. +- SQL Editor is available by default for [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters. To use SQL Editor and Chat2Query on [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md). -## Access Chat2Query +## Access SQL Editor 1. Go to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. @@ -37,21 +31,21 @@ The recommended use cases of Chat2Query are as follows: > > If you have multiple projects, you can click in the lower-left corner and switch to another project. -2. Click your cluster name, and then click **Chat2Query** in the left navigation pane. +2. Click your cluster name, and then click **SQL Editor** in the left navigation pane. > **Note:** > - > In the following cases, the **Chat2Query** entry is displayed in grey and not clickable. + > In the following cases, the **SQL Editor** entry is displayed in gray and not clickable. > - > - Your TiDB Dedicated cluster is earlier than v6.5.0. To use Chat2Query, you need to contract [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md) to upgrade your clusters. - > - Your TiDB Dedicated cluster is just created and the running environment for Chat2Query is still in preparation. In this case, wait for a few minutes and Chat2Query will be available. - > - Your TiDB Dedicated cluster is [paused](/tidb-cloud/pause-or-resume-tidb-cluster.md). + > - Your TiDB Cloud Dedicated cluster is earlier than v6.5.0. To use SQL Editor, you need to contract [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md) to upgrade your clusters. + > - Your TiDB Cloud Dedicated cluster is just created and the running environment for SQL Editor is still in preparation. In this case, wait for a few minutes and Chat2Query will be available. + > - Your TiDB Cloud Dedicated cluster is [paused](/tidb-cloud/pause-or-resume-tidb-cluster.md). ## Enable or disable AI to generate SQL queries -PingCAP takes the privacy and security of users' data as a top priority. The AI capacity of Chat2Query only needs to access database schemas to generate SQL queries, not your data itself. For more information, see [Chat2Query Privacy FAQ](https://www.pingcap.com/privacy-policy/privacy-chat2query). +PingCAP takes the privacy and security of users' data as a top priority. The AI capacity of Chat2Query in SQL Editor only needs to access database schemas to generate SQL queries, not your data itself. For more information, see [Chat2Query Privacy FAQ](https://www.pingcap.com/privacy-policy/privacy-chat2query). -When you access Chat2Query for the first time, you will be prompted with a dialog about whether to allow PingCAP and OpenAI to use your code snippets to research and improve the services. +When you access Chat2Query for the first time, you will be prompted with a dialog about whether to allow PingCAP and Amazon Bedrock to use your code snippets to research and improve the services. - To enable the AI to generate SQL queries, select the checkbox and click **Save and Get Started**. - To disable the AI to generate SQL queries, close this dialog directly. @@ -63,16 +57,36 @@ After the first-time access, you can still change the AI setting as follows: ## Write and run SQL queries -In Chat2Query, you can write and run SQL queries using your own dataset. +In SQL Editor, you can write and run SQL queries using your own dataset. 1. Write SQL queries. - - If AI is enabled, simply type `--` followed by your instructions to let AI generate SQL queries automatically or write SQL queries manually. + +
+ + For macOS: + + - If AI is enabled, simply press **⌘ + I** followed by your instructions and press **Enter** to let AI generate SQL queries automatically. - For a SQL query generated by AI, you can accept it by pressing Tab and then further edit it if needed, or reject it by pressing Esc. + For a SQL query generated by Chat2Query, click **Accept** to accept the query and continue editing. If the query does not meet your requirements, click **Discard** to reject it. Alternatively, click **Regenerate** to request a new query from Chat2Query. - If AI is disabled, write SQL queries manually. +
+ +
+ + For Windows or Linux: + + - If AI is enabled, simply press **Ctrl + I** followed by your instructions and press **Enter** to let AI generate SQL queries automatically. + + For a SQL query generated by Chat2Query, click **Accept** to accept the query and continue editing. If the query does not meet your requirements, click **Discard** to reject it. Alternatively, click **Regenerate** to request a new query from Chat2Query. + + - If AI is disabled, write SQL queries manually. + +
+ + 2. Run SQL queries. @@ -103,9 +117,36 @@ In Chat2Query, you can write and run SQL queries using your own dataset. After running the queries, you can see the query logs and results immediately at the bottom of the page. +> **Note:** +> +> The returned result has a size limit of 8 MiB. + +## Rewrite SQL queries using Chat2Query + +In SQL Editor, you can use Chat2Query to rewrite existing SQL queries to optimize performance, fix errors, or meet other specific requirements. + +1. Select the lines of SQL queries you want to rewrite with your cursor. + +2. Invoke Chat2Query for rewriting by using the keyboard shortcut for your operating system: + + - ⌘ + I on macOS + - Control + I on Windows or Linux + + Press **Enter** after providing your instructions to let AI handle the rewrite. + +3. After invoking Chat2Query, you can see the suggested rewrite and the following options: + + - **Accept**: click this to accept the suggested rewrite and continue editing. + - **Discard**: click this if the suggested rewrite does not meet your expectations. + - **Regenerate**: click this to request another rewrite from Chat2Query based on your feedback or additional instructions. + +> **Note:** +> +> Chat2Query uses AI algorithms to suggest optimizations and corrections. It is recommended that you review these suggestions carefully before finalizing the queries. + ## Manage SQL files -In Chat2Query, you can save your SQL queries in different SQL files and manage SQL files as follows: +In SQL Editor, you can save your SQL queries in different SQL files and manage SQL files as follows: - To add a SQL file, click **+** on the **SQL Files** tab. - To rename a SQL file, move your cursor on the filename, click **...** next to the filename, and then select **Rename**. @@ -127,7 +168,7 @@ For more information, see [Get started with Chat2Query API](/tidb-cloud/use-chat ## Generate an endpoint from a SQL file -For TiDB clusters, TiDB Cloud provides a [Data Service (beta)](/tidb-cloud/data-service-overview.md) feature that enables you to access TiDB Cloud data via an HTTPS request using a custom API endpoint. In Chat2Query, you can generate an endpoint of Data Service (beta) from a SQL file by taking the following steps: +For TiDB clusters, TiDB Cloud provides a [Data Service (beta)](/tidb-cloud/data-service-overview.md) feature that enables you to access TiDB Cloud data via an HTTPS request using a custom API endpoint. In SQL Editor, you can generate an endpoint of Data Service (beta) from a SQL file by taking the following steps: 1. Move your cursor on the filename, click **...** next to the filename, and then select **Generate endpoint**. 2. In the **Generate endpoint** dialog box, select the Data App you want to generate the endpoint for and enter the endpoint name. @@ -135,15 +176,15 @@ For TiDB clusters, TiDB Cloud provides a [Data Service (beta)](/tidb-cloud/data- For more information, see [Manage an endpoint](/tidb-cloud/data-service-manage-endpoint.md). -## Manage Chat2Query settings +## Manage SQL Editor settings -In Chat2Query, you can change the following settings: +In SQL Editor, you can change the following settings: - The maximum number of rows in query results - Whether to show system database schemas on the **Schemas** tab To change the settings, take the following steps: -1. In the upper-right corner of Chat2Query, click **...** and select **Settings**. +1. In the upper-right corner of **SQL Editor**, click **...** and select **Settings**. 2. Change the settings according to your need. 3. Click **Save**. diff --git a/tidb-cloud/get-started-with-cli.md b/tidb-cloud/get-started-with-cli.md index c5d86ed720727..db21975b117e2 100644 --- a/tidb-cloud/get-started-with-cli.md +++ b/tidb-cloud/get-started-with-cli.md @@ -8,12 +8,16 @@ summary: Learn how to manage TiDB Cloud resources through the TiDB Cloud CLI. TiDB Cloud provides a command-line interface (CLI) [`ticloud`](https://github.com/tidbcloud/tidbcloud-cli) for you to interact with TiDB Cloud from your terminal with a few lines of commands. For example, you can easily perform the following operations using `ticloud`: - Create, delete, and list your clusters. -- Import data from Amazon S3 or local files to your clusters. +- Import data to your clusters. +- Export data from your clusters. + +> **Note:** +> +> TiDB Cloud CLI is in beta. ## Before you begin - Have a TiDB Cloud account. If you do not have one, [sign up for a free trial](https://tidbcloud.com/free-trial). -- [Create a TiDB Cloud API Key](https://docs.pingcap.com/tidbcloud/api/v1beta#section/Authentication/API-Key-Management). ## Installation @@ -81,6 +85,44 @@ Install the MySQL command-line client if you do not have it. You can refer to th
+## Quick start + +[TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) is the best way to get started with TiDB Cloud. In this section, you will learn how to create a TiDB Cloud Serverless cluster with TiDB Cloud CLI. + +### Create a user profile or log into TiDB Cloud + +Before creating a cluster with TiDB Cloud CLI, you need to either create a user profile or log into TiDB Cloud. + +- Create a user profile with your [TiDB Cloud API key](https://docs.pingcap.com/tidbcloud/api/v1beta#section/Authentication/API-Key-Management): + + ```shell + ticloud config create + ``` + + > **Warning:** + > + > The profile name **MUST NOT** contain `.`. + +- Log into TiDB Cloud with authentication: + + ```shell + ticloud auth login + ``` + + After successful login, an OAuth token will be assigned to the current profile. If no profiles exist, the token will be assigned to a profile named `default`. + +> **Note:** +> +> In the preceding two methods, the TiDB Cloud API key takes precedence over the OAuth token. If both are available, the API key will be used. + +### Create a TiDB Cloud Serverless cluster + +To create a TiDB Cloud Serverless cluster, enter the following command, and then follow the CLI prompts to provide the required information: + +```shell +ticloud serverless create +``` + ## Use the TiDB Cloud CLI View all commands available: @@ -114,7 +156,7 @@ tiup cloud --help Run commands with `tiup cloud `. For example: ```shell -tiup cloud cluster create +tiup cloud serverless create ``` Update to the latest version by TiUP: @@ -123,40 +165,6 @@ Update to the latest version by TiUP: tiup update cloud ``` -## Quick start - -[TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) is the best way to get started with TiDB Cloud. In this section, you will learn how to create a TiDB Serverless cluster with TiDB Cloud CLI. - -### Create a user profile - -Before creating a cluster, you need to create a user profile with your TiDB Cloud API Key: - -```shell -ticloud config create -``` - -> **Warning:** -> -> The profile name **MUST NOT** contain `.`. - -### Create a TiDB Serverless cluster - -To create a TiDB Serverless cluster, enter the following command, and then follow the CLI prompts to provide the required information and set the password: - -```shell -ticloud cluster create -``` - -### Connect to the cluster - -After the cluster is created, you can connect to the cluster: - -```shell -ticloud connect -``` - -When you are prompted about whether to use the default user, choose `Y` and enter the password that you set when creating the cluster. - ## What's next Check out [CLI reference](/tidb-cloud/cli-reference.md) to explore more features of TiDB Cloud CLI. diff --git a/tidb-cloud/high-availability-with-multi-az.md b/tidb-cloud/high-availability-with-multi-az.md index eb4b0ecb3987b..3b81cf35043d4 100644 --- a/tidb-cloud/high-availability-with-multi-az.md +++ b/tidb-cloud/high-availability-with-multi-az.md @@ -1,22 +1,22 @@ --- -title: High Availability with Multi-AZ Deployments -summary: TiDB Cloud supports high availability with Multi-AZ deployments. +title: High Availability in TiDB Cloud Dedicated +summary: TiDB Cloud Dedicated supports high availability with Multi-AZ deployments. --- -# High Availability with Multi-AZ Deployments +# High Availability in TiDB Cloud Dedicated TiDB uses the Raft consensus algorithm to ensure that data is highly available and safely replicated throughout storage in Raft Groups. Data is redundantly copied between storage nodes and placed in different availability zones to protect against machine or data center failures. With automatic failover, TiDB ensures that your service is always on. -TiDB Cloud clusters consist of three major components: TiDB node, TiKV node, and TiFlash node. The highly availability implementation of each component for TiDB Dedicated is as follows: +TiDB Cloud Dedicated clusters consist of three major components: TiDB node, TiKV node, and TiFlash node. The high availability implementation of each component for TiDB Cloud Dedicated is as follows: * **TiDB node** - TiDB is for computing only and does not store data. It is horizontally scalable. TiDB Cloud deploys TiDB nodes evenly to different availability zones in a region. When a user executes a SQL request, the request first passes through a load balancer deployed across availability zones, and then the load balancer distributes the request to different TiDB nodes for execution. It is recommended that each TiDB Cloud cluster has at least two TiDB nodes for high availability. + TiDB is for computing only and does not store data. It is horizontally scalable. TiDB Cloud Dedicated deploys TiDB nodes evenly to different availability zones in a region. When a user executes a SQL request, the request first passes through a load balancer deployed across availability zones, and then the load balancer distributes the request to different TiDB nodes for execution. It is recommended that each TiDB Cloud Dedicated cluster has at least two TiDB nodes for high availability. * **TiKV node** - [TiKV](https://docs.pingcap.com/tidb/stable/tikv-overview) is the row-based storage layer of TiDB Cloud cluster with horizontal scalability. On TiDB Cloud, the minimum number of TiKV nodes for a cluster is 3. TiDB Cloud deploys TiKV nodes evenly to all availability zones (at least 3) in the region you select to achieve durability and high availability. In a typical 3-replica setup, your data is distributed evenly among the TiKV nodes across all availability zones and is persisted to the disk of each TiKV node. + [TiKV](https://docs.pingcap.com/tidb/stable/tikv-overview) is the row-based storage layer of TiDB Cloud Dedicated cluster with horizontal scalability. The minimum number of TiKV nodes for a TiDB Cloud Dedicated cluster is 3. TiDB Cloud Dedicated deploys TiKV nodes evenly to all availability zones (at least 3) in the region you select to achieve durability and high availability. In a typical 3-replica setup, your data is distributed evenly among the TiKV nodes across all availability zones and is persisted to the disk of each TiKV node. * **TiFlash node** - [TiFlash](https://docs.pingcap.com/tidb/stable/tiflash-overview), as a columnar storage extension of TiKV, is the key component that makes TiDB essentially a Hybrid Transactional/Analytical Processing (HTAP) database. In TiFlash, the columnar replicas are asynchronously replicated according to the Raft Learner consensus algorithm. TiDB Cloud deploys TiFlash nodes evenly to different availability zones in a region. It is recommended that you configure at least two TiFlash nodes in each TiDB Cloud cluster and create at least two replicas of the data for high availability in your production environment. + [TiFlash](https://docs.pingcap.com/tidb/stable/tiflash-overview), as a columnar storage extension of TiKV, is the key component that makes TiDB essentially a Hybrid Transactional/Analytical Processing (HTAP) database. In TiFlash, the columnar replicas are asynchronously replicated according to the Raft Learner consensus algorithm. TiDB Cloud Dedicated deploys TiFlash nodes evenly to different availability zones in a region. It is recommended that you configure at least two TiFlash nodes in each TiDB Cloud Dedicated cluster and create at least two replicas of the data for high availability in your production environment. diff --git a/tidb-cloud/import-csv-files-serverless.md b/tidb-cloud/import-csv-files-serverless.md new file mode 100644 index 0000000000000..7bce6389ed826 --- /dev/null +++ b/tidb-cloud/import-csv-files-serverless.md @@ -0,0 +1,313 @@ +--- +title: Import CSV Files from Amazon S3, GCS, Azure Blob Storage, or Alibaba Cloud OSS into TiDB Cloud Serverless +summary: Learn how to import CSV files from Amazon S3, GCS, Azure Blob Storage, or Alibaba Cloud Object Storage Service (OSS) into TiDB Cloud Serverless. +--- + +# Import CSV Files from Amazon S3, GCS, Azure Blob Storage, or Alibaba Cloud OSS into TiDB Cloud Serverless + +This document describes how to import CSV files from Amazon Simple Storage Service (Amazon S3), Google Cloud Storage (GCS), Azure Blob Storage, or Alibaba Cloud Object Storage Service (OSS) into TiDB Cloud Serverless. + +## Limitations + +- To ensure data consistency, TiDB Cloud Serverless allows to import CSV files into empty tables only. To import data into an existing table that already contains data, you can use TiDB Cloud Serverless to import the data into a temporary empty table by following this document, and then use the `INSERT SELECT` statement to copy the data to the target existing table. + +## Step 1. Prepare the CSV files + +1. If a CSV file is larger than 256 MB, consider splitting it into smaller files, each with a size around 256 MB. + + TiDB Cloud Serverless supports importing very large CSV files but performs best with multiple input files around 256 MB in size. This is because TiDB Cloud Serverless can process multiple files in parallel, which can greatly improve the import speed. + +2. Name the CSV files as follows: + + - If a CSV file contains all data of an entire table, name the file in the `${db_name}.${table_name}.csv` format, which maps to the `${db_name}.${table_name}` table when you import the data. + - If the data of one table is separated into multiple CSV files, append a numeric suffix to these CSV files. For example, `${db_name}.${table_name}.000001.csv` and `${db_name}.${table_name}.000002.csv`. The numeric suffixes can be inconsecutive but must be in ascending order. You also need to add extra zeros before the number to ensure all the suffixes are in the same length. + - TiDB Cloud Serverless supports importing compressed files in the following formats: `.gzip`, `.gz`, `.zstd`, `.zst` and `.snappy`. If you want to import compressed CSV files, name the files in the `${db_name}.${table_name}.${suffix}.csv.${compress}` format, in which `${suffix}` is optional and can be any integer such as '000001'. For example, if you want to import the `trips.000001.csv.gz` file to the `bikeshare.trips` table, you need to rename the file as `bikeshare.trips.000001.csv.gz`. + + > **Note:** + > + > - To achieve better performance, it is recommended to limit the size of each compressed file to 100 MiB. + > - The Snappy compressed file must be in the [official Snappy format](https://github.com/google/snappy). Other variants of Snappy compression are not supported. + > - For uncompressed files, if you cannot update the CSV filenames according to the preceding rules in some cases (for example, the CSV file links are also used by your other programs), you can keep the filenames unchanged and use the **Mapping Settings** in [Step 4](#step-4-import-csv-files-to-tidb-cloud-serverless) to import your source data to a single target table. + +## Step 2. Create the target table schemas + +Because CSV files do not contain schema information, before importing data from CSV files into TiDB Cloud Serverless, you need to create the table schemas using either of the following methods: + +- Method 1: In TiDB Cloud Serverless, create the target databases and tables for your source data. + +- Method 2: In the Amazon S3, GCS, Azure Blob Storage, or Alibaba Cloud Object Storage Service directory where the CSV files are located, create the target table schema files for your source data as follows: + + 1. Create database schema files for your source data. + + If your CSV files follow the naming rules in [Step 1](#step-1-prepare-the-csv-files), the database schema files are optional for the data import. Otherwise, the database schema files are mandatory. + + Each database schema file must be in the `${db_name}-schema-create.sql` format and contain a `CREATE DATABASE` DDL statement. With this file, TiDB Cloud Serverless will create the `${db_name}` database to store your data when you import the data. + + For example, if you create a `mydb-scehma-create.sql` file that contains the following statement, TiDB Cloud Serverless will create the `mydb` database when you import the data. + + ```sql + CREATE DATABASE mydb; + ``` + + 2. Create table schema files for your source data. + + If you do not include the table schema files in the Amazon S3, GCS, Azure Blob Storage, or Alibaba Cloud Object Storage Service directory where the CSV files are located, TiDB Cloud Serverless will not create the corresponding tables for you when you import the data. + + Each table schema file must be in the `${db_name}.${table_name}-schema.sql` format and contain a `CREATE TABLE` DDL statement. With this file, TiDB Cloud Serverless will create the `${db_table}` table in the `${db_name}` database when you import the data. + + For example, if you create a `mydb.mytable-schema.sql` file that contains the following statement, TiDB Cloud Serverless will create the `mytable` table in the `mydb` database when you import the data. + + ```sql + CREATE TABLE mytable ( + ID INT, + REGION VARCHAR(20), + COUNT INT ); + ``` + + > **Note:** + > + > Each `${db_name}.${table_name}-schema.sql` file should only contain a single DDL statement. If the file contains multiple DDL statements, only the first one takes effect. + +## Step 3. Configure cross-account access + +To allow TiDB Cloud Serverless to access the CSV files in the Amazon S3, GCS, Azure Blob Storage, or Alibaba Cloud Object Storage Service bucket, do one of the following: + +- If your CSV files are located in Amazon S3, [configure external storage access for TiDB Cloud Serverless](/tidb-cloud/serverless-external-storage.md#configure-amazon-s3-access). + + You can use either an AWS access key or a Role ARN to access your bucket. Once finished, make a note of the access key (including the access key ID and secret access key) or the Role ARN value as you will need it in [Step 4](#step-4-import-csv-files-to-tidb-cloud-serverless). + +- If your CSV files are located in GCS, [configure external storage access for TiDB Cloud Serverless](/tidb-cloud/serverless-external-storage.md#configure-gcs-access). + +- If your CSV files are located in Azure Blob Storage, [configure external storage access for TiDB Cloud Serverless](/tidb-cloud/serverless-external-storage.md#configure-azure-blob-storage-access). + +- If your CSV files are located in Alibaba Cloud Object Storage Service (OSS), [configure external storage access for TiDB Cloud Serverless](/tidb-cloud/serverless-external-storage.md#configure-alibaba-cloud-object-storage-service-oss-access). + +## Step 4. Import CSV files to TiDB Cloud Serverless + +To import the CSV files to TiDB Cloud Serverless, take the following steps: + + +
+ +1. Open the **Import** page for your target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +2. Select **Import data from Cloud Storage**, and then click **Amazon S3**. + +3. On the **Import Data from Amazon S3** page, provide the following information for the source CSV files: + + - **Import File Count**: select **One file** or **Multiple files** as needed. + - **Included Schema Files**: this field is only visible when importing multiple files. If the source folder contains the target table schemas, select **Yes**. Otherwise, select **No**. + - **Data Format**: select **CSV**. + - **File URI** or **Folder URI**: + - When importing one file, enter the source file URI and name in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `s3://sampledata/ingest/TableName.01.csv`. + - When importing multiple files, enter the source file URI and name in the following format `s3://[bucket_name]/[data_source_folder]/`. For example, `s3://sampledata/ingest/`. + - **Bucket Access**: you can use either an AWS Role ARN or an AWS access key to access your bucket. For more information, see [Configure Amazon S3 access](/tidb-cloud/serverless-external-storage.md#configure-amazon-s3-access). + - **AWS Role ARN**: enter the AWS Role ARN value. + - **AWS Access Key**: enter the AWS access key ID and AWS secret access key. + +4. Click **Connect**. + +5. In the **Destination** section, select the target database and table. + + When importing multiple files, you can use **Advanced Settings** > **Mapping Settings** to define a custom mapping rule for each target table and its corresponding CSV file. After that, the data source files will be re-scanned using the provided custom mapping rule. + + When you enter the source file URI and name in **Source File URIs and Names**, make sure it is in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `s3://sampledata/ingest/TableName.01.csv`. + + You can also use wildcards to match the source files. For example: + + - `s3://[bucket_name]/[data_source_folder]/my-data?.csv`: all CSV files starting with `my-data` followed by one character (such as `my-data1.csv` and `my-data2.csv`) in that folder will be imported into the same target table. + + - `s3://[bucket_name]/[data_source_folder]/my-data*.csv`: all CSV files in the folder starting with `my-data` will be imported into the same target table. + + Note that only `?` and `*` are supported. + + > **Note:** + > + > The URI must contain the data source folder. + +6. Click **Start Import**. + +7. When the import progress shows **Completed**, check the imported tables. + +
+ +
+ +1. Open the **Import** page for your target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +2. Select **Import data from Cloud Storage**, and then click **Google Cloud Storage**. + +3. On the **Import Data from Google Cloud Storage** page, provide the following information for the source CSV files: + + - **Import File Count**: select **One file** or **Multiple files** as needed. + - **Included Schema Files**: this field is only visible when importing multiple files. If the source folder contains the target table schemas, select **Yes**. Otherwise, select **No**. + - **Data Format**: select **CSV**. + - **File URI** or **Folder URI**: + - When importing one file, enter the source file URI and name in the following format `[gcs|gs]://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `[gcs|gs]://sampledata/ingest/TableName.01.csv`. + - When importing multiple files, enter the source file URI and name in the following format `[gcs|gs]://[bucket_name]/[data_source_folder]/`. For example, `[gcs|gs]://sampledata/ingest/`. + - **Bucket Access**: you can use a service account key to access your bucket. For more information, see [Configure GCS access](/tidb-cloud/serverless-external-storage.md#configure-gcs-access). + +4. Click **Connect**. + +5. In the **Destination** section, select the target database and table. + + When importing multiple files, you can use **Advanced Settings** > **Mapping Settings** to define a custom mapping rule for each target table and its corresponding CSV file. After that, the data source files will be re-scanned using the provided custom mapping rule. + + When you enter the source file URI and name in **Source File URIs and Names**, make sure it is in the following format `[gcs|gs]://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `[gcs|gs]://sampledata/ingest/TableName.01.csv`. + + You can also use wildcards to match the source files. For example: + + - `[gcs|gs]://[bucket_name]/[data_source_folder]/my-data?.csv`: all CSV files starting with `my-data` followed by one character (such as `my-data1.csv` and `my-data2.csv`) in that folder will be imported into the same target table. + + - `[gcs|gs]://[bucket_name]/[data_source_folder]/my-data*.csv`: all CSV files in the folder starting with `my-data` will be imported into the same target table. + + Note that only `?` and `*` are supported. + + > **Note:** + > + > The URI must contain the data source folder. + +6. Click **Start Import**. + +7. When the import progress shows **Completed**, check the imported tables. + +
+ +
+ +1. Open the **Import** page for your target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +2. Select **Import data from Cloud Storage**, and then click **Azure Blob Storage**. + +3. On the **Import Data from Azure Blob Storage** page, provide the following information for the source CSV files: + + - **Import File Count**: select **One file** or **Multiple files** as needed. + - **Included Schema Files**: this field is only visible when importing multiple files. If the source folder contains the target table schemas, select **Yes**. Otherwise, select **No**. + - **Data Format**: select **CSV**. + - **File URI** or **Folder URI**: + - When importing one file, enter the source file URI and name in the following format `[azure|https]://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `[azure|https]://sampledata/ingest/TableName.01.csv`. + - When importing multiple files, enter the source file URI and name in the following format `[azure|https]://[bucket_name]/[data_source_folder]/`. For example, `[azure|https]://sampledata/ingest/`. + - **Bucket Access**: you can use a shared access signature (SAS) token to access your bucket. For more information, see [Configure Azure Blob Storage access](/tidb-cloud/serverless-external-storage.md#configure-azure-blob-storage-access). + +4. Click **Connect**. + +5. In the **Destination** section, select the target database and table. + + When importing multiple files, you can use **Advanced Settings** > **Mapping Settings** to define a custom mapping rule for each target table and its corresponding CSV file. After that, the data source files will be re-scanned using the provided custom mapping rule. + + When you enter the source file URI and name in **Source File URIs and Names**, make sure it is in the following format `[azure|https]://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `[azure|https]://sampledata/ingest/TableName.01.csv`. + + You can also use wildcards to match the source files. For example: + + - `[azure|https]://[bucket_name]/[data_source_folder]/my-data?.csv`: all CSV files starting with `my-data` followed by one character (such as `my-data1.csv` and `my-data2.csv`) in that folder will be imported into the same target table. + + - `[azure|https]://[bucket_name]/[data_source_folder]/my-data*.csv`: all CSV files in the folder starting with `my-data` will be imported into the same target table. + + Note that only `?` and `*` are supported. + + > **Note:** + > + > The URI must contain the data source folder. + +6. Click **Start Import**. + +7. When the import progress shows **Completed**, check the imported tables. + +
+ +
+ +1. Open the **Import** page for your target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +2. Select **Import data from Cloud Storage**, and then click **Alibaba Cloud OSS**. + +3. On the **Import Data from Alibaba Cloud OSS** page, provide the following information for the source CSV files: + + - **Import File Count**: select **One file** or **Multiple files** as needed. + - **Included Schema Files**: this field is only visible when importing multiple files. If the source folder contains the target table schemas, select **Yes**. Otherwise, select **No**. + - **Data Format**: select **CSV**. + - **File URI** or **Folder URI**: + - When importing one file, enter the source file URI and name in the following format `oss://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `oss://sampledata/ingest/TableName.01.csv`. + - When importing multiple files, enter the source file URI and name in the following format `oss://[bucket_name]/[data_source_folder]/`. For example, `oss://sampledata/ingest/`. + - **Bucket Access**: you can use an AccessKey pair to access your bucket. For more information, see [Configure Alibaba Cloud Object Storage Service (OSS) access](/tidb-cloud/serverless-external-storage.md#configure-alibaba-cloud-object-storage-service-oss-access). + +4. Click **Connect**. + +5. In the **Destination** section, select the target database and table. + + When importing multiple files, you can use **Advanced Settings** > **Mapping Settings** to define a custom mapping rule for each target table and its corresponding CSV file. After that, the data source files will be re-scanned using the provided custom mapping rule. + + When you enter the source file URI and name in **Source File URIs and Names**, make sure it is in the following format `oss://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `oss://sampledata/ingest/TableName.01.csv`. + + You can also use wildcards to match the source files. For example: + + - `oss://[bucket_name]/[data_source_folder]/my-data?.csv`: all CSV files starting with `my-data` followed by one character (such as `my-data1.csv` and `my-data2.csv`) in that folder will be imported into the same target table. + + - `oss://[bucket_name]/[data_source_folder]/my-data*.csv`: all CSV files in the folder starting with `my-data` will be imported into the same target table. + + Note that only `?` and `*` are supported. + + > **Note:** + > + > The URI must contain the data source folder. + +6. Click **Start Import**. + +7. When the import progress shows **Completed**, check the imported tables. + +
+ + + +When you run an import task, if any unsupported or invalid conversions are detected, TiDB Cloud Serverless terminates the import job automatically and reports an importing error. + +If you get an importing error, do the following: + +1. Drop the partially imported table. +2. Check the table schema file. If there are any errors, correct the table schema file. +3. Check the data types in the CSV files. +4. Try the import task again. + +## Troubleshooting + +### Resolve warnings during data import + +After clicking **Start Import**, if you see a warning message such as `can't find the corresponding source files`, resolve this by providing the correct source file, renaming the existing one according to [Naming Conventions for Data Import](/tidb-cloud/naming-conventions-for-data-import.md), or using **Advanced Settings** to make changes. + +After resolving these issues, you need to import the data again. + +### Zero rows in the imported tables + +After the import progress shows **Completed**, check the imported tables. If the number of rows is zero, it means no data files matched the Bucket URI that you entered. In this case, resolve this issue by providing the correct source file, renaming the existing one according to [Naming Conventions for Data Import](/tidb-cloud/naming-conventions-for-data-import.md), or using **Advanced Settings** to make changes. After that, import those tables again. diff --git a/tidb-cloud/import-csv-files.md b/tidb-cloud/import-csv-files.md index b1833a993e70e..a0320489169b9 100644 --- a/tidb-cloud/import-csv-files.md +++ b/tidb-cloud/import-csv-files.md @@ -1,18 +1,18 @@ --- -title: Import CSV Files from Amazon S3 or GCS into TiDB Cloud -summary: Learn how to import CSV files from Amazon S3 or GCS into TiDB Cloud. +title: Import CSV Files from Amazon S3 or GCS into TiDB Cloud Dedicated +summary: Learn how to import CSV files from Amazon S3 or GCS into TiDB Cloud Dedicated. aliases: ['/tidbcloud/migrate-from-amazon-s3-or-gcs','/tidbcloud/migrate-from-aurora-bulk-import'] --- -# Import CSV Files from Amazon S3 or GCS into TiDB Cloud +# Import CSV Files from Amazon S3 or GCS into TiDB Cloud Dedicated -This document describes how to import CSV files from Amazon Simple Storage Service (Amazon S3) or Google Cloud Storage (GCS) into TiDB Cloud. +This document describes how to import CSV files from Amazon Simple Storage Service (Amazon S3) or Google Cloud Storage (GCS) into TiDB Cloud Dedicated. ## Limitations - To ensure data consistency, TiDB Cloud allows to import CSV files into empty tables only. To import data into an existing table that already contains data, you can use TiDB Cloud to import the data into a temporary empty table by following this document, and then use the `INSERT SELECT` statement to copy the data to the target existing table. -- If a TiDB Dedicated cluster has a [changefeed](/tidb-cloud/changefeed-overview.md) or has [Point-in-time Restore](/tidb-cloud/backup-and-restore.md#turn-on-point-in-time-restore) enabled, you cannot import data to the cluster (the **Import Data** button will be disabled), because the current import data feature uses the [physical import mode](https://docs.pingcap.com/tidb/stable/tidb-lightning-physical-import-mode). In this mode, the imported data does not generate change logs, so the changefeed and Point-in-time Restore cannot detect the imported data. +- If a TiDB Cloud Dedicated cluster has a [changefeed](/tidb-cloud/changefeed-overview.md) or has [Point-in-time Restore](/tidb-cloud/backup-and-restore.md#turn-on-point-in-time-restore) enabled, you cannot import data to the cluster (the **Import Data** button will be disabled), because the current import data feature uses the [physical import mode](https://docs.pingcap.com/tidb/stable/tidb-lightning-physical-import-mode). In this mode, the imported data does not generate change logs, so the changefeed and Point-in-time Restore cannot detect the imported data. ## Step 1. Prepare the CSV files @@ -90,6 +90,9 @@ To allow TiDB Cloud to access the CSV files in the Amazon S3 or GCS bucket, do o To import the CSV files to TiDB Cloud, take the following steps: + +
+ 1. Open the **Import** page for your target cluster. 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. @@ -100,64 +103,102 @@ To import the CSV files to TiDB Cloud, take the following steps: 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. -2. On the **Import** page: +2. Select **Import data from S3**. - - For a TiDB Dedicated cluster, click **Import Data** in the upper-right corner. - - For a TiDB Serverless cluster, click the **import data from S3** link above the upload area. + If this is your first time importing data into this cluster, select **Import From Amazon S3**. -3. Provide the following information for the source CSV files: +3. On the **Import Data from Amazon S3** page, provide the following information for the source CSV files: - Depending on where you cluster is located, you can choose to import data from Amazon S3 or GCS. + - **Import File Count**: select **One file** or **Multiple files** as needed. + - **Included Schema Files**: this field is only visible when importing multiple files. If the source folder contains the target table schemas, select **Yes**. Otherwise, select **No**. + - **Data Format**: select **CSV**. + - **File URI** or **Folder URI**: + - When importing one file, enter the source file URI and name in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `s3://sampledata/ingest/TableName.01.csv`. + - When importing multiple files, enter the source file URI and name in the following format `s3://[bucket_name]/[data_source_folder]/`. For example, `s3://sampledata/ingest/`. + - **Bucket Access**: you can use either an AWS Role ARN or an AWS access key to access your bucket. For more information, see [Configure Amazon S3 access](/tidb-cloud/config-s3-and-gcs-access.md#configure-amazon-s3-access). + - **AWS Role ARN**: enter the AWS Role ARN value. + - **AWS Access Key**: enter the AWS access key ID and AWS secret access key. - -
+4. Click **Connect**. - - **Location**: select **Amazon S3**. - - **Data Format**: select **CSV**. If you need to edit the CSV configurations, click **Edit CSV configuration** to update the CSV-specific configurations. For more information, see [CSV Configurations for Importing Data](/tidb-cloud/csv-config-for-import-data.md). +5. In the **Destination** section, select the target database and table. - > **Note:** - > - > For the configurations of separator and delimiter, you can use both alphanumeric characters and certain special characters. The supported special characters include `\t`, `\b`, `\n`, `\r`, `\f`, and `\u0001`. + When importing multiple files, you can use **Advanced Settings** > **Mapping Settings** to define a custom mapping rule for each target table and its corresponding CSV file. After that, the data source files will be re-scanned using the provided custom mapping rule. - - **Bucket URI**: select the bucket URI where your CSV files are located. Note that you must include `/` at the end of the URI, for example, `s3://sampledate/ingest/`. - - **Bucket Access** (This field is visible only for AWS S3): you can use either an AWS access key or an AWS Role ARN to access your bucket. For more information, see [Configure Amazon S3 access](/tidb-cloud/config-s3-and-gcs-access.md#configure-amazon-s3-access). - - **AWS Access Keys**: enter the AWS access key ID and AWS secret access key. - - **AWS Role ARN**: enter the AWS Role ARN value. + When you enter the source file URI and name in **Source File URIs and Names**, make sure it is in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `s3://sampledata/ingest/TableName.01.csv`. -
-
+ You can also use wildcards to match the source files. For example: - - **Location**: use **Google Cloud**. - - **Data Format**: select **CSV**. If you need to edit the CSV configurations, click **Edit CSV configuration** to update the CSV-specific configurations. For more information, see [CSV Configurations for Importing Data](/tidb-cloud/csv-config-for-import-data.md). + - `s3://[bucket_name]/[data_source_folder]/my-data?.csv`: all CSV files starting with `my-data` followed by one character (such as `my-data1.csv` and `my-data2.csv`) in that folder will be imported into the same target table. - > **Note:** + - `s3://[bucket_name]/[data_source_folder]/my-data*.csv`: all CSV files in the folder starting with `my-data` will be imported into the same target table. + + Note that only `?` and `*` are supported. + + > **Note:** + > + > The URI must contain the data source folder. + +6. Click **Start Import**. + +7. When the import progress shows **Completed**, check the imported tables. + +
+ +
+ +1. Open the **Import** page for your target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** > - > For the configurations of separator and delimiter, you can use both alphanumeric characters and certain special characters. The supported special characters include `\t`, `\b`, `\n`, `\r`, `\f`, and `\u0001`. + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +2. Click **Import Data** in the upper-right corner. + + If this is your first time importing data into this cluster, select **Import From GCS**. - - **Bucket gsutil URI**: select the bucket gsutil URI where your CSV files are located. Note that you must include `/` at the end of the URI, for example, `gs://sampledate/ingest/`. +3. On the **Import Data from GCS** page, provide the following information for the source CSV files: + + - **Import File Count**: select **One file** or **Multiple files** as needed. + - **Included Schema Files**: this field is only visible when importing multiple files. If the source folder contains the target table schemas, select **Yes**. Otherwise, select **No**. + - **Data Format**: select **CSV**. + - **File URI** or **Folder URI**: + - When importing one file, enter the source file URI and name in the following format `gs://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `gs://sampledata/ingest/TableName.01.csv`. + - When importing multiple files, enter the source file URI and name in the following format `gs://[bucket_name]/[data_source_folder]/`. For example, `gs://sampledata/ingest/`. - **Bucket Access**: you can use a GCS IAM Role to access your bucket. For more information, see [Configure GCS access](/tidb-cloud/config-s3-and-gcs-access.md#configure-gcs-access). -
- +4. Click **Connect**. + +5. In the **Destination** section, select the target database and table. + + When importing multiple files, you can use **Advanced Settings** > **Mapping Settings** to define a custom mapping rule for each target table and its corresponding CSV file. After that, the data source files will be re-scanned using the provided custom mapping rule. -4. You can choose to import into pre-created tables, or import schema and data from the source. + When you enter the source file URI and name in **Source File URIs and Names**, make sure it is in the following format `gs://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `gs://sampledata/ingest/TableName.01.csv`. - - **Import into pre-created tables** allows you to create tables in TiDB in advance and select the tables that you want to import data into. In this case, you can choose up to 1000 tables to import. To create tables, click **Chat2Query** in the left navigation pane. For more information about how to use Chat2Query, see [Explore Your Data with AI-Powered Chat2Query](/tidb-cloud/explore-data-with-chat2query.md). - - **Import schema and data from S3** (This field is visible only for AWS S3) allows you to import SQL scripts that create a table along with its corresponding data stored in S3 directly into TiDB. - - **Import schema and data from GCS** (This field is visible only for GCS) allows you to import SQL scripts that create a table along with its corresponding data stored in GCS directly into TiDB. + You can also use wildcards to match the source files. For example: -5. If the source files do not meet the naming conventions, you can define a custom mapping rule for each target table and its corresponding CSV file. After that, the data source files will be re-scanned using the provided custom mapping rule. To modify the mapping, go to **Advanced Settings** and then click **Mapping Settings**. Note that **Mapping Settings** is available only when you choose to import into pre-created tables. + - `gs://[bucket_name]/[data_source_folder]/my-data?.csv`: all CSV files starting with `my-data` followed by one character (such as `my-data1.csv` and `my-data2.csv`) in that folder will be imported into the same target table. - - **Target Database**: enter the name of the target database you select. + - `gs://[bucket_name]/[data_source_folder]/my-data*.csv`: all CSV files in the folder starting with `my-data` will be imported into the same target table. - - **Target Tables**: enter the name of the target table you select. Note that this field only accepts one specific table name, so wildcards are not supported. + Note that only `?` and `*` are supported. - - **Source file URIs and names**: enter the source file URI and name in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `s3://sampledate/ingest/TableName.01.csv`. You can also use wildcards to match the source files. For more information, see [Mapping Settings](#mapping-settings). + > **Note:** + > + > The URI must contain the data source folder. 6. Click **Start Import**. 7. When the import progress shows **Completed**, check the imported tables. +
+ + + When you run an import task, if any unsupported or invalid conversions are detected, TiDB Cloud terminates the import job automatically and reports an importing error. If you get an importing error, do the following: @@ -167,24 +208,6 @@ If you get an importing error, do the following: 3. Check the data types in the CSV files. 4. Try the import task again. -## Mapping Settings - -If the source files do not meet the naming conventions, you can define a custom mapping rule for each target table and its corresponding CSV file. After that, the data source files will be re-scanned using the provided custom mapping rule. To modify the mapping, go to **Advanced Settings** and then click **Mapping Settings**. Note that **Mapping Settings** is available only when you choose **Import into Pre-created Tables**. - -When you enter the source file URI and name in **Source file URIs and names**, make sure it is in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].csv`. For example, `s3://sampledate/ingest/TableName.01.csv`. - -You can also use wildcards to match the source files. For example: - -- `s3://[bucket_name]/[data_source_folder]/my-data?.csv`: all CSV files starting with `my-data` followed by one character (such as `my-data1.csv` and `my-data2.csv`) in that folder will be imported into the same target table. - -- `s3://[bucket_name]/[data_source_folder]/my-data*.csv`: all CSV files in the folder starting with `my-data` will be imported into the same target table. - -Note that only `?` and `*` are supported. - -> **Note:** -> -> The URI must contain the data source folder. - ## Troubleshooting ### Resolve warnings during data import diff --git a/tidb-cloud/import-parquet-files-serverless.md b/tidb-cloud/import-parquet-files-serverless.md new file mode 100644 index 0000000000000..0501df45d6b27 --- /dev/null +++ b/tidb-cloud/import-parquet-files-serverless.md @@ -0,0 +1,345 @@ +--- +title: Import Apache Parquet Files from Amazon S3, GCS, Azure Blob Storage, or Alibaba Cloud OSS into TiDB Cloud Serverless +summary: Learn how to import Apache Parquet files from Amazon S3, GCS, Azure Blob Storage, or Alibaba Cloud Object Storage Service (OSS) into TiDB Cloud Serverless. +--- + +# Import Apache Parquet Files from Amazon S3, GCS, Azure Blob Storage, or Alibaba Cloud OSS into TiDB Cloud Serverless + +You can import both uncompressed and Snappy compressed [Apache Parquet](https://parquet.apache.org/) format data files to TiDB Cloud Serverless. This document describes how to import Parquet files from Amazon Simple Storage Service (Amazon S3), Google Cloud Storage (GCS), Azure Blob Storage, or Alibaba Cloud Object Storage Service (OSS) into TiDB Cloud Serverless. + +> **Note:** +> +> - TiDB Cloud Serverless only supports importing Parquet files into empty tables. To import data into an existing table that already contains data, you can use TiDB Cloud Serverless to import the data into a temporary empty table by following this document, and then use the `INSERT SELECT` statement to copy the data to the target existing table. +> - The Snappy compressed file must be in the [official Snappy format](https://github.com/google/snappy). Other variants of Snappy compression are not supported. + +## Step 1. Prepare the Parquet files + +> **Note:** +> +> Currently, TiDB Cloud Serverless does not support importing Parquet files that contain any of the following data types. If Parquet files to be imported contain such data types, you need to first regenerate the Parquet files using the [supported data types](#supported-data-types) (for example, `STRING`). Alternatively, you could use a service such as AWS Glue to transform data types easily. +> +> - `LIST` +> - `NEST STRUCT` +> - `BOOL` +> - `ARRAY` +> - `MAP` + +1. If a Parquet file is larger than 256 MB, consider splitting it into smaller files, each with a size around 256 MB. + + TiDB Cloud Serverless supports importing very large Parquet files but performs best with multiple input files around 256 MB in size. This is because TiDB Cloud Serverless can process multiple files in parallel, which can greatly improve the import speed. + +2. Name the Parquet files as follows: + + - If a Parquet file contains all data of an entire table, name the file in the `${db_name}.${table_name}.parquet` format, which maps to the `${db_name}.${table_name}` table when you import the data. + - If the data of one table is separated into multiple Parquet files, append a numeric suffix to these Parquet files. For example, `${db_name}.${table_name}.000001.parquet` and `${db_name}.${table_name}.000002.parquet`. The numeric suffixes can be inconsecutive but must be in ascending order. You also need to add extra zeros before the number to ensure all the suffixes are in the same length. + + > **Note:** + > + > If you cannot update the Parquet filenames according to the preceding rules in some cases (for example, the Parquet file links are also used by your other programs), you can keep the filenames unchanged and use the **Mapping Settings** in [Step 4](#step-4-import-parquet-files-to-tidb-cloud-serverless) to import your source data to a single target table. + +## Step 2. Create the target table schemas + +Because Parquet files do not contain schema information, before importing data from Parquet files into TiDB Cloud Serverless, you need to create the table schemas using either of the following methods: + +- Method 1: In TiDB Cloud Serverless, create the target databases and tables for your source data. + +- Method 2: In the Amazon S3, GCS, Azure Blob Storage, or Alibaba Cloud Object Storage Service directory where the Parquet files are located, create the target table schema files for your source data as follows: + + 1. Create database schema files for your source data. + + If your Parquet files follow the naming rules in [Step 1](#step-1-prepare-the-parquet-files), the database schema files are optional for the data import. Otherwise, the database schema files are mandatory. + + Each database schema file must be in the `${db_name}-schema-create.sql` format and contain a `CREATE DATABASE` DDL statement. With this file, TiDB Cloud Serverless will create the `${db_name}` database to store your data when you import the data. + + For example, if you create a `mydb-scehma-create.sql` file that contains the following statement, TiDB Cloud Serverless will create the `mydb` database when you import the data. + + ```sql + CREATE DATABASE mydb; + ``` + + 2. Create table schema files for your source data. + + If you do not include the table schema files in the Amazon S3, GCS, Azure Blob Storage, or Alibaba Cloud Object Storage Service directory where the Parquet files are located, TiDB Cloud Serverless will not create the corresponding tables for you when you import the data. + + Each table schema file must be in the `${db_name}.${table_name}-schema.sql` format and contain a `CREATE TABLE` DDL statement. With this file, TiDB Cloud Serverless will create the `${db_table}` table in the `${db_name}` database when you import the data. + + For example, if you create a `mydb.mytable-schema.sql` file that contains the following statement, TiDB Cloud Serverless will create the `mytable` table in the `mydb` database when you import the data. + + ```sql + CREATE TABLE mytable ( + ID INT, + REGION VARCHAR(20), + COUNT INT ); + ``` + + > **Note:** + > + > Each `${db_name}.${table_name}-schema.sql` file should only contain a single DDL statement. If the file contains multiple DDL statements, only the first one takes effect. + +## Step 3. Configure cross-account access + +To allow TiDB Cloud Serverless to access the Parquet files in the Amazon S3, GCS, Azure Blob Storage, or Alibaba Cloud Object Storage Service bucket, do one of the following: + +- If your Parquet files are located in Amazon S3, [configure external storage access for TiDB Cloud Serverless](/tidb-cloud/serverless-external-storage.md#configure-amazon-s3-access). + + You can use either an AWS access key or a Role ARN to access your bucket. Once finished, make a note of the access key (including the access key ID and secret access key) or the Role ARN value as you will need it in [Step 4](#step-4-import-parquet-files-to-tidb-cloud-serverless). + +- If your Parquet files are located in GCS, [configure external storage access for TiDB Cloud Serverless](/tidb-cloud/serverless-external-storage.md#configure-gcs-access). + +- If your Parquet files are located in Azure Blob Storage, [configure external storage access for TiDB Cloud Serverless](/tidb-cloud/serverless-external-storage.md#configure-azure-blob-storage-access). + +- If your Parquet files are located in Alibaba Cloud Object Storage Service (OSS), [configure external storage access for TiDB Cloud Serverless](/tidb-cloud/serverless-external-storage.md#configure-alibaba-cloud-object-storage-service-oss-access). + +## Step 4. Import Parquet files to TiDB Cloud Serverless + +To import the Parquet files to TiDB Cloud Serverless, take the following steps: + + +
+ +1. Open the **Import** page for your target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +2. Select **Import data from Cloud Storage**, and then click **Amazon S3**. + +3. On the **Import Data from Amazon S3** page, provide the following information for the source Parquet files: + + - **Import File Count**: select **One file** or **Multiple files** as needed. + - **Included Schema Files**: this field is only visible when importing multiple files. If the source folder contains the target table schemas, select **Yes**. Otherwise, select **No**. + - **Data Format**: select **Parquet**. + - **File URI** or **Folder URI**: + - When importing one file, enter the source file URI and name in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `s3://sampledata/ingest/TableName.01.parquet`. + - When importing multiple files, enter the source file URI and name in the following format `s3://[bucket_name]/[data_source_folder]/`. For example, `s3://sampledata/ingest/`. + - **Bucket Access**: you can use either an AWS Role ARN or an AWS access key to access your bucket. For more information, see [Configure Amazon S3 access](/tidb-cloud/serverless-external-storage.md#configure-amazon-s3-access). + - **AWS Role ARN**: enter the AWS Role ARN value. + - **AWS Access Key**: enter the AWS access key ID and AWS secret access key. + +4. Click **Connect**. + +5. In the **Destination** section, select the target database and table. + + When importing multiple files, you can use **Advanced Settings** > **Mapping Settings** to define a custom mapping rule for each target table and its corresponding Parquet file. After that, the data source files will be re-scanned using the provided custom mapping rule. + + When you enter the source file URI and name in **Source File URIs and Names**, make sure it is in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `s3://sampledata/ingest/TableName.01.parquet`. + + You can also use wildcards to match the source files. For example: + + - `s3://[bucket_name]/[data_source_folder]/my-data?.parquet`: all Parquet files starting with `my-data` followed by one character (such as `my-data1.parquet` and `my-data2.parquet`) in that folder will be imported into the same target table. + + - `s3://[bucket_name]/[data_source_folder]/my-data*.parquet`: all Parquet files in the folder starting with `my-data` will be imported into the same target table. + + Note that only `?` and `*` are supported. + + > **Note:** + > + > The URI must contain the data source folder. + +6. Click **Start Import**. + +7. When the import progress shows **Completed**, check the imported tables. + +
+ +
+ +1. Open the **Import** page for your target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +2. Select **Import data from Cloud Storage**, and then click **Google Cloud Storage**. + +3. On the **Import Data from Google Cloud Storage** page, provide the following information for the source Parquet files: + + - **Import File Count**: select **One file** or **Multiple files** as needed. + - **Included Schema Files**: this field is only visible when importing multiple files. If the source folder contains the target table schemas, select **Yes**. Otherwise, select **No**. + - **Data Format**: select **Parquet**. + - **File URI** or **Folder URI**: + - When importing one file, enter the source file URI and name in the following format `[gcs|gs]://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `[gcs|gs]://sampledata/ingest/TableName.01.parquet`. + - When importing multiple files, enter the source file URI and name in the following format `[gcs|gs]://[bucket_name]/[data_source_folder]/`. For example, `[gcs|gs]://sampledata/ingest/`. + - **Bucket Access**: you can use a GCS IAM Role to access your bucket. For more information, see [Configure GCS access](/tidb-cloud/serverless-external-storage.md#configure-gcs-access). + +4. Click **Connect**. + +5. In the **Destination** section, select the target database and table. + + When importing multiple files, you can use **Advanced Settings** > **Mapping Settings** to define a custom mapping rule for each target table and its corresponding Parquet file. After that, the data source files will be re-scanned using the provided custom mapping rule. + + When you enter the source file URI and name in **Source File URIs and Names**, make sure it is in the following format `[gcs|gs]://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `[gcs|gs]://sampledata/ingest/TableName.01.parquet`. + + You can also use wildcards to match the source files. For example: + + - `[gcs|gs]://[bucket_name]/[data_source_folder]/my-data?.parquet`: all Parquet files starting with `my-data` followed by one character (such as `my-data1.parquet` and `my-data2.parquet`) in that folder will be imported into the same target table. + + - `[gcs|gs]://[bucket_name]/[data_source_folder]/my-data*.parquet`: all Parquet files in the folder starting with `my-data` will be imported into the same target table. + + Note that only `?` and `*` are supported. + + > **Note:** + > + > The URI must contain the data source folder. + +6. Click **Start Import**. + +7. When the import progress shows **Completed**, check the imported tables. + +
+ +
+ +1. Open the **Import** page for your target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +2. Select **Import data from Cloud Storage**, and then click **Azure Blob Storage**. + +3. On the **Import Data from Azure Blob Storage** page, provide the following information for the source Parquet files: + + - **Import File Count**: select **One file** or **Multiple files** as needed. + - **Included Schema Files**: this field is only visible when importing multiple files. If the source folder contains the target table schemas, select **Yes**. Otherwise, select **No**. + - **Data Format**: select **Parquet**. + - **File URI** or **Folder URI**: + - When importing one file, enter the source file URI and name in the following format `[azure|https]://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `[azure|https]://sampledata/ingest/TableName.01.parquet`. + - When importing multiple files, enter the source file URI and name in the following format `[azure|https]://[bucket_name]/[data_source_folder]/`. For example, `[azure|https]://sampledata/ingest/`. + - **Bucket Access**: you can use a shared access signature (SAS) token to access your bucket. For more information, see [Configure Azure Blob Storage access](/tidb-cloud/serverless-external-storage.md#configure-azure-blob-storage-access). + +4. Click **Connect**. + +5. In the **Destination** section, select the target database and table. + + When importing multiple files, you can use **Advanced Settings** > **Mapping Settings** to define a custom mapping rule for each target table and its corresponding Parquet file. After that, the data source files will be re-scanned using the provided custom mapping rule. + + When you enter the source file URI and name in **Source File URIs and Names**, make sure it is in the following format `[azure|https]://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `[azure|https]://sampledata/ingest/TableName.01.parquet`. + + You can also use wildcards to match the source files. For example: + + - `[azure|https]://[bucket_name]/[data_source_folder]/my-data?.parquet`: all Parquet files starting with `my-data` followed by one character (such as `my-data1.parquet` and `my-data2.parquet`) in that folder will be imported into the same target table. + + - `[azure|https]://[bucket_name]/[data_source_folder]/my-data*.parquet`: all Parquet files in the folder starting with `my-data` will be imported into the same target table. + + Note that only `?` and `*` are supported. + + > **Note:** + > + > The URI must contain the data source folder. + +6. Click **Start Import**. + +7. When the import progress shows **Completed**, check the imported tables. + +
+ +
+ +1. Open the **Import** page for your target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +2. Select **Import data from Cloud Storage**, and then click **Alibaba Cloud OSS**. + +3. On the **Import Data from Alibaba Cloud OSS** page, provide the following information for the source Parquet files: + + - **Import File Count**: select **One file** or **Multiple files** as needed. + - **Included Schema Files**: this field is only visible when importing multiple files. If the source folder contains the target table schemas, select **Yes**. Otherwise, select **No**. + - **Data Format**: select **Parquet**. + - **File URI** or **Folder URI**: + - When importing one file, enter the source file URI and name in the following format `oss://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `oss://sampledata/ingest/TableName.01.parquet`. + - When importing multiple files, enter the source file URI and name in the following format `oss://[bucket_name]/[data_source_folder]/`. For example, `oss://sampledata/ingest/`. + - **Bucket Access**: you can use an AccessKey pair to access your bucket. For more information, see [Configure Alibaba Cloud Object Storage Service (OSS) access](/tidb-cloud/serverless-external-storage.md#configure-alibaba-cloud-object-storage-service-oss-access). + +4. Click **Connect**. + +5. In the **Destination** section, select the target database and table. + + When importing multiple files, you can use **Advanced Settings** > **Mapping Settings** to define a custom mapping rule for each target table and its corresponding Parquet file. After that, the data source files will be re-scanned using the provided custom mapping rule. + + When you enter the source file URI and name in **Source File URIs and Names**, make sure it is in the following format `oss://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `oss://sampledata/ingest/TableName.01.parquet`. + + You can also use wildcards to match the source files. For example: + + - `oss://[bucket_name]/[data_source_folder]/my-data?.parquet`: all Parquet files starting with `my-data` followed by one character (such as `my-data1.parquet` and `my-data2.parquet`) in that folder will be imported into the same target table. + + - `oss://[bucket_name]/[data_source_folder]/my-data*.parquet`: all Parquet files in the folder starting with `my-data` will be imported into the same target table. + + Note that only `?` and `*` are supported. + + > **Note:** + > + > The URI must contain the data source folder. + +6. Click **Start Import**. + +7. When the import progress shows **Completed**, check the imported tables. + +
+ + + +When you run an import task, if any unsupported or invalid conversions are detected, TiDB Cloud Serverless terminates the import job automatically and reports an importing error. + +If you get an importing error, do the following: + +1. Drop the partially imported table. +2. Check the table schema file. If there are any errors, correct the table schema file. +3. Check the data types in the Parquet files. + + If the Parquet files contain any unsupported data types (for example, `NEST STRUCT`, `ARRAY`, or `MAP`), you need to regenerate the Parquet files using [supported data types](#supported-data-types) (for example, `STRING`). + +4. Try the import task again. + +## Supported data types + +The following table lists the supported Parquet data types that can be imported to TiDB Cloud Serverless. + +| Parquet Primitive Type | Parquet Logical Type | Types in TiDB or MySQL | +|---|---|---| +| DOUBLE | DOUBLE | DOUBLE
FLOAT | +| FIXED_LEN_BYTE_ARRAY(9) | DECIMAL(20,0) | BIGINT UNSIGNED | +| FIXED_LEN_BYTE_ARRAY(N) | DECIMAL(p,s) | DECIMAL
NUMERIC | +| INT32 | DECIMAL(p,s) | DECIMAL
NUMERIC | +| INT32 | N/A | INT
MEDIUMINT
YEAR | +| INT64 | DECIMAL(p,s) | DECIMAL
NUMERIC | +| INT64 | N/A | BIGINT
INT UNSIGNED
MEDIUMINT UNSIGNED | +| INT64 | TIMESTAMP_MICROS | DATETIME
TIMESTAMP | +| BYTE_ARRAY | N/A | BINARY
BIT
BLOB
CHAR
LINESTRING
LONGBLOB
MEDIUMBLOB
MULTILINESTRING
TINYBLOB
VARBINARY | +| BYTE_ARRAY | STRING | ENUM
DATE
DECIMAL
GEOMETRY
GEOMETRYCOLLECTION
JSON
LONGTEXT
MEDIUMTEXT
MULTIPOINT
MULTIPOLYGON
NUMERIC
POINT
POLYGON
SET
TEXT
TIME
TINYTEXT
VARCHAR | +| SMALLINT | N/A | INT32 | +| SMALLINT UNSIGNED | N/A | INT32 | +| TINYINT | N/A | INT32 | +| TINYINT UNSIGNED | N/A | INT32 | + +## Troubleshooting + +### Resolve warnings during data import + +After clicking **Start Import**, if you see a warning message such as `can't find the corresponding source files`, resolve this by providing the correct source file, renaming the existing one according to [Naming Conventions for Data Import](/tidb-cloud/naming-conventions-for-data-import.md), or using **Advanced Settings** to make changes. + +After resolving these issues, you need to import the data again. + +### Zero rows in the imported tables + +After the import progress shows **Completed**, check the imported tables. If the number of rows is zero, it means no data files matched the Bucket URI that you entered. In this case, resolve this issue by providing the correct source file, renaming the existing one according to [Naming Conventions for Data Import](/tidb-cloud/naming-conventions-for-data-import.md), or using **Advanced Settings** to make changes. After that, import those tables again. diff --git a/tidb-cloud/import-parquet-files.md b/tidb-cloud/import-parquet-files.md index bf96136f3dda6..4504ceefe68ae 100644 --- a/tidb-cloud/import-parquet-files.md +++ b/tidb-cloud/import-parquet-files.md @@ -1,17 +1,16 @@ --- -title: Import Apache Parquet Files from Amazon S3 or GCS into TiDB Cloud -summary: Learn how to import Apache Parquet files from Amazon S3 or GCS into TiDB Cloud. +title: Import Apache Parquet Files from Amazon S3 or GCS into TiDB Cloud Dedicated +summary: Learn how to import Apache Parquet files from Amazon S3 or GCS into TiDB Cloud Dedicated. --- -# Import Apache Parquet Files from Amazon S3 or GCS into TiDB Cloud +# Import Apache Parquet Files from Amazon S3 or GCS into TiDB Cloud Dedicated -You can import both uncompressed and Snappy compressed [Apache Parquet](https://parquet.apache.org/) format data files to TiDB Cloud. This document describes how to import Parquet files from Amazon Simple Storage Service (Amazon S3) or Google Cloud Storage (GCS) into TiDB Cloud. +You can import both uncompressed and Snappy compressed [Apache Parquet](https://parquet.apache.org/) format data files to TiDB Cloud Dedicated. This document describes how to import Parquet files from Amazon Simple Storage Service (Amazon S3) or Google Cloud Storage (GCS) into TiDB Cloud Dedicated. > **Note:** > > - TiDB Cloud only supports importing Parquet files into empty tables. To import data into an existing table that already contains data, you can use TiDB Cloud to import the data into a temporary empty table by following this document, and then use the `INSERT SELECT` statement to copy the data to the target existing table. -> - If there is a changefeed in a TiDB Dedicated cluster, you cannot import data to the cluster (the **Import Data** button will be disabled), because the current import data feature uses the [physical import mode](https://docs.pingcap.com/tidb/stable/tidb-lightning-physical-import-mode). In this mode, the imported data does not generate change logs, so the changefeed cannot detect the imported data. -> - Only TiDB Dedicated clusters support importing Parquet files from GCS. +> - If there is a changefeed in a TiDB Cloud Dedicated cluster, you cannot import data to the cluster (the **Import Data** button will be disabled), because the current import data feature uses the [physical import mode](https://docs.pingcap.com/tidb/stable/tidb-lightning-physical-import-mode). In this mode, the imported data does not generate change logs, so the changefeed cannot detect the imported data. > - The Snappy compressed file must be in the [official Snappy format](https://github.com/google/snappy). Other variants of Snappy compression are not supported. ## Step 1. Prepare the Parquet files @@ -96,6 +95,9 @@ To allow TiDB Cloud to access the Parquet files in the Amazon S3 or GCS bucket, To import the Parquet files to TiDB Cloud, take the following steps: + +
+ 1. Open the **Import** page for your target cluster. 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. @@ -106,62 +108,102 @@ To import the Parquet files to TiDB Cloud, take the following steps: 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. -2. On the **Import** page: - - For a TiDB Dedicated cluster, click **Import Data** in the upper-right corner. - - For a TiDB Serverless cluster, click the **import data from S3** link above the upload area. +2. Select **Import data from S3**. -3. Provide the following information for the source Parquet files: + If this is your first time importing data into this cluster, select **Import From Amazon S3**. - Depending on where you cluster is located, you can choose to import data from Amazon S3 or GCS. +3. On the **Import Data from Amazon S3** page, provide the following information for the source Parquet files: - -
+ - **Import File Count**: select **One file** or **Multiple files** as needed. + - **Included Schema Files**: this field is only visible when importing multiple files. If the source folder contains the target table schemas, select **Yes**. Otherwise, select **No**. + - **Data Format**: select **Parquet**. + - **File URI** or **Folder URI**: + - When importing one file, enter the source file URI and name in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `s3://sampledata/ingest/TableName.01.parquet`. + - When importing multiple files, enter the source file URI and name in the following format `s3://[bucket_name]/[data_source_folder]/`. For example, `s3://sampledata/ingest/`. + - **Bucket Access**: you can use either an AWS Role ARN or an AWS access key to access your bucket. For more information, see [Configure Amazon S3 access](/tidb-cloud/config-s3-and-gcs-access.md#configure-amazon-s3-access). + - **AWS Role ARN**: enter the AWS Role ARN value. + - **AWS Access Key**: enter the AWS access key ID and AWS secret access key. - - **Location**: select **Amazon S3**. - - **Data format**: select **Parquet**. - - **Bucket URI**: select the bucket URI where your Parquet files are located. Note that you must include `/` at the end of the URI, for example, `s3://sampledate/ingest/`. - - **Bucket Access** (This field is visible only for AWS S3): you can use either an AWS access key or a Role ARN to access your bucket. For more information, see [Configure Amazon S3 access](/tidb-cloud/config-s3-and-gcs-access.md#configure-amazon-s3-access). - - **AWS Access Keys**: enter the AWS access key ID and AWS secret access key. - - **AWS Role ARN**: enter the Role ARN value. +4. Click **Connect**. -
-
+5. In the **Destination** section, select the target database and table. - - **Location**: use **Google Cloud**.. - - **Data format**: select **Parquet**. - - **Bucket gsutil URI**: select the bucket URI where your Parquet files are located. Note that you must include `/` at the end of the URI, for example, `gs://sampledate/ingest/`. - - **Bucket Access**: you can use a GCS IAM Role to access your bucket. For more information, see [Configure GCS access](/tidb-cloud/config-s3-and-gcs-access.md#configure-gcs-access). + When importing multiple files, you can use **Advanced Settings** > **Mapping Settings** to define a custom mapping rule for each target table and its corresponding Parquet file. After that, the data source files will be re-scanned using the provided custom mapping rule. -
- + When you enter the source file URI and name in **Source File URIs and Names**, make sure it is in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `s3://sampledata/ingest/TableName.01.parquet`. -4. You can choose to import into pre-created tables, or import schema and data from the source. + You can also use wildcards to match the source files. For example: - - **Import into pre-created tables** allows you to create tables in TiDB in advance and select the tables that you want to import data into. In this case, you can choose up to 1000 tables to import. You can click **Chat2Query** in the left navigation pane to create tables. For more information about how to use Chat2Query, see [Explore Your Data with AI-Powered Chat2Query](/tidb-cloud/explore-data-with-chat2query.md). - - **Import schema and data from S3** (This field is visible only for AWS S3) allows you to import SQL scripts for creating a table and import corresponding table data stored in S3 into TiDB. - - **Import schema and data from GCS** (This field is visible only for GCS) allows you to import SQL scripts that create a table along with its corresponding data stored in GCS directly into TiDB. + - `s3://[bucket_name]/[data_source_folder]/my-data?.parquet`: all Parquet files starting with `my-data` followed by one character (such as `my-data1.parquet` and `my-data2.parquet`) in that folder will be imported into the same target table. -5. If the source files do not meet the naming conventions, you can specify a custom mapping rule between a single target table and the CSV file. After that, the data source files will be re-scanned using the provided custom mapping rule. To modify the mapping, click **Advanced Settings** and then click **Mapping Settings**. Note that **Mapping Settings** is available only when you choose to import into pre-created tables**. + - `s3://[bucket_name]/[data_source_folder]/my-data*.parquet`: all Parquet files in the folder starting with `my-data` will be imported into the same target table. - - **Target Database**: enter the name of the target database you select. + Note that only `?` and `*` are supported. - - **Target Tables**: enter the name of the target table you select. Note that this field only accepts one specific table name, so wildcards are not supported. + > **Note:** + > + > The URI must contain the data source folder. - - **Source file URIs and names**: enter the source file URI and name in the following format `s3://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `s3://sampledate/ingest/TableName.01.parquet`. You can also use wildcards to match the source files. For example: +6. Click **Start Import**. - - `s3://[bucket_name]/[data_source_folder]/my-data?.parquet`: all Parquet files starting with `my-data` and one character (such as `my-data1.parquet` and `my-data2.parquet`) in that folder will be imported into the same target table. - - `s3://[bucket_name]/[data_source_folder]/my-data*.parquet`: all Parquet files in the folder starting with `my-data` will be imported into the same target table. +7. When the import progress shows **Completed**, check the imported tables. - Note that only `?` and `*` are supported. +
- > **Note:** +
+ +1. Open the **Import** page for your target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** > - > The URI must contain the data source folder. + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +2. Click **Import Data** in the upper-right corner. + + If this is your first time importing data into this cluster, select **Import From GCS**. + +3. On the **Import Data from GCS** page, provide the following information for the source Parquet files: + + - **Import File Count**: select **One file** or **Multiple files** as needed. + - **Included Schema Files**: this field is only visible when importing multiple files. If the source folder contains the target table schemas, select **Yes**. Otherwise, select **No**. + - **Data Format**: select **Parquet**. + - **File URI** or **Folder URI**: + - When importing one file, enter the source file URI and name in the following format `gs://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `gs://sampledata/ingest/TableName.01.parquet`. + - When importing multiple files, enter the source file URI and name in the following format `gs://[bucket_name]/[data_source_folder]/`. For example, `gs://sampledata/ingest/`. + - **Bucket Access**: you can use a GCS IAM Role to access your bucket. For more information, see [Configure GCS access](/tidb-cloud/config-s3-and-gcs-access.md#configure-gcs-access). + +4. Click **Connect**. + +5. In the **Destination** section, select the target database and table. + + When importing multiple files, you can use **Advanced Settings** > **Mapping Settings** to define a custom mapping rule for each target table and its corresponding Parquet file. After that, the data source files will be re-scanned using the provided custom mapping rule. + + When you enter the source file URI and name in **Source File URIs and Names**, make sure it is in the following format `gs://[bucket_name]/[data_source_folder]/[file_name].parquet`. For example, `gs://sampledata/ingest/TableName.01.parquet`. + + You can also use wildcards to match the source files. For example: + + - `gs://[bucket_name]/[data_source_folder]/my-data?.parquet`: all Parquet files starting with `my-data` followed by one character (such as `my-data1.parquet` and `my-data2.parquet`) in that folder will be imported into the same target table. + + - `gs://[bucket_name]/[data_source_folder]/my-data*.parquet`: all Parquet files in the folder starting with `my-data` will be imported into the same target table. + + Note that only `?` and `*` are supported. + + > **Note:** + > + > The URI must contain the data source folder. 6. Click **Start Import**. 7. When the import progress shows **Completed**, check the imported tables. +
+ + + When you run an import task, if any unsupported or invalid conversions are detected, TiDB Cloud terminates the import job automatically and reports an importing error. If you get an importing error, do the following: diff --git a/tidb-cloud/import-sample-data-serverless.md b/tidb-cloud/import-sample-data-serverless.md new file mode 100644 index 0000000000000..d824fb7816406 --- /dev/null +++ b/tidb-cloud/import-sample-data-serverless.md @@ -0,0 +1,96 @@ +--- +title: Import Sample Data into TiDB Cloud Serverless +summary: Learn how to import sample data into TiDB Cloud Serverless via the UI. +--- + +# Import Sample Data into TiDB Cloud Serverless + +This document describes how to import the sample data into TiDB Cloud Serverless via the UI. The sample data used is the system data from Capital Bikeshare, released under the Capital Bikeshare Data License Agreement. Before importing the sample data, you need to have one TiDB cluster. + +> **Note:** +> +> TiDB Cloud Serverless currently only supports importing sample data from Amazon S3. + +1. Open the **Import** page for your target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +2. Select **Import data from S3**. + + If this is your first time importing data into this cluster, select **Import From Amazon S3**. + +3. On the **Import Data from Amazon S3** page, configure the following source data information: + + - **Import File Count**: for the sample data, select **Multiple files**. + - **Included Schema Files**: for the sample data, select **Yes**. + - **Data Format**: select **SQL**. + - **Folder URI** or **File URI**: enter the sample data URI `s3://tidbcloud-sample-data/data-ingestion/`. + - **Bucket Access**: for the sample data, you can only use a Role ARN to access its bucket. For your own data, you can use either an AWS access key or a Role ARN to access your bucket. + - **AWS Role ARN**: enter `arn:aws:iam::801626783489:role/import-sample-access`. + - **AWS Access Key**: skip this option for the sample data. + +4. Click **Connect** > **Start Import**. + +When the data import progress shows **Completed**, you have successfully imported the sample data and the database schema to your database in TiDB Cloud Serverless. + +After connecting to the cluster, you can run some queries in your terminal to check the result, for example: + +1. Get the trip records starting at "12th & U St NW": + + ```sql + use bikeshare; + ``` + + ```sql + select * from `trips` where start_station_name='12th & U St NW' limit 10; + ``` + + ```sql + +-----------------+---------------+---------------------+---------------------+--------------------+------------------+-------------------------------------------+----------------+-----------+------------+-----------+------------+---------------+ + | ride_id | rideable_type | started_at | ended_at | start_station_name | start_station_id | end_station_name | end_station_id | start_lat | start_lng | end_lat | end_lng | member_casual | + +-----------------+---------------+---------------------+---------------------+--------------------+------------------+-------------------------------------------+----------------+-----------+------------+-----------+------------+---------------+ + | E291FF5018 | classic_bike | 2021-01-02 11:12:38 | 2021-01-02 11:23:47 | 12th & U St NW | 31268 | 7th & F St NW / National Portrait Gallery | 31232 | 38.916786 | -77.02814 | 38.89728 | -77.022194 | member | + | E76F3605D0 | docked_bike | 2020-09-13 00:44:11 | 2020-09-13 00:59:38 | 12th & U St NW | 31268 | 17th St & Massachusetts Ave NW | 31267 | 38.916786 | -77.02814 | 38.908142 | -77.03836 | casual | + | FFF0B75414 | docked_bike | 2020-09-28 16:47:53 | 2020-09-28 16:57:30 | 12th & U St NW | 31268 | 17th St & Massachusetts Ave NW | 31267 | 38.916786 | -77.02814 | 38.908142 | -77.03836 | casual | + | C3F2C16949 | docked_bike | 2020-09-13 00:42:03 | 2020-09-13 00:59:43 | 12th & U St NW | 31268 | 17th St & Massachusetts Ave NW | 31267 | 38.916786 | -77.02814 | 38.908142 | -77.03836 | casual | + | 1C7EC91629 | docked_bike | 2020-09-28 16:47:49 | 2020-09-28 16:57:26 | 12th & U St NW | 31268 | 17th St & Massachusetts Ave NW | 31267 | 38.916786 | -77.02814 | 38.908142 | -77.03836 | member | + | A3A38BCACA | classic_bike | 2021-01-14 09:52:53 | 2021-01-14 10:00:51 | 12th & U St NW | 31268 | 10th & E St NW | 31256 | 38.916786 | -77.02814 | 38.895912 | -77.02606 | member | + | EC4943257E | electric_bike | 2021-01-28 10:06:52 | 2021-01-28 10:16:28 | 12th & U St NW | 31268 | 10th & E St NW | 31256 | 38.916843 | -77.028206 | 38.89607 | -77.02608 | member | + | D4070FBFA7 | classic_bike | 2021-01-12 09:50:51 | 2021-01-12 09:59:41 | 12th & U St NW | 31268 | 10th & E St NW | 31256 | 38.916786 | -77.02814 | 38.895912 | -77.02606 | member | + | 6EABEF3CAB | classic_bike | 2021-01-09 15:00:43 | 2021-01-09 15:18:30 | 12th & U St NW | 31268 | 1st & M St NE | 31603 | 38.916786 | -77.02814 | 38.905697 | -77.005486 | member | + | 2F5CC89018 | electric_bike | 2021-01-02 01:47:07 | 2021-01-02 01:58:29 | 12th & U St NW | 31268 | 3rd & H St NE | 31616 | 38.916836 | -77.02815 | 38.90074 | -77.00219 | member | + +-----------------+---------------+---------------------+---------------------+--------------------+------------------+-------------------------------------------+----------------+-----------+------------+-----------+------------+---------------+ + ``` + +2. Get the trip records with electric bikes: + + ```sql + use bikeshare; + ``` + + ```sql + select * from `trips` where rideable_type="electric_bike" limit 10; + ``` + + ```sql + +------------------+---------------+---------------------+---------------------+----------------------------------------+------------------+-------------------------------------------------------+----------------+-----------+------------+-----------+------------+---------------+ + | ride_id | rideable_type | started_at | ended_at | start_station_name | start_station_id | end_station_name | end_station_id | start_lat | start_lng | end_lat | end_lng | member_casual | + +------------------+---------------+---------------------+---------------------+----------------------------------------+------------------+-------------------------------------------------------+----------------+-----------+------------+-----------+------------+---------------+ + | AF15B12839DA4367 | electric_bike | 2021-01-23 14:50:46 | 2021-01-23 14:59:55 | Columbus Circle / Union Station | 31623 | 15th & East Capitol St NE | 31630 | 38.8974 | -77.00481 | 38.890 | 76.98354 | member | + | 7173E217338C4752 | electric_bike | 2021-01-15 08:28:38 | 2021-01-15 08:33:49 | 37th & O St NW / Georgetown University | 31236 | 34th St & Wisconsin Ave NW | 31226 | 38.907825 | -77.071655 | 38.916 | -77.0683 | member | + | E665505ED621D1AB | electric_bike | 2021-01-05 13:25:47 | 2021-01-05 13:35:58 | N Lynn St & Fairfax Dr | 31917 | 34th St & Wisconsin Ave NW | 31226 | 38.89359 | -77.07089 | 38.916 | 77.06829 | member | + | 646AFE266A6375AF | electric_bike | 2021-01-16 00:08:10 | 2021-01-16 00:35:58 | 7th St & Massachusetts Ave NE | 31647 | 34th St & Wisconsin Ave NW | 31226 | 38.892235 | -76.996025 | 38.91 | 7.068245 | member | + | 40CDDA0378E45736 | electric_bike | 2021-01-03 11:14:50 | 2021-01-03 11:26:04 | N Lynn St & Fairfax Dr | 31917 | 34th St & Wisconsin Ave NW | 31226 | 38.893734 | -77.07096 | 38.916 | 7.068275 | member | + | E0A7DDB0CE680C01 | electric_bike | 2021-01-05 18:18:17 | 2021-01-05 19:04:11 | Maine Ave & 7th St SW | 31609 | Smithsonian-National Mall / Jefferson Dr & 12th St SW | 31248 | 38.878727 | -77.02304 | 38.8 | 7.028755 | casual | + | 71BDF35029AF0039 | electric_bike | 2021-01-07 10:23:57 | 2021-01-07 10:59:43 | 10th & K St NW | 31263 | East West Hwy & Blair Mill Rd | 32019 | 38.90279 | -77.02633 | 38.990 | 77.02937 | member | + | D5EACDF488260A61 | electric_bike | 2021-01-13 20:57:23 | 2021-01-13 21:04:19 | 8th & H St NE | 31661 | 15th & East Capitol St NE | 31630 | 38.89985 | -76.994835 | 38.88 | 76.98345 | member | + | 211D449363FB7EE3 | electric_bike | 2021-01-15 17:22:02 | 2021-01-15 17:35:49 | 7th & K St NW | 31653 | 15th & East Capitol St NE | 31630 | 38.90216 | -77.0211 | 38.88 | 76.98357 | casual | + | CE667578A7291701 | electric_bike | 2021-01-15 16:55:12 | 2021-01-15 17:38:26 | East West Hwy & 16th St | 32056 | East West Hwy & Blair Mill Rd | 32019 | 38.995674 | -77.03868 | 38.990 | 77.02953 | casual | + +------------------+---------------+---------------------+---------------------+----------------------------------------+------------------+-------------------------------------------------------+----------------+-----------+------------+-----------+------------+---------------+ + ``` diff --git a/tidb-cloud/import-sample-data.md b/tidb-cloud/import-sample-data.md index aabf06d9bf15c..acfa8c0484ed8 100644 --- a/tidb-cloud/import-sample-data.md +++ b/tidb-cloud/import-sample-data.md @@ -1,11 +1,14 @@ --- -title: Import Sample Data -summary: Learn how to import sample data into TiDB Cloud via UI. +title: Import Sample Data into TiDB Cloud Dedicated +summary: Learn how to import sample data into TiDB Cloud Dedicated via the UI. --- -# Import Sample Data +# Import Sample Data into TiDB Cloud Dedicated -This document describes how to import the sample data into TiDB Cloud via the UI. The sample data used is the system data from Capital Bikeshare, released under the Capital Bikeshare Data License Agreement. Before importing the sample data, you need to have one TiDB cluster. +This document describes how to import the sample data into TiDB Cloud Dedicated via the UI. The sample data used is the system data from Capital Bikeshare, released under the Capital Bikeshare Data License Agreement. Before importing the sample data, you need to have one TiDB cluster. + + +
1. Open the **Import** page for your target cluster. @@ -17,52 +20,55 @@ This document describes how to import the sample data into TiDB Cloud via the UI 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. -2. Configure the source data information. +2. Select **Import data from S3**. - -
+ If this is your first time importing data into this cluster, select **Import From Amazon S3**. - On the **Import** page: +3. On the **Import Data from Amazon S3** page, configure the following source data information: - - For a TiDB Dedicated cluster, click **Import Data** in the upper-right corner. - - For a TiDB Serverless cluster, click the **import data from S3** link above the upload area. + - **Import File Count**: for the sample data, select **Multiple files**. + - **Included Schema Files**: for the sample data, select **Yes**. + - **Data Format**: select **SQL**. + - **Folder URI** or **File URI**: enter the sample data URI `s3://tidbcloud-sample-data/data-ingestion/`. + - **Bucket Access**: for the sample data, you can only use a Role ARN to access its bucket. For your own data, you can use either an AWS access key or a Role ARN to access your bucket. + - **AWS Role ARN**: enter `arn:aws:iam::801626783489:role/import-sample-access`. + - **AWS Access Key**: skip this option for the sample data. - Fill in the following parameters: +4. Click **Connect** > **Start Import**. - - **Data format**: select **SQL File**. TiDB Cloud supports importing compressed files in the following formats: `.gzip`, `.gz`, `.zstd`, `.zst` and `.snappy`. If you want to import compressed SQL files, name the files in the `${db_name}.${table_name}.${suffix}.sql.${compress}` format, in which `${suffix}` is optional and can be any integer such as '000001'. For example, if you want to import the `trips.000001.sql.gz` file to the `bikeshare.trips` table, you can rename the file as `bikeshare.trips.000001.sql.gz`. Note that you only need to compress the data files, not the database or table schema files. The Snappy compressed file must be in the [official Snappy format](https://github.com/google/snappy). Other variants of Snappy compression are not supported. - - **Bucket URI**: enter the sample data URI `s3://tidbcloud-sample-data/data-ingestion/` - - **Bucket Access**: for the sample data, you can only use a Role ARN to access its bucket. For your own data, you can use either an AWS access key or a Role ARN to access your bucket. - - **AWS Access Keys**: skip this option for the sample data. - - **AWS Role ARN**: enter `arn:aws:iam::801626783489:role/import-sample-access` +
+
- If the region of the bucket is different from your cluster, confirm the compliance of cross region. Click **Next**. +1. Open the **Import** page for your target cluster. -
+ 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. -
+ > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. - If your TiDB cluster is hosted by Google Cloud, click **Import Data** in the upper-right corner, and then fill in the following parameters: + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. - - **Data Format**: select **SQL File**. TiDB Cloud supports importing compressed files in the following formats: `.gzip`, `.gz`, `.zstd`, `.zst` and `.snappy`. If you want to import compressed SQL files, name the files in the `${db_name}.${table_name}.${suffix}.sql.${compress}` format, in which `${suffix}` is optional and can be any integer such as '000001'. For example, if you want to import the `trips.000001.sql.gz` file to the `bikeshare.trips` table, you can rename the file as `bikeshare.trips.000001.sql.gz`. Note that you only need to compress the data files, not the database or table schema files. Note that you only need to compress the data files, not the database or table schema files. The Snappy compressed file must be in the [official Snappy format](https://github.com/google/snappy). Other variants of Snappy compression are not supported. - - **Bucket gsutil URI**: enter the sample data URI `gs://tidbcloud-samples-us-west1/`. - - **Bucket Access**: you can use a GCS IAM Role to access your bucket. For more information, see [Configure GCS access](/tidb-cloud/config-s3-and-gcs-access.md#configure-gcs-access). +2. Click **Import Data** in the upper-right corner. - If the region of the bucket is different from your cluster, confirm the compliance of cross region. Click **Next**. + If this is your first time importing data into this cluster, select **Import From GCS**. -
- +3. On the **Import Data from GCS** page, configure the following source data information: -3. You can choose to import into pre-created tables, or import schema and data from the source. + - **Import File Count**: for the sample data, select **Multiple files**. + - **Included Schema Files**: for the sample data, select **Yes**. + - **Data Format**: select **SQL**. + - **Folder URI** or **File URI**: enter the sample data URI `gs://tidbcloud-samples-us-west1/`. + - **Bucket Access**: you can use a GCS IAM Role to access your bucket. For more information, see [Configure GCS access](/tidb-cloud/config-s3-and-gcs-access.md#configure-gcs-access). - - **Import into pre-created tables** allows you to create tables in TiDB in advance and select the tables that you want to import data into. In this case, you can choose up to 1000 tables to import. You can click **Chat2Query** in the left navigation pane to create tables. For more information about how to use Chat2Query, see [Explore Your Data with AI-Powered Chat2Query](/tidb-cloud/explore-data-with-chat2query.md). - - **Import schema and data from S3** (This field is visible only for AWS S3) allows you to import SQL scripts for creating a table and import corresponding table data stored in S3 into TiDB. - - **Import schema and data from GCS** (This field is visible only for GCS) allows you to import SQL scripts that create a table along with its corresponding data stored in GCS directly into TiDB. + If the region of the bucket is different from your cluster, confirm the compliance of cross region. -4. Click **Start Import**. +4. Click **Connect** > **Start Import**. -When the data import progress shows **Completed**, you have successfully imported the sample data and the database schema to your database in TiDB Cloud. +
+ -Once the cluster finishes the data importing process, you will get the sample data in your database. +When the data import progress shows **Completed**, you have successfully imported the sample data and the database schema to your database in TiDB Cloud. After connecting to the cluster, you can run some queries in your terminal to check the result, for example: diff --git a/tidb-cloud/import-with-mysql-cli-serverless.md b/tidb-cloud/import-with-mysql-cli-serverless.md new file mode 100644 index 0000000000000..4818233915b69 --- /dev/null +++ b/tidb-cloud/import-with-mysql-cli-serverless.md @@ -0,0 +1,111 @@ +--- +title: Import Data into TiDB Cloud Serverless via MySQL CLI +summary: Learn how to import Data into TiDB Cloud Serverless via MySQL CLI. +--- + +# Import Data into TiDB Cloud Serverless via MySQL CLI + +This document describes how to import data into TiDB Cloud Serverless via the [MySQL Command-Line Client](https://dev.mysql.com/doc/refman/8.0/en/mysql.html). You can import data from an SQL file or a CSV file. The following sections provide step-by-step instructions for importing data from each type of file. + +## Prerequisites + +Before you can import data via MySQL CLI to TiDB Cloud Serverless, you need the following prerequisites: + +- You have access to your TiDB Cloud Serverless cluster. If you do not have, create one following the instructions in [Build a TiDB Cloud Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). +- Install MySQL CLI on your local computer. + +## Step 1. Connect to your TiDB Cloud Serverless cluster + +Connect to your TiDB cluster. + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public`. + - **Connect With** is set to `MySQL CLI`. + - **Operating System** matches your environment. + +4. Click **Generate Password** to create a random password. + + > **Tip:** + > + > If you have created a password before, either use the original password or click **Reset Password** to generate a new one. + +## Step 2. Define the table and insert sample data + +Before importing data, you need to prepare the table structure and insert real sample data into it. The following is an example SQL file (`product_data.sql`) that you can use to create a table and insert sample data: + +```sql +-- Create a table in your TiDB database +CREATE TABLE products ( + product_id INT PRIMARY KEY, + product_name VARCHAR(255), + price DECIMAL(10, 2) +); + +-- Insert sample data into the table +INSERT INTO products (product_id, product_name, price) VALUES + (1, 'Laptop', 999.99), + (2, 'Smartphone', 499.99), + (3, 'Tablet', 299.99); +``` + +## Step 3. Import data from a SQL or CSV file + +You can import data from an SQL file or a CSV file. The following sections provide step-by-step instructions for importing data from each type. + + +
+ +Do the following to import data from an SQL file: + +1. Provide a real SQL file (for example, `product_data.sql`) that contains the data you want to import. This SQL file must contain `INSERT` statements with real data. + +2. Use the following command to import data from the SQL file: + + ```bash + mysql --comments --connect-timeout 150 -u '' -h -P 4000 -D test --ssl-mode=VERIFY_IDENTITY --ssl-ca= -p < product_data.sql + ``` + +> **Note:** +> +> The default database name used here is `test`, and you can either manually create your own database or use the `CREATE DATABASE` command in an SQL file. + +
+
+ +Do the following to import data from a CSV file: + +1. Create a database and schema in TiDB to match your data import needs. + +2. Provide a sample CSV file (for example, `product_data.csv`) that contains the data you want to import. The following is an example of a CSV file: + + **product_data.csv:** + + ```csv + product_id,product_name,price + 4,Laptop,999.99 + 5,Smartphone,499.99 + 6,Tablet,299.99 + ``` + +3. Use the following command to import data from the CSV file: + + ```bash + mysql --comments --connect-timeout 150 -u '' -h -P 4000 -D test --ssl-mode=VERIFY_IDENTITY --ssl-ca= -p -e "LOAD DATA LOCAL INFILE '' INTO TABLE products + FIELDS TERMINATED BY ',' + LINES TERMINATED BY '\n' + IGNORE 1 LINES (product_id, product_name, price);" + ``` + + Make sure to replace the paths, table name (`products` in this example), ``, ``, ``, ``, ``, and other placeholders with your actual information, and replace the sample CSV data with your real dataset as needed. + +> **Note:** +> +> For more syntax details about `LOAD DATA LOCAL INFILE`, see [`LOAD DATA`](/sql-statements/sql-statement-load-data.md). + +
+ diff --git a/tidb-cloud/import-with-mysql-cli.md b/tidb-cloud/import-with-mysql-cli.md index 749efd0697c31..314783d437da4 100644 --- a/tidb-cloud/import-with-mysql-cli.md +++ b/tidb-cloud/import-with-mysql-cli.md @@ -1,55 +1,34 @@ --- -title: Import Data into TiDB Cloud via MySQL CLI -summary: Learn how to import Data into TiDB Cloud via MySQL CLI. +title: Import Data into TiDB Cloud Dedicated via MySQL CLI +summary: Learn how to import Data into TiDB Cloud Dedicated via MySQL CLI. --- -# Import Data into TiDB Cloud via MySQL CLI +# Import Data into TiDB Cloud Dedicated via MySQL CLI -This document describes how to import data into TiDB Cloud via the [MySQL Command-Line Client](https://dev.mysql.com/doc/refman/8.0/en/mysql.html). You can import data from an SQL file or a CSV file. The following sections provide step-by-step instructions for importing data from each type of file. +This document describes how to import data into TiDB Cloud Dedicated via the [MySQL Command-Line Client](https://dev.mysql.com/doc/refman/8.0/en/mysql.html). You can import data from an SQL file or a CSV file. The following sections provide step-by-step instructions for importing data from each type of file. ## Prerequisites -Before you can import data via MySQL CLI to TiDB Cloud, you need the following prerequisites: +Before you can import data via MySQL CLI to TiDB Cloud Dedicated, you need the following prerequisites: -- You have access to your TiDB Cloud cluster. If you do not have a TiDB cluster, create one following the instructions in [Build a TiDB Serverless Cluster](/develop/dev-guide-build-cluster-in-cloud.md). +- You have access to your TiDB Cloud Dedicated cluster. If you do not have, create one following the instructions in [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). - Install MySQL CLI on your local computer. -## Step 1. Connect to your TiDB Cloud cluster +## Step 1. Connect to your TiDB Cloud Dedicated cluster -Connect to your TiDB cluster depending on the TiDB deployment option you have selected. - - -
+Connect to your TiDB cluster. 1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. -2. Click **Connect** in the upper-right corner. A connection dialog is displayed. - -3. Ensure the configurations in the connection dialog match your operating environment. - - - **Endpoint Type** is set to `Public`. - - **Connect With** is set to `MySQL CLI`. - - **Operating System** matches your environment. - -4. Click **Generate Password** to create a random password. - - > **Tip:** - > - > If you have created a password before, either use the original password or click **Reset Password** to generate a new one. +2. In the left navigation pane, click **Networking**. -
-
- -1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. +3. On the **Networking** page, click **Add IP Address** in the **IP Access List** area. -2. Click **Connect** in the upper-right corner. A connection dialog is displayed. +4. In the dialog, choose **Allow access from anywhere**, and then click **Confirm**. -3. Click **Allow Access from Anywhere**. +5. In the upper-right corner, click **Connect** to open the dialog for connection information. - For more details about how to obtain the connection string, see [Connect to TiDB Dedicated via Standard Connection](/tidb-cloud/connect-via-standard-connection.md). - -
- + For more details about how to obtain the connection string, see [Connect to TiDB Cloud Dedicated via Public Connection](/tidb-cloud/connect-via-standard-connection.md). ## Step 2. Define the table and insert sample data @@ -84,7 +63,7 @@ Do the following to import data from an SQL file: 2. Use the following command to import data from the SQL file: ```bash - mysql --comments --connect-timeout 150 -u '' -h -P 4000 -D test --ssl-mode=VERIFY_IDENTITY --ssl-ca= -p < product_data.sql + mysql --comments --connect-timeout 150 -u '' -h -P 4000 -D test --ssl-mode=VERIFY_IDENTITY --ssl-ca= -p < product_data.sql ``` > **Note:** diff --git a/tidb-cloud/index-insight.md b/tidb-cloud/index-insight.md index 5557e1e51f107..4aadef56e36da 100644 --- a/tidb-cloud/index-insight.md +++ b/tidb-cloud/index-insight.md @@ -9,7 +9,7 @@ The Index Insight (beta) feature in TiDB Cloud provides powerful capabilities to > **Note:** > -> Index Insight is currently in beta and only available for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. +> Index Insight is currently in beta and only available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters. ## Introduction @@ -26,11 +26,11 @@ This section introduces how to enable the Index Insight feature and obtain recom ### Before you begin -Before enabling the Index Insight feature, make sure that you have created a TiDB Dedicated cluster. If you do not have one, follow the steps in [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) to create one. +Before enabling the Index Insight feature, make sure that you have created a TiDB Cloud Dedicated cluster. If you do not have one, follow the steps in [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) to create one. ### Step 1: Enable Index Insight -1. In the [TiDB Cloud console](https://tidbcloud.com), navigate to the cluster overview page of your TiDB Dedicated cluster, and then click **Diagnosis** in the left navigation pane. +1. In the [TiDB Cloud console](https://tidbcloud.com), navigate to the cluster overview page of your TiDB Cloud Dedicated cluster, and then click **Diagnosis** in the left navigation pane. 2. Click the **Index Insight BETA** tab. The **Index Insight overview** page is displayed. @@ -46,7 +46,7 @@ Before enabling the Index Insight feature, make sure that you have created a TiD > **Note:** > - > To connect to your TiDB Dedicated cluster, see [Connect to a TiDB Dedicated cluster](/tidb-cloud/connect-to-tidb-cluster.md). + > To connect to your TiDB Cloud Dedicated cluster, see [Connect to a TiDB Cloud Dedicated cluster](/tidb-cloud/connect-to-tidb-cluster.md). 4. Enter the username and password of the SQL user created in the preceding step. Then, click **Activate** to initiate the activation process. diff --git a/tidb-cloud/integrate-tidbcloud-with-airbyte.md b/tidb-cloud/integrate-tidbcloud-with-airbyte.md index 41064f01eeefa..eb21250caa2d8 100644 --- a/tidb-cloud/integrate-tidbcloud-with-airbyte.md +++ b/tidb-cloud/integrate-tidbcloud-with-airbyte.md @@ -62,7 +62,7 @@ Conveniently, the steps are the same for setting TiDB as the source and the dest > > - TiDB Cloud supports TLS connection. You can choose your TLS protocols in **TLSv1.2** and **TLSv1.3**, for example, `enabledTLSProtocols=TLSv1.2`. > - If you want to disable TLS connection to TiDB Cloud via JDBC, you need to set useSSL to `false` in JDBC URL Params specifically and close SSL connection, for example, `useSSL=false`. - > - TiDB Serverless only supports TLS connections. + > - TiDB Cloud Serverless only supports TLS connections. 4. Click **Set up source** or **destination** to complete creating the connector. The following screenshot shows the configuration of TiDB as the source. @@ -87,7 +87,7 @@ The following steps use TiDB as both a source and a destination. Other connector > **Tip:** > - > The TiDB connector supports both Incremental and Full Refresh syncs. + > The TiDB connector supports both [Incremental and Full Refresh syncs](https://airbyte.com/blog/understanding-data-replication-modes). > > - In Incremental mode, Airbyte only reads records added to the source since the last sync job. The first sync using Incremental mode is equivalent to Full Refresh mode. > - In Full Refresh mode, Airbyte reads all records in the source and replicates to the destination in every sync task. You can set the sync mode for every table named **Namespace** in Airbyte individually. @@ -102,7 +102,7 @@ The following steps use TiDB as both a source and a destination. Other connector ## Limitations -- The TiDB connector does not support the Change Data Capture (CDC) feature. +- The TiDB connector cannot use the Change Data Capture (CDC) feature provided by TiCDC. The incremental sync is performed based on a cursor mechanism. - TiDB destination converts the `timestamp` type to the `varchar` type in default normalization mode. It happens because Airbyte converts the timestamp type to string during transmission, and TiDB does not support `cast ('2020-07-28 14:50:15+1:00' as timestamp)`. - For some large ELT missions, you need to increase the parameters of [transaction restrictions](/develop/dev-guide-transaction-restraints.md#large-transaction-restrictions) in TiDB. diff --git a/tidb-cloud/integrate-tidbcloud-with-aws-lambda.md b/tidb-cloud/integrate-tidbcloud-with-aws-lambda.md new file mode 100644 index 0000000000000..db4a2c62bcf2d --- /dev/null +++ b/tidb-cloud/integrate-tidbcloud-with-aws-lambda.md @@ -0,0 +1,202 @@ +--- +title: Integrate TiDB Cloud Serverless with Amazon Lambda Using AWS CloudFormation +summary: Introduce how to integrate TiDB Cloud Serverless with Amazon Lambda and CloudFormation step by step. +--- + +# Integrate TiDB Cloud Serverless with Amazon Lambda Using AWS CloudFormation + +This document provides a step-by-step guide on how to use [AWS CloudFormation](https://aws.amazon.com/cloudformation/) to integrate [TiDB Cloud Serverless](https://www.pingcap.com/tidb-cloud/), a cloud-native distributed SQL database, with [AWS Lambda](https://aws.amazon.com/lambda/), a serverless and event-driven compute service. By integrating TiDB Cloud Serverless with Amazon Lambda, you can leverage the scalability and cost-efficiency of microservices through TiDB Cloud Serverless and AWS Lambda. AWS CloudFormation automates the creation and management of AWS resources, including Lambda functions, API Gateway, and Secrets Manager. + +## Solution overview + +In this guide, you will create a fully functional online bookshop with the following components: + +- AWS Lambda Function: handles requests and queries data from a TiDB Cloud Serverless cluster using Sequelize ORM and Fastify API framework. +- AWS Secrets Manager SDK: retrieves and manages connection configurations for the TiDB Cloud Serverless cluster. +- AWS API Gateway: handles HTTP request routes. +- TiDB Cloud Serverless: a cloud-native distributed SQL database. + +AWS CloudFormation is used to create the necessary resources for the project, including the Secrets Manager, API Gateway, and Lambda Functions. + +The structure of the bookshop project is as follows: + +![AWS Lambda structure overview](/media/develop/aws-lambda-structure-overview.png) + +## Prerequisites + +Before getting started, ensure that you have the following: + +- An AWS account with access to the following AWS services: + - [AWS CloudFormation](https://aws.amazon.com/cloudformation/) + - [Secrets Manager](https://aws.amazon.com/secrets-manager/) + - [API Gateway](https://aws.amazon.com/api-gateway/) + - [Lambda services](https://aws.amazon.com/lambda/) + - [S3](https://aws.amazon.com/s3/) + - [IAM Roles](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles.html) +- A [TiDB Cloud](https://tidbcloud.com) account and a TiDB Cloud Serverless cluster. Get the connection information for your TiDB Cloud Serverless cluster: + + ![TiDB Cloud connection information](/media/develop/aws-lambda-tidbcloud-connection-info.png) + +- API test tools such as [Postman](https://www.postman.com/) and [cURL](https://curl.se/). Most examples in this document use cURL. For Windows users, Postman is recommended. +- Download the [latest release assets](https://github.com/pingcap/TiDB-Lambda-integration/releases/latest) of the project to your local machine, which includes `cloudformation_template.yml` and `cloudformation_template.json` files. + +> **Note:** +> +> - When you create the AWS resources, it is recommended to use `us-east-1` as your cluster region. This is because the Lambda function code in this demo hardcodes the region as `us-east-1`, and the code bundle is stored in the `us-east-1` region. +> - If you use a different region, you need to follow the following instructions to modify the Lambda function code, rebuild it, and upload the code bundle to your own S3 bucket. + +
+Modify and rebuild the Lambda function code if you use a region other than us-east-1 + +If you use `us-east-1` as your cluster region, skip this section and go to [Step 1: Set up the project using AWS CloudFormation](#step-1-set-up-the-bookshop-project-using-aws-cloudformation). + +If you use a different AWS region other than `us-east-1` to create the AWS resources, you need to modify the Lambda function code, rebuild it, and upload the code bundle to your own S3 bucket. + +To avoid local development environment issues, it is recommended that you use a cloud-native development environment, such as [Gitpod](https://www.gitpod.io/). + +To rebuild and upload the code bundle to your own S3 bucket, do the following: + +1. Initialize the development environment. + + - Open the [Gitpod](https://gitpod.io/#/https://github.com/pingcap/TiDB-Lambda-integration) workspace and log in with your GitHub account. + +2. Modify the Lambda function code. + + 1. Open the `aws-lambda-cloudformation/src/secretManager.ts` file in the left sidebar. + 2. Locate the line 22 and then modify the `region` variable to match your own region. + +3. Rebuild the code bundle. + + 1. Install the dependencies. + + 1. Open a terminal in Gitpod. + 2. Enter the working directory: + + ```shell + cd aws-lambda-cloudformation + ``` + + 3. Install the dependencies: + + ```shell + yarn + ``` + + 2. Rebuild the code bundle. + + 1. Build the code bundle. + + ```shell + yarn build + ``` + + 2. Check the `aws-lambda-cloudformation/dist/index.zip` file. + 3. Right-click the `index.zip` file and select **Download**. + +4. Upload the rebuilt code bundle to your own S3 bucket. + + 1. Visit the [S3 service](https://console.aws.amazon.com/s3) in the AWS Management Console. + 2. Create a new bucket in your selected region. + 3. Upload the `index.zip` file to the bucket. + 4. Note down the S3 bucket name and region for later use. + +
+ +## Step 1. Set up the bookshop project using AWS CloudFormation + +To set up the bookshop project using AWS CloudFormation, do the following: + +1. Navigate to the AWS Management Console and access the [AWS CloudFormation service](https://console.aws.amazon.com/cloudformation). +2. Click **Create Stack** > **With new resources (standard)**. +3. On the **Create Stack** page, complete the stack creation process. + + 1. In the **Prerequisite** area, select **Choose an existing template**. + 2. In the **Specify template** area, select **Upload a template file**, click **Choose file** to upload the template file (either YAML or JSON), and click **Next**. + + If you do not have the file yet, download it from [GitHub](https://github.com/pingcap/TiDB-Lambda-integration/releases/latest). The file contains the AWS CloudFormation template that creates the necessary resources for the project. + + ![Create a stack](/media/develop/aws-lambda-cf-create-stack.png) + + 3. Specify stack details. + + - If you use `us-east-1` as your cluster region, fill in the fields as in the following screenshot: + + ![Specify AWS Lambda stack details](/media/develop/aws-lambda-cf-stack-config.png) + + - **Stack name**: enter the stack name. + - **S3Bucket**: enter the S3 bucket where you store the zip file. + - **S3Key**: enter the S3 key. + - **TiDBDatabase**: enter the TiDB Cloud cluster name. + - **TiDBHost**: enter the host URL for TiDB Cloud database access. Enter `localhost`. + - **TiDBPassword**: enter the password for TiDB Cloud database access. + - **TiDBPort**: enter the port for TiDB Cloud database access. + - **TiDBUser**: enter the user name for TiDB Cloud database access. + + - If you use a different AWS region other than `us-east-1`, follow these steps: + + 1. Refer to [Modify and rebuild the Lambda function code if you use a region other than `us-east-1`](#prerequisites) to modify the Lambda function code, rebuild it, and upload the code bundle to your own S3 bucket. + 2. In the stack details fields, specify the S3 bucket name and region in the `S3Bucket` and `S3Key` parameters according to your own configuration. + 3. Fill in other fields as in the preceding screenshot. + + 4. Configure stack options. You can use the default configurations. + + ![Configure stack options](/media/develop/aws-lambda-cf-stack-config-option.png) + + 5. Review and create the stack. + + ![Review and create the stack](/media/develop/aws-lambda-cf-stack-config-review.png) + +## Step 2. Use the bookshop project + +After the stack has been created, you can use the project as follows: + +1. Visit the [API Gateway service](https://console.aws.amazon.com/apigateway) in the AWS Management Console, click the `TiDBCloudApiGatewayV2` API, and then click **API: TiDBCloudApiGatewayV2** in the left pane. + +2. Copy the `Invoke URL` from the **Overview** page. This URL serves as the API endpoint. + + ![API Gateway Invoke URL](/media/develop/aws-lambda-get-apigateway-invoke-url.png) + +3. Use API test tools such as Postman and cURL to test the API: + + - Init mock books: + + ```shell + curl -X POST -H "Content-Type: application/json" -d '{"count":100}' https:///book/init + ``` + + - Get all books: + + ```shell + curl https:///book + ``` + + - Get a book by the book ID: + + ```shell + curl https:///book/ + ``` + + - Create a book: + + ```shell + curl -X POST -H "Content-Type: application/json" -d '{ "title": "Book Title", "type": "Test", "publishAt": "2022-12-15T21:01:49.000Z", "stock": 123, "price": 12.34, "authors": "Test Test" }' https:// /book + ``` + + - Update a book: + + ```shell + curl -X PUT -H "Content-Type: application/json" -d '{ "title": "Book Title(updated)" }' https:///book/ + ``` + + - Delete a book: + + ```shell + curl -X DELETE https:///book/ + ``` + +## Step 3. Clean up resources + +To avoid unnecessary charges, clean up all resources that have been created. + +1. Access the [AWS Management Console](https://console.aws.amazon.com/cloudformation). +2. Delete the AWS CloudFormation stack that you created. diff --git a/tidb-cloud/integrate-tidbcloud-with-cloudflare.md b/tidb-cloud/integrate-tidbcloud-with-cloudflare.md index 40bde4ecd0c7a..c9d78aaea21c9 100644 --- a/tidb-cloud/integrate-tidbcloud-with-cloudflare.md +++ b/tidb-cloud/integrate-tidbcloud-with-cloudflare.md @@ -13,13 +13,13 @@ This document shows how to connect to Cloudflare Workers with TiDB Cloud serverl > **Note:** > -> TiDB Cloud serverless driver can only be used in TiDB Serverless. +> TiDB Cloud serverless driver can only be used in TiDB Cloud Serverless. ## Before you begin Before you try the steps in this article, you need to prepare the following things: -- A TiDB Cloud account and a TiDB Serverless cluster on TiDB Cloud. For more details, see [TiDB Cloud Quick Start](/tidb-cloud/tidb-cloud-quickstart.md#step-1-create-a-tidb-cluster). +- A TiDB Cloud account and a TiDB Cloud Serverless cluster on TiDB Cloud. For more details, see [TiDB Cloud Quick Start](/tidb-cloud/tidb-cloud-quickstart.md#step-1-create-a-tidb-cluster). - A [Cloudflare Workers account](https://dash.cloudflare.com/login). - [npm](https://docs.npmjs.com/about-npm) is installed. diff --git a/tidb-cloud/integrate-tidbcloud-with-n8n.md b/tidb-cloud/integrate-tidbcloud-with-n8n.md index e815286686217..eb4a615bb2070 100644 --- a/tidb-cloud/integrate-tidbcloud-with-n8n.md +++ b/tidb-cloud/integrate-tidbcloud-with-n8n.md @@ -7,7 +7,7 @@ summary: Learn the use of TiDB Cloud node in n8n. [n8n](https://n8n.io/) is an extendable workflow automation tool. With a [fair-code](https://faircode.io/) distribution model, n8n will always have visible source code, be available to self-host, and allow you to add your custom functions, logic, and apps. -This document introduces how to build an auto-workflow: create a TiDB Serverless cluster, gather Hacker News RSS, store it to TiDB and send a briefing email. +This document introduces how to build an auto-workflow: create a TiDB Cloud Serverless cluster, gather Hacker News RSS, store it to TiDB and send a briefing email. ## Prerequisites: Get TiDB Cloud API key @@ -76,9 +76,9 @@ The final workflow should look like the following image. ![img](/media/tidb-cloud/integration-n8n-workflow-rss.jpg) -### (Optional) Create a TiDB Serverless cluster +### (Optional) Create a TiDB Cloud Serverless cluster -If you don't have a TiDB Serverless cluster, you can use this node to create one. Otherwise, feel free to skip this operation. +If you don't have a TiDB Cloud Serverless cluster, you can use this node to create one. Otherwise, feel free to skip this operation. 1. Navigate to **Workflows** panel, and click **Add workflow**. 2. In new workflow workspace, click **+** in the top right corner and choose **All** field. @@ -93,7 +93,7 @@ If you don't have a TiDB Serverless cluster, you can use this node to create one > **Note:** > -> It takes several seconds to create a new TiDB Serverless cluster. +> It takes several seconds to create a new TiDB Cloud Serverless cluster. ### Create a workflow @@ -140,7 +140,7 @@ This trigger will execute your workflow every morning at 8 AM. 1. Click **+** to the right of the RSS Read node. 2. Search `TiDB Cloud` and add it to the workspace. -3. Select the credentials that you entered in the previous TiDB Cloud node. +3. Select the credentials that you entered the previous TiDB Cloud node. 4. In the **Project** list, select your project. 5. In the **Operation** list, select `Insert`. 6. In **Cluster**, **User**, **Database** and **Password** boxes, enter the corresponding values. @@ -203,7 +203,7 @@ This trigger will execute your workflow every morning at 8 AM. After building up the workflow, you can click **Execute Workflow** to test run it. -If the workflow runs as expected, you'll get Hacker News briefing emails. These news contents will be logged to your TiDB Serverless cluster, so you don't have to worry about losing them. +If the workflow runs as expected, you'll get Hacker News briefing emails. These news contents will be logged to your TiDB Cloud Serverless cluster, so you don't have to worry about losing them. Now you can activate this workflow in the **Workflows** panel. This workflow will help you get the front-page articles on Hacker News every day. @@ -213,7 +213,7 @@ Now you can activate this workflow in the **Workflows** panel. This workflow wil TiDB Cloud node acts as a [regular node](https://docs.n8n.io/workflows/nodes/#regular-nodes) and only supports the following five operations: -- **Create Serverless Cluster**: creates a TiDB Serverless cluster. +- **Create Serverless Cluster**: creates a TiDB Cloud Serverless cluster. - **Execute SQL**: executes an SQL statement in TiDB. - **Delete**: deletes rows in TiDB. - **Insert**: inserts rows in TiDB. diff --git a/tidb-cloud/integrate-tidbcloud-with-netlify.md b/tidb-cloud/integrate-tidbcloud-with-netlify.md index 0f52e3ff72c74..ab13a4267cdd8 100644 --- a/tidb-cloud/integrate-tidbcloud-with-netlify.md +++ b/tidb-cloud/integrate-tidbcloud-with-netlify.md @@ -24,16 +24,16 @@ You are expected to have a Netlify account and CLI. If you do not have any, refe You are expected to have an account and a cluster in TiDB Cloud. If you do not have any, refer to the following to create one: -- [Create a TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) -- [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) +- [Create a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) +- [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) One TiDB Cloud cluster can connect to multiple Netlify sites. ### All IP addresses allowed for traffic filter in TiDB Cloud -For TiDB Dedicated clusters, make sure that the traffic filter of the cluster allows all IP addresses (set to `0.0.0.0/0`) for connection. This is because Netlify deployments use dynamic IP addresses. +For TiDB Cloud Dedicated clusters, make sure that the traffic filter of the cluster allows all IP addresses (set to `0.0.0.0/0`) for connection. This is because Netlify deployments use dynamic IP addresses. -TiDB Serverless clusters allow all IP addresses for connection by default, so you do not need to configure any traffic filter. +TiDB Cloud Serverless clusters allow all IP addresses for connection by default, so you do not need to configure any traffic filter. ## Step 1. Get the example project and the connection string @@ -52,9 +52,9 @@ To help you get started quickly, TiDB Cloud provides a fullstack example app in ### Get the TiDB Cloud connection string -For a TiDB Serverless cluster, you can get the connection string either from [TiDB Cloud CLI](/tidb-cloud/cli-reference.md) or from [TiDB Cloud console](https://tidbcloud.com/). +For a TiDB Cloud Serverless cluster, you can get the connection string either from [TiDB Cloud CLI](/tidb-cloud/cli-reference.md) or from [TiDB Cloud console](https://tidbcloud.com/). -For a TiDB Dedicated cluster, you can get the connection string only from the TiDB Cloud console. +For a TiDB Cloud Dedicated cluster, you can get the connection string only from the TiDB Cloud console.
diff --git a/tidb-cloud/integrate-tidbcloud-with-vercel.md b/tidb-cloud/integrate-tidbcloud-with-vercel.md index e6d3544cfdd95..f576c97d633e5 100644 --- a/tidb-cloud/integrate-tidbcloud-with-vercel.md +++ b/tidb-cloud/integrate-tidbcloud-with-vercel.md @@ -38,17 +38,17 @@ One Vercel project can only connect to one TiDB Cloud cluster. To change the int You are expected to have an account and a cluster in TiDB Cloud. If you do not have any, refer to the following to create one: -- [Create a TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) +- [Create a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) > **Note:** > - > The TiDB Cloud Vercel integration supports creating TiDB Serverless clusters. You can also create one later during the integration process. + > The TiDB Cloud Vercel integration supports creating TiDB Cloud Serverless clusters. You can also create one later during the integration process. -- [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) +- [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) > **Note:** > - > For TiDB Dedicated clusters, make sure that the traffic filter of the cluster allows all IP addresses (set to `0.0.0.0/0`) for connection, because Vercel deployments use [dynamic IP addresses](https://vercel.com/guides/how-to-allowlist-deployment-ip-address). + > For TiDB Cloud Dedicated clusters, make sure that the traffic filter of the cluster allows all IP addresses (set to `0.0.0.0/0`) for connection, because Vercel deployments use [dynamic IP addresses](https://vercel.com/guides/how-to-allowlist-deployment-ip-address). To [integrate with Vercel via the TiDB Cloud Vercel Integration](#connect-via-the-tidb-cloud-vercel-integration), you are expected to be in the `Organization Owner` role of your organization or the `Project Owner` role of the target project in TiDB Cloud. For more information, see [User roles](/tidb-cloud/manage-user-access.md#user-roles). @@ -71,7 +71,7 @@ To connect via the TiDB Cloud Vercel integration, go to the [TiDB Cloud integrat > **Note:** > -> This method is only available for TiDB Serverless clusters. If you want to connect to a TiDB Dedicated cluster, use the [manual method](#connect-via-manually-setting-environment-variables). +> This method is only available for TiDB Cloud Serverless clusters. If you want to connect to a TiDB Cloud Dedicated cluster, use the [manual method](#connect-via-manually-setting-environment-variables). ### Integration workflow @@ -89,7 +89,7 @@ The detailed steps are as follows: 1. Select your target Vercel projects and click **Next**. 2. Select your target TiDB Cloud organization and project. 3. Select **Cluster** as your connection type. - 4. Select your target TiDB Cloud cluster. If the **Cluster** drop-down list is empty or you want to select a new TiDB Serverless cluster, click **+ Create Cluster** in the list to create one. + 4. Select your target TiDB Cloud cluster. If the **Cluster** drop-down list is empty or you want to select a new TiDB Cloud Serverless cluster, click **+ Create Cluster** in the list to create one. 5. Select the database that you want to connect to. If the **Database** drop-down list is empty or you want to select a new Database, click **+ Create Database** in the list to create one. 6. Select the framework that your Vercel projects are using. If the target framework is not listed, select **General**. Different frameworks determine different environment variables. 7. Choose whether to enable **Branching** to create new branches for preview environments. @@ -165,22 +165,22 @@ If you have installed [TiDB Cloud Vercel integration](https://vercel.com/integra ![Vercel Integration Configuration Page](/media/tidb-cloud/vercel/integration-vercel-configuration-page.png) - When you remove a connection, the environment variables set by the integration workflow are removed from the Vercel project, too. However, this action does not affect the data of the TiDB Serverless cluster. + When you remove a connection, the environment variables set by the integration workflow are removed from the Vercel project, too. However, this action does not affect the data of the TiDB Cloud Serverless cluster. -### Connect with TiDB Serverless branching +### Connect with TiDB Cloud Serverless branching -Vercel's [Preview Deployments](https://vercel.com/docs/deployments/preview-deployments) feature allows you to preview changes to your app in a live deployment without merging those changes to your Git project's production branch. With [TiDB Serverless Branching](/tidb-cloud/branch-overview.md), you can create a new instance for each branch of your Vercel project. This allows you to preview app changes in a live deployment without affecting your production data. +Vercel's [Preview Deployments](https://vercel.com/docs/deployments/preview-deployments) feature allows you to preview changes to your app in a live deployment without merging those changes to your Git project's production branch. With [TiDB Cloud Serverless Branching](/tidb-cloud/branch-overview.md), you can create a new instance for each branch of your Vercel project. This allows you to preview app changes in a live deployment without affecting your production data. > **Note:** > -> Currently, TiDB Serverless branching only supports [Vercel projects associated with GitHub repositories](https://vercel.com/docs/deployments/git/vercel-for-github). +> Currently, TiDB Cloud Serverless branching only supports [Vercel projects associated with GitHub repositories](https://vercel.com/docs/deployments/git/vercel-for-github). -To enable TiDB Serverless Branching, you need to ensure the following in the [TiDB Cloud Vercel integration workflow](#integration-workflow): +To enable TiDB Cloud Serverless Branching, you need to ensure the following in the [TiDB Cloud Vercel integration workflow](#integration-workflow): 1. Select **Cluster** as your connection type. 2. Enable **Branching** to create new branches for preview environments. -After you push changes to the Git repository, Vercel will trigger a preview deployment. TiDB Cloud integration will automatically create a TiDB Serverless branch for the Git branch and set environment variables. The detailed steps are as follows: +After you push changes to the Git repository, Vercel will trigger a preview deployment. TiDB Cloud integration will automatically create a TiDB Cloud Serverless branch for the Git branch and set environment variables. The detailed steps are as follows: 1. Create a new branch in your Git repository. @@ -194,15 +194,15 @@ After you push changes to the Git repository, Vercel will trigger a preview depl ![Vercel Preview_Deployment](/media/tidb-cloud/vercel/vercel-preview-deployment.png) - 1. During the deployment, TiDB Cloud integration will automatically create a TiDB Serverless branch with the same name as the Git branch. If the TiDB Serverless branch already exists, TiDB Cloud integration will skip this step. + 1. During the deployment, TiDB Cloud integration will automatically create a TiDB Cloud Serverless branch with the same name as the Git branch. If the TiDB Cloud Serverless branch already exists, TiDB Cloud integration will skip this step. ![TiDB_Cloud_Branch_Check](/media/tidb-cloud/vercel/tidbcloud-branch-check.png) - 2. After the TiDB Serverless branch is ready, TiDB Cloud integration will set environment variables in the preview deployment for the Vercel project. + 2. After the TiDB Cloud Serverless branch is ready, TiDB Cloud integration will set environment variables in the preview deployment for the Vercel project. ![Preview_Envs](/media/tidb-cloud/vercel/preview-envs.png) - 3. TiDB Cloud integration will also register a blocking check to wait for the TiDB Serverless branch to be ready. You can rerun the check manually. + 3. TiDB Cloud integration will also register a blocking check to wait for the TiDB Cloud Serverless branch to be ready. You can rerun the check manually. 4. After the check is passed, you can visit the preview deployment to see the changes. > **Note:** @@ -211,7 +211,7 @@ After you push changes to the Git repository, Vercel will trigger a preview depl > **Note:** > -> For each organization in TiDB Cloud, you can create a maximum of five TiDB Serverless branches by default. To avoid exceeding the limit, you can delete the TiDB Serverless branches that are no longer needed. For more information, see [Manage TiDB Serverless branches](/tidb-cloud/branch-manage.md). +> For each organization in TiDB Cloud, you can create a maximum of five TiDB Cloud Serverless branches by default. To avoid exceeding the limit, you can delete the TiDB Cloud Serverless branches that are no longer needed. For more information, see [Manage TiDB Cloud Serverless branches](/tidb-cloud/branch-manage.md). ## Connect via manually setting environment variables @@ -226,7 +226,7 @@ After you push changes to the Git repository, Vercel will trigger a preview depl ![Vercel Environment Variables](/media/tidb-cloud/vercel/integration-vercel-environment-variables.png) -Here we use a Prisma application as an example. The following is a datasource setting in the Prisma schema file for a TiDB Serverless cluster: +Here we use a Prisma application as an example. The following is a datasource setting in the Prisma schema file for a TiDB Cloud Serverless cluster: ``` datasource db { diff --git a/tidb-cloud/integrate-tidbcloud-with-zapier.md b/tidb-cloud/integrate-tidbcloud-with-zapier.md index 57b107dc1548d..b31a1b17d0e94 100644 --- a/tidb-cloud/integrate-tidbcloud-with-zapier.md +++ b/tidb-cloud/integrate-tidbcloud-with-zapier.md @@ -27,7 +27,7 @@ Before you start, you need: - A [Zapier account](https://zapier.com/app/login). - A [GitHub account](https://github.com/login). -- A [TiDB Cloud account](https://tidbcloud.com/signup) and a TiDB Serverless cluster on TiDB Cloud. For more details, see [TiDB Cloud Quick Start](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart#step-1-create-a-tidb-cluster). +- A [TiDB Cloud account](https://tidbcloud.com/signup) and a TiDB Cloud Serverless cluster on TiDB Cloud. For more details, see [TiDB Cloud Quick Start](https://docs.pingcap.com/tidbcloud/tidb-cloud-quickstart#step-1-create-a-tidb-cluster). ### Step 1: Get the template @@ -160,16 +160,16 @@ The following table lists the actions supported by TiDB Cloud App. Note that som | Action | Description | Resource | |---|---|---| -| Find Cluster | Finds an existing TiDB Serverless or TiDB Dedicated cluster. | None | -| Create Cluster | Creates a new cluster. Only supports creating a TiDB Serverless cluster. | None | -| Find Database | Finds an existing database. | A TiDB Serverless cluster | -| Create Database | Creates a new database. | A TiDB Serverless cluster | -| Find Table | Finds an existing Table. | A TiDB Serverless cluster and a database | -| Create Table | Creates a new table. | A TiDB Serverless cluster and a database | -| Create Row | Creates a new row. | A TiDB Serverless cluster, a database, and a table | -| Update Row | Updates an existing row. | A TiDB Serverless cluster, a database, and a table | -| Find Row | Finds a row in a table via a lookup column. | A TiDB Serverless cluster, a database, and a table | -| Find Row (Custom Query) | Finds a row in a table via a custom query the you provide. | A TiDB Serverless cluster, a database, and a table | +| Find Cluster | Finds an existing TiDB Cloud Serverless or TiDB Cloud Dedicated cluster. | None | +| Create Cluster | Creates a new cluster. Only supports creating a TiDB Cloud Serverless cluster. | None | +| Find Database | Finds an existing database. | A TiDB Cloud Serverless cluster | +| Create Database | Creates a new database. | A TiDB Cloud Serverless cluster | +| Find Table | Finds an existing Table. | A TiDB Cloud Serverless cluster and a database | +| Create Table | Creates a new table. | A TiDB Cloud Serverless cluster and a database | +| Create Row | Creates a new row. | A TiDB Cloud Serverless cluster, a database, and a table | +| Update Row | Updates an existing row. | A TiDB Cloud Serverless cluster, a database, and a table | +| Find Row | Finds a row in a table via a lookup column. | A TiDB Cloud Serverless cluster, a database, and a table | +| Find Row (Custom Query) | Finds a row in a table via a custom query the you provide. | A TiDB Cloud Serverless cluster, a database, and a table | ## TiDB Cloud App templates diff --git a/tidb-cloud/key-concepts.md b/tidb-cloud/key-concepts.md new file mode 100644 index 0000000000000..436cb5b068e6f --- /dev/null +++ b/tidb-cloud/key-concepts.md @@ -0,0 +1,59 @@ +--- +title: Overview of Key Concepts +summary: Learn about the key concepts in TiDB Cloud. +--- + +# Overview of Key Concepts + +This document provides an overview of the key concepts in [TiDB Cloud](https://www.pingcap.com/tidb-cloud/). Understanding these concepts helps you better use TiDB Cloud features and capabilities. + +## Architecture + +TiDB Cloud is built on a cloud-native distributed architecture that separates computing from storage, enabling elastic scaling and high availability. [Learn more about TiDB Cloud architecture](/tidb-cloud/architecture-concepts.md). + +## Database schema + +TiDB Cloud enables you to organize and structure your data using objects such as databases, tables, columns, indexes, and constraints. It also supports advanced features such as temporary tables, vector indexes, and cached tables. [Learn more about database schema](/tidb-cloud/database-schema-concepts.md). + +## Transactions + +TiDB provides complete distributed transactions, and the model has some optimizations on the basis of [Google Percolator](https://research.google.com/pubs/pub36726.html). [Learn more about transactions](/tidb-cloud/transaction-concepts.md). + +## SQL + +TiDB is highly compatible with the MySQL protocol and the common features and syntax of MySQL 5.7 and MySQL 8.0. [Learn more about SQL in TiDB Cloud](/tidb-cloud/sql-concepts.md). + +## AI features + +The AI features in TiDB Cloud enable you to fully leverage advanced technologies for data exploration, search, and integration. [Learn more about AI features](/tidb-cloud/ai-feature-concepts.md). + +## Data Service (Beta) + +Data Service enables you to access TiDB Cloud data via an HTTPS request using a custom API endpoint. [Learn more about Data Service](/tidb-cloud/data-service-concepts.md). + +## Scalability + +TiDB Cloud Dedicated lets you adjust its compute and storage resources separately to match your data volume or workload changes. [Learn more about scalability](/tidb-cloud/scalability-concepts.md). + +## High availability + +TiDB Cloud ensures high availability in both TiDB Cloud Serverless and TiDB Cloud Dedicated clusters: + +- [High Availability in TiDB Cloud Serverless](/tidb-cloud/serverless-high-availability.md) +- [High Availability in TiDB Cloud Dedicated](/tidb-cloud/high-availability-with-multi-az.md) + +## Monitoring + +TiDB Cloud provides comprehensive monitoring capabilities for cluster performance and health. [Learn more about monitoring](/tidb-cloud/monitoring-concepts.md). + +## Data streaming + +TiDB Cloud lets you stream data changes from your TiDB Cluster to other systems such as Kafka, MySQL, and object storage. [Learn more about data streaming](/tidb-cloud/data-streaming-concepts.md). + +## Backup & Restore + +TiDB Cloud offers automated backup solutions and point-in-time recovery (PITR) capabilities. [Learn more about backup and restore](/tidb-cloud/backup-and-restore-concepts.md). + +## Security + +TiDB Cloud provides a robust and flexible security framework designed to protect data, enforce access control, and meet modern compliance standards. [Learn more about security](/tidb-cloud/security-concepts.md). \ No newline at end of file diff --git a/tidb-cloud/limitations-and-quotas.md b/tidb-cloud/limitations-and-quotas.md index 77956a759c0e3..04b52ccaff92b 100644 --- a/tidb-cloud/limitations-and-quotas.md +++ b/tidb-cloud/limitations-and-quotas.md @@ -1,11 +1,11 @@ --- -title: TiDB Dedicated Limitations and Quotas +title: TiDB Cloud Dedicated Limitations and Quotas summary: Learn the limitations and quotas in TiDB Cloud. --- -# TiDB Dedicated Limitations and Quotas +# TiDB Cloud Dedicated Limitations and Quotas -TiDB Cloud limits how many of each kind of component you can create in a [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) cluster, and the common usage limitations of TiDB. In addition, there are some organization-level quotas to limit the amount of resources created by users to prevent from creating more resources than you actually need. These tables outline limits and quotas. +TiDB Cloud limits how many of each kind of component you can create in a [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) cluster, and the common usage limitations of TiDB. In addition, there are some organization-level quotas to limit the amount of resources created by users to prevent from creating more resources than you actually need. These tables outline limits and quotas. > **Note:** > @@ -15,7 +15,7 @@ TiDB Cloud limits how many of each kind of component you can create in a [TiDB D | Component | Limit | |:-|:-| -| Number of data replicas | 3 | +| Number of copies for each [data region](/tidb-cloud/tidb-cloud-glossary.md#region) | 3 | | Number of Availability Zones for a cross-zone deployment | 3 | > **Note:** @@ -29,3 +29,7 @@ TiDB Cloud limits how many of each kind of component you can create in a [TiDB D | Maximum number of total TiDB nodes for all clusters in your organization | 10 | | Maximum number of total TiKV nodes for all clusters in your organization | 15 | | Maximum number of total TiFlash nodes for all clusters in your organization | 5 | + +> **Note:** +> +> If any of these limits or quotas present a problem for your organization, please contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). diff --git a/tidb-cloud/limited-sql-features.md b/tidb-cloud/limited-sql-features.md index e5b521bd3b427..537186a13a923 100644 --- a/tidb-cloud/limited-sql-features.md +++ b/tidb-cloud/limited-sql-features.md @@ -5,13 +5,13 @@ summary: Learn about the limited SQL features on TiDB Cloud. # Limited SQL features on TiDB Cloud -TiDB Cloud works with almost all workloads that TiDB supports, but there are some feature differences between TiDB Self-Hosted and TiDB Dedicated/Serverless. This document describes the limitations of SQL features on TiDB Cloud. We are constantly filling in the feature gaps between TiDB Self-Hosted and TiDB Dedicated/Serverless. If you require these features or capabilities in the gap, [contact us](/tidb-cloud/tidb-cloud-support.md) for a feature request. +TiDB Cloud works with almost all workloads that TiDB supports, but there are some feature differences between TiDB Self-Managed and TiDB Cloud Dedicated/Serverless. This document describes the limitations of SQL features on TiDB Cloud. We are constantly filling in the feature gaps between TiDB Self-Managed and TiDB Cloud Dedicated/Serverless. If you require these features or capabilities in the gap, [contact us](/tidb-cloud/tidb-cloud-support.md) for a feature request. ## Statements ### Placement and range management -| Statement | TiDB Dedicated | TiDB Serverless | +| Statement | TiDB Cloud Dedicated | TiDB Cloud Serverless | |:-|:-|:-| | `ALTER PLACEMENT POLICY` | Supported | Not supported [^1] | | `CREATE PLACEMENT POLICY` | Supported | Not supported [^1] | @@ -25,7 +25,7 @@ TiDB Cloud works with almost all workloads that TiDB supports, but there are som ### Resource groups -| Statement | TiDB Dedicated | TiDB Serverless | +| Statement | TiDB Cloud Dedicated | TiDB Cloud Serverless | |:-|:-|:-| | `ALTER RESOURCE GROUP` | Supported | Not supported [^2] | | `CALIBRATE RESOURCE` | Not supported | Not supported [^2] | @@ -36,19 +36,19 @@ TiDB Cloud works with almost all workloads that TiDB supports, but there are som ### Others -| Statement | TiDB Dedicated | TiDB Serverless | +| Statement | TiDB Cloud Dedicated | TiDB Cloud Serverless | |:-|:-|:-| | `BACKUP` | Supported | Not supported [^3] | | `SHOW BACKUPS` | Supported | Not supported [^3] | | `RESTORE` | Supported | Not supported [^3] | | `SHOW RESTORES` | Supported | Not supported [^3] | -| `ADMIN RESET TELEMETRY_ID` | Supported | Telemetry is not supported on TiDB Serverless. | +| `ADMIN RESET TELEMETRY_ID` | Supported | Telemetry is not supported on TiDB Cloud Serverless. | | `ADMIN SHOW TELEMETRY` | Not supported [^4] | Not supported [^4] | | `ADMIN SHOW SLOW` | Supported | Not supported [^5] | | `ADMIN PLUGINS ENABLE` | Supported | Not supported [^8] | | `ADMIN PLUGINS DISABLE` | Supported | Not supported [^8] | -| `ALTER INSTANCE RELOAD TLS` | Supported | TiDB Serverless automatically refreshes the TLS certificate. | -| `LOAD DATA INFILE` | Only supports `LOAD DATA LOCAL INFILE` | Only supports `LOAD DATA LOCAL INFILE` | +| `ALTER INSTANCE RELOAD TLS` | Supported | TiDB Cloud Serverless automatically refreshes the TLS certificate. | +| `LOAD DATA INFILE` | Supports `LOAD DATA LOCAL INFILE`, and `LOAD DATA INFILE` from Amazon S3 or Google Cloud Storage | Only supports `LOAD DATA LOCAL INFILE` | | `CHANGE DRAINER` | Not supported [^7] | Not supported [^7] | | `CHANGE PUMP` | Not supported [^7] | Not supported [^7] | | `FLASHBACK CLUSTER` | Supported | Not supported [^3] | @@ -60,16 +60,17 @@ TiDB Cloud works with almost all workloads that TiDB supports, but there are som | `SHOW PLUGINS` | Supported | Not supported [^8] | | `SHOW PUMP STATUS` | Not supported [^7] | Not supported [^7] | | `SHUTDOWN` | Not supported [^4] | Not supported [^4] | +| `PLAN REPLAYER` | Supported | Supported in a different way[^11] | ## Functions and operators -| Function and operator | TiDB Dedicated | TiDB Serverless | +| Function and operator | TiDB Cloud Dedicated | TiDB Cloud Serverless | |:-|:-|:-| | `SLEEP` | No Limitation | The [`SLEEP()` function](https://docs.pingcap.com/tidbcloud/miscellaneous-functions) has a limitation wherein it can only support a maximum sleep time of 300 seconds.| ## System tables -| Database | Table | TiDB Dedicated | TiDB Serverless | +| Database | Table | TiDB Cloud Dedicated | TiDB Cloud Serverless | |:-|:-|:-|:-| | `information_schema` | `ATTRIBUTES` | Supported | Not supported [^1] | | `information_schema` | `CLUSTER_CONFIG` | Not supported [^4] | Not supported [^4] | @@ -97,8 +98,6 @@ TiDB Cloud works with almost all workloads that TiDB supports, but there are som | `information_schema` | `TIDB_HOT_REGIONS` | Not supported [^4] | Not supported [^4] | | `information_schema` | `TIDB_HOT_REGIONS_HISTORY` | Supported | Not supported [^1] | | `information_schema` | `TIDB_SERVERS_INFO` | Supported | Not supported [^1] | -| `information_schema` | `TIFLASH_SEGMENTS` | Supported | Not supported [^1] | -| `information_schema` | `TIFLASH_TABLES` | Supported | Not supported [^1] | | `information_schema` | `TIKV_REGION_PEERS` | Supported | Not supported [^1] | | `information_schema` | `TIKV_REGION_STATUS` | Supported | Not supported [^1] | | `information_schema` | `TIKV_STORE_STATUS` | Supported | Not supported [^1] | @@ -120,129 +119,124 @@ TiDB Cloud works with almost all workloads that TiDB supports, but there are som | `mysql` | `gc_delete_range_done` | Not supported [^4] | Not supported [^4] | | `mysql` | `opt_rule_blacklist` | Not supported [^4] | Not supported [^4] | | `mysql` | `tidb` | Not supported [^4] | Not supported [^4] | -| `mysql` | `tidb_ttl_job_history` | Supported | Not supported [^9] | -| `mysql` | `tidb_ttl_table_status` | Supported | Not supported [^9] | -| `mysql` | `tidb_ttl_task` | Supported | Not supported [^9] | ## System variables -| Variable | TiDB Dedicated | TiDB Serverless | +| Variable | TiDB Cloud Dedicated | TiDB Cloud Serverless | |:-|:-|:-| | `datadir` | No limitation | Not supported [^1] | -| `interactive_timeout` | No limitation | Read-only [^11] | -| `max_allowed_packet` | No limitation | Read-only [^11] | +| `interactive_timeout` | No limitation | Read-only [^10] | +| `max_allowed_packet` | No limitation | Read-only [^10] | | `plugin_dir` | No limitation | Not supported [^8] | | `plugin_load` | No limitation | Not supported [^8] | -| `require_secure_transport` | Not supported [^12] | Read-only [^11] | -| `skip_name_resolve` | No limitation | Read-only [^11] | -| `sql_log_bin` | No limitation | Read-only [^11] | -| `tidb_cdc_write_source` | No limitation | Read-only [^11] | +| `require_secure_transport` | Not supported [^12] | Read-only [^10] | +| `skip_name_resolve` | No limitation | Read-only [^10] | +| `sql_log_bin` | No limitation | Read-only [^10] | +| `tidb_cdc_write_source` | No limitation | Read-only [^10] | | `tidb_check_mb4_value_in_utf8` | Not supported [^4] | Not supported [^4] | | `tidb_config` | Not supported [^4] | Not supported [^4] | -| `tidb_ddl_disk_quota` | No limitation | Read-only [^11] | -| `tidb_ddl_enable_fast_reorg` | No limitation | Read-only [^11] | -| `tidb_ddl_error_count_limit` | No limitation | Read-only [^11] | -| `tidb_ddl_flashback_concurrency` | No limitation | Read-only [^11] | -| `tidb_ddl_reorg_batch_size` | No limitation | Read-only [^11] | -| `tidb_ddl_reorg_priority` | No limitation | Read-only [^11] | -| `tidb_ddl_reorg_worker_cnt` | No limitation | Read-only [^11] | -| `tidb_enable_1pc` | No limitation | Read-only [^11] | -| `tidb_enable_async_commit` | No limitation | Read-only [^11] | -| `tidb_enable_auto_analyze` | No limitation | Read-only [^11] | +| `tidb_ddl_disk_quota` | No limitation | Read-only [^10] | +| `tidb_ddl_enable_fast_reorg` | No limitation | Read-only [^10] | +| `tidb_ddl_error_count_limit` | No limitation | Read-only [^10] | +| `tidb_ddl_flashback_concurrency` | No limitation | Read-only [^10] | +| `tidb_ddl_reorg_batch_size` | No limitation | Read-only [^10] | +| `tidb_ddl_reorg_priority` | No limitation | Read-only [^10] | +| `tidb_ddl_reorg_worker_cnt` | No limitation | Read-only [^10] | +| `tidb_enable_1pc` | No limitation | Read-only [^10] | +| `tidb_enable_async_commit` | No limitation | Read-only [^10] | +| `tidb_enable_auto_analyze` | No limitation | Read-only [^10] | | `tidb_enable_collect_execution_info` | Not supported [^4] | Not supported [^4] | -| `tidb_enable_ddl` | No limitation | Read-only [^11] | -| `tidb_enable_gc_aware_memory_track` | No limitation | Read-only [^11] | -| `tidb_enable_gogc_tuner` | No limitation | Read-only [^11] | -| `tidb_enable_local_txn` | No limitation | Read-only [^11] | -| `tidb_enable_resource_control` | No limitation | Read-only [^11] | +| `tidb_enable_ddl` | No limitation | Read-only [^10] | +| `tidb_enable_gc_aware_memory_track` | No limitation | Read-only [^10] | +| `tidb_enable_gogc_tuner` | No limitation | Read-only [^10] | +| `tidb_enable_local_txn` | No limitation | Read-only [^10] | +| `tidb_enable_resource_control` | No limitation | Read-only [^10] | | `tidb_enable_slow_log` | Not supported [^4] | Not supported [^4] | -| `tidb_enable_stmt_summary` | No limitation | Read-only [^11] | +| `tidb_enable_stmt_summary` | No limitation | Read-only [^10] | | `tidb_enable_telemetry` | Not supported [^4] | Not supported [^4] | -| `tidb_enable_top_sql` | No limitation | Read-only [^11] | -| `tidb_enable_tso_follower_proxy` | No limitation | Read-only [^11] | +| `tidb_enable_top_sql` | No limitation | Read-only [^10] | +| `tidb_enable_tso_follower_proxy` | No limitation | Read-only [^10] | | `tidb_expensive_query_time_threshold` | Not supported [^4] | Not supported [^4] | | `tidb_force_priority` | Not supported [^4] | Not supported [^4] | -| `tidb_gc_concurrency` | No limitation | Read-only [^11] | -| `tidb_gc_enable` | No limitation | Read-only [^11] | -| `tidb_gc_life_time` | No limitation | Read-only [^11] | -| `tidb_gc_max_wait_time` | No limitation | Read-only [^11] | -| `tidb_gc_run_interval` | No limitation | Read-only [^11] | -| `tidb_gc_scan_lock_mode` | No limitation | Read-only [^11] | +| `tidb_gc_concurrency` | No limitation | Read-only [^10] | +| `tidb_gc_enable` | No limitation | Read-only [^10] | +| `tidb_gc_max_wait_time` | No limitation | Read-only [^10] | +| `tidb_gc_run_interval` | No limitation | Read-only [^10] | +| `tidb_gc_scan_lock_mode` | No limitation | Read-only [^10] | | `tidb_general_log` | Not supported [^4] | Not supported [^4] | -| `tidb_generate_binary_plan` | No limitation | Read-only [^11] | -| `tidb_gogc_tuner_threshold` | No limitation | Read-only [^11] | -| `tidb_guarantee_linearizability` | No limitation | Read-only [^11] | -| `tidb_isolation_read_engines` | No limitation | Read-only [^11] | -| `tidb_log_file_max_days` | No limitation | Read-only [^11] | +| `tidb_generate_binary_plan` | No limitation | Read-only [^10] | +| `tidb_gogc_tuner_threshold` | No limitation | Read-only [^10] | +| `tidb_guarantee_linearizability` | No limitation | Read-only [^10] | +| `tidb_isolation_read_engines` | No limitation | Read-only [^10] | +| `tidb_log_file_max_days` | No limitation | Read-only [^10] | | `tidb_memory_usage_alarm_ratio` | Not supported [^4] | Not supported [^4] | | `tidb_metric_query_range_duration` | Not supported [^4] | Not supported [^4] | | `tidb_metric_query_step` | Not supported [^4] | Not supported [^4] | | `tidb_opt_write_row_id` | Not supported [^4] | Not supported [^4] | -| `tidb_placement_mode` | No limitation | Read-only [^11] | +| `tidb_placement_mode` | No limitation | Read-only [^10] | | `tidb_pprof_sql_cpu` | Not supported [^4] | Not supported [^4] | | `tidb_record_plan_in_slow_log` | Not supported [^4] | Not supported [^4] | | `tidb_redact_log` | Not supported [^4] | Not supported [^4] | | `tidb_restricted_read_only` | Not supported [^4] | Not supported [^4] | | `tidb_row_format_version` | Not supported [^4] | Not supported [^4] | -| `tidb_scatter_region` | No limitation | Read-only [^11] | -| `tidb_server_memory_limit` | No limitation | Read-only [^11] | -| `tidb_server_memory_limit_gc_trigger` | No limitation | Read-only [^11] | -| `tidb_server_memory_limit_sess_min_size` | No limitation | Read-only [^11] | -| `tidb_simplified_metrics` | No limitation | Read-only [^11] | +| `tidb_scatter_region` | No limitation | Read-only [^10] | +| `tidb_server_memory_limit` | No limitation | Read-only [^10] | +| `tidb_server_memory_limit_gc_trigger` | No limitation | Read-only [^10] | +| `tidb_server_memory_limit_sess_min_size` | No limitation | Read-only [^10] | +| `tidb_simplified_metrics` | No limitation | Read-only [^10] | | `tidb_slow_query_file` | Not supported [^4] | Not supported [^4] | | `tidb_slow_log_threshold` | Not supported [^4] | Not supported [^4] | | `tidb_slow_txn_log_threshold` | Not supported [^4] | Not supported [^4] | -| `tidb_stats_load_sync_wait` | No limitation | Read-only [^11] | -| `tidb_stmt_summary_history_size` | No limitation | Read-only [^11] | -| `tidb_stmt_summary_internal_query` | No limitation | Read-only [^11] | -| `tidb_stmt_summary_max_sql_length` | No limitation | Read-only [^11] | -| `tidb_stmt_summary_max_stmt_count` | No limitation | Read-only [^11] | -| `tidb_stmt_summary_refresh_interval` | No limitation | Read-only [^11] | -| `tidb_sysproc_scan_concurrency` | No limitation | Read-only [^11] | +| `tidb_stats_load_sync_wait` | No limitation | Read-only [^10] | +| `tidb_stmt_summary_history_size` | No limitation | Read-only [^10] | +| `tidb_stmt_summary_internal_query` | No limitation | Read-only [^10] | +| `tidb_stmt_summary_max_sql_length` | No limitation | Read-only [^10] | +| `tidb_stmt_summary_max_stmt_count` | No limitation | Read-only [^10] | +| `tidb_stmt_summary_refresh_interval` | No limitation | Read-only [^10] | +| `tidb_sysproc_scan_concurrency` | No limitation | Read-only [^10] | | `tidb_top_sql_max_meta_count` | Not supported [^4] | Not supported [^4] | | `tidb_top_sql_max_time_series_count` | Not supported [^4] | Not supported [^4] | -| `tidb_tso_client_batch_max_wait_time` | No limitation | Read-only [^11] | -| `tidb_ttl_delete_batch_size` | No limitation | Read-only [^9] | -| `tidb_ttl_delete_rate_limit` | No limitation | Read-only [^9] | -| `tidb_ttl_delete_worker_count` | No limitation | Read-only [^9] | -| `tidb_ttl_job_enable` | No limitation | Read-only [^9] | -| `tidb_ttl_job_schedule_window_end_time` | No limitation | Read-only [^9] | -| `tidb_ttl_job_schedule_window_start_time` | No limitation | Read-only [^9] | -| `tidb_ttl_running_tasks` | No limitation | Read-only [^9] | -| `tidb_ttl_scan_batch_size` | No limitation | Read-only [^9] | -| `tidb_ttl_scan_worker_count` | No limitation | Read-only [^9] | -| `tidb_txn_mode` | No limitation | Read-only [^11] | -| `tidb_wait_split_region_finish` | No limitation | Read-only [^11] | -| `tidb_wait_split_region_timeout` | No limitation | Read-only [^11] | -| `txn_scope` | No limitation | Read-only [^11] | -| `validate_password.enable` | No limitation | Always enabled [^10] | -| `validate_password.length` | No limitation | At least `8` [^10] | -| `validate_password.mixed_case_count` | No limitation | At least `1` [^10] | -| `validate_password.number_count` | No limitation | At least `1` [^10] | -| `validate_password.policy` | No limitation | Can only be `MEDIUM` or `STRONG` [^10] | -| `validate_password.special_char_count` | No limitation | At least `1` [^10] | -| `wait_timeout` | No limitation | Read-only [^11] | - -[^1]: Configuring data placement is not supported on TiDB Serverless. - -[^2]: Configuring resource groups is not supported on TiDB Serverless. - -[^3]: To perform [Back up and Restore](/tidb-cloud/backup-and-restore-serverless.md) operations on TiDB Serverless, you can use the TiDB Cloud console instead. +| `tidb_tso_client_batch_max_wait_time` | No limitation | Read-only [^10] | +| `tidb_ttl_delete_batch_size` | No limitation | Read-only [^10] | +| `tidb_ttl_delete_rate_limit` | No limitation | Read-only [^10] | +| `tidb_ttl_delete_worker_count` | No limitation | Read-only [^10] | +| `tidb_ttl_job_schedule_window_end_time` | No limitation | Read-only [^10] | +| `tidb_ttl_job_schedule_window_start_time` | No limitation | Read-only [^10] | +| `tidb_ttl_running_tasks` | No limitation | Read-only [^10] | +| `tidb_ttl_scan_batch_size` | No limitation | Read-only [^10] | +| `tidb_ttl_scan_worker_count` | No limitation | Read-only [^10] | +| `tidb_txn_mode` | No limitation | Read-only [^10] | +| `tidb_wait_split_region_finish` | No limitation | Read-only [^10] | +| `tidb_wait_split_region_timeout` | No limitation | Read-only [^10] | +| `txn_scope` | No limitation | Read-only [^10] | +| `validate_password.enable` | No limitation | Always enabled [^9] | +| `validate_password.length` | No limitation | At least `8` [^9] | +| `validate_password.mixed_case_count` | No limitation | At least `1` [^9] | +| `validate_password.number_count` | No limitation | At least `1` [^9] | +| `validate_password.policy` | No limitation | Can only be `MEDIUM` or `STRONG` [^9] | +| `validate_password.special_char_count` | No limitation | At least `1` [^9] | +| `wait_timeout` | No limitation | Read-only [^10] | + +[^1]: Configuring data placement is not supported on TiDB Cloud Serverless. + +[^2]: Configuring resource groups is not supported on TiDB Cloud Serverless. + +[^3]: To perform [Back up and Restore](/tidb-cloud/backup-and-restore-serverless.md) operations on TiDB Cloud Serverless, you can use the TiDB Cloud console instead. [^4]: The feature is unavailable in [Security Enhanced Mode (SEM)](/system-variables.md#tidb_enable_enhanced_security). -[^5]: To track [Slow Query](/tidb-cloud/tune-performance.md#slow-query) on TiDB Serverless, you can use the TiDB Cloud console instead. +[^5]: To track [Slow Query](/tidb-cloud/tune-performance.md#slow-query) on TiDB Cloud Serverless, you can use the TiDB Cloud console instead. -[^6]: To perform [Statement Analysis](/tidb-cloud/tune-performance.md#statement-analysis) on TiDB Serverless, you can use the TiDB Cloud console instead. +[^6]: To perform [Statement Analysis](/tidb-cloud/tune-performance.md#statement-analysis) on TiDB Cloud Serverless, you can use the TiDB Cloud console instead. [^7]: Drainer and Pump are not supported on TiDB Cloud. -[^8]: Plugin is not supported on TiDB Serverless. +[^8]: Plugin is not supported on TiDB Cloud Serverless. -[^9]: [Time to live (TTL)](/time-to-live.md) is currently unavailable on TiDB Serverless. +[^9]: TiDB Cloud Serverless enforces strong password policy. -[^10]: TiDB Serverless enforces strong password policy. +[^10]: The variable is read-only on TiDB Cloud Serverless. -[^11]: The variable is read-only on TiDB Serverless. +[^11]: TiDB Cloud Serverless does not support downloading the file exported by `PLAN REPLAYER` through `${tidb-server-status-port}` as in the [example](https://docs.pingcap.com/tidb/stable/sql-plan-replayer#examples-of-exporting-cluster-information). Instead, TiDB Cloud Serverless generates a [presigned URL](https://docs.aws.amazon.com/AmazonS3/latest/userguide/ShareObjectPreSignedURL.html) for you to download the file. Note that this URL remains valid for 10 hours after generation. -[^12]: Not supported. Enabling `require_secure_transport` for TiDB Dedicated clusters will result in SQL client connection failures. +[^12]: Not supported. Enabling `require_secure_transport` for TiDB Cloud Dedicated clusters will result in SQL client connection failures. diff --git a/tidb-cloud/manage-serverless-spend-limit.md b/tidb-cloud/manage-serverless-spend-limit.md index 1a3e130056d03..7e79ddc8c4eff 100644 --- a/tidb-cloud/manage-serverless-spend-limit.md +++ b/tidb-cloud/manage-serverless-spend-limit.md @@ -1,34 +1,37 @@ --- -title: Manage Spending Limit for TiDB Serverless clusters -summary: Learn how to manage spending limit for your TiDB Serverless clusters. +title: Manage Spending Limit for TiDB Cloud Serverless Scalable Clusters +summary: Learn how to manage spending limit for your TiDB Cloud Serverless scalable clusters. --- -# Manage Spending Limit for TiDB Serverless Clusters +# Manage Spending Limit for TiDB Cloud Serverless Scalable Clusters > **Note:** > -> The spending limit is only applicable to TiDB Serverless clusters. +> The spending limit is only applicable to TiDB Cloud Serverless [scalable clusters](/tidb-cloud/select-cluster-tier.md#scalable-cluster-plan). -Spending limit refers to the maximum amount of money that you are willing to spend on a particular workload in a month. It is a cost-control mechanism that allows you to set a budget for your TiDB Serverless clusters. +Spending limit refers to the maximum amount of money that you are willing to spend on a particular workload in a month. It is a cost-control mechanism that allows you to set a budget for your TiDB Cloud Serverless scalable clusters. -For each organization in TiDB Cloud, you can create a maximum of five TiDB Serverless clusters by default. To create more TiDB Serverless clusters, you need to add a credit card and set a spending limit for the usage. But if you delete some of your previous clusters before creating more, the new cluster can still be created without a credit card. +For each organization in TiDB Cloud, you can create a maximum of five [free clusters](/tidb-cloud/select-cluster-tier.md#free-cluster-plan) by default. To create more TiDB Cloud Serverless clusters, you need to add a credit card and create scalable clusters for the usage. But if you delete some of your previous clusters before creating more, the new cluster can still be created without a credit card. ## Usage quota -For the first five TiDB Serverless clusters in your organization, TiDB Cloud provides a free usage quota for each of them as follows: +For the first five TiDB Cloud Serverless clusters in your organization, whether they are free or scalable, TiDB Cloud provides a free usage quota for each of them as follows: - Row-based storage: 5 GiB +- Columnar storage: 5 GiB - [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit): 50 million RUs per month -Once the free quota of a cluster is reached, the read and write operations on this cluster will be throttled until you [increase the quota](#update-spending-limit) or the usage is reset upon the start of a new month. For example, when the storage of a cluster exceeds 5 GiB, the maximum size limit of a single transaction is reduced from 10 MiB to 1 MiB. +Once a cluster reaches its usage quota, it immediately denies any new connection attempts until you [increase the quota](#update-spending-limit) or the usage is reset upon the start of a new month. Existing connections established before reaching the quota will remain active but will experience throttling. For example, when the row-based storage of a cluster exceeds 5 GiB for a free cluster, the cluster automatically restricts any new connection attempts. -To learn more about the RU consumption of different resources (including read, write, SQL CPU, and network egress), the pricing details, and the throttled information, see [TiDB Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). +To learn more about the RU consumption of different resources (including read, write, SQL CPU, and network egress), the pricing details, and the throttled information, see [TiDB Cloud Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). -If you want to create a TiDB Serverless cluster with an additional quota, you can edit the spending limit on the cluster creation page. For more information, see [Create a TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md). +If you want to create a TiDB Cloud Serverless cluster with an additional quota, you can edit the spending limit on the cluster creation page. For more information, see [Create a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md). ## Update spending limit -For an existing TiDB Serverless cluster, you can increase the usage quota by updating the spending limit as follows: +For a TiDB Cloud Serverless free cluster, you can increase the usage quota by upgrading it to a scalable cluster. For an existing scalable cluster, you can adjust the monthly spending limit directly. + +To update the spending limit for a TiDB Cloud Serverless cluster, perform the following steps: 1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, click the name of your target cluster to go to its overview page. @@ -36,9 +39,9 @@ For an existing TiDB Serverless cluster, you can increase the usage quota by upd > > If you have multiple projects, you can click in the lower-left corner and switch to another project. -2. In the **Usage This Month** area, click **Get more usage quota**. +2. In the **Usage This Month** area, click **Upgrade to Scalable Cluster**. - If you have previously updated the spending limit for a cluster and want to increase it further, click **Edit**. + To adjust the spending limit for an existing scalable cluster, click **Edit**. 3. Edit the monthly spending limit as needed. If you have not added a payment method, you will need to add a credit card after editing the limit. -4. Click **Update Spending Limit**. +4. Click **Update Cluster Plan**. diff --git a/tidb-cloud/manage-user-access.md b/tidb-cloud/manage-user-access.md index d3f87b9a27728..effc466b8bc4f 100644 --- a/tidb-cloud/manage-user-access.md +++ b/tidb-cloud/manage-user-access.md @@ -72,19 +72,21 @@ You can grant roles to a user at the organization level or at the project level. At the organization level, TiDB Cloud defines four roles, in which `Organization Owner` can invite members and grant organization roles to members. -| Permission | `Organization Owner` | `Organization Billing Admin` | `Organization Console Audit Admin` | `Organization Member` | -|---|---|---|---|---| -| Manage organization settings, such as projects, API keys, and time zones. | ✅ | ❌ | ❌ | ❌ | -| Invite users to or remove users from an organization, and edit organization roles of users. | ✅ | ❌ | ❌ | ❌ | -| All the permissions of `Project Owner` for all projects in the organization. | ✅ | ❌ | ❌ | ❌ | -| Create projects with Customer-Managed Encryption Key (CMEK) enabled | ✅ | ❌ | ❌ | ❌ | -| View bills and edit payment information for the organization. | ✅ | ✅ | ❌ | ❌ | -| Manage TiDB Cloud [console audit logging](/tidb-cloud/tidb-cloud-console-auditing.md) for the organization. | ✅ | ❌ | ✅ | ❌ | -| View users in the organization and projects in which the member belong to. | ✅ | ✅ | ✅ | ✅ | +| Permission | `Organization Owner` | `Organization Billing Manager` | `Organization Billing Viewer` | `Organization Console Audit Manager` | `Organization Viewer` | +|---|---|---|---|---|---| +| Manage organization settings, such as projects, API keys, and time zones. | ✅ | ❌ | ❌ | ❌ | ❌ | +| Invite users to or remove users from an organization, and edit organization roles of users. | ✅ | ❌ | ❌ | ❌ | ❌ | +| All the permissions of `Project Owner` for all projects in the organization. | ✅ | ❌ | ❌ | ❌ | ❌ | +| Create projects with Customer-Managed Encryption Key (CMEK) enabled. | ✅ | ❌ | ❌ | ❌ | ❌ | +| Edit payment information for the organization. | ✅ | ✅ | ❌ | ❌ | ❌ | +| View bills and use [cost explorer](/tidb-cloud/tidb-cloud-billing.md#cost-explorer). | ✅ | ✅ | ✅ | ❌ | ❌ | +| Manage TiDB Cloud [console audit logging](/tidb-cloud/tidb-cloud-console-auditing.md) for the organization. | ✅ | ❌ | ❌ | ✅ | ❌ | +| View users in the organization and projects in which the member belong to. | ✅ | ✅ | ✅ | ✅ | ✅ | > **Note:** > -> The `Organization Console Audit Admin` role is used to manage audit logging in the TiDB Cloud console, instead of database audit logging. To manage database auditing, use the `Project Owner` role at the project level. +> - The `Organization Console Audit Manager` role (renamed from `Organization Console Audit Admin`) is used to manage audit logging in the TiDB Cloud console, instead of database audit logging. To manage database auditing, use the `Project Owner` role at the project level. +> - The `Organization Billing Manager` role is renamed from `Organization Billing Admin`, and the `Organization Viewer` role is renamed from `Organization Member`. ### Project roles @@ -93,23 +95,26 @@ At the project level, TiDB Cloud defines three roles, in which `Project Owner` c > **Note:** > > - `Organization Owner` has all the permissions of Project Owner for all projects so `Organization Owner` can invite project members and grant project roles to members too. -> - Each project role has all the permissions of Organization Member by default. +> - Each project role has all the permissions of Organization Viewer by default. > - If a user in your organization does not belong to any projects, the user does not have any project permissions. -| Permission | `Project Owner` | `Project Data Access Read-Write` | `Project Data Access Read-Only` | -|---|---|---|---| -| Manage project settings | ✅ | ❌ | ❌ | -| Invite users to or remove users from a project, and edit project roles of users. | ✅ | ❌ | ❌ | -| Manage [database audit logging](/tidb-cloud/tidb-cloud-auditing.md) of the project. | ✅ | ❌ | ❌ | -| Manage [spending limit](/tidb-cloud/manage-serverless-spend-limit.md) for all TiDB Serverless clusters in the project. | ✅ | ❌ | ❌ | -| Manage cluster operations in the project, such as cluster creation, modification, and deletion. | ✅ | ❌ | ❌ | -| Manage branches for TiDB Serverless clusters in the project, such as branch creation, connection, and deletion. | ✅ | ❌ | ❌ | -| Manage cluster data such as data import, data backup and restore, and data migration. | ✅ | ✅ | ❌ | -| Manage [Data Service](/tidb-cloud/data-service-overview.md) for data read-only operations such as using or creating endpoints to read data. | ✅ | ✅ | ✅ | -| Manage [Data Service](/tidb-cloud/data-service-overview.md) for data read and write operations. | ✅ | ✅ | ❌ | -| View cluster data using [Chat2Query](/tidb-cloud/explore-data-with-chat2query.md). | ✅ | ✅ | ✅ | -| Modify and delete cluster data using [Chat2Query](/tidb-cloud/explore-data-with-chat2query.md). | ✅ | ✅ | ❌ | -| View clusters in the project, view cluster backup records, and manage [changefeeds](/tidb-cloud/changefeed-overview.md). | ✅ | ✅ | ✅ | +| Permission | `Project Owner` | `Project Data Access Read-Write` | `Project Data Access Read-Only` | `Project Viewer` | +|---|---|---|---|---| +| Manage project settings | ✅ | ❌ | ❌ | ❌ | +| Invite users to or remove users from a project, and edit project roles of users. | ✅ | ❌ | ❌ | ❌ | +| Manage [database audit logging](/tidb-cloud/tidb-cloud-auditing.md) of the project. | ✅ | ❌ | ❌ | ❌ | +| Manage [spending limit](/tidb-cloud/manage-serverless-spend-limit.md) for all TiDB Cloud Serverless clusters in the project. | ✅ | ❌ | ❌ | ❌ | +| Manage cluster operations in the project, such as cluster creation, modification, and deletion. | ✅ | ❌ | ❌ | ❌ | +| Manage branches for TiDB Cloud Serverless clusters in the project, such as branch creation, connection, and deletion. | ✅ | ❌ | ❌ | ❌ | +| Manage [recovery groups](/tidb-cloud/recovery-group-overview.md) for TiDB Cloud Dedicated clusters in the project, such as recovery group creation and deletion. | ✅ | ❌ | ❌ | ❌ | +| Manage cluster data such as data import, data backup and restore, and data migration. | ✅ | ✅ | ❌ | ❌ | +| Manage [Data Service](/tidb-cloud/data-service-overview.md) for data read-only operations such as using or creating endpoints to read data. | ✅ | ✅ | ✅ | ❌ | +| Manage [Data Service](/tidb-cloud/data-service-overview.md) for data read and write operations. | ✅ | ✅ | ❌ | ❌ | +| View cluster data using [SQL Editor](/tidb-cloud/explore-data-with-chat2query.md). | ✅ | ✅ | ✅ | ❌ | +| Modify and delete cluster data using [SQL Editor](/tidb-cloud/explore-data-with-chat2query.md). | ✅ | ✅ | ❌ | ❌ | +| Manage [changefeeds](/tidb-cloud/changefeed-overview.md). | ✅ | ✅ | ✅ | ❌ | +| Review and reset cluster passwords. | ✅ | ✅ | ✅ | ❌ | +| View cluster overview, backup records, metrics, events, and [changefeeds](/tidb-cloud/changefeed-overview.md) in the project. | ✅ | ✅ | ✅ | ✅ | ## Manage organization access @@ -139,11 +144,9 @@ To change the local timezone setting, take the following steps: 2. Click **Organization Settings**. The organization settings page is displayed. -3. Click the **Time Zone** tab. - -4. Click the drop-down list and select your time zone. +3. In the **Time Zone** section, select your time zone from the drop-down list. -5. Click **Save**. +4. Click **Update**. ### Invite an organization member @@ -159,7 +162,7 @@ To invite a member to an organization, take the following steps: 2. Click **Organization Settings**. The organization settings page is displayed. -3. Click the **User Management** tab, and then select **By Organization**. +3. Click the **Users** tab in the left navigation pane, and then select **By Organization**. 4. Click **Invite**. @@ -190,7 +193,7 @@ To modify the organization role of a member, take the following steps: 2. Click **Organization Settings**. The organization settings page is displayed. -3. Click the **User Management** tab, and then select **By Organization**. +3. Click the **Users** tab in the left navigation pane, and then select **By Organization**. 4. Click the role of the target member, and then modify the role. @@ -208,7 +211,7 @@ To remove a member from an organization, take the following steps: 2. Click **Organization Settings**. The organization settings page is displayed. -3. Click the **User Management** tab, and then select **By Organization**. +3. Click the **Users** tab in the left navigation pane, and then select **By Organization**. 4. Click **Delete** in the user row that you want to delete. @@ -220,7 +223,7 @@ To check which project you belong to, take the following steps: 1. Click in the lower-left corner of the TiDB Cloud console. -2. Click **Organization Settings**. The **Projects** tab is displayed by default. +2. Click **Organization Settings**, and then click the **Projects** tab in the left navigation pane. The **Projects** tab is displayed. > **Tip:** > @@ -238,7 +241,7 @@ To create a new project, take the following steps: 1. Click in the lower-left corner of the TiDB Cloud console. -2. Click **Organization Settings**. The **Projects** tab is displayed by default. +2. Click **Organization Settings**, and then click the **Projects** tab in the left navigation pane. The **Projects** tab is displayed. 3. Click **Create New Project**. @@ -254,7 +257,7 @@ To rename a project, take the following steps: 1. Click in the lower-left corner of the TiDB Cloud console. -2. Click **Organization Settings**. The **Projects** tab is displayed by default. +2. Click **Organization Settings**, and then click the **Projects** tab in the left navigation pane. The **Projects** tab is displayed. 3. In the row of your project to be renamed, click **Rename**. diff --git a/tidb-cloud/managed-service-provider-customer.md b/tidb-cloud/managed-service-provider-customer.md new file mode 100644 index 0000000000000..7fd199970f6c3 --- /dev/null +++ b/tidb-cloud/managed-service-provider-customer.md @@ -0,0 +1,43 @@ +--- +title: Managed Service Provider Customer +summary: Learn how to become a Managed Service Provider (MSP) customer. +--- + +# Managed Service Provider Customer + +A Managed Service Provider (MSP) customer is a customer who uses the TiDB Cloud Services provided by the Managed Service Provider. + +Compared to the direct TiDB Cloud customer, there are several differences for sign-up and invoice payment: + +- The MSP customer needs to sign up for a TiDB Cloud account from the dedicated sign-up page provided by the MSP. +- The MSP customer pays invoices through the MSP channel, instead of paying directly to PingCAP. + +Other daily operations on the TiDB Cloud console are the same for both direct TiDB Cloud customers and MSP customers. + +This document describes how to become an MSP customer and how to check history and future bills for MSP customers. + +## Create a new MSP customer account + +To create a new MSP customer account, visit the MSP dedicated sign-up page. Each MSP has a unique dedicated sign-up page. Contact your MSP to get the URL of the sign-up page. + +## Migrate from a direct TiDB Cloud account to an MSP customer account + +> **Tip:** +> +> Direct customers are the end customers who purchase TiDB Cloud and pay invoices directly from PingCAP. + +If you are currently a direct customer with a TiDB Cloud account, you can ask your MSP to migrate your account to an MSP customer account. + +The migration will take effect on the first day of a future month. Discuss with your MSP to determine the specific effective date. + +On the effective day of migration, you will receive an email notification. + +## Check your future bills + +Once you successfully become an MSP customer, you will pay invoices through your MSP. Ask your MSP where you can check your bills and invoices. + +PingCAP does not send any bills and invoices to MSP customers. + +## Check your history bills + +If you have migrated from a direct TiDB Cloud account to an MSP customer account, you can view your history bills prior to the migration by visiting **Billing** > **Bills** > **History** in the TiDB Cloud console. diff --git a/tidb-cloud/migrate-from-mysql-using-aws-dms.md b/tidb-cloud/migrate-from-mysql-using-aws-dms.md index 48de275322666..4a7fc78e0ddd7 100644 --- a/tidb-cloud/migrate-from-mysql-using-aws-dms.md +++ b/tidb-cloud/migrate-from-mysql-using-aws-dms.md @@ -11,7 +11,7 @@ AWS DMS is a cloud service that makes it easy to migrate relational databases, d This document uses Amazon RDS as an example to show how to migrate data to TiDB Cloud using AWS DMS. The procedure also applies to migrating data from self-hosted MySQL databases or Amazon Aurora to TiDB Cloud. -In this example, the data source is Amazon RDS, and the data destination is a TiDB Dedicated cluster in TiDB Cloud. Both upstream and downstream databases are in the same region. +In this example, the data source is Amazon RDS, and the data destination is a TiDB Cloud Dedicated cluster in TiDB Cloud. Both upstream and downstream databases are in the same region. ## Prerequisites @@ -181,8 +181,8 @@ If you encounter any issues or failures during the migration, you can check the ## See also -- If you want to learn more about how to connect AWS DMS to TiDB Serverless or TiDB Dedicated, see [Connect AWS DMS to TiDB Cloud clusters](/tidb-cloud/tidb-cloud-connect-aws-dms.md). +- If you want to learn more about how to connect AWS DMS to TiDB Cloud Serverless or TiDB Cloud Dedicated, see [Connect AWS DMS to TiDB Cloud clusters](/tidb-cloud/tidb-cloud-connect-aws-dms.md). - If you want to migrate from MySQL-compatible databases, such as Aurora MySQL and Amazon Relational Database Service (RDS), to TiDB Cloud, it is recommended to use [Data Migration on TiDB Cloud](/tidb-cloud/migrate-from-mysql-using-data-migration.md). -- If you want to migrate from Amazon RDS for Oracle to TiDB Serverless Using AWS DMS, see [Migrate from Amazon RDS for Oracle to TiDB Serverless Using AWS DMS](/tidb-cloud/migrate-from-oracle-using-aws-dms.md). +- If you want to migrate from Amazon RDS for Oracle to TiDB Cloud Serverless Using AWS DMS, see [Migrate from Amazon RDS for Oracle to TiDB Cloud Serverless Using AWS DMS](/tidb-cloud/migrate-from-oracle-using-aws-dms.md). diff --git a/tidb-cloud/migrate-from-mysql-using-data-migration.md b/tidb-cloud/migrate-from-mysql-using-data-migration.md index 44a88b8998c19..f72c93472e6fe 100644 --- a/tidb-cloud/migrate-from-mysql-using-data-migration.md +++ b/tidb-cloud/migrate-from-mysql-using-data-migration.md @@ -16,7 +16,7 @@ If you want to migrate incremental data only, see [Migrate Incremental Data from ### Availability -- The Data Migration feature is available only for **TiDB Dedicated** clusters. +- The Data Migration feature is available only for **TiDB Cloud Dedicated** clusters. - The Data Migration feature is only available to clusters that are created in [certain regions](https://www.pingcap.com/tidb-cloud-pricing-details/#dm-cost) after November 9, 2022. If your **project** was created before the date or if your cluster is in another region, this feature is not available to your cluster and the **Data Migration** tab will not be displayed on the cluster overview page in the TiDB Cloud console. @@ -93,21 +93,20 @@ The username you use for the downstream TiDB Cloud cluster must have the followi | `ALTER` | Tables | | `DROP` | Databases, Tables | | `INDEX` | Tables | -| `TRUNCATE` | Tables | For example, you can execute the following `GRANT` statement to grant corresponding privileges: ```sql -GRANT CREATE,SELECT,INSERT,UPDATE,DELETE,ALTER,TRUNCATE,DROP,INDEX ON *.* TO 'your_user'@'your_IP_address_of_host' +GRANT CREATE,SELECT,INSERT,UPDATE,DELETE,ALTER,DROP,INDEX ON *.* TO 'your_user'@'your_IP_address_of_host' ``` To quickly test a migration job, you can use the `root` account of the TiDB Cloud cluster. ### Set up network connection -Before creating a migration job, set up the network connection according to your connection methods. See [Connect to Your TiDB Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md). +Before creating a migration job, set up the network connection according to your connection methods. See [Connect to Your TiDB Cloud Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md). -- If you use public IP (this is, standard connection) for network connection, make sure that the upstream database can be connected through the public network. +- If you use public IP (this is, public connection) for network connection, make sure that the upstream database can be connected through the public network. - If you use AWS VPC Peering or Google Cloud VPC Network Peering, see the following instructions to configure the network. @@ -120,7 +119,7 @@ If your MySQL service is in an AWS VPC, take the following steps: 2. Modify the inbound rules of the security group that the MySQL service is associated with. - You must add [the CIDR of the region where your TiDB Cloud cluster is located](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-project-cidr) to the inbound rules. Doing so allows the traffic to flow from your TiDB cluster to the MySQL instance. + You must add [the CIDR of the region where your TiDB Cloud cluster is located](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-cidr-for-a-region) to the inbound rules. Doing so allows the traffic to flow from your TiDB cluster to the MySQL instance. 3. If the MySQL URL contains a DNS hostname, you need to allow TiDB Cloud to be able to resolve the hostname of the MySQL service. @@ -140,13 +139,20 @@ If your MySQL service is in a Google Cloud VPC, take the following steps: 3. Modify the ingress firewall rules of the VPC where MySQL is located. - You must add [the CIDR of the region where your TiDB Cloud cluster is located](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-project-cidr) to the ingress firewall rules. This allows the traffic to flow from your TiDB cluster to the MySQL endpoint. + You must add [the CIDR of the region where your TiDB Cloud cluster is located](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-cidr-for-a-region) to the ingress firewall rules. This allows the traffic to flow from your TiDB cluster to the MySQL endpoint. ### Enable binary logs -To perform incremental data migration, make sure you have enabled binary logs of the upstream database, and the binary logs have been kept for more than 24 hours. +To perform incremental data migration, make sure the following requirements are met: + +- Binary logs are enabled for the upstream database. +- The binary logs are retained for at least 24 hours. +- The binlog format for the upstream database is set to `ROW`. If not, update the format to `ROW` as follows to avoid the [format error](/tidb-cloud/tidb-cloud-dm-precheck-and-troubleshooting.md#error-message-check-whether-mysql-binlog_format-is-row): + + - MySQL: execute the `SET GLOBAL binlog_format=ROW;` statement. If you want to persist this change across reboots, you can execute the `SET PERSIST binlog_format=ROW;` statement. + - Amazon Aurora MySQL or RDS for MySQL: follow the instructions in [AWS documentation](https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/USER_WorkingWithDBInstanceParamGroups.html) to create a new DB parameter group. Set the `binlog_format=row` parameter in the new DB parameter group, modify the instance to use the new DB parameter group, and then restart the instance to take effect. ## Step 1: Go to the **Data Migration** page @@ -201,7 +207,7 @@ In the **Choose the objects to be migrated** step, you can choose existing data To migrate data to TiDB Cloud once and for all, choose both **Existing data migration** and **Incremental data migration**, which ensures data consistency between the source and target databases. -You can use **physical mode** or **logical mode** to migrate **existing data**. +You can use **physical mode** or **logical mode** to migrate **existing data** and **incremental data**. - The default mode is **logical mode**. This mode exports data from upstream databases as SQL statements, and then executes them on TiDB. In this mode, the target tables before migration can be either empty or non-empty. But the performance is slower than physical mode. @@ -227,7 +233,7 @@ Physical mode exports the upstream data as fast as possible, so [different speci To migrate only existing data of the source database to TiDB Cloud, choose **Existing data migration**. -You can choose to use physical mode or logical mode to migrate existing data. For more information, see [Migrate existing data and incremental data](#migrate-existing-data-and-incremental-data). +You can only use logical mode to migrate existing data. For more information, see [Migrate existing data and incremental data](#migrate-existing-data-and-incremental-data). ### Migrate only incremental data @@ -241,15 +247,15 @@ For detailed instructions about incremental data migration, see [Migrate Only In - If you click **All**, the migration job will migrate the existing data from the whole source database instance to TiDB Cloud and migrate ongoing changes after the full migration. Note that it happens only if you have selected the **Existing data migration** and **Incremental data migration** checkboxes in the previous step. - + - If you click **Customize** and select some databases, the migration job will migrate the existing data and migrate ongoing changes of the selected databases to TiDB Cloud. Note that it happens only if you have selected the **Existing data migration** and **Incremental data migration** checkboxes in the previous step. - + - If you click **Customize** and select some tables under a dataset name, the migration job only will migrate the existing data and migrate ongoing changes of the selected tables. Tables created afterwards in the same database will not be migrated. - + -TiDB Serverless is a fully managed, multi-tenant TiDB offering. It delivers an instant, autoscaling MySQL-compatible database and offers a generous free tier and consumption based billing once free limits are exceeded. +TiDB Cloud Serverless is a fully managed, multi-tenant TiDB offering. It delivers an instant, autoscaling MySQL-compatible database and offers a generous free tier and consumption based billing once free limits are exceeded. + +### Cluster plans + +TiDB Cloud Serverless offers two service plans to meet different user requirements. Whether you are just getting started or scaling to meet the increasing application demands, these service plans provide the flexibility and capability you need. + +#### Free cluster plan + +The free cluster plan is ideal for those who are getting started with TiDB Cloud Serverless. It provides developers and small teams with the following essential features: + +- **No cost**: This plan is completely free, with no credit card required to get started. +- **Storage**: Provides an initial 5 GiB of row-based storage and 5 GiB of columnar storage. +- **Request Units**: Includes 50 million [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit) for database operations. +- **Easy upgrade**: Offers a smooth transition to the [scalable cluster plan](#scalable-cluster-plan) as your needs grow. + +#### Scalable cluster plan + +For applications experiencing growing workloads and needing scalability in real time, the scalable cluster plan provides the flexibility and performance to keep pace with your business growth with the following features: + +- **Enhanced capabilities**: Includes all capabilities of the free cluster plan, along with the capacity to handle larger and more complex workloads, as well as advanced security features. +- **Automatic scaling**: Automatically adjusts storage and computing resources to efficiently meet changing workload demands. +- **Predictable pricing**: Although this plan requires a credit card, you are only charged for the resources you use, ensuring cost-effective scalability. ### Usage quota -For each organization in TiDB Cloud, you can create a maximum of five TiDB Serverless clusters by default. To create more TiDB Serverless clusters, you need to add a credit card and set a [spending limit](/tidb-cloud/tidb-cloud-glossary.md#spending-limit) for the usage. +For each organization in TiDB Cloud, you can create a maximum of five [free clusters](#free-cluster-plan) by default. To create more TiDB Cloud Serverless clusters, you need to add a credit card and create [scalable clusters](#scalable-cluster-plan) for the usage. -For the first five TiDB Serverless clusters in your organization, TiDB Cloud provides a free usage quota for each of them as follows: +For the first five TiDB Cloud Serverless clusters in your organization, whether they are free or scalable, TiDB Cloud provides a free usage quota for each of them as follows: - Row-based storage: 5 GiB -- [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit): 50 million RUs per month +- Columnar storage: 5 GiB +- Request Units (RUs): 50 million RUs per month A Request Unit (RU) is a unit of measure used to represent the amount of resources consumed by a single request to the database. The amount of RUs consumed by a request depends on various factors, such as the operation type or the amount of data being retrieved or modified. -Once the free quota of a cluster is reached, the read and write operations on this cluster will be throttled until you [increase the quota](/tidb-cloud/manage-serverless-spend-limit.md#update-spending-limit) or the usage is reset upon the start of a new month. For example, when the storage of a cluster exceeds 5 GiB, the maximum size limit of a single transaction is reduced from 10 MiB to 1 MiB. +Once a cluster reaches its usage quota, it immediately denies any new connection attempts until you [increase the quota](/tidb-cloud/manage-serverless-spend-limit.md#update-spending-limit) or the usage is reset upon the start of a new month. Existing connections established before reaching the quota will remain active but will experience throttling. For example, when the row-based storage of a cluster exceeds 5 GiB for a free cluster, the cluster automatically restricts any new connection attempts. -To learn more about the RU consumption of different resources (including read, write, SQL CPU, and network egress), the pricing details, and the throttled information, see [TiDB Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). +To learn more about the RU consumption of different resources (including read, write, SQL CPU, and network egress), the pricing details, and the throttled information, see [TiDB Cloud Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). ### User name prefix -For each TiDB Serverless cluster, TiDB Cloud generates a unique prefix to distinguish it from other clusters. +For each TiDB Cloud Serverless cluster, TiDB Cloud generates a unique prefix to distinguish it from other clusters. Whenever you use or set a database user name, you must include the prefix in the user name. For example, assume that the prefix of your cluster is `3pTAoNNegb47Uc8`. @@ -49,7 +71,7 @@ Whenever you use or set a database user name, you must include the prefix in the > **Note:** > - > TiDB Serverless requires TLS connection. To find the CA root path on your system, see [Root certificate default path](/tidb-cloud/secure-connections-to-serverless-clusters.md#root-certificate-default-path). + > TiDB Cloud Serverless requires TLS connection. To find the CA root path on your system, see [Root certificate default path](/tidb-cloud/secure-connections-to-serverless-clusters.md#root-certificate-default-path). - To create a database user: @@ -63,17 +85,17 @@ To get the prefix for your cluster, take the following steps: 2. Click the name of your target cluster to go to its overview page, and then click **Connect** in the upper-right corner. A connection dialog is displayed. 3. In the dialog, get the prefix from the connection string. -### TiDB Serverless special terms and conditions +### TiDB Cloud Serverless special terms and conditions -Some of TiDB Cloud features are partially supported or not supported on TiDB Serverless. See [TiDB Serverless Limitations](/tidb-cloud/serverless-limitations.md) for details. +Some of TiDB Cloud features are partially supported or not supported on TiDB Cloud Serverless. See [TiDB Cloud Serverless Limitations](/tidb-cloud/serverless-limitations.md) for details. -## TiDB Dedicated +## TiDB Cloud Dedicated -TiDB Dedicated is for production use with the benefits of cross-zone high availability, horizontal scaling, and [HTAP](https://en.wikipedia.org/wiki/Hybrid_transactional/analytical_processing). +TiDB Cloud Dedicated is for production use with the benefits of cross-zone high availability, horizontal scaling, and [HTAP](https://en.wikipedia.org/wiki/Hybrid_transactional/analytical_processing). -For TiDB Dedicated clusters, you can customize the cluster size of TiDB, TiKV, and TiFlash easily according to your business need. For each TiKV node and TiFlash node, the data on the node is replicated and distributed in different availability zones for [high availability](/tidb-cloud/high-availability-with-multi-az.md). +For TiDB Cloud Dedicated clusters, you can customize the cluster size of TiDB, TiKV, and TiFlash easily according to your business need. For each TiKV node and TiFlash node, the data on the node is replicated and distributed in different availability zones for [high availability](/tidb-cloud/high-availability-with-multi-az.md). -To create a TiDB Dedicated cluster, you need to [add a payment method](/tidb-cloud/tidb-cloud-billing.md#payment-method) or [apply for a Proof of Concept (PoC) trial](/tidb-cloud/tidb-cloud-poc.md). +To create a TiDB Cloud Dedicated cluster, you need to [add a payment method](/tidb-cloud/tidb-cloud-billing.md#payment-method) or [apply for a Proof of Concept (PoC) trial](/tidb-cloud/tidb-cloud-poc.md). > **Note:** > diff --git a/tidb-cloud/serverless-driver-drizzle-example.md b/tidb-cloud/serverless-driver-drizzle-example.md new file mode 100644 index 0000000000000..cae90ab0ec24b --- /dev/null +++ b/tidb-cloud/serverless-driver-drizzle-example.md @@ -0,0 +1,272 @@ +--- +title: TiDB Cloud Serverless Driver Drizzle Tutorial +summary: Learn how to use TiDB Cloud serverless driver with Drizzle. +--- + +# TiDB Cloud Serverless Driver Drizzle Tutorial + +[Drizzle ORM](https://orm.drizzle.team/) is a lightweight and performant TypeScript ORM with developer experience in mind. Starting from `drizzle-orm@0.31.2`, it supports [drizzle-orm/tidb-serverless](https://orm.drizzle.team/docs/get-started-mysql#tidb-serverless), enabling you to use Drizzle over HTTPS with [TiDB Cloud serverless driver](/tidb-cloud/serverless-driver.md). + +This tutorial describes how to use TiDB Cloud serverless driver with Drizzle in Node.js environments and edge environments. + +## Use Drizzle and TiDB Cloud serverless driver in Node.js environments + +This section describes how to use TiDB Cloud serverless driver with Drizzle in Node.js environments. + +### Before you begin + +To complete this tutorial, you need the following: + +- [Node.js](https://nodejs.org/en) >= 18.0.0. +- [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) or your preferred package manager. +- A TiDB Cloud Serverless cluster. If you don't have any, you can [create a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md). + +### Step 1. Create a project + +1. Create a project named `drizzle-node-example`: + + ```shell + mkdir drizzle-node-example + cd drizzle-node-example + ``` + +2. Install the `drizzle-orm` and `@tidbcloud/serverless` packages: + + ```shell + npm install drizzle-orm @tidbcloud/serverless + ``` + +3. In the root directory of your project, locate the `package.json` file, and then specify the ES module by adding `"type": "module"` to the file: + + ```json + { + "type": "module", + "dependencies": { + "@tidbcloud/serverless": "^0.1.1", + "drizzle-orm": "^0.31.2" + } + } + ``` + +4. In the root directory of your project, add a `tsconfig.json` file to define the TypeScript compiler options. Here is an example file: + + ```json + { + "compilerOptions": { + "module": "ES2022", + "target": "ES2022", + "moduleResolution": "node", + "strict": false, + "declaration": true, + "outDir": "dist", + "removeComments": true, + "allowJs": true, + "esModuleInterop": true, + "resolveJsonModule": true + } + } + ``` + +### Step 2. Set the environment + +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. + +2. On the overview page, click **Connect** in the upper-right corner, select `Serverless Driver` in the **Connect With** drop-down box, and then click **Generate Password** to create a random password. + + > **Tip:** + > + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. + + The connection string looks like this: + + ``` + mysql://[username]:[password]@[host]/[database] + ``` + +3. Set the environment variable `DATABASE_URL` in your local environment. For example, in Linux or macOS, you can run the following command: + + ```shell + export DATABASE_URL='mysql://[username]:[password]@[host]/[database]' + ``` + +### Step 3. Use Drizzle to query data + +1. Create a table in your TiDB Cloud Serverless cluster. + + You can use [SQL Editor in the TiDB Cloud console](/tidb-cloud/explore-data-with-chat2query.md) to execute SQL statements. Here is an example: + + ```sql + CREATE TABLE `test`.`users` ( + `id` BIGINT PRIMARY KEY auto_increment, + `full_name` TEXT, + `phone` VARCHAR(256) + ); + ``` + +2. In the root directory of your project, create a file named `hello-world.ts` and add the following code: + + ```ts + import { connect } from '@tidbcloud/serverless'; + import { drizzle } from 'drizzle-orm/tidb-serverless'; + import { mysqlTable, serial, text, varchar } from 'drizzle-orm/mysql-core'; + + // Initialize + const client = connect({ url: process.env.DATABASE_URL }); + const db = drizzle(client); + + // Define schema + export const users = mysqlTable('users', { + id: serial("id").primaryKey(), + fullName: text('full_name'), + phone: varchar('phone', { length: 256 }), + }); + export type User = typeof users.$inferSelect; // return type when queried + export type NewUser = typeof users.$inferInsert; // insert type + + // Insert and select data + const user: NewUser = { fullName: 'John Doe', phone: '123-456-7890' }; + await db.insert(users).values(user) + const result: User[] = await db.select().from(users); + console.log(result); + ``` + +### Step 4. Run the Typescript code + +1. Install `ts-node` to transform TypeScript into JavaScript, and then install `@types/node` to provide TypeScript type definitions for Node.js. + + ```shell + npm install -g ts-node + npm i --save-dev @types/node + ``` + +2. Run the Typescript code with the following command: + + ```shell + ts-node --esm hello-world.ts + ``` + +## Use Drizzle and TiDB Cloud serverless driver in edge environments + +This section takes the Vercel Edge Function as an example. + +### Before you begin + +To complete this tutorial, you need the following: + +- A [Vercel](https://vercel.com/docs) account that provides edge environment. +- [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) or your preferred package manager. +- A TiDB Cloud Serverless cluster. If you don't have any, you can [create a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md). + +### Step 1. Create a project + +1. Install the Vercel CLI: + + ```shell + npm i -g vercel@latest + ``` + +2. Create a [Next.js](https://nextjs.org/) project called `drizzle-example` using the following terminal command: + + ```shell + npx create-next-app@latest drizzle-example --ts --no-eslint --tailwind --no-src-dir --app --import-alias "@/*" + ``` + +3. Navigate to the `drizzle-example` directory: + + ```shell + cd drizzle-example + ``` + +4. Install the `drizzle-orm` and `@tidbcloud/serverless` packages: + + ```shell + npm install drizzle-orm @tidbcloud/serverless --force + ``` + +### Step 2. Set the environment + +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target TiDB Cloud Serverless cluster to go to its overview page. + +2. On the overview page, click **Connect** in the upper-right corner, select `Serverless Driver` in the **Connect With** drop-down box, and then click **Generate Password** to create a random password. + + > **Tip:** + > + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. + + The connection string looks like this: + + ``` + mysql://[username]:[password]@[host]/[database] + ``` + +### Step 3. Create an edge function + +1. Create a table in your TiDB Cloud Serverless cluster. + + You can use [SQL Editor in the TiDB Cloud console](/tidb-cloud/explore-data-with-chat2query.md) to execute SQL statements. Here is an example: + + ```sql + CREATE TABLE `test`.`users` ( + `id` BIGINT PRIMARY KEY auto_increment, + `full_name` TEXT, + `phone` VARCHAR(256) + ); + ``` + +2. In the `app` directory of your project, create a file `/api/edge-function-example/route.ts` and add the following code: + + ```ts + import { NextResponse } from 'next/server'; + import type { NextRequest } from 'next/server'; + import { connect } from '@tidbcloud/serverless'; + import { drizzle } from 'drizzle-orm/tidb-serverless'; + import { mysqlTable, serial, text, varchar } from 'drizzle-orm/mysql-core'; + export const runtime = 'edge'; + + // Initialize + const client = connect({ url: process.env.DATABASE_URL }); + const db = drizzle(client); + + // Define schema + export const users = mysqlTable('users', { + id: serial("id").primaryKey(), + fullName: text('full_name'), + phone: varchar('phone', { length: 256 }), + }); + export type User = typeof users.$inferSelect; // return type when queried + export type NewUser = typeof users.$inferInsert; // insert type + + export async function GET(request: NextRequest) { + // Insert and select data + const user: NewUser = { fullName: 'John Doe', phone: '123-456-7890' }; + await db.insert(users).values(user) + const result: User[] = await db.select().from(users); + return NextResponse.json(result); + } + ``` + +3. Test your code locally: + + ```shell + export DATABASE_URL='mysql://[username]:[password]@[host]/[database]' + next dev + ``` + +4. Navigate to `http://localhost:3000/api/edge-function-example` to get the response from your route. + +### Step 4. Deploy your code to Vercel + +1. Deploy your code to Vercel with the `DATABASE_URL` environment variable: + + ```shell + vercel -e DATABASE_URL='mysql://[username]:[password]@[host]/[database]' --prod + ``` + + After the deployment is complete, you will get the URL of your project. + +2. Navigate to the `${Your-URL}/api/edge-function-example` page to get the response from your route. + +## What's next + +- Learn more about [Drizzle](https://orm.drizzle.team/docs/overview) and [drizzle-orm/tidb-serverless](https://orm.drizzle.team/docs/get-started-mysql#tidb-serverless). +- Learn how to [integrate TiDB Cloud with Vercel](/tidb-cloud/integrate-tidbcloud-with-vercel.md). diff --git a/tidb-cloud/serverless-driver-kysely-example.md b/tidb-cloud/serverless-driver-kysely-example.md index ded6989ccf6af..38a9763ababeb 100644 --- a/tidb-cloud/serverless-driver-kysely-example.md +++ b/tidb-cloud/serverless-driver-kysely-example.md @@ -22,7 +22,7 @@ To complete this tutorial, you need the following: - [Node.js](https://nodejs.org/en) >= 18.0.0. - [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) or your preferred package manager. -- A TiDB Serverless cluster. If you don't have any, you can [create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md). +- A TiDB Cloud Serverless cluster. If you don't have any, you can [create a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md). ### Step 1. Create a project @@ -39,7 +39,7 @@ To complete this tutorial, you need the following: npm install kysely @tidbcloud/kysely @tidbcloud/serverless ``` -3. In the root directory of your project, locate the `package.json` file, and then specify the ES module by adding `type: "module"` to the file: +3. In the root directory of your project, locate the `package.json` file, and then specify the ES module by adding `"type": "module"` to the file: ```json { @@ -73,7 +73,7 @@ To complete this tutorial, you need the following: ### Step 2. Set the environment -1. On the overview page of your TiDB Serverless cluster, click **Connect** in the upper-right corner, and then get the connection string for your database from the displayed dialog. The connection string looks like this: +1. On the overview page of your TiDB Cloud Serverless cluster, click **Connect** in the upper-right corner, and then get the connection string for your database from the displayed dialog. The connection string looks like this: ``` mysql://[username]:[password]@[host]/[database] @@ -87,9 +87,9 @@ To complete this tutorial, you need the following: ### Step 3. Use Kysely to query data -1. Create a table in your TiDB Serverless cluster and insert some data. +1. Create a table in your TiDB Cloud Serverless cluster and insert some data. - You can use [Chat2Query in the TiDB Cloud console](/tidb-cloud/explore-data-with-chat2query.md) to execute SQL statements. Here is an example: + You can use [SQL Editor in the TiDB Cloud console](/tidb-cloud/explore-data-with-chat2query.md) to execute SQL statements. Here is an example: ```sql CREATE TABLE `test`.`person` ( @@ -102,7 +102,7 @@ To complete this tutorial, you need the following: insert into test.person values (1,'pingcap','male') ``` -2. In the root directory of your project, create a file named `hello-word.ts` and add the following code: +2. In the root directory of your project, create a file named `hello-world.ts` and add the following code: ```ts import { Kysely,GeneratedAlways,Selectable } from 'kysely' @@ -166,7 +166,7 @@ To complete this tutorial, you need the following: - A [Vercel](https://vercel.com/docs) account that provides edge environment. - [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) or your preferred package manager. -- A TiDB Serverless cluster. If you don't have any, you can [create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md). +- A TiDB Cloud Serverless cluster. If you don't have any, you can [create a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md). ### Step 1. Create a project @@ -191,7 +191,7 @@ To complete this tutorial, you need the following: ### Step 2. Set the environment -On the overview page of your TiDB Serverless cluster, click **Connect** in the upper-right corner, and then get the connection string for your database from the displayed dialog. The connection string looks like this: +On the overview page of your TiDB Cloud Serverless cluster, click **Connect** in the upper-right corner, and then get the connection string for your database from the displayed dialog. The connection string looks like this: ``` mysql://[username]:[password]@[host]/[database] @@ -199,9 +199,9 @@ mysql://[username]:[password]@[host]/[database] ### Step 3. Create an edge function -1. Create a table in your TiDB Serverless cluster and insert some data. +1. Create a table in your TiDB Cloud Serverless cluster and insert some data. - You can use [Chat2Query in the TiDB Cloud console](/tidb-cloud/explore-data-with-chat2query.md) to execute SQL statements. Here is an example: + You can use [SQL Editor in the TiDB Cloud console](/tidb-cloud/explore-data-with-chat2query.md) to execute SQL statements. Here is an example: ```sql CREATE TABLE `test`.`person` ( @@ -296,4 +296,4 @@ mysql://[username]:[password]@[host]/[database] ## What's next - Learn more about [Kysely](https://kysely.dev/docs/intro) and [@tidbcloud/kysely](https://github.com/tidbcloud/kysely) -- Learn how to [integrate TiDB Cloud with Vercel](/tidb-cloud/integrate-tidbcloud-with-vercel.md) \ No newline at end of file +- Learn how to [integrate TiDB Cloud with Vercel](/tidb-cloud/integrate-tidbcloud-with-vercel.md) diff --git a/tidb-cloud/serverless-driver-node-example.md b/tidb-cloud/serverless-driver-node-example.md index b6a9fb7509012..e900f653bd4e9 100644 --- a/tidb-cloud/serverless-driver-node-example.md +++ b/tidb-cloud/serverless-driver-node-example.md @@ -9,7 +9,7 @@ This tutorial describes how to use TiDB Cloud serverless driver in a local Node. > **Note:** > -> - This tutorial is applicable to TiDB Serverless clusters only. +> - This tutorial is applicable to TiDB Cloud Serverless clusters only. > - To learn how to use TiDB Cloud serverless driver with Cloudflare Workers, Vercel Edge Functions, and Netlify Edge Functions, check out our [Insights into Automotive Sales](https://car-sales-insight.vercel.app/) and the [sample repository](https://github.com/tidbcloud/car-sales-insight). ## Before you begin @@ -18,7 +18,7 @@ To complete this step-by-step tutorial, you need the following: - [Node.js](https://nodejs.org/en) >= 18.0.0. - [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) or your preferred package manager. -- A TiDB Serverless cluster. If you don't have any, you can [create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md). +- A TiDB Cloud Serverless cluster. If you don't have any, you can [create a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md). ## Step 1. Create a local Node.js project @@ -65,7 +65,7 @@ The serverless driver supports both CommonJS and ES modules. The following steps ```js import { connect } from '@tidbcloud/serverless' - const conn = connect({url: 'mysql://[username]:[password]@[host]/[database]'}) // replace with your TiDB Serverless cluster information + const conn = connect({url: 'mysql://[username]:[password]@[host]/[database]'}) // replace with your TiDB Cloud Serverless cluster information console.log(await conn.execute("show tables")) ``` diff --git a/tidb-cloud/serverless-driver-prisma-example.md b/tidb-cloud/serverless-driver-prisma-example.md index 49b3ffd9a7c88..056985f603b4d 100644 --- a/tidb-cloud/serverless-driver-prisma-example.md +++ b/tidb-cloud/serverless-driver-prisma-example.md @@ -7,20 +7,79 @@ summary: Learn how to use TiDB Cloud serverless driver with Prisma ORM. [Prisma](https://www.prisma.io/docs) is an open source next-generation ORM (Object-Relational Mapping) that helps developers interact with their database in an intuitive, efficient, and safe way. TiDB Cloud offers [@tidbcloud/prisma-adapter](https://github.com/tidbcloud/prisma-adapter), enabling you to use [Prisma Client](https://www.prisma.io/docs/concepts/components/prisma-client) over HTTPS with [TiDB Cloud serverless driver](/tidb-cloud/serverless-driver.md). Compared with the traditional TCP way, [@tidbcloud/prisma-adapter](https://github.com/tidbcloud/prisma-adapter) brings the following benefits: -- Better performance in serverless environments -- Possibility of using Prisma client in the edge environments (see [#21394](https://github.com/prisma/prisma/issues/21394) for more information) +- Better performance of Prisma Client in serverless environments +- Ability to use Prisma Client in edge environments -This tutorial describes how to use TiDB Cloud serverless driver with the Prisma adapter. +This tutorial describes how to use [@tidbcloud/prisma-adapter](https://github.com/tidbcloud/prisma-adapter) in serverless environments and edge environments. + +## Install + +You need to install both [@tidbcloud/prisma-adapter](https://github.com/tidbcloud/prisma-adapter) and [TiDB Cloud serverless driver](/tidb-cloud/serverless-driver.md). You can install them using [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) or your preferred package manager. + +Taking npm as an example, you can run the following commands for installation: + +```shell +npm install @tidbcloud/prisma-adapter +npm install @tidbcloud/serverless +``` + +## Enable `driverAdapters` + +To use the Prisma adapter, you need to enable the `driverAdapters` feature in the `schema.prisma` file. For example: + +```prisma +generator client { + provider = "prisma-client-js" + previewFeatures = ["driverAdapters"] +} + +datasource db { + provider = "mysql" + url = env("DATABASE_URL") +} +``` + +## Initialize Prisma Client + +Before using Prisma Client, you need to initialize it with `@tidbcloud/prisma-adapter`. + +For `@tidbcloud/prisma-adapter` earlier than v6.6.0: + +```js +import { connect } from '@tidbcloud/serverless'; +import { PrismaTiDBCloud } from '@tidbcloud/prisma-adapter'; +import { PrismaClient } from '@prisma/client'; + +// Initialize Prisma Client +const connection = connect({ url: ${DATABASE_URL} }); +const adapter = new PrismaTiDBCloud(connection); +const prisma = new PrismaClient({ adapter }); +``` + +For `@tidbcloud/prisma-adapter` v6.6.0 or a later version: + +```js +import { PrismaTiDBCloud } from '@tidbcloud/prisma-adapter'; +import { PrismaClient } from '@prisma/client'; + +// Initialize Prisma Client +const adapter = new PrismaTiDBCloud({ url: ${DATABASE_URL} }); +const prisma = new PrismaClient({ adapter }); +``` + +Then, queries from Prisma Client can be sent to the TiDB Cloud serverless driver for processing. ## Use the Prisma adapter in Node.js environments +This section provides an example of how to use `@tidbcloud/prisma-adapter` in Node.js environments. + ### Before you begin To complete this tutorial, you need the following: - [Node.js](https://nodejs.org/en) >= 18.0.0. - [npm](https://docs.npmjs.com/downloading-and-installing-node-js-and-npm) or your preferred package manager. -- A TiDB Serverless cluster. If you don't have any, you can [create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md). +- A TiDB Cloud Serverless cluster. If you don't have any, you can [create a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md). ### Step 1. Create a project @@ -40,26 +99,26 @@ To complete this tutorial, you need the following: npm install @tidbcloud/serverless npm install prisma --save-dev ``` - + 3. In the `package.json` file, specify the ES module by adding `type: "module"`: ```json { "type": "module", "dependencies": { - "@prisma/client": "^5.5.2", - "@tidbcloud/prisma-adapter": "^5.5.2", - "@tidbcloud/serverless": "^0.0.7" + "@prisma/client": "^6.6.0", + "@tidbcloud/prisma-adapter": "^6.6.0", + "@tidbcloud/serverless": "^0.1.0" }, "devDependencies": { - "prisma": "^5.5.2" + "prisma": "^6.6.0" } } ``` ### Step 2. Set the environment -1. On the overview page of your TiDB Serverless cluster, click **Connect** in the upper-right corner, and then get the connection string for your database from the displayed dialog. The connection string looks like this: +1. On the overview page of your TiDB Cloud Serverless cluster, click **Connect** in the upper-right corner, and then get the connection string for your database from the displayed dialog. The connection string looks like this: ``` mysql://[username]:[password]@[host]:4000/[database]?sslaccept=strict @@ -120,13 +179,13 @@ To complete this tutorial, you need the following: } ``` -3. Synchronize your database with the Prisma schema. You can either manually create the database tables in your TiDB Serverless cluster or use the Prisma CLI to create them automatically as follows: +3. Synchronize your database with the Prisma schema. You can either manually create the database tables in your TiDB Cloud Serverless cluster or use the Prisma CLI to create them automatically as follows: ``` npx prisma db push ``` - - This command will create the `user` table in your TiDB Serverless cluster through the traditional TCP connection, rather than through the HTTPS connection using `@tidbcloud/prisma-adapter`. This is because it uses the same engine as Prisma Migrate. For more information about this command, see [Prototype your schema](https://www.prisma.io/docs/concepts/components/prisma-migrate/db-push). + + This command will create the `user` table in your TiDB Cloud Serverless cluster through the traditional TCP connection, rather than through the HTTPS connection using `@tidbcloud/prisma-adapter`. This is because it uses the same engine as Prisma Migrate. For more information about this command, see [Prototype your schema](https://www.prisma.io/docs/concepts/components/prisma-migrate/db-push). 4. Generate Prisma Client: @@ -134,14 +193,13 @@ To complete this tutorial, you need the following: npx prisma generate ``` - This command will generate Prisma Client based on the Prisma schema. + This command will generate Prisma Client based on the Prisma schema. ### Step 4. Execute CRUD operations 1. Create a file named `hello-word.js` and add the following code to initialize Prisma Client: ```js - import { connect } from '@tidbcloud/serverless'; import { PrismaTiDBCloud } from '@tidbcloud/prisma-adapter'; import { PrismaClient } from '@prisma/client'; import dotenv from 'dotenv'; @@ -151,8 +209,7 @@ To complete this tutorial, you need the following: const connectionString = `${process.env.DATABASE_URL}`; // Initialize Prisma Client - const connection = connect({ url: connectionString }); - const adapter = new PrismaTiDBCloud(connection); + const adapter = new PrismaTiDBCloud({ url: connectionString }); const prisma = new PrismaClient({ adapter }); ``` @@ -178,7 +235,7 @@ To complete this tutorial, you need the following: }, }) ``` - + 3. Execute some transaction operations with Prisma Client. For example: ```js @@ -213,7 +270,10 @@ To complete this tutorial, you need the following: console.log(e) } ``` - + ## Use the Prisma adapter in edge environments -Currently, `@tidbcloud/prisma-adapter` is not compatible with edge environments such as Vercel Edge Function and Cloudflare Workers. However, there are plans to support these environments. For more information, see [#21394](https://github.com/prisma/prisma/issues/21394). \ No newline at end of file +You can use `@tidbcloud/prisma-adapter` v5.11.0 or a later version in edge environments such as Vercel Edge Functions and Cloudflare Workers. + +- [Vercel Edge Function example](https://github.com/tidbcloud/serverless-driver-example/tree/main/prisma/prisma-vercel-example) +- [Cloudflare Workers example](https://github.com/tidbcloud/serverless-driver-example/tree/main/prisma/prisma-cloudflare-worker-example) diff --git a/tidb-cloud/serverless-driver.md b/tidb-cloud/serverless-driver.md index 04904e50075d9..c3ebd151db135 100644 --- a/tidb-cloud/serverless-driver.md +++ b/tidb-cloud/serverless-driver.md @@ -1,6 +1,6 @@ --- title: TiDB Cloud Serverless Driver (Beta) -summary: Learn how to connect to TiDB Serverless from serverless and edge environments. +summary: Learn how to connect to TiDB Cloud Serverless from serverless and edge environments. aliases: ['/tidbcloud/serverless-driver-config'] --- @@ -10,7 +10,7 @@ aliases: ['/tidbcloud/serverless-driver-config'] Traditional TCP-based MySQL drivers are not suitable for serverless functions due to their expectation of long-lived, persistent TCP connections, which contradict the short-lived nature of serverless functions. Moreover, in edge environments such as [Vercel Edge Functions](https://vercel.com/docs/functions/edge-functions) and [Cloudflare Workers](https://workers.cloudflare.com/), where comprehensive TCP support and full Node.js compatibility may be lacking, these drivers may not work at all. -[TiDB Cloud serverless driver (Beta)](https://github.com/tidbcloud/serverless-js) for JavaScript allows you to connect to your TiDB Serverless cluster over HTTP, which is generally supported by serverless environments. With it, it is now possible to connect to TiDB Serverless clusters from edge environments and reduce connection overhead with TCP while keeping the similar development experience of traditional TCP-based MySQL drivers. +[TiDB Cloud serverless driver (Beta)](https://github.com/tidbcloud/serverless-js) for JavaScript allows you to connect to your TiDB Cloud Serverless cluster over HTTP, which is generally supported by serverless environments. With it, it is now possible to connect to TiDB Cloud Serverless clusters from edge environments and reduce connection overhead with TCP while keeping the similar development experience of traditional TCP-based MySQL drivers. > **Note:** > @@ -26,11 +26,11 @@ npm install @tidbcloud/serverless ## Use the serverless driver -You can use the serverless driver to query data of a TiDB Serverless cluster or perform interactive transactions. +You can use the serverless driver to query data of a TiDB Cloud Serverless cluster or perform interactive transactions. ### Query -To query data from a TiDB Serverless cluster, you need to create a connection first. Then you can use the connection to execute raw SQL queries. For example: +To query data from a TiDB Cloud Serverless cluster, you need to create a connection first. Then you can use the connection to execute raw SQL queries. For example: ```ts import { connect } from '@tidbcloud/serverless' @@ -123,7 +123,7 @@ Learn more about [using TiDB Cloud serverless driver in Netlify](/tidb-cloud/int
```ts -import { connect } from "npm:@tidbcloud/serverless-js" +import { connect } from "npm:@tidbcloud/serverless" const conn = connect({url: Deno.env.get('DATABASE_URL')}) const result = await conn.execute('show tables') @@ -134,7 +134,7 @@ const result = await conn.execute('show tables')
```ts -import { connect } from "@tidbcloud/serverless-js" +import { connect } from "@tidbcloud/serverless" const conn = connect({url: Bun.env.DATABASE_URL}) const result = await conn.execute('show tables') @@ -152,16 +152,17 @@ You can configure TiDB Cloud serverless driver at both the connection level and At the connection level, you can make the following configurations: -| Name | Type | Default value | Description | -|--------------|------------|---------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `username` | string | N/A | Username of TiDB Serverless | -| `password` | string | N/A | Password of TiDB Serverless | -| `host` | string | N/A | Hostname of TiDB Serverless | -| `database` | string | `test` | Database of TiDB Serverless | -| `url` | string | N/A | The URL for the database, in the `mysql://[username]:[password]@[host]/[database]` format, where `database` can be skipped if you intend to connect to the default database. | -| `fetch` | function | global fetch | Custom fetch function. For example, you can use the `undici` fetch in node.js. | -| `arrayMode` | bool | `false` | Whether to return results as arrays instead of objects. To get better performance, set it to `true`. | -| `fullResult` | bool | `false` | Whether to return full result object instead of just rows. To get more detailed results, set it to `true`. | +| Name | Type | Default value | Description | +|--------------|----------|---------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `username` | string | N/A | Username of TiDB Cloud Serverless | +| `password` | string | N/A | Password of TiDB Cloud Serverless | +| `host` | string | N/A | Hostname of TiDB Cloud Serverless | +| `database` | string | `test` | Database of TiDB Cloud Serverless | +| `url` | string | N/A | The URL for the database, in the `mysql://[username]:[password]@[host]/[database]` format, where `database` can be skipped if you intend to connect to the default database. | +| `fetch` | function | global fetch | Custom fetch function. For example, you can use the `undici` fetch in node.js. | +| `arrayMode` | bool | `false` | Whether to return results as arrays instead of objects. To get better performance, set it to `true`. | +| `fullResult` | bool | `false` | Whether to return full result object instead of just rows. To get more detailed results, set it to `true`. | +| `decoders` | object | `{}` | A collection of key-value pairs, which enables you to customize the decoding process for different column types. In each pair, you can specify a column type as the key and specify a corresponding function as the value. This function takes the raw string value received from TiDB Cloud serverless driver as an argument and returns the decoded value. | **Database URL** @@ -200,31 +201,64 @@ const conn = connect(config) At the SQL level, you can configure the following options: -| Option | Type | Default value | Description | -|--------------|------|---------------|------------------------------------------------------------------------------------------------------------| -| `arrayMode` | bool | `false` | Whether to return results as arrays instead of objects. To get better performance, set it to `true`. | -| `fullResult` | bool | `false` | Whether to return full result object instead of just rows. To get more detailed results, set it to `true`. | +| Option | Type | Default value | Description | +|--------------|--------|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `arrayMode` | bool | `false` | Whether to return results as arrays instead of objects. To get better performance, set it to `true`. | +| `fullResult` | bool | `false` | Whether to return the full result object instead of just rows. To get more detailed results, set it to `true`. | +| `isolation` | string | `REPEATABLE READ` | The transaction isolation level, which can be set to `READ COMMITTED` or `REPEATABLE READ`. | +| `decoders` | object | `{}` | A collection of key-value pairs, which enables you to customize the decoding process for different column types. In each pair, you can specify a column type as the key and specify a corresponding function as the value. This function takes the raw string value received from TiDB Cloud serverless driver as an argument and returns the decoded value. If you have configured `decoders` at both the connection and SQL levels, the key-value pairs with different keys configured at the connection level will be merged to the SQL level to take effect. If the same key (this is, column type) is specified at both levels, the value at the SQL level takes precedence. | -For example: +**arrayMode and fullResult** + +To return the full result object as arrays, you can configure the `arrayMode` and `fullResult` options as follows: ```ts const conn = connect({url: process.env['DATABASE_URL'] || 'mysql://[username]:[password]@[host]/[database]'}) const results = await conn.execute('select * from test',null,{arrayMode:true,fullResult:true}) ``` -Starting from TiDB Cloud serverless driver v0.0.7, you can also configure the following SQL level option when you use transactions: - -| Option | Type | Default value | Description | -|--------------|--------|-------------------|------------------------------------------------------------------------------------| -| `isolation` | string | `REPEATABLE READ` | The transaction isolation level, which can be set to `READ COMMITTED` or `REPEATABLE READ`. | +**isolation** -The `isolation` option can only be used in the `begin` function. Here is an example: +The `isolation` option can only be used in the `begin` function. ```ts const conn = connect({url: 'mysql://[username]:[password]@[host]/[database]'}) const tx = await conn.begin({isolation:"READ COMMITTED"}) ``` +**decoders** + +To customize the format of returned column values, you can configure the `decoder` option in the `connect()` method as follows: + +```ts +import { connect, ColumnType } from '@tidbcloud/serverless'; + +const conn = connect({ + url: 'mysql://[username]:[password]@[host]/[database]', + decoders: { + // By default, TiDB Cloud serverless driver returns the BIGINT type as text value. This decoder converts BIGINT to the JavaScript built-in BigInt type. + [ColumnType.BIGINT]: (rawValue: string) => BigInt(rawValue), + + // By default, TiDB Cloud serverless driver returns the DATETIME type as the text value in the 'yyyy-MM-dd HH:mm:ss' format. This decoder converts the DATETIME text to the JavaScript native Date object. + [ColumnType.DATETIME]: (rawValue: string) => new Date(rawValue), + } +}) + +// You can also configure the decoder option at the SQL level to override the decoders with the same keys at the connection level. +conn.execute(`select ...`, [], { + decoders: { + // ... + } +}) +``` + +> **Note:** +> +> TiDB Cloud serverless driver configuration changes: +> +> - v0.0.7: add the SQL level option `isolation`. +> - v0.0.10: add the connection level configuration `decoders` and the SQL level option `decoders`. + ## Features ### Supported SQL statements @@ -233,9 +267,9 @@ DDL is supported and the following SQL statements are supported: `SELECT`, `SHO ### Data type mapping -The type mapping between TiDB Serverless and Javascript is as follows: +The type mapping between TiDB Cloud Serverless and Javascript is as follows: -| TiDB Serverless type | Javascript type | +| TiDB Cloud Serverless type | Javascript type | |----------------------|-----------------| | TINYINT | number | | UNSIGNED TINYINT | number | @@ -253,27 +287,37 @@ The type mapping between TiDB Serverless and Javascript is as follows: | DECIMAL | string | | CHAR | string | | VARCHAR | string | -| BINARY | string | -| VARBINARY | string | +| BINARY | Uint8Array | +| VARBINARY | Uint8Array | | TINYTEXT | string | | TEXT | string | | MEDIUMTEXT | string | | LONGTEXT | string | -| TINYBLOB | string | -| BLOB | string | -| MEDIUMBLOB | string | -| LONGBLOB | string | +| TINYBLOB | Uint8Array | +| BLOB | Uint8Array | +| MEDIUMBLOB | Uint8Array | +| LONGBLOB | Uint8Array | | DATE | string | | TIME | string | | DATETIME | string | | TIMESTAMP | string | | ENUM | string | | SET | string | -| BIT | string | +| BIT | Uint8Array | | JSON | object | | NULL | null | | Others | string | +> **Note:** +> +> Make sure to use the default `utf8mb4` character set in TiDB Cloud Serverless for the type conversion to JavaScript strings, because TiDB Cloud serverless driver uses the UTF-8 encoding to decode them to strings. + +> **Note:** +> +> TiDB Cloud serverless driver data type mapping changes: +> +> - v0.1.0: the `BINARY`, `VARBINARY`, `TINYBLOB`, `BLOB`, `MEDIUMBLOB`, `LONGBLOB`, and `BIT` types are now returned as a `Uint8Array` instead of a `string`. + ### ORM integrations TiDB Cloud serverless driver has been integrated with the following ORMs: @@ -283,7 +327,7 @@ TiDB Cloud serverless driver has been integrated with the following ORMs: ## Pricing -The serverless driver itself is free, but accessing data with the driver generates [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit) and storage usage. The pricing follows the [TiDB Serverless pricing](https://www.pingcap.com/tidb-serverless-pricing-details/) model. +The serverless driver itself is free, but accessing data with the driver generates [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit) and storage usage. The pricing follows the [TiDB Cloud Serverless pricing](https://www.pingcap.com/tidb-serverless-pricing-details/) model. ## Limitations diff --git a/tidb-cloud/serverless-export.md b/tidb-cloud/serverless-export.md new file mode 100644 index 0000000000000..d3f6f185daf8e --- /dev/null +++ b/tidb-cloud/serverless-export.md @@ -0,0 +1,428 @@ +--- +title: Export Data from TiDB Cloud Serverless +summary: Learn how to export data from TiDB Cloud Serverless clusters. +--- + +# Export Data from TiDB Cloud Serverless + +TiDB Cloud Serverless Export (Beta) is a service that enables you to export data from a TiDB Cloud Serverless cluster to a local file or an external storage service. You can use the exported data for backup, migration, data analysis, or other purposes. + +While you can also export data using tools such as [mysqldump](https://dev.mysql.com/doc/refman/8.0/en/mysqldump.html) and TiDB [Dumpling](https://docs.pingcap.com/tidb/dev/dumpling-overview), TiDB Cloud Serverless Export offers a more convenient and efficient way to export data from a TiDB Cloud Serverless cluster. It brings the following benefits: + +- Convenience: the export service provides a simple and easy-to-use way to export data from a TiDB Cloud Serverless cluster, eliminating the need for additional tools or resources. +- Isolation: the export service uses separate computing resources, ensuring isolation from the resources used by your online services. +- Consistency: the export service ensures the consistency of the exported data without causing locks, which does not affect your online services. + +> **Note:** +> +> The maximum export size is currently 1 TiB. To export more data or request a higher export speed, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). + +## Export locations + +You can export data to the following locations: + +- A local file +- An external storage, including: + + - [Amazon S3](https://aws.amazon.com/s3/) + - [Google Cloud Storage](https://cloud.google.com/storage) + - [Azure Blob Storage](https://azure.microsoft.com/en-us/services/storage/blobs/) + - [Alibaba Cloud Object Storage Service (OSS)](https://www.alibabacloud.com/product/oss) + +> **Note:** +> +> If the size of the data to be exported is large (more than 100 GiB), it is recommended that you export it to an external storage. + +### A local file + +To export data from a TiDB Cloud Serverless cluster to a local file, you need to export data [using the TiDB Cloud console](#export-data-to-a-local-file) or [using the TiDB Cloud CLI](/tidb-cloud/ticloud-serverless-export-create.md), and then download the exported data using the TiDB Cloud CLI. + +Exporting data to a local file has the following limitations: + +- Downloading exported data using the TiDB Cloud console is not supported. +- Exported data is saved in the stashing area of TiDB Cloud and will expire after two days. You need to download the exported data in time. +- If the storage space of stashing area is full, you will not be able to export data to the local file. + +### Amazon S3 + +To export data to Amazon S3, you need to provide the following information: + +- URI: `s3:////` +- One of the following access credentials: + - [An access key](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html): make sure the access key has the `s3:PutObject` and `s3:ListBucket` permissions. + - [A role ARN](https://docs.aws.amazon.com/IAM/latest/UserGuide/reference-arns.html): make sure the role ARN (Amazon Resource Name) has the `s3:PutObject` and `s3:ListBucket` permissions. Note that only clusters hosted on AWS support the role ARN. + +For more information, see [Configure Amazon S3 access](/tidb-cloud/serverless-external-storage.md#configure-amazon-s3-access). + +### Google Cloud Storage + +To export data to Google Cloud Storage, you need to provide the following information: + +- URI: `gs:////` +- Access credential: a **base64 encoded** [service account key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) for your bucket. Make sure the service account key has the `storage.objects.create` permission. + +For more information, see [Configure GCS access](/tidb-cloud/serverless-external-storage.md#configure-gcs-access). + +### Azure Blob Storage + +To export data to Azure Blob Storage, you need to provide the following information: + +- URI: `azure://.blob.core.windows.net///` or `https://.blob.core.windows.net///` +- Access credential: a [shared access signature (SAS) token](https://docs.microsoft.com/en-us/azure/storage/common/storage-sas-overview) for your Azure Blob Storage container. Make sure the SAS token has the `Read` and `Write` permissions on the `Container` and `Object` resources. + +For more information, see [Configure Azure Blob Storage access](/tidb-cloud/serverless-external-storage.md#configure-azure-blob-storage-access). + +### Alibaba Cloud OSS + +To export data to Alibaba Cloud OSS, you need to provide the following information: + +- URI: `oss:////` +- Access credential: An [AccessKey pair](https://www.alibabacloud.com/help/en/ram/user-guide/create-an-accesskey-pair) for your Alibaba Cloud account. Make sure the AccessKey pair has the `oss:PutObject`, `oss:ListBuckets` and `oss:GetBucketInfo` permissions to allow data export to the OSS bucket. + +For more information, see [Configure Alibaba Cloud Object Storage Service (OSS) access](/tidb-cloud/serverless-external-storage.md#configure-alibaba-cloud-object-storage-service-oss-access). + +## Export options + +### Data filtering + +- TiDB Cloud console supports exporting data with the selected databases and tables. +- TiDB Cloud CLI supports exporting data with SQL statements and [table filters](/table-filter.md). + +### Data formats + +You can export data in the following formats: + +- `SQL`: export data in SQL format. +- `CSV`: export data in CSV format. You can specify the following options: + - `delimiter`: specify the delimiter used in the exported data. The default delimiter is `"`. + - `separator`: specify the character used to separate fields in the exported data. The default separator is `,`. + - `header`: specify whether to include a header row in the exported data. The default value is `true`. + - `null-value`: specify the string that represents a NULL value in the exported data. The default value is `\N`. +- `Parquet`: export data in Parquet format. + +The schema and data are exported according to the following naming conventions: + +| Item | Not compressed | Compressed | +|-----------------|-------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------| +| Database schema | {database}-schema-create.sql | {database}-schema-create.sql.{compression-type} | +| Table schema | {database}.{table}-schema.sql | {database}.{table}-schema.sql.{compression-type} | +| Data | {database}.{table}.{0001}.{csv|parquet|sql} | {database}.{table}.{0001}.{csv|sql}.{compression-type}
{database}.{table}.{0001}.{compression-type}.parquet | + +### Data compression + +You can compress the exported CSV and SQL data using the following algorithms: + +- `gzip` (default): compress the exported data with `gzip`. +- `snappy`: compress the exported data with `snappy`. +- `zstd`: compress the exported data with `zstd`. +- `none`: do not compress the exported `data`. + +You can compress the exported Parquet data using the following algorithms: + +- `zstd` (default): compress the Parquet file with `zstd`. +- `gzip`: compress the Parquet file with `gzip`. +- `snappy`: compress the Parquet file with `snappy`. +- `none`: do not compress the Parquet file. + +### Data conversion + +When exporting data to the Parquet format, the data conversion between TiDB Cloud Serverless and Parquet is as follows: + +| TiDB Cloud Serverless Type | Parquest primitive type | Parquet logical type | +|----------------------------|-------------------------|----------------------------------------------| +| VARCHAR | BYTE_ARRAY | String(UTF8) | +| TIME | BYTE_ARRAY | String(UTF8) | +| TINYTEXT | BYTE_ARRAY | String(UTF8) | +| MEDIUMTEXT | BYTE_ARRAY | String(UTF8) | +| TEXT | BYTE_ARRAY | String(UTF8) | +| LONGTEXT | BYTE_ARRAY | String(UTF8) | +| SET | BYTE_ARRAY | String(UTF8) | +| JSON | BYTE_ARRAY | String(UTF8) | +| DATE | BYTE_ARRAY | String(UTF8) | +| CHAR | BYTE_ARRAY | String(UTF8) | +| VECTOR | BYTE_ARRAY | String(UTF8) | +| DECIMAL(1<=p<=9) | INT32 | DECIMAL(p,s) | +| DECIMAL(10<=p<=18) | INT64 | DECIMAL(p,s) | +| DECIMAL(p>=19) | BYTE_ARRAY | String(UTF8) | +| ENUM | BYTE_ARRAY | String(UTF8) | +| TIMESTAMP | INT64 | TIMESTAMP(unit=MICROS,isAdjustedToUTC=false) | +| DATETIME | INT64 | TIMESTAMP(unit=MICROS,isAdjustedToUTC=false) | +| YEAR | INT32 | / | +| TINYINT | INT32 | / | +| UNSIGNED TINYINT | INT32 | / | +| SMALLINT | INT32 | / | +| UNSIGNED SMALLINT | INT32 | / | +| MEDIUMINT | INT32 | / | +| UNSIGNED MEDIUMINT | INT32 | / | +| INT | INT32 | / | +| UNSIGNED INT | FIXED_LEN_BYTE_ARRAY(9) | DECIMAL(20,0) | +| BIGINT | FIXED_LEN_BYTE_ARRAY(9) | DECIMAL(20,0) | +| UNSIGNED BIGINT | BYTE_ARRAY | String(UTF8) | +| FLOAT | FLOAT | / | +| DOUBLE | DOUBLE | / | +| BLOB | BYTE_ARRAY | / | +| TINYBLOB | BYTE_ARRAY | / | +| MEDIUMBLOB | BYTE_ARRAY | / | +| LONGBLOB | BYTE_ARRAY | / | +| BINARY | BYTE_ARRAY | / | +| VARBINARY | BYTE_ARRAY | / | +| BIT | BYTE_ARRAY | / | + +## Examples + +### Export data to a local file + + +
+ +1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + +2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +3. On the **Import** page, click **Export Data to** in the upper-right corner, then choose **Local File** from the drop-down list. Fill in the following parameters: + + - **Task Name**: enter a name for the export task. The default value is `SNAPSHOT_{snapshot_time}`. + - **Exported Data**: choose the databases and tables you want to export. + - **Data Format**: choose **SQL**, **CSV**, or **Parquet**. + - **Compression**: choose **Gzip**, **Snappy**, **Zstd**, or **None**. + + > **Tip:** + > + > If your cluster has neither imported nor exported any data before, you need to click **Click here to export data to...** at the bottom of the page to export data. + +4. Click **Export**. + +5. After the export task is successful, you can copy the download command displayed in the export task detail, and then download the exported data by running the command in the [TiDB Cloud CLI](/tidb-cloud/cli-reference.md). + +
+ +
+ +1. Create an export task: + + ```shell + ticloud serverless export create -c + ``` + + You will get an export ID from the output. + +2. After the export task is successful, download the exported data to your local file: + + ```shell + ticloud serverless export download -c -e + ``` + + For more information about the download command, see [ticloud serverless export download](/tidb-cloud/ticloud-serverless-export-download.md). + +
+ + +### Export data to Amazon S3 + + +
+ +1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + +2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +3. On the **Import** page, click **Export Data to** in the upper-right corner, then choose **Amazon S3** from the drop-down list. Fill in the following parameters: + + - **Task Name**: enter a name for the export task. The default value is `SNAPSHOT_{snapshot_time}`. + - **Exported Data**: choose the databases and tables you want to export. + - **Data Format**: choose **SQL**, **CSV**, or **Parquet**. + - **Compression**: choose **Gzip**, **Snappy**, **Zstd**, or **None**. + - **Folder URI**: enter the URI of the Amazon S3 with the `s3:////` format. + - **Bucket Access**: choose one of the following access credentials and then fill in the credential information: + - **AWS Role ARN**: enter the role ARN that has the permission to access the bucket. It is recommended to create the role ARN with AWS CloudFormation. For more information, see [Configure External Storage Access for TiDB Cloud Serverless](/tidb-cloud/serverless-external-storage.md#configure-amazon-s3-access). + - **AWS Access Key**: enter the access key ID and access key secret that have the permission to access the bucket. + +4. Click **Export**. + +
+ +
+ +```shell +ticloud serverless export create -c --target-type S3 --s3.uri --s3.access-key-id --s3.secret-access-key --filter "database.table" + +ticloud serverless export create -c --target-type S3 --s3.uri --s3.role-arn --filter "database.table" +``` + +- `s3.uri`: the Amazon S3 URI with the `s3:////` format. +- `s3.access-key-id`: the access key ID of the user who has the permission to access the bucket. +- `s3.secret-access-key`: the access key secret of the user who has the permission to access the bucket. +- `s3.role-arn`: the role ARN that has the permission to access the bucket. + +
+ + +### Export data to Google Cloud Storage + + +
+ +1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + +2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +3. On the **Import** page, click **Export Data to** in the upper-right corner, and then choose **Google Cloud Storage** from the drop-down list. Fill in the following parameters: + + - **Task Name**: enter a name for the export task. The default value is `SNAPSHOT_{snapshot_time}`. + - **Exported Data**: choose the databases and tables you want to export. + - **Data Format**: choose **SQL**, **CSV**, or **Parquet**. + - **Compression**: choose **Gzip**, **Snappy**, **Zstd**, or **None**. + - **Folder URI**: enter the URI of the Google Cloud Storage with the `gs:////` format. + - **Bucket Access**: upload the Google Cloud credentials file that has permission to access the bucket. + +4. Click **Export**. + +
+ +
+ +```shell +ticloud serverless export create -c --target-type GCS --gcs.uri --gcs.service-account-key --filter "database.table" +``` + +- `gcs.uri`: the URI of the Google Cloud Storage bucket in the `gs:////` format. +- `gcs.service-account-key`: the base64 encoded service account key. + +
+ + +### Export data to Azure Blob Storage + + +
+ +1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + +2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +3. On the **Import** page, click **Export Data to** in the upper-right corner, and then choose **Azure Blob Storage** from the drop-down list. Fill in the following parameters: + + - **Task Name**: enter a name for the export task. The default value is `SNAPSHOT_{snapshot_time}`. + - **Exported Data**: choose the databases and tables you want to export. + - **Data Format**: choose **SQL**, **CSV**, or **Parquet**. + - **Compression**: choose **Gzip**, **Snappy**, **Zstd**, or **None**. + - **Folder URI**: enter the URI of Azure Blob Storage with the `azure://.blob.core.windows.net///` format. + - **SAS Token**: enter the SAS token that has the permission to access the container. It is recommended to create a SAS token with the [Azure ARM template](https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/). For more information, see [Configure External Storage Access for TiDB Cloud Serverless](/tidb-cloud/serverless-external-storage.md#configure-azure-blob-storage-access). + +4. Click **Export**. + +
+ +
+ +```shell +ticloud serverless export create -c --target-type AZURE_BLOB --azblob.uri --azblob.sas-token --filter "database.table" +``` + +- `azblob.uri`: the URI of the Azure Blob Storage in the `(azure|https)://.blob.core.windows.net///` format. +- `azblob.sas-token`: the account SAS token of the Azure Blob Storage. + +
+ + +### Export data to Alibaba Cloud OSS + + +
+ +1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + +2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +3. On the **Import** page, click **Export Data to** in the upper-right corner, and then choose **Alibaba Cloud OSS** from the drop-down list. + +4. Fill in the following parameters: + + - **Task Name**: enter a name for the export task. The default value is `SNAPSHOT_{snapshot_time}`. + - **Exported Data**: choose the databases and tables you want to export. + - **Data Format**: choose **SQL**, **CSV**, or **Parquet**. + - **Compression**: choose **Gzip**, **Snappy**, **Zstd**, or **None**. + - **Folder URI**: enter the Alibaba Cloud OSS URI where you want to export the data, in the `oss:////` format. + - **AccessKey ID** and **AccessKey Secret**: enter the AccessKey ID and AccessKey Secret that have the permission to access the bucket. + +5. Click **Export**. + +
+ +
+ +```shell +ticloud serverless export create -c --target-type OSS --oss.uri --oss.access-key-id --oss.access-key-secret --filter "database.table" +``` + +- `oss.uri`: the Alibaba Cloud OSS URI where you want to export the data, in the `oss:////` format. +- `oss.access-key-id`: the AccessKey ID of the user who has the permission to access the bucket. +- `oss.access-key-secret`: the AccessKey secret of the user who has the permission to access the bucket. + +
+ + +### Cancel an export task + +To cancel an ongoing export task, take the following steps: + + +
+ +1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + +2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +3. On the **Import** page, click **Export** to view the export task list. + +4. Choose the export task you want to cancel, and then click **Action**. + +5. Choose **Cancel** in the drop-down list. Note that you can only cancel the export task that is in the **Running** status. + +
+ +
+ +```shell +ticloud serverless export cancel -c -e +``` + +
+ + +## Export speed + +The export speed depends on your [cluster plan](/tidb-cloud/select-cluster-tier.md#cluster-plans). For details, see the following table: + +| Plan | Export speed | +|:-------------------|:-------------------| +| Free cluster plan | Up to 25 MiB/s | +| Scalable cluster plan | Up to 100 MiB/s | + +## Pricing + +The export service is free during the beta period. You only need to pay for the [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit) generated during the export process of successful or canceled tasks. For failed export tasks, you will not be charged. diff --git a/tidb-cloud/serverless-external-storage.md b/tidb-cloud/serverless-external-storage.md new file mode 100644 index 0000000000000..e63b5cd151f3e --- /dev/null +++ b/tidb-cloud/serverless-external-storage.md @@ -0,0 +1,294 @@ +--- +title: Configure TiDB Cloud Serverless External Storage Access +summary: Learn how to configure Amazon Simple Storage Service (Amazon S3) access. +--- + +# Configure External Storage Access for TiDB Cloud Serverless + +If you want to import data from or export data to an external storage in a TiDB Cloud Serverless cluster, you need to configure cross-account access. This document describes how to configure access to an external storage for TiDB Cloud Serverless clusters. + +If you need to configure these external storages for a TiDB Cloud Dedicated cluster, see [Configure External Storage Access for TiDB Cloud Dedicated](/tidb-cloud/config-s3-and-gcs-access.md). + +## Configure Amazon S3 access + +To allow a TiDB Cloud Serverless cluster to access the source data in your Amazon S3 bucket, configure the bucket access for the cluster using either of the following methods: + +- [Use a Role ARN](#configure-amazon-s3-access-using-a-role-arn): use a Role ARN to access your Amazon S3 bucket. +- [Use an AWS access key](#configure-amazon-s3-access-using-an-aws-access-key): use the access key of an IAM user to access your Amazon S3 bucket. + +### Configure Amazon S3 access using a Role ARN + +It is recommended that you use [AWS CloudFormation](https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/Welcome.html) to create a role ARN. Take the following steps to create one: + +> **Note:** +> +> Role ARN access to Amazon S3 is only supported for clusters with AWS as the cloud provider. If you use a different cloud provider, use an AWS access key instead. For more information, see [Configure Amazon S3 access using an AWS access key](#configure-amazon-s3-access-using-an-aws-access-key). + +1. Open the **Import** page for your target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +2. Open the **Add New ARN** dialog. + + - If you want to import data from Amazon S3, open the **Add New ARN** dialog as follows: + + 1. Click **Import from S3**. + 2. Fill in the **File URI** field. + 3. Choose **AWS Role ARN** and click **Click here to create new one with AWS CloudFormation**. + + - If you want to export data to Amazon S3, open the **Add New ARN** dialog as follows: + + 1. Click **Export data to...** > **Amazon S3**. If your cluster has neither imported nor exported any data before, click **Click here to export data to...** > **Amazon S3** at the bottom of the page. + 2. Fill in the **Folder URI** field. + 3. Choose **AWS Role ARN** and click **Click here to create new one with AWS CloudFormation**. + +3. Create a role ARN with an AWS CloudFormation template. + + 1. In the **Add New ARN** dialog, click **AWS Console with CloudFormation Template**. + + 2. Log in to the [AWS Management Console](https://console.aws.amazon.com) and you will be redirected to the AWS CloudFormation **Quick create stack** page. + + 3. Fill in the **Role Name**. + + 4. Acknowledge to create a new role and click **Create stack** to create the role ARN. + + 5. After the CloudFormation stack is executed, you can click the **Outputs** tab and find the Role ARN value in the **Value** column. + + ![img.png](/media/tidb-cloud/serverless-external-storage/serverless-role-arn.png) + +If you have any trouble creating a role ARN with AWS CloudFormation, you can take the following steps to create one manually: + +
+Click here to see details + +1. In the **Add New ARN** dialog described in previous instructions, click **Having trouble? Create Role ARN manually**. You will get the **TiDB Cloud Account ID** and **TiDB Cloud External ID**. + +2. In the AWS Management Console, create a managed policy for your Amazon S3 bucket. + + 1. Sign in to the [AWS Management Console](https://console.aws.amazon.com/) and open the [Amazon S3 console](https://console.aws.amazon.com/s3/). + + 2. In the **Buckets** list, choose the name of your bucket with the source data, and then click **Copy ARN** to get your S3 bucket ARN (for example, `arn:aws:s3:::tidb-cloud-source-data`). Take a note of the bucket ARN for later use. + + ![Copy bucket ARN](/media/tidb-cloud/copy-bucket-arn.png) + + 3. Open the [IAM console](https://console.aws.amazon.com/iam/), click **Policies** in the left navigation pane, and then click **Create Policy**. + + ![Create a policy](/media/tidb-cloud/aws-create-policy.png) + + 4. On the **Create policy** page, click the **JSON** tab. + + 5. Configure the policy in the policy text field according to your needs. The following is an example that you can use to export data from and import data into a TiDB Cloud Serverless cluster. + + - Exporting data from a TiDB Cloud Serverless cluster needs the **s3:PutObject** and **s3:ListBucket** permissions. + - Importing data into a TiDB Cloud Serverless cluster needs the **s3:GetObject**, **s3:GetObjectVersion**, and **s3:ListBucket** permissions. + + ```json + { + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "VisualEditor0", + "Effect": "Allow", + "Action": [ + "s3:GetObject", + "s3:GetObjectVersion", + "s3:PutObject" + ], + "Resource": "//*" + }, + { + "Sid": "VisualEditor1", + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": "" + } + ] + } + ``` + + In the policy text field, replace the following configurations with your own values. + + - `"Resource": "//*"`. For example: + + - If your source data is stored in the root directory of the `tidb-cloud-source-data` bucket, use `"Resource": "arn:aws:s3:::tidb-cloud-source-data/*"`. + - If your source data is stored in the `mydata` directory of the bucket, use `"Resource": "arn:aws:s3:::tidb-cloud-source-data/mydata/*"`. + + Make sure that `/*` is added to the end of the directory so TiDB Cloud can access all files in this directory. + + - `"Resource": ""`, for example, `"Resource": "arn:aws:s3:::tidb-cloud-source-data"`. + + - If you have enabled AWS Key Management Service key (SSE-KMS) with customer-managed key encryption, make sure the following configuration is included in the policy. `"arn:aws:kms:ap-northeast-1:105880447796:key/c3046e91-fdfc-4f3a-acff-00597dd3801f"` is a sample KMS key of the bucket. + + ``` + { + "Sid": "AllowKMSkey", + "Effect": "Allow", + "Action": [ + "kms:Decrypt" + ], + "Resource": "arn:aws:kms:ap-northeast-1:105880447796:key/c3046e91-fdfc-4f3a-acff-00597dd3801f" + } + ``` + + - If the objects in your bucket have been copied from another encrypted bucket, the KMS key value needs to include the keys of both buckets. For example, `"Resource": ["arn:aws:kms:ap-northeast-1:105880447796:key/c3046e91-fdfc-4f3a-acff-00597dd3801f","arn:aws:kms:ap-northeast-1:495580073302:key/0d7926a7-6ecc-4bf7-a9c1-a38f0faec0cd"]`. + + 6. Click **Next**. + + 7. Set a policy name, add a tag of the policy (optional), and then click **Create policy**. + +3. In the AWS Management Console, create an access role for TiDB Cloud and get the role ARN. + + 1. In the [IAM console](https://console.aws.amazon.com/iam/), click **Roles** in the left navigation pane, and then click **Create role**. + + ![Create a role](/media/tidb-cloud/aws-create-role.png) + + 2. To create a role, fill in the following information: + + - In **Trusted entity type**, select **AWS account**. + - In **An AWS account**, select **Another AWS account**, and then paste the TiDB Cloud account ID to the **Account ID** field. + - In **Options**, click **Require external ID (Best practice when a third party will assume this role)**, and then paste the TiDB Cloud External ID to the **External ID** field. If the role is created without a Require external ID, once the configuration is done for one TiDB cluster in a project, all TiDB clusters in that project can use the same Role ARN to access your Amazon S3 bucket. If the role is created with the account ID and external ID, only the corresponding TiDB cluster can access the bucket. + + 3. Click **Next** to open the policy list, choose the policy you just created, and then click **Next**. + + 4. In **Role details**, set a name for the role, and then click **Create role** in the lower-right corner. After the role is created, the list of roles is displayed. + + 5. In the list of roles, click the name of the role that you just created to go to its summary page, and then you can get the role ARN. + + ![Copy AWS role ARN](/media/tidb-cloud/aws-role-arn.png) + +
+ +### Configure Amazon S3 access using an AWS access key + +It is recommended that you use an IAM user (instead of the AWS account root user) to create an access key. + +Take the following steps to configure an access key: + +1. Create an IAM user. For more information, see [creating an IAM user](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users_create.html#id_users_create_console). + +2. Use your AWS account ID or account alias, and your IAM user name and password to sign in to [the IAM console](https://console.aws.amazon.com/iam). + +3. Create an access key. For more information, see [creating an access key for an IAM user](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey). + +> **Note:** +> +> TiDB Cloud does not store your access keys. It is recommended that you [delete the access key](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_credentials_access-keys.html#Using_CreateAccessKey) after the import or export is complete. + +## Configure GCS access + +To allow a TiDB Cloud Serverless cluster to access your GCS bucket, you need to configure the GCS access for the bucket. You can use a service account key to configure the bucket access: + +Take the following steps to configure a service account key: + +1. On the Google Cloud [service account page](https://console.cloud.google.com/iam-admin/serviceaccounts), click **CREATE SERVICE ACCOUNT** to create a service account. For more information, see [Creating a service account](https://cloud.google.com/iam/docs/creating-managing-service-accounts). + + 1. Enter a service account name. + 2. Optional: Enter a description of the service account. + 3. Click **CREATE AND CONTINUE** to create the service account. + 4. In the `Grant this service account access to project`, choose the [IAM roles](https://cloud.google.com/iam/docs/understanding-roles) with the needed permission. + + - Exporting data from a TiDB Cloud Serverless cluster needs a role with `storage.objects.create` permission. + - Importing data into a TiDB Cloud Serverless cluster needs a role with `storage.buckets.get`, `storage.objects.get`, and `storage.objects.list` permissions. + + 5. Click **Continue** to go to the next step. + 6. Optional: In the `Grant users access to this service account`, choose members that need to [attach the service account to other resources](https://cloud.google.com/iam/docs/attach-service-accounts). + 7. Click **Done** to finish creating the service account. + + ![service-account](/media/tidb-cloud/serverless-external-storage/gcs-service-account.png) + +2. Click the service account, and then click **ADD KEY** on the `KEYS` page to create a service account key. + + ![service-account-key](/media/tidb-cloud/serverless-external-storage/gcs-service-account-key.png) + +3. Choose the default `JSON` key type, and then click **CREATE** to download the Google Cloud credentials file. The file contains the service account key that you need to use when configuring the GCS access for the TiDB Cloud Serverless cluster. + +## Configure Azure Blob Storage access + +To allow TiDB Cloud Serverless to access your Azure Blob container, you need to create a service SAS token for the container. + +You can create a SAS token either using an [Azure ARM template](https://learn.microsoft.com/en-us/azure/azure-resource-manager/templates/overview) (recommended) or manual configuration. + +To create a SAS token using an Azure ARM template, take the following steps: + +1. Open the **Import** page for your target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. + +2. Open the **Generate New SAS Token via ARM Template Deployment** dialog. + + 1. Click **Export data to...** > **Azure Blob Storage**. If your cluster has neither imported nor exported any data before, click **Click here to export data to...** > **Azure Blob Storage** at the bottom of the page. + + 2. Scroll down to the **Azure Blob Storage Settings** area, and then click **Click here to create a new one with Azure ARM template** under the SAS Token field. + +3. Create a SAS token with the Azure ARM template. + + 1. In the **Generate New SAS Token via ARM Template Deployment** dialog, click **Click to open the Azure Portal with the pre-configured ARM template**. + + 2. After logging in to Azure, you will be redirected to the Azure **Custom deployment** page. + + 3. Fill in the **Resource group** and **Storage Account Name** in the **Custom deployment** page. You can get all the information from the storage account overview page where the container is located. + + ![azure-storage-account-overview](/media/tidb-cloud/serverless-external-storage/azure-storage-account-overview.png) + + 4. Click **Review + create** or **Next** to review the deployment. Click **Create** to start the deployment. + + 5. After it completes, you will be redirected to the deployment overview page. Navigate to the **Outputs** section to get the SAS token. + +If you have any trouble creating a SAS token with the Azure ARM template, take the following steps to create one manually: + +
+Click here to see details + +1. On the [Azure Storage account](https://portal.azure.com/#browse/Microsoft.Storage%2FStorageAccounts) page, click your storage account to which the container belongs. + +2. On your **Storage account** page, click the **Security+network**, and then click **Shared access signature**. + + ![sas-position](/media/tidb-cloud/serverless-external-storage/azure-sas-position.png) + +3. On the **Shared access signature** page, create a service SAS token with needed permissions as follows. For more information, see [Create a service SAS token](https://docs.microsoft.com/en-us/azure/storage/common/storage-sas-overview). + + 1. In the **Allowed services** section, choose the **Blob** service. + 2. In the **Allowed Resource types** section, choose **Container** and **Object**. + 3. In the **Allowed permissions** section, choose the permission as needed. + + - Exporting data from a TiDB Cloud Serverless cluster needs the **Read** and **Write** permissions. + - Importing data into a TiDB Cloud Serverless cluster needs the **Read** and **List** permissions. + + 4. Adjust **Start and expiry date/time** as needed. + 5. You can keep the default values for other settings. + + ![sas-create](/media/tidb-cloud/serverless-external-storage/azure-sas-create.png) + +4. Click **Generate SAS and connection string** to generate the SAS token. + +
+ +## Configure Alibaba Cloud Object Storage Service (OSS) access + +To allow TiDB Cloud Serverless to access your Alibaba Cloud OSS bucket, you need to create an AccessKey pair for the bucket. + +Take the following steps to configure an AccessKey pair: + +1. Create a RAM user and get the AccessKey pair. For more information, see [Create a RAM user](https://www.alibabacloud.com/help/en/ram/user-guide/create-a-ram-user). + + In the **Access Mode** section, select **Using permanent AccessKey to access**. + +2. Create a custom policy with the required permissions. For more information, see [Create custom policies](https://www.alibabacloud.com/help/en/ram/user-guide/create-a-custom-policy). + + - In the **Effect** section, select **Allow**. + - In the **Service** section, select **Object Storage Service**. + - In the **Action** section, select the permissions as needed. + + To import data into a TiDB Cloud Serverless cluster, grant **oss:GetObject**, **oss:GetBucketInfo**, and **oss:ListObjects** permissions. + + To export data from a TiDB Cloud Serverless cluster, grant **oss:PutObject**, **oss:GetBucketInfo**, and **oss:ListBuckets** permissions. + + - In the **Resource** section, select the bucket and the objects in the bucket. + +3. Attach the custom policies to the RAM user. For more information, see [Grant permissions to a RAM user](https://www.alibabacloud.com/help/en/ram/user-guide/grant-permissions-to-the-ram-user). diff --git a/tidb-cloud/serverless-faqs.md b/tidb-cloud/serverless-faqs.md index 8aa876ddc0b61..4675b76fd11cf 100644 --- a/tidb-cloud/serverless-faqs.md +++ b/tidb-cloud/serverless-faqs.md @@ -1,87 +1,105 @@ --- -title: TiDB Serverless FAQs -summary: Learn about the most frequently asked questions (FAQs) relating to TiDB Serverless. +title: TiDB Cloud Serverless FAQs +summary: Learn about the most frequently asked questions (FAQs) relating to TiDB Cloud Serverless. aliases: ['/tidbcloud/serverless-tier-faqs'] --- -# TiDB Serverless FAQs +# TiDB Cloud Serverless FAQs -This document lists the most frequently asked questions about TiDB Serverless. +This document lists the most frequently asked questions about TiDB Cloud Serverless. ## General FAQs -### What is TiDB Serverless? +### What is TiDB Cloud Serverless? -TiDB Serverless offers the TiDB database with full HTAP capabilities for you and your organization. It is a fully managed, auto-scaling deployment of TiDB that lets you start using your database immediately, develop and run your application without caring about the underlying nodes, and automatically scale based on your application's workload changes. +TiDB Cloud Serverless offers the TiDB database with full HTAP capabilities for you and your organization. It is a fully managed, auto-scaling deployment of TiDB that lets you start using your database immediately, develop and run your application without caring about the underlying nodes, and automatically scale based on your application's workload changes. -### How do I get started with TiDB Serverless? +### How do I get started with TiDB Cloud Serverless? Get started with the 5-minute [TiDB Cloud Quick Start](/tidb-cloud/tidb-cloud-quickstart.md). -### How many TiDB Serverless clusters can I create in TiDB Cloud? +### How many TiDB Cloud Serverless clusters can I create in TiDB Cloud? -For each organization in TiDB Cloud, you can create a maximum of five TiDB Serverless clusters by default. To create more TiDB Serverless clusters, you need to add a credit card and set a [spending limit](/tidb-cloud/tidb-cloud-glossary.md#spending-limit) for usage. +For each organization in TiDB Cloud, you can create a maximum of five [free clusters](/tidb-cloud/select-cluster-tier.md#free-cluster-plan) by default. To create more TiDB Cloud Serverless clusters, you need to add a credit card and create [scalable clusters](/tidb-cloud/select-cluster-tier.md#scalable-cluster-plan) for the usage. -### Are all TiDB Cloud features fully supported on TiDB Serverless? +### Are all TiDB Cloud features fully supported on TiDB Cloud Serverless? -Some of TiDB Cloud features are partially supported or not supported on TiDB Serverless. For more information, see [TiDB Serverless Limitations and Quotas](/tidb-cloud/serverless-limitations.md). +Some of TiDB Cloud features are partially supported or not supported on TiDB Cloud Serverless. For more information, see [TiDB Cloud Serverless Limitations and Quotas](/tidb-cloud/serverless-limitations.md). -### When will TiDB serverless be available on cloud platforms other than AWS, such as Google Cloud or Azure? +### When will TiDB Cloud Serverless be available on cloud platforms other than AWS, such as Google Cloud or Azure? -We are actively working on expanding TiDB Serverless to other cloud platforms, including Google Cloud and Azure. However, we do not have an exact timeline for now as we currently focus on filling gaps and ensuring seamless functionality across all environments. Rest assured, we are working hard to make TiDB Serverless available on more cloud platforms, and we will keep our community updated as we progress. +We are actively working on expanding TiDB Cloud Serverless to other cloud platforms, including Google Cloud and Azure. However, we do not have an exact timeline for now as we currently focus on filling gaps and ensuring seamless functionality across all environments. Rest assured, we are working hard to make TiDB Cloud Serverless available on more cloud platforms, and we will keep our community updated as we progress. -### I created a Developer Tier cluster before TiDB Serverless was available. Can I still use my cluster? +### I created a Developer Tier cluster before TiDB Cloud Serverless was available. Can I still use my cluster? -Yes, your Developer Tier cluster has been automatically migrated to the TiDB Serverless cluster, providing you with an improved user experience without any disruptions to your prior usage. +Yes, your Developer Tier cluster has been automatically migrated to the TiDB Cloud Serverless cluster, providing you with an improved user experience without any disruptions to your prior usage. -## Billing and metering FAQs +### What is columnar storage in TiDB Cloud Serverless? -### What are Request Units? +Columnar storage in TiDB Cloud Serverless acts as an additional replica of row-based storage, ensuring strong consistency. Unlike traditional row-based storage, which stores data in rows, columnar storage organizes data in columns, optimizing it for data analytics tasks. + +Columnar storage is a key feature that enables the Hybrid Transactional and Analytical Processing (HTAP) capabilities of TiDB by seamlessly blending transactional and analytical workloads. + +To efficiently manage columnar storage data, TiDB Cloud Serverless uses a separate elastic TiFlash engine. During query execution, the optimizer guides the cluster to automatically decide whether to retrieve data from row-based or columnar storage. + +### When should I use columnar storage in TiDB Cloud Serverless? + +Consider using columnar storage in TiDB Cloud Serverless in the following scenarios: + +- Your workload involves analytical tasks that require efficient data scanning and aggregation. +- You prioritize improved performance, especially for analytics workloads. +- You want to isolate analytical processing from transactional processing to prevent performance impact on your transactional processing (TP) workload. The separate columnar storage helps optimize these distinct workload patterns. + +In these scenarios, columnar storage can significantly improve query performance and provide a seamless experience for mixed workloads in your system. + +### How to use columnar storage in TiDB Cloud Serverless? -TiDB Serverless adopts a pay-as-you-go model, meaning that you only pay for the storage space and cluster usage. In this model, all cluster activities such as SQL queries, bulk operations, and background jobs are quantified in [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit). RU is an abstract measurement for the size and intricacy of requests initiated on your cluster. For more information, see [TiDB Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details/). +Using columnar storage in TiDB Cloud Serverless is similar to using it in TiFlash. You can enable columnar storage at both the table and database levels: -### How can I view the RU costs for my SQL statements in TiDB Serverless? +- Table level: Assign a TiFlash replica to a table to enable columnar storage for that specific table. +- Database level: Configure TiFlash replicas for all tables in a database to use columnar storage across the entire database. -You can view both the **Total RU** and **Mean RU** costs per SQL statement in [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless). This feature helps in identifying and analyzing RU costs, which can lead to potential cost savings in your operations. +Once a TiFlash replica is set up for a table, TiDB automatically replicates data from the row-based storage to the columnar storage for that table. This ensures data consistency and optimizes performance for analytical queries. -To check your SQL statement RU details, perform the following steps: +For more information about how to set up TiFlash replicas, see [Create TiFlash replicas](/tiflash/create-tiflash-replicas.md). -1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. +## Billing and metering FAQs -2. Navigate to the **Diagnosis** page of [your TiDB Serverless cluster](https://tidbcloud.com/console/clusters). +### What are Request Units? -3. Click the **SQL Statement** tab. +TiDB Cloud Serverless adopts a pay-as-you-go model, meaning that you only pay for the storage space and cluster usage. In this model, all cluster activities such as SQL queries, bulk operations, and background jobs are quantified in [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit). RU is an abstract measurement for the size and intricacy of requests initiated on your cluster. For more information, see [TiDB Cloud Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details/). -### Is there any free plan available for TiDB Serverless? +### Is there any free plan available for TiDB Cloud Serverless? -For the first five TiDB Serverless clusters in your organization, TiDB Cloud provides a free usage quota for each of them as follows: +For the first five TiDB Cloud Serverless clusters in your organization, TiDB Cloud provides a free usage quota for each of them as follows: - Row-based storage: 5 GiB +- Columnar storage: 5 GiB - [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit): 50 million RUs per month -Usage beyond the free quota will be charged. Once the free quota of a cluster is reached, the read and write operations on this cluster will be throttled until you [increase the quota](/tidb-cloud/manage-serverless-spend-limit.md#update-spending-limit) or the usage is reset upon the start of a new month. +If you are using a scalable cluster, usage beyond the free quota will be charged. For a free cluster, once the free quota is reached, the read and write operations on this cluster will be throttled until you upgrade to a scalable cluster or the usage is reset upon the start of a new month. -For more information, see [TiDB Serverless usage quota](/tidb-cloud/select-cluster-tier.md#usage-quota). +For more information, see [TiDB Cloud Serverless usage quota](/tidb-cloud/select-cluster-tier.md#usage-quota). ### What are the limitations of the free plan? -Under the free plan, cluster performance is capped at a maximum of 10,000 RUs per second based on actual workload. Additionally, memory allocation per query is limited to 256 MiB. To maximize cluster performance, you can choose to enable the commercial offering by [increasing your spending limit](/tidb-cloud/manage-serverless-spend-limit.md#update-spending-limit). +Under the free plan, cluster performance is limited due to non-scalable resources. This results in a restriction on memory allocation per query to 256 MiB and might cause observable bottlenecks in request units (RUs) per second. To maximize cluster performance and avoid these limitations, you can upgrade to a [scalable cluster](/tidb-cloud/select-cluster-tier.md#scalable-cluster-plan). ### How can I estimate the number of RUs required by my workloads and plan my monthly budget? To get the RU consumption of individual SQL statements, you can use the [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md#ru-request-unit-consumption) SQL statement. However, it is important to note that the RUs usage returned in `EXPLAIN ANALYZE` does not incorporate egress RUs, as egress usage is measured separately in the gateway, which is unknown to the TiDB server. -To get the RUs and storage used by your cluster, view the **Usage this month** pane on your cluster overview page. With your past resource usage data and real-time resource usage in this pane, you can track your cluster's resource consumption and estimate a reasonable spending limit. If the free quota cannot meet your requirement, you can edit the spending limit easily. For more information, see [Manage Spending Limit for TiDB Serverless clusters](/tidb-cloud/manage-serverless-spend-limit.md). +To get the RUs and storage used by your cluster, view the **Usage this month** pane on your cluster overview page. With your past resource usage data and real-time resource usage in this pane, you can track your cluster's resource consumption and estimate a reasonable spending limit. If the free quota cannot meet your requirement, you can upgrade to a [scalable cluster](/tidb-cloud/select-cluster-tier.md#scalable-cluster-plan) and edit the spending limit. For more information, see [TiDB Cloud Serverless usage quota](/tidb-cloud/select-cluster-tier.md#usage-quota). ### How can I optimize my workload to minimize the number of RUs consumed? -Ensure that your queries have been carefully optimized for optimal performance by following the guidelines in [Optimizing SQL Performance](/develop/dev-guide-optimize-sql-overview.md). In addition, minimizing the amount of egress traffic is also crucial for reducing RUs consumption. To achieve this, it is recommended to return only the necessary columns and rows in your query, which in turn helps reduce network egress traffic. This can be achieved by carefully selecting and filtering the columns and rows to be returned, thereby optimizing network utilization. +Ensure that your queries have been carefully optimized for optimal performance by following the guidelines in [Optimizing SQL Performance](/develop/dev-guide-optimize-sql-overview.md). To identify the SQL statements that consume the most RUs, navigate to **Diagnosis > SQL Statements** on your cluster overview page, where you can observe SQL execution and view the top statements sorted by **Total RU** or **Mean RU**. For more information, see [Statement Analysis](/tidb-cloud/tune-performance.md#statement-analysis). In addition, minimizing the amount of egress traffic is also crucial for reducing RUs consumption. To achieve this, it is recommended to return only the necessary columns and rows in your query, which in turn helps reduce network egress traffic. This can be achieved by carefully selecting and filtering the columns and rows to be returned, thereby optimizing network utilization. -### How storage is metered for TiDB Serverless？ +### How storage is metered for TiDB Cloud Serverless？ -The storage is metered based on the amount of data stored in a TiDB Serverless cluster, measured in GiB per month. It is calculated by multiplying the total size of all the tables and indexes (excluding data compression or replicas) with the number of hours the data is stored in that month. +The storage is metered based on the amount of data stored in a TiDB Cloud Serverless cluster, measured in GiB per month. It is calculated by multiplying the total size of all the tables and indexes (excluding data compression or replicas) with the number of hours the data is stored in that month. ### Why does the storage usage size remain unchanged after dropping a table or database immediately? @@ -97,25 +115,39 @@ A spike in RU usage can occur due to necessary background jobs in TiDB. These jo ### What happens when my cluster exhausts its free quota or exceeds its spending limit? -Once a cluster reaches its free quota or spending limit, the cluster will enforce throttling measures on its read and write operations. These operations will be limited until the quota is increased or the usage is reset at the start of a new month. For more information, see [TiDB Serverless Limitations and Quotas](/tidb-cloud/serverless-limitations.md#usage-quota). +Once a cluster reaches its free quota or spending limit, the cluster immediately denies any new connection attempts until the quota is increased or the usage is reset at the start of a new month. Existing connections established before reaching the quota will remain active but will experience throttling. For more information, see [TiDB Cloud Serverless Limitations and Quotas](/tidb-cloud/serverless-limitations.md#usage-quota). ### Why do I observe spikes in RU usage while importing data? -During the data import process of a TiDB Serverless cluster, RU consumption occurs only when the data is successfully imported, which leads to spikes in RU usage. +During the data import process of a TiDB Cloud Serverless cluster, RU consumption occurs only when the data is successfully imported, which leads to spikes in RU usage. + +### What costs are involved when using columnar storage in TiDB Cloud Serverless? + +The pricing for columnar storage in TiDB Cloud Serverless is similar to that for row-based storage. When you use columnar storage, an additional replica is created to store your data (without indexes). The replication of data from row-based to columnar storage does not incur extra charges. + +For detailed pricing information, see [TiDB Cloud Serverless pricing details](https://www.pingcap.com/tidb-serverless-pricing-details/). + +### Is using columnar storage more expensive? + +Columnar storage in TiDB Cloud Serverless incurs additional costs due to the extra replica, which requires more storage and resources for data replication. However, columnar storage becomes more cost-effective when running analytical queries. + +According to the TPC-H benchmark test, the cost of running analytic queries on columnar storage is about one-third of the cost when using row-based storage. + +Therefore, while there might be an initial cost due to the extra replica, the reduced computational costs during analytics can make it more cost-effective for specific use cases. Especially for users with analytical demands, columnar storage can significantly reduce costs, offering considerable cost savings opportunities. ## Security FAQs -### Is my TiDB Serverless shared or dedicated? +### Is my TiDB Cloud Serverless shared or dedicated? -The serverless technology is designed for multi-tenancy and the resources used by all clusters are shared. To get managed TiDB service with isolated infrastructure and resources, you can upgrade it to [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated). +The serverless technology is designed for multi-tenancy and the resources used by all clusters are shared. To get managed TiDB service with isolated infrastructure and resources, you can upgrade it to [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). -### How does TiDB Serverless ensure security? +### How does TiDB Cloud Serverless ensure security? -- Your connections are encrypted by Transport Layer Security (TLS). For more information about using TLS to connect to TiDB Serverless, see [TLS Connection to TiDB Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md). -- All persisted data on TiDB Serverless is encrypted-at-rest using the tool of the cloud provider that your cluster is running in. +- Your connections are encrypted by Transport Layer Security (TLS). For more information about using TLS to connect to TiDB Cloud Serverless, see [TLS Connection to TiDB Cloud Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md). +- All persisted data on TiDB Cloud Serverless is encrypted-at-rest using the tool of the cloud provider that your cluster is running in. ## Maintenance FAQ ### Can I upgrade the version of TiDB that my cluster is running on? -No. TiDB Serverless clusters are upgraded automatically as we roll out new TiDB versions on TiDB Cloud. You can see what version of TiDB your cluster is running in the [TiDB Cloud console](https://tidbcloud.com/console/clusters) or in the latest [release note](https://docs.pingcap.com/tidbcloud/tidb-cloud-release-notes). Alternatively, you can also connect to your cluster and use `SELECT version()` or `SELECT tidb_version()` to check the TiDB version. +No. TiDB Cloud Serverless clusters are upgraded automatically as we roll out new TiDB versions on TiDB Cloud. You can see what version of TiDB your cluster is running in the [TiDB Cloud console](https://tidbcloud.com/console/clusters) or in the latest [release note](https://docs.pingcap.com/tidbcloud/tidb-cloud-release-notes). Alternatively, you can also connect to your cluster and use `SELECT version()` or `SELECT tidb_version()` to check the TiDB version. diff --git a/tidb-cloud/serverless-high-availability.md b/tidb-cloud/serverless-high-availability.md new file mode 100644 index 0000000000000..e288e9329f771 --- /dev/null +++ b/tidb-cloud/serverless-high-availability.md @@ -0,0 +1,114 @@ +--- +title: High Availability in TiDB Cloud Serverless +summary: Learn about the high availability architecture of TiDB Cloud Serverless. Discover Zonal and Regional High Availability options, automated backups, failover processes, and how TiDB ensures data durability and business continuity. +--- + +# High Availability in TiDB Cloud Serverless + +TiDB Cloud Serverless is designed with robust mechanisms to maintain high availability and data durability by default, preventing single points of failure and ensuring continuous service even in the face of disruptions. As a fully managed service based on the battle-tested TiDB Open Source product, it inherits TiDB's core high availability (HA) features and augments them with additional cloud-native capabilities. + +## Overview + +TiDB ensures high availability and data durability using the Raft consensus algorithm. This algorithm consistently replicates data changes across multiple nodes, allowing TiDB to handle read and write requests even in the event of node failures or network partitions. This approach provides both high data durability and fault tolerance. + +TiDB Cloud Serverless extends these capabilities with two types of high availability to meet different operational requirements: + +- **Zonal high availability (default)**: This option places all nodes within a single availability zone, reducing network latency. It ensures high availability without requiring application-level redundancy across zones, making it suitable for applications that prioritize low latency within a single zone. Zonal high availability is available in all regions that support TiDB Cloud Serverless. For more information, see [Zonal high availability architecture](#zonal-high-availability-architecture). + +- **Regional high availability (beta)**: This option distributes nodes across multiple availability zones, offering maximum infrastructure isolation and redundancy. It provides the highest level of availability but requires application-level redundancy across zones. It is recommended to choose this option if you need maximum availability protection against infrastructure failures within a zone. Note that it increases latency and might incur cross-zone data transfer fees. This feature is available in selected regions with multi-availability zone support and can only be enabled during cluster creation. For more information, see [Regional high availability architecture](#regional-high-availability-architecture). + +## Zonal high availability architecture + +> **Note:** +> +> Zonal high availability is the default option and is available in all AWS regions that support TiDB Cloud Serverless. + +When you create a cluster with the default zonal high availability, all components, including Gateway, TiDB, TiKV, and TiFlash compute/write nodes, run in the same availability zone. The placement of these components in the data plane offer infrastructure redundancy with virtual machine pools, which minimizes failover time and network latency due to colocation. + +![TiDB Cloud Serverless zonal high availability](/media/tidb-cloud/serverless-zonal-high-avaliability-aws.png) + +In zonal high availability architecture: + +- The Placement Driver (PD) is deployed across multiple availability zones, ensuring high availability by replicating data redundantly across zones. +- Data is replicated across TiKV servers and TiFlash write nodes within the local availability zone. +- TiDB servers and TiFlash compute nodes read from and write to TiKV and TiFlash write nodes, which are safeguarded by storage-level replication. + +### Failover process + +TiDB Cloud Serverless ensures a transparent failover process for your applications. During a failover: + +- A new replica is created to replace the failed one. + +- Servers providing storage services recover local caches from persisted data on Amazon S3, restoring the system to a consistent state with the replicas. + +In the storage layer, persisted data is regularly pushed to Amazon S3 for high durability. Moreover, immediate updates are not only replicated across multiple TiKV servers but also stored on the EBS of each server, which further replicates the data for additional durability. TiDB automatically resolves issues by backing off and retrying in milliseconds, ensuring the failover process remains seamless for client applications. + +The gateway and computing layers are stateless, so failover involves restarting them elsewhere immediately. Applications should implement retry logic for their connections. While the zonal setup provides high availability, it cannot handle an entire zone failure. If the zone becomes unavailable, downtime will occur until the zone and its dependent services are restored. + +## Regional high availability architecture + +When you create a cluster with regional high availability, critical OLTP (Online Transactional Processing) workload components, such as PD and TiKV, are deployed across multiple availability zones to ensure redundant replication and maximizing availability. During normal operations, components like Gateway, TiDB, and TiFlash compute/write nodes are hosted in the primary availability zone. These components in data plane offer infrastructure redundancy through virtual machine pools, which minimizes failover time and network latency due to colocation. + +> **Note:** +> +> - Regional high availability is currently in beta and only available in the AWS Tokyo (`ap-northeast-1`) region. +> - You can enable regional high availability only during cluster creation. + +![TiDB Cloud Serverless regional high availability](/media/tidb-cloud/serverless-regional-high-avaliability-aws.png) + +In regional high availability architecture: + +- The Placement Driver (PD) and TiKV are deployed across multiple availability zones, and data is always replicated redundantly across zones to ensure the highest level of availability. +- Data is replicated across TiFlash write nodes within the primary availability zone. +- TiDB servers and TiFlash compute nodes read from and write to these TiKV and TiFlash write nodes, which are safeguarded by storage-level replication. + +### Failover process + +In the rare event of a primary zone failure scenario, which could be caused by a natural disaster, configuration change, software issue, or hardware failure, critical OLTP workload components, including Gateway and TiDB, are automatically launched in the standby availability zone. Traffic is automatically redirected to the standby zone to ensure swift recovery and maintain business continuity. + +TiDB Cloud Serverless minimizes service disruption and ensures business continuity during a primary zone failure by performing the following actions: + +- Automatically create new replicas of Gateway and TiDB in the standby availability zone. +- Use the elastic load balancer to detect active gateway replicas in the standby availability zone and redirect OLTP traffic from the failed primary zone. + +In addition to providing high availability through TiKV replication, TiKV instances are deployed and configured to place each data replica in a different availability zone. The system remains available as long as two availability zones are operating normally. For high durability, data persistence is ensured by regularly backing up data to S3. Even if two zones fail, data stored in S3 remains accessible and recoverable. + +Applications are unaffected by failures in non-primary zones and remain unaware of such events. During a primary zone failure, Gateway and TiDB are launched in the standby availability zone to handle workloads. Ensure that your applications implement retry logic to redirect new requests to active servers in the standby availability zone. + +## Automatic backups and durability + +Database backups are essential for business continuity and disaster recovery, helping to protect your data from corruption or accidental deletion. With backups, you can restore your database to a specific point in time within the retention period, minimizing data loss and downtime. + +TiDB Cloud Serverless provides robust automated backup mechanisms to ensure continuous data protection: + +- **Daily full backups**: A full backup of your database is created once a day, capturing the entire database state. +- **Continuous transaction log backups**: Transaction logs are backed up continuously, approximately every 5 minutes, though the exact frequency depends on database activity. + +These automated backups enable you to restore your database either from a full backup or from a specific point in time by combining full backups with continuous transaction logs. This flexibility ensures that you can recover your database to a precise point just before an incident occurs. + +> **Note:** +> +> Automatic backups, including snapshot-based and continuous backups for Point-in-Time Recovery (PITR), are performed on Amazon S3, which provides regional-level high durability. + +## Impact on sessions during failures + +During a failure, ongoing transactions on the failed server might be interrupted. Although failover is transparent to applications, you must implement logic to handle recoverable failures during active transactions. Different failure scenarios are handled as follows: + +- **TiDB failures**: If a TiDB instance fails, client connections are unaffected because TiDB Cloud Serverless automatically reroutes traffic through the gateway. While transactions on the failed TiDB instance might be interrupted, the system ensures that committed data is preserved, and new transactions are handled by another available TiDB instance. +- **Gateway failures**: If the Gateway fails, client connections are disrupted. However, TiDB Cloud Serverless gateways are stateless and can restart immediately in a new zone or server. Traffic is automatically redirected to the new gateway, minimizing downtime. + +It is recommended to implement retry logic in your application to handle recoverable failures. For implementation details, refer to your driver or ORM documentation (for example, [JDBC](https://dev.mysql.com/doc/connector-j/en/connector-j-config-failover.html)). + +## RTO and RPO + +When creating your business continuity plan, consider these two key metrics: + +- Recovery Time Objective (RTO): The maximum acceptable time it takes for the application to fully recover after a disruptive event. +- Recovery Point Objective (RPO): The maximum acceptable time interval of recent data updates that the application can tolerate losing during recovery from an unplanned disruptive event. + +The following table compares the RTO and RPO for each high availability option: + +| High availability architecture | RTO (downtime) | RPO (data loss) | +|--------------------------------|-------------------------------|-----------------| +| Zonal high availability | Near 0 seconds | 0 | +| Regional high availability | Typically less than 600 seconds | 0 | diff --git a/tidb-cloud/serverless-limitations.md b/tidb-cloud/serverless-limitations.md index dce4940d0f72d..40bf6a078bfbc 100644 --- a/tidb-cloud/serverless-limitations.md +++ b/tidb-cloud/serverless-limitations.md @@ -1,16 +1,16 @@ --- -title: TiDB Serverless Limitations and Quotas -summary: Learn about the limitations of TiDB Serverless. +title: TiDB Cloud Serverless Limitations and Quotas +summary: Learn about the limitations of TiDB Cloud Serverless. aliases: ['/tidbcloud/serverless-tier-limitations'] --- -# TiDB Serverless Limitations and Quotas +# TiDB Cloud Serverless Limitations and Quotas -TiDB Serverless works with almost all workloads that TiDB supports, but there are some feature differences between TiDB Self-Hosted or TiDB Dedicated clusters and TiDB Serverless clusters. This document describes the limitations of TiDB Serverless. +TiDB Cloud Serverless works with almost all workloads that TiDB supports, but there are some feature differences between TiDB Self-Managed or TiDB Cloud Dedicated clusters and TiDB Cloud Serverless clusters. This document describes the limitations of TiDB Cloud Serverless. -We are constantly filling in the feature gaps between TiDB Serverless and TiDB Dedicated. If you require these features or capabilities in the gap, use [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) or [contact us](https://www.pingcap.com/contact-us/?from=en) for a feature request. +We are constantly filling in the feature gaps between TiDB Cloud Serverless and TiDB Cloud Dedicated. If you require these features or capabilities in the gap, use [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) or [contact us](https://www.pingcap.com/contact-us/?from=en) for a feature request. ## Limitations @@ -20,12 +20,12 @@ We are constantly filling in the feature gaps between TiDB Serverless and TiDB D ### Connection -- Only [Public Endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md) and [Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md) can be used. You cannot use [VPC Peering](/tidb-cloud/set-up-vpc-peering-connections.md) to connect to TiDB Serverless clusters.  +- Only [Public Endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md) and [Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md) can be used. You cannot use [VPC Peering](/tidb-cloud/set-up-vpc-peering-connections.md) to connect to TiDB Cloud Serverless clusters.  - No [IP Access list](/tidb-cloud/configure-ip-access-list.md) support. ### Encryption -- Data persisted in your TiDB Serverless cluster is encrypted using the encryption tool provided by the cloud provider that manages your cluster. However, TiDB Serverless does not provide any additional optional measures for protecting data at-rest on disks beyond infrastructure-level encryption. +- Data persisted in your TiDB Cloud Serverless cluster is encrypted using the encryption tool provided by the cloud provider that manages your cluster. For [scalable clusters](/tidb-cloud/select-cluster-tier.md#scalable-cluster-plan), an optional second layer of encryption is available during the cluster creation process, providing an additional level of security beyond the default encryption at rest. - Using [customer-managed encryption keys (CMEK)](/tidb-cloud/tidb-cloud-encrypt-cmek.md) is currently unavailable. ### Maintenance window @@ -41,34 +41,38 @@ We are constantly filling in the feature gaps between TiDB Serverless and TiDB D ### Self-service upgrades -- TiDB Serverless is a fully managed deployment of TiDB. Major and minor version upgrades of TiDB Serverless are handled by TiDB Cloud and therefore cannot be initiated by users. +- TiDB Cloud Serverless is a fully managed deployment of TiDB. Major and minor version upgrades of TiDB Cloud Serverless are handled by TiDB Cloud and therefore cannot be initiated by users. ### Stream data -- [Changefeed](/tidb-cloud/changefeed-overview.md) is not supported for TiDB Serverless currently. -- [Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md) is not supported for TiDB Serverless currently. +- [Changefeed](/tidb-cloud/changefeed-overview.md) is not supported for TiDB Cloud Serverless currently. +- [Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md) is not supported for TiDB Cloud Serverless currently. + +### Time to live (TTL) + +- In TiDB Cloud Serverless, the [`TTL_JOB_INTERVAL`](/time-to-live.md#ttl-job) attribute for a table is fixed at `15m` and cannot be modified. This means that TiDB Cloud Serverless schedules a background job every 15 minutes to clean up expired data. ### Others -- [Time to live (TTL)](/time-to-live.md) is currently unavailable. - Transaction can not last longer than 30 minutes. - For more details about SQL limitations, refer to [Limited SQL Features](/tidb-cloud/limited-sql-features.md). ## Usage quota -For each organization in TiDB Cloud, you can create a maximum of five TiDB Serverless clusters by default. To create more TiDB Serverless clusters, you need to add a credit card and set a [spending limit](/tidb-cloud/tidb-cloud-glossary.md#spending-limit) for the usage. +For each organization in TiDB Cloud, you can create a maximum of five [free clusters](/tidb-cloud/select-cluster-tier.md#free-cluster-plan) by default. To create more TiDB Cloud Serverless clusters, you need to add a credit card and create [scalable clusters](/tidb-cloud/select-cluster-tier.md#scalable-cluster-plan) for the usage. -For the first five TiDB Serverless clusters in your organization, TiDB Cloud provides a free usage quota for each of them as follows: +For the first five TiDB Cloud Serverless clusters in your organization, whether they are free or scalable, TiDB Cloud provides a free usage quota for each of them as follows: - Row-based storage: 5 GiB +- Columnar storage: 5 GiB - [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit): 50 million RUs per month The Request Unit (RU) is a unit of measurement used to track the resource consumption of a query or transaction. It is a metric that allows you to estimate the computational resources required to process a specific request in the database. The request unit is also the billing unit for TiDB Cloud Serverless service. -Once the free quota of a cluster is reached, the read and write operations on this cluster will be throttled until you [increase the quota](/tidb-cloud/manage-serverless-spend-limit.md#update-spending-limit) or the usage is reset upon the start of a new month. +Once a cluster reaches its usage quota, it immediately denies any new connection attempts until you [increase the quota](/tidb-cloud/manage-serverless-spend-limit.md#update-spending-limit) or the usage is reset upon the start of a new month. Existing connections established before reaching the quota will remain active but will experience throttling. -To learn more about the RU consumption of different resources (including read, write, SQL CPU, and network egress), the pricing details, and the throttled information, see [TiDB Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). +To learn more about the RU consumption of different resources (including read, write, SQL CPU, and network egress), the pricing details, and the throttled information, see [TiDB Cloud Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). -If you want to create a TiDB Serverless cluster with an additional quota, you can edit the spending limit on the cluster creation page. For more information, see [Create a TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md). +If you want to create a TiDB Cloud Serverless cluster with an additional quota, you can select the scalable cluster plan and edit the spending limit on the cluster creation page. For more information, see [Create a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md). -After creating a TiDB Serverless, you can still check and edit the spending limit on your cluster overview page. For more information, see [Manage Spending Limit for TiDB Serverless Clusters](/tidb-cloud/manage-serverless-spend-limit.md). +After creating a TiDB Cloud Serverless cluster, you can still check and edit the spending limit on your cluster overview page. For more information, see [Manage Spending Limit for TiDB Cloud Serverless Clusters](/tidb-cloud/manage-serverless-spend-limit.md). diff --git a/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md b/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md index ad728d54d0071..2ae40e9d64ff9 100644 --- a/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md +++ b/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md @@ -1,16 +1,16 @@ --- -title: Connect to a TiDB Dedicated Cluster via Google Cloud Private Service Connect +title: Connect to a TiDB Cloud Dedicated Cluster via Google Cloud Private Service Connect summary: Learn how to connect to your TiDB Cloud cluster via Google Cloud Private Service Connect. --- -# Connect to a TiDB Dedicated Cluster via Google Cloud Private Service Connect +# Connect to a TiDB Cloud Dedicated Cluster via Google Cloud Private Service Connect -This document describes how to connect to your TiDB Dedicated cluster via Google Cloud Private Service Connect. Google Cloud Private Service Connect is a private endpoint service provided by Google Cloud. +This document describes how to connect to your TiDB Cloud Dedicated cluster via Google Cloud Private Service Connect. Google Cloud Private Service Connect is a private endpoint service provided by Google Cloud. > **Tip:** > -> - To learn how to connect to a TiDB Dedicated cluster via private endpoint with AWS, see [Connect to TiDB Dedicated via Private Endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md). -> - To learn how to connect to a TiDB Serverless cluster via private endpoint, see [Connect to TiDB Serverless via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md). +> - To learn how to connect to a TiDB Cloud Dedicated cluster via private endpoint with AWS, see [Connect to TiDB Cloud Dedicated via Private Endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md). +> - To learn how to connect to a TiDB Cloud Serverless cluster via private endpoint, see [Connect to TiDB Cloud Serverless via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md). TiDB Cloud supports highly secure and one-way access to the TiDB Cloud service hosted in a Google Cloud VPC via the [Private Service Connect](https://cloud.google.com/vpc/docs/private-service-connect). You can create an endpoint and use it to connect to the TiDB Cloud service . @@ -27,11 +27,11 @@ For more detailed definitions of the private endpoint and endpoint service, see ## Restrictions -- This feature is applicable to TiDB Dedicated clusters created after April 13, 2023. For older clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md) for assistance. +- This feature is applicable to TiDB Cloud Dedicated clusters created after April 13, 2023. For older clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md) for assistance. - Only the `Organization Owner` and `Project Owner` roles can create Google Cloud Private Service Connect endpoints. - Each TiDB cluster can handle connections from up to 10 endpoints. - Each Google Cloud project can have up to 10 endpoints connecting to a TiDB Cluster. -- You can create up to 8 TiDB Dedicated clusters hosted on Google Cloud in a project with the endpoint service configured. +- You can create up to 8 TiDB Cloud Dedicated clusters hosted on Google Cloud in a project with the endpoint service configured. - The private endpoint and the TiDB cluster to be connected must be located in the same region. - Egress firewall rules must permit traffic to the internal IP address of the endpoint. The [implied allow egress firewall rule](https://cloud.google.com/firewall/docs/firewalls#default_firewall_rules) permits egress to any destination IP address. - If you have created egress deny firewall rules in your VPC network, or if you have created hierarchical firewall policies that modify the implied allowed egress behavior, access to the endpoint might be affected. In this case, you need to create a specific egress allow firewall rule or policy to permit traffic to the internal destination IP address of the endpoint. @@ -43,10 +43,10 @@ In most scenarios, it is recommended that you use private endpoint connection ov ## Set up a private endpoint with Google Cloud Private Service Connect -To connect to your TiDB Dedicated cluster via a private endpoint, complete the [prerequisites](#prerequisites) and follow these steps: +To connect to your TiDB Cloud Dedicated cluster via a private endpoint, complete the [prerequisites](#prerequisites) and follow these steps: -1. [Choose a TiDB cluster](#step-1-choose-a-tidb-cluster) -2. [Provide the information for creating an endpoint](#step-2-provide-the-information-for-creating-an-endpoint) +1. [Select a TiDB cluster](#step-1-select-a-tidb-cluster) +2. [Create a Google Cloud private endpoint](#step-2-create-a-google-cloud-private-endpoint) 3. [Accept endpoint access](#step-3-accept-endpoint-access) 4. [Connect to your TiDB cluster](#step-4-connect-to-your-tidb-cluster) @@ -70,25 +70,24 @@ Before you begin to create an endpoint: - [Compute Network Admin](https://cloud.google.com/iam/docs/understanding-roles#compute.networkAdmin) (roles/compute.networkAdmin) - [Service Directory Editor](https://cloud.google.com/iam/docs/understanding-roles#servicedirectory.editor) (roles/servicedirectory.editor) -Perform the following steps to go to the **Google Cloud Private Endpoint** page: +### Step 1. Select a TiDB cluster -1. Log in to the [TiDB Cloud console](https://tidbcloud.com). -2. Click in the lower-left corner, switch to the target project if you have multiple projects, and then click **Project Settings**. -3. On the **Project Settings** page of your project, click **Network Access** in the left navigation pane, and click the **Private Endpoint** tab. -4. Click **Create Private Endpoint** in the upper-right corner, and then select **Google Cloud Private Endpoint**. +1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click the name of your target TiDB cluster to go to its overview page. You can select a cluster with any of the following statuses: -### Step 1. Choose a TiDB cluster + - **Available** + - **Restoring** + - **Modifying** + - **Importing** -Click the drop-down list and choose an available TiDB Dedicated cluster. +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -You can select a cluster with any of the following statuses: +3. In the **Connection Type** drop-down list, select **Private Endpoint**, and then click **Create Private Endpoint Connection**. -- **Available** -- **Restoring** -- **Modifying** -- **Importing** + > **Note:** + > + > If you have already created a private endpoint connection, the active endpoint will appear in the connection dialog. To create additional private endpoint connections, navigate to the **Networking** page in the left navigation pane. -### Step 2. Provide the information for creating an endpoint +### Step 2. Create a Google Cloud private endpoint 1. Provide the following information to generate the command for private endpoint creation: - **Google Cloud Project ID**: the Project ID associated with your Google Cloud account. You can find the ID on the [Google Cloud **Dashboard** page](https://console.cloud.google.com/home/dashboard). @@ -96,8 +95,8 @@ You can select a cluster with any of the following statuses: - **Google Cloud Subnet Name**: the name of the subnet in the specified VPC. You can find it on the **VPC network details** page. - **Private Service Connect Endpoint Name**: enter a unique name for the private endpoint that will be created. 2. After entering the information, click **Generate Command**. -3. Copy the command. -4. Go to [Google Cloud Shell](https://console.cloud.google.com/home/dashboard) to execute the command. +3. Copy the generated command. +4. Open [Google Cloud Shell](https://console.cloud.google.com/home/dashboard) and execute the command to create the private endpoint. ### Step 3. Accept endpoint access @@ -107,11 +106,11 @@ If you see an error `not received connection request from endpoint`, make sure t ### Step 4. Connect to your TiDB cluster -After you have accepted the endpoint connection, take the following steps to connect to your TiDB cluster: +After you have accepted the private endpoint connection, you are redirected back to the connection dialog. -1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click **...** in the **Action** column. -2. Click **Connect**. A connection dialog is displayed. -3. Select the **Private Endpoint** tab. The private endpoint you just created is displayed. Copy the command to connect to the TiDB cluster. +1. Wait for the private endpoint connection status to change from **System Checking** to **Active** (approximately 5 minutes). +2. In the **Connect With** drop-down list, select your preferred connection method. The corresponding connection string is displayed at the bottom of the dialog. +3. Connect to your cluster with the connection string. ### Private endpoint status reference @@ -133,7 +132,7 @@ The possible statuses of a private endpoint service are explained as follows: ### TiDB Cloud fails to create an endpoint service. What should I do? -The endpoint service is created automatically after you open the **Create Google Cloud Private Endpoint** page and choose the TiDB cluster. If it shows as failed or remains in the **Creating** state for a long time, submit a [support ticket](/tidb-cloud/tidb-cloud-support.md) for assistance. +The endpoint service is created automatically after you open the **Create Google Cloud Private Endpoint Connection** page and choose the TiDB cluster. If it shows as failed or remains in the **Creating** state for a long time, submit a [support ticket](/tidb-cloud/tidb-cloud-support.md) for assistance. ### Fail to create an endpoint in Google Cloud. What should I do? @@ -147,6 +146,6 @@ If you have already executed the command to create a private endpoint in Google ### Why can't I see the endpoints generated by directly copying the service attachment in the TiDB Cloud console? -In the TiDB Cloud console, you can only view endpoints that are created through the command generated on the **Create Google Cloud Private Endpoint** page. +In the TiDB Cloud console, you can only view endpoints that are created through the command generated on the **Create Google Cloud Private Endpoint Connection** page. However, endpoints generated by directly copying the service attachment (that is, not created through the command generated in the TiDB Cloud console) are not displayed in the TiDB Cloud console. diff --git a/tidb-cloud/set-up-private-endpoint-connections-serverless.md b/tidb-cloud/set-up-private-endpoint-connections-serverless.md index 0195f5ae81b2a..b921157258f37 100644 --- a/tidb-cloud/set-up-private-endpoint-connections-serverless.md +++ b/tidb-cloud/set-up-private-endpoint-connections-serverless.md @@ -1,16 +1,16 @@ --- -title: Connect to TiDB Serverless via Private Endpoint +title: Connect to TiDB Cloud Serverless via Private Endpoint summary: Learn how to connect to your TiDB Cloud cluster via private endpoint. --- -# Connect to TiDB Serverless via Private Endpoint +# Connect to TiDB Cloud Serverless via Private Endpoint -This document describes how to connect to your TiDB Serverless cluster via private endpoint. +This document describes how to connect to your TiDB Cloud Serverless cluster via private endpoint. > **Tip:** > -> To learn how to connect to a TiDB Dedicated cluster via private endpoint with AWS, see [Connect to TiDB Dedicated via Private Endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md). -> To learn how to connect to a TiDB Dedicated cluster via private endpoint with Google Cloud, see [Connect to TiDB Dedicated via Private Service Connect with Google Cloud](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md). +> To learn how to connect to a TiDB Cloud Dedicated cluster via private endpoint with AWS, see [Connect to TiDB Cloud Dedicated via Private Endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md). +> To learn how to connect to a TiDB Cloud Dedicated cluster via private endpoint with Google Cloud, see [Connect to TiDB Cloud Dedicated via Private Service Connect with Google Cloud](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md). TiDB Cloud supports highly secure and one-way access to the TiDB Cloud service hosted in an AWS VPC via the [AWS PrivateLink](https://aws.amazon.com/privatelink/?privatelink-blogs.sort-by=item.additionalFields.createdDate&privatelink-blogs.sort-order=desc), as if the service were in your own VPC. A private endpoint is exposed in your VPC and you can create a connection to the TiDB Cloud service via the endpoint with permission. @@ -27,12 +27,16 @@ For more detailed definitions of the private endpoint and endpoint service, see ## Restrictions -- Currently, TiDB Cloud supports private endpoint connection to TiDB Serverless only when the endpoint service is hosted in AWS. If the service is hosted in Google Cloud, the private endpoint is not applicable. +- Currently, TiDB Cloud supports private endpoint connection to TiDB Cloud Serverless only when the endpoint service is hosted in AWS. If the service is hosted in Google Cloud, the private endpoint is not applicable. - Private endpoint connection across regions is not supported. +## Prerequisites + +Make sure that DNS hostnames and DNS resolution are both enabled in your AWS VPC settings. They are disabled by default when you create a VPC in the [AWS Management Console](https://console.aws.amazon.com/). + ## Set up a private endpoint with AWS -To connect to your TiDB Serverless cluster via a private endpoint, follow these steps: +To connect to your TiDB Cloud Serverless cluster via a private endpoint, follow these steps: 1. [Choose a TiDB cluster](#step-1-choose-a-tidb-cluster) 2. [Create an AWS interface endpoint](#step-2-create-an-aws-interface-endpoint) @@ -40,14 +44,14 @@ To connect to your TiDB Serverless cluster via a private endpoint, follow these ### Step 1. Choose a TiDB cluster -1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click the name of your target TiDB Serverless cluster to go to its overview page. +1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click the name of your target TiDB Cloud Serverless cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. In the **Endpoint Type** drop-down list, select **Private**. +3. In the **Connection Type** drop-down list, select **Private Endpoint**. 4. Take a note of **Service Name**, **Availability Zone ID**, and **Region ID**. > **Note:** > - > You only need to create one private endpoint per AWS region, which can be shared by all TiDB Serverless clusters located in the same region. + > You only need to create one private endpoint per AWS region, which can be shared by all TiDB Cloud Serverless clusters located in the same region. ### Step 2. Create an AWS interface endpoint @@ -63,7 +67,7 @@ To use the AWS Management Console to create a VPC interface endpoint, perform th ![Verify endpoint service](/media/tidb-cloud/private-endpoint/create-endpoint-2.png) -3. Select **Other endpoint services**. +3. Select **Endpoint services that use NLBs and GWLBs**. 4. Enter the service name that you found in [step 1](#step-1-choose-a-tidb-cluster). 5. Click **Verify service**. 6. Select your VPC in the drop-down list. Expand **Additional settings** and select the **Enable DNS name** checkbox. @@ -72,7 +76,7 @@ To use the AWS Management Console to create a VPC interface endpoint, perform th > **Note:** > - > Make sure the selected security group allows inbound access from your EC2 instances on port 4000. + > Make sure the selected security group allows inbound access from your EC2 instances on port 4000. 9. Click **Create endpoint**. @@ -103,7 +107,7 @@ After you have created the interface endpoint, go back to the TiDB Cloud console 1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click the name of your target cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A connection dialog is displayed. -3. In the **Endpoint Type** drop-down list, select **Private**. +3. In the **Connection Type** drop-down list, select **Private Endpoint**. 4. In the **Connect With** drop-down list, select your preferred connection method. The corresponding connection string is displayed at the bottom of the dialog. 5. Connect to your cluster with the connection string. @@ -120,7 +124,3 @@ After you have created the interface endpoint, go back to the TiDB Cloud console You might need to properly set the security group for your VPC endpoint in the AWS Management Console. Go to **VPC** > **Endpoints**. Right-click your VPC endpoint and select the proper **Manage security groups**. A proper security group within your VPC that allows inbound access from your EC2 instances on Port 4000 or a customer-defined port. ![Manage security groups](/media/tidb-cloud/private-endpoint/manage-security-groups.png) - -### I cannot enable private DNS. An error is reported indicating that the `enableDnsSupport` and `enableDnsHostnames` VPC attributes are not enabled - -Make sure that DNS hostname and DNS resolution are both enabled in your VPC setting. They are disabled by default when you create a VPC in the AWS Management Console. diff --git a/tidb-cloud/set-up-private-endpoint-connections.md b/tidb-cloud/set-up-private-endpoint-connections.md index ff75084628416..6dd3ab38fa3de 100644 --- a/tidb-cloud/set-up-private-endpoint-connections.md +++ b/tidb-cloud/set-up-private-endpoint-connections.md @@ -1,16 +1,16 @@ --- -title: Connect to a TiDB Dedicated Cluster via Private Endpoint with AWS +title: Connect to a TiDB Cloud Dedicated Cluster via Private Endpoint with AWS summary: Learn how to connect to your TiDB Cloud cluster via private endpoint with AWS. --- -# Connect to a TiDB Dedicated Cluster via Private Endpoint with AWS +# Connect to a TiDB Cloud Dedicated Cluster via Private Endpoint with AWS -This document describes how to connect to your TiDB Dedicated cluster via private endpoint with AWS. +This document describes how to connect to your TiDB Cloud Dedicated cluster via private endpoint with AWS. > **Tip:** > -> To learn how to connect to a TiDB Serverless cluster via private endpoint, see [Connect to TiDB Serverless via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md). -> To learn how to connect to a TiDB Dedicated cluster via private endpoint with Google Cloud, see [Connect to TiDB Dedicated via Private Service Connect with Google Cloud](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md). +> To learn how to connect to a TiDB Cloud Serverless cluster via private endpoint, see [Connect to TiDB Cloud Serverless via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md). +> To learn how to connect to a TiDB Cloud Dedicated cluster via private endpoint with Google Cloud, see [Connect to TiDB Cloud Dedicated via Private Service Connect with Google Cloud](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md). TiDB Cloud supports highly secure and one-way access to the TiDB Cloud service hosted in an AWS VPC via the [AWS PrivateLink](https://aws.amazon.com/privatelink/?privatelink-blogs.sort-by=item.additionalFields.createdDate&privatelink-blogs.sort-order=desc), as if the service were in your own VPC. A private endpoint is exposed in your VPC and you can create a connection to the TiDB Cloud service via the endpoint with permission. @@ -36,54 +36,64 @@ In most scenarios, you are recommended to use private endpoint connection over V - You are using a TiCDC cluster to replicate data to a downstream cluster (such as Amazon Aurora, MySQL, and Kafka) but you cannot maintain the endpoint service on your own. - You are connecting to PD or TiKV nodes directly. -## Set up a private endpoint with AWS +## Prerequisites -To connect to your TiDB Dedicated cluster via a private endpoint, complete the [prerequisites](#prerequisites) and follow these steps: +Make sure that DNS hostnames and DNS resolution are both enabled in your AWS VPC settings. They are disabled by default when you create a VPC in the [AWS Management Console](https://console.aws.amazon.com/). -1. [Choose a TiDB cluster](#step-1-choose-a-tidb-cluster) -2. [Check the service endpoint region](#step-2-check-the-service-endpoint-region) -3. [Create an AWS interface endpoint](#step-3-create-an-aws-interface-endpoint) -4. [Accept the endpoint connection](#step-4-accept-the-endpoint-connection) -5. [Enable private DNS](#step-5-enable-private-dns) -6. [Connect to your TiDB cluster](#step-6-connect-to-your-tidb-cluster) +## Set up a private endpoint connection and connect to your cluster -If you have multiple clusters, you need to repeat these steps for each cluster that you want to connect to using AWS PrivateLink. - -### Prerequisites +To connect to your TiDB Cloud Dedicated cluster via a private endpoint, complete the follow these steps: -1. Log in to the [TiDB Cloud console](https://tidbcloud.com). -2. Click in the lower-left corner, switch to the target project if you have multiple projects, and then click **Project Settings**. -3. On the **Project Settings** page of your project, click **Network Access** in the left navigation pane, and click the **Private Endpoint** tab. -4. Click **Create Private Endpoint** in the upper-right corner, and then select **AWS Private Endpoint**. +1. [Select a TiDB cluster](#step-1-select-a-tidb-cluster) +2. [Create an AWS interface endpoint](#step-2-create-an-aws-interface-endpoint) +3. [Create a private endpoint connection](#step-3-create-a-private-endpoint-connection) +4. [Enable private DNS](#step-4-enable-private-dns) +5. [Connect to your TiDB cluster](#step-5-connect-to-your-tidb-cluster) -### Step 1. Choose a TiDB cluster - -1. Click the drop-down list and choose an available TiDB Dedicated cluster. -2. Click **Next**. +If you have multiple clusters, you need to repeat these steps for each cluster that you want to connect to using AWS PrivateLink. -### Step 2. Check the service endpoint region +### Step 1. Select a TiDB cluster -Your service endpoint region is selected by default. Have a quick check and click **Next**. +1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click the name of your target TiDB cluster to go to its overview page. +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. +3. In the **Connection Type** drop-down list, select **Private Endpoint**, and then click **Create Private Endpoint Connection**. > **Note:** > -> The default region is where your cluster is located. Do not change it. Cross-region private endpoint is currently not supported. +> If you have already created a private endpoint connection, the active endpoint will appear in the connection dialog. To create additional private endpoint connections, navigate to the **Networking** page in the left navigation pane. -### Step 3. Create an AWS interface endpoint +### Step 2. Create an AWS interface endpoint > **Note:** > -> For each TiDB Dedicated cluster created after March 28, 2023, the corresponding endpoint service is automatically created 3 to 4 minutes after the cluster creation. +> For each TiDB Cloud Dedicated cluster created after March 28, 2023, the corresponding endpoint service is automatically created 3 to 4 minutes after the cluster creation. -If you see the `Endpoint Service Ready` message, take note of your endpoint service name from the command in the lower area of the console for later use. Otherwise, wait 3 to 4 minutes to let TiDB Cloud create an endpoint service for your cluster. +If you see the `TiDB Private Link Service is ready` message, the corresponding endpoint service is ready. You can provide the following information to create the endpoint. -```bash -aws ec2 create-vpc-endpoint --vpc-id ${your_vpc_id} --region ${your_region} --service-name ${your_endpoint_service_name} --vpc-endpoint-type Interface --subnet-ids ${your_application_subnet_ids} -``` +1. Fill in the **Your VPC ID** and **Your Subnet IDs** fields. You can find these IDs from your [AWS Management Console](https://console.aws.amazon.com/). For multiple subnets, enter the IDs separated by spaces. +2. Click **Generate Command** to get the following endpoint creation command. + + ```bash + aws ec2 create-vpc-endpoint --vpc-id ${your_vpc_id} --region ${your_region} --service-name ${your_endpoint_service_name} --vpc-endpoint-type Interface --subnet-ids ${your_application_subnet_ids} + ``` -Then create an AWS interface endpoint either using the AWS Management Console or using the AWS CLI. +Then, you can create an AWS interface endpoint either using the AWS CLI or using the [AWS Management Console](https://aws.amazon.com/console/). +
+ +To use the AWS CLI to create a VPC interface endpoint, perform the following steps: + +1. Copy the generated command and run it in your terminal. +2. Record the VPC endpoint ID you just created. + +> **Tip:** +> +> - Before running the command, you need to have AWS CLI installed and configured. See [AWS CLI configuration basics](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html) for details. +> +> - If your service is spanning across more than three availability zones (AZs), you will get an error message indicating that the VPC endpoint service does not support the AZ of the subnet. This issue occurs when there is an extra AZ in your selected region in addition to the AZs where your TiDB cluster is located. In this case, you can contact [PingCAP Technical Support](https://docs.pingcap.com/tidbcloud/tidb-cloud-support). + +
To use the AWS Management Console to create a VPC interface endpoint, perform the following steps: @@ -95,54 +105,56 @@ To use the AWS Management Console to create a VPC interface endpoint, perform th ![Verify endpoint service](/media/tidb-cloud/private-endpoint/create-endpoint-2.png) -3. Select **Other endpoint services**. -4. Enter the service name that you found in the TiDB Cloud console. +3. In the **Endpoint settings** area, fill in a name tag if needed, and then select the **Endpoint services that use NLBs and GWLBs** option. +4. In the **Service settings** area, enter the service name `${your_endpoint_service_name}` from the generated command (`--service-name ${your_endpoint_service_name}`). 5. Click **Verify service**. -6. Select your VPC in the drop-down list. -7. Select the availability zones where your TiDB cluster is located in the **Subnets** area. +6. In the **Network settings** area, select your VPC in the drop-down list. +7. In the **Subnets** area, select the availability zones where your TiDB cluster is located. > **Tip:** > > If your service is spanning across more than three availability zones (AZs), you might not be able to select AZs in the **Subnets** area. This issue occurs when there is an extra AZ in your selected region in addition to the AZs where your TiDB cluster is located. In this case, contact [PingCAP Technical Support](https://docs.pingcap.com/tidbcloud/tidb-cloud-support). -8. Select your security group properly in the **Security groups** area. +8. In the **Security groups** area, select your security group properly. > **Note:** > - > Make sure the selected security group allows inbound access from your EC2 instances on Port 4000 or a customer-defined port. + > Make sure the selected security group allows inbound access from your EC2 instances on Port 4000 or a customer-defined port. 9. Click **Create endpoint**.
-
+ -To use the AWS CLI to create a VPC interface endpoint, perform the following steps: +### Step 3. Create a private endpoint connection -1. Fill in the **VPC ID** and **Subnet IDs** fields on the private endpoint creation page. You can get the IDs from your AWS Management Console. -2. Copy the command in the lower area of the page and run it in your terminal. Then click **Next**. +1. Go back to the TiDB Cloud console. +2. On the **Create AWS Private Endpoint Connection** page, enter your VPC endpoint ID. +3. Click **Create Private Endpoint Connection**. > **Tip:** > -> - Before running the command, you need to have AWS CLI installed and configured. See [AWS CLI configuration basics](https://docs.aws.amazon.com/cli/latest/userguide/cli-configure-quickstart.html) for details. -> -> - If your service is spanning across more than three availability zones (AZs), you will get an error message indicating that the VPC endpoint service does not support the AZ of the subnet. This issue occurs when there is an extra AZ in your selected region in addition to the AZs where your TiDB cluster is located. In this case, you can contact [PingCAP Technical Support](https://docs.pingcap.com/tidbcloud/tidb-cloud-support). +> You can view and manage private endpoint connections on two pages: > -> - You cannot copy the command until TiDB Cloud finishes creating an endpoint service in the background. +> - Cluster-level **Networking** page: click **Networking** in the left navigation pane of the cluster overview page. +> - Project-level **Network Access** page: click **Network Access** in the left navigation pane of the **Project Settings** page. -
- +### Step 4. Enable private DNS -### Step 4. Accept the endpoint connection +Enable private DNS in AWS. You can either use the AWS CLI or the AWS Management Console. -1. Go back to the TiDB Cloud console. -2. Fill in the box with your VPC endpoint ID on the **Create Private Endpoint** page. -3. Click **Next**. + +
-### Step 5. Enable private DNS +To enable private DNS using your AWS CLI, copy the following `aws ec2 modify-vpc-endpoint` command from the **Create Private Endpoint Connection** page and run it in your AWS CLI. -Enable private DNS in AWS. You can either use the AWS Management Console or the AWS CLI. +```bash +aws ec2 modify-vpc-endpoint --vpc-endpoint-id ${your_vpc_endpoint_id} --private-dns-enabled +``` - +Alternatively, you can find the command on the **Networking** page of your cluster. Locate the private endpoint and click **...*** > **Enable DNS** in the **Action** column. + +
To enable private DNS in your AWS Management Console: @@ -154,30 +166,16 @@ To enable private DNS in your AWS Management Console: ![Enable private DNS](/media/tidb-cloud/private-endpoint/enable-private-dns.png) -
-
- -To enable private DNS using your AWS CLI, copy the command and run it in your AWS CLI. - -```bash -aws ec2 modify-vpc-endpoint --vpc-endpoint-id ${your_vpc_endpoint_id} --private-dns-enabled -``` -
-Click **Create** in the TiDB Cloud console to finalize the creation of the private endpoint. - -Then you can connect to the endpoint service. - -### Step 6. Connect to your TiDB cluster +### Step 5. Connect to your TiDB cluster -After you have enabled the private DNS, go back to the TiDB Cloud console and take the following steps: +After you have accepted the private endpoint connection, you are redirected back to the connection dialog. -1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click **...** in the **Action** column. -2. Click **Connect**. A connection dialog is displayed. -3. Select the **Private Endpoint** tab. The private endpoint you just created is displayed under **Step 1: Create Private Endpoint**. -4. Under **Step 2: Connect your connection**, click **Connect**, click the tab of your preferred connection method, and then connect to your cluster with the connection string. The placeholders `:` in the connection string are automatically replaced with the real values. +1. Wait for the private endpoint connection status to change from **System Checking** to **Active** (approximately 5 minutes). +2. In the **Connect With** drop-down list, select your preferred connection method. The corresponding connection string is displayed at the bottom of the dialog. +3. Connect to your cluster with the connection string. > **Tip:** > @@ -185,7 +183,10 @@ After you have enabled the private DNS, go back to the TiDB Cloud console and ta ### Private endpoint status reference -When you use private endpoint connections, the statuses of private endpoints or private endpoint services are displayed on the [**Private Endpoint** page](#prerequisites). +When you use private endpoint connections, the statuses of private endpoints or private endpoint services are displayed on the following pages: + +- Cluster-level **Networking** page: click **Networking** in the left navigation pane of the cluster overview page. +- Project-level **Network Access** page: click **Network Access** in the left navigation pane of the **Project Settings** page. The possible statuses of a private endpoint are explained as follows: @@ -208,7 +209,3 @@ The possible statuses of a private endpoint service are explained as follows: You might need to properly set the security group for your VPC endpoint in the AWS Management Console. Go to **VPC** > **Endpoints**. Right-click your VPC endpoint and select the proper **Manage security groups**. A proper security group within your VPC that allows inbound access from your EC2 instances on Port 4000 or a customer-defined port. ![Manage security groups](/media/tidb-cloud/private-endpoint/manage-security-groups.png) - -### I cannot enable private DNS. An error is reported indicating that the `enableDnsSupport` and `enableDnsHostnames` VPC attributes are not enabled - -Make sure that DNS hostname and DNS resolution are both enabled in your VPC setting. They are disabled by default when you create a VPC in the AWS Management Console. diff --git a/tidb-cloud/set-up-vpc-peering-connections.md b/tidb-cloud/set-up-vpc-peering-connections.md index 23f6986e2b777..2bdc751469c46 100644 --- a/tidb-cloud/set-up-vpc-peering-connections.md +++ b/tidb-cloud/set-up-vpc-peering-connections.md @@ -1,53 +1,55 @@ --- -title: Connect to TiDB Dedicated via VPC Peering -summary: Learn how to connect to TiDB Dedicated via VPC peering. +title: Connect to TiDB Cloud Dedicated via VPC Peering +summary: Learn how to connect to TiDB Cloud Dedicated via VPC peering. --- -# Connect to TiDB Dedicated via VPC Peering +# Connect to TiDB Cloud Dedicated via VPC Peering > **Note:** > -> VPC peering connection is only available for TiDB Dedicated clusters. You cannot connect to [TiDB Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-serverless) using VPC peering. +> VPC peering connection is only available for TiDB Cloud Dedicated clusters. You cannot connect to [TiDB Cloud Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) using VPC peering. To connect your application to TiDB Cloud via VPC peering, you need to set up [VPC peering](/tidb-cloud/tidb-cloud-glossary.md#vpc-peering) with TiDB Cloud. This document walks you through setting up VPC peering connections [on AWS](#set-up-vpc-peering-on-aws) and [on Google Cloud](#set-up-vpc-peering-on-google-cloud) and connecting to TiDB Cloud via a VPC peering. VPC peering connection is a networking connection between two VPCs that enables you to route traffic between them using private IP addresses. Instances in either VPC can communicate with each other as if they are within the same network. -Currently, TiDB Cloud only supports VPC peering in the same region for the same project. TiDB clusters of the same project in the same region are created in the same VPC. Therefore, once VPC peering is set up in a region of a project, all the TiDB clusters created in the same region of this project can be connected in your VPC. VPC peering setup differs among cloud providers. +Currently, TiDB clusters of the same project in the same region are created in the same VPC. Therefore, once VPC peering is set up in a region of a project, all the TiDB clusters created in the same region of this project can be connected in your VPC. VPC peering setup differs among cloud providers. > **Tip:** > > To connect your application to TiDB Cloud, you can also set up [private endpoint connection](/tidb-cloud/set-up-private-endpoint-connections.md) with TiDB Cloud, which is secure and private, and does not expose your data to the public internet. It is recommended to use private endpoints over VPC peering connections. -## Prerequisite: Set a Project CIDR +## Prerequisite: Set a CIDR for a region -Project CIDR (Classless Inter-Domain Routing) is the CIDR block used for network peering in a project. +CIDR (Classless Inter-Domain Routing) is the CIDR block used for creating VPC for TiDB Cloud Dedicated clusters. -Before adding VPC Peering requests to a region, you need to set a project CIDR for your project's cloud provider (AWS or Google Cloud) to establish a peering link to your application's VPC. +Before adding VPC Peering requests to a region, you must set a CIDR for that region and create an initial TiDB Cloud Dedicated cluster in that region. Once the first Dedicated cluster is created, TiDB Cloud will create the VPC of the cluster, allowing you to establish a peering link to your application's VPC. -You can set the project CIDR when creating the first TiDB Dedicated of your project. If you want to set the project CIDR before creating the cluster, perform the following operations: +You can set the CIDR when creating the first TiDB Cloud Dedicated cluster. If you want to set the CIDR before creating the cluster, perform the following operations: 1. Log in to the [TiDB Cloud console](https://tidbcloud.com). 2. Click in the lower-left corner, switch to the target project if you have multiple projects, and then click **Project Settings**. -3. On the **Project Settings** page of your project, click **Network Access** in the left navigation pane, and then click the **Project CIDR** tab. -4. Click **Add a project CIDR for AWS** or **Add a project CIDR for Google Cloud** according to your cloud provider, specify one of the following network addresses in the **Project CIDR** field, and then click **Confirm**. +3. On the **Project Settings** page of your project, click **Network Access** in the left navigation pane, click the **Project CIDR** tab, and then select **AWS** or **Google Cloud** according to your cloud provider. +4. In the upper-right corner, click **Create CIDR**. Specify the region and CIDR value in the **Create AWS CIDR** or **Create Google Cloud CIDR** dialog, and then click **Confirm**. + + ![Project-CIDR4](/media/tidb-cloud/Project-CIDR4.png) > **Note:** > - > To avoid any conflicts with the CIDR of the VPC where your application is located, you need to set a different project CIDR in this field. - - - 10.250.0.0/16 - - 10.250.0.0/17 - - 10.250.128.0/17 - - 172.30.0.0/16 - - 172.30.0.0/17 - - 172.30.128.0/17 - - ![Project-CIDR4](/media/tidb-cloud/Project-CIDR4.png) + > - To avoid any conflicts with the CIDR of the VPC where your application is located, you need to set a different project CIDR in this field. + > - For AWS Region, it is recommended to configure an IP range size between `/16` and `/23`. Supported network addresses include: + > - 10.250.0.0 - 10.251.255.255 + > - 172.16.0.0 - 172.31.255.255 + > - 192.168.0.0 - 192.168.255.255 + > - For Google Cloud Region, it is recommended to configure an IP range size between `/19` and `/20`. If you want to configure an IP range size between `/16` and `/18`, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). Supported network addresses include: + > - 10.250.0.0 - 10.251.255.255 + > - 172.16.0.0 - 172.17.255.255 + > - 172.30.0.0 - 172.31.255.255 + > - TiDB Cloud limits the number of TiDB Cloud nodes in a region of a project based on the CIDR block size of the region. 5. View the CIDR of the cloud provider and the specific region. - The region CIDR is inactive by default. To activate the region CIDR, you need to create a cluster in the target region. When the region CIDR is active, you can create VPC Peering for the region. + The CIDR is inactive by default. To activate the CIDR, you need to create a cluster in the target region. When the region CIDR is active, you can create VPC Peering for the region. ![Project-CIDR2](/media/tidb-cloud/Project-CIDR2.png) @@ -57,33 +59,71 @@ This section describes how to set up VPC peering connections on AWS. For Google ### Step 1. Add VPC peering requests +You can add VPC peering requests on either the project-level **Network Access** page or the cluster-level **Networking** page in the TiDB Cloud console. + + +
+ 1. Log in to the [TiDB Cloud console](https://tidbcloud.com). 2. Click in the lower-left corner, switch to the target project if you have multiple projects, and then click **Project Settings**. -3. On the **Project Settings** page of your project, click **Network Access** in the left navigation pane, and then click the **VPC Peering** tab. +3. On the **Project Settings** page of your project, click **Network Access** in the left navigation pane, and click the **VPC Peering** > **AWS** tab. The **VPC Peering** configuration is displayed by default. -4. Click **Add**, choose the AWS icon, and then fill in the required information of your existing AWS VPC: +4. In the upper-right corner, click **Create VPC Peering**, select the **TiDB Cloud VPC Region**, and then fill in the required information of your existing AWS VPC: - - Region + - Your VPC Region - AWS Account ID - VPC ID - VPC CIDR - You can get these information from your VPC details on the VPC dashboard. + You can get such information from your VPC details page of the [AWS Management Console](https://console.aws.amazon.com/). TiDB Cloud supports creating VPC peerings between VPCs in the same region or from two different regions. ![VPC peering](/media/tidb-cloud/vpc-peering/vpc-peering-creating-infos.png) -5. Click **Initialize**. The **Approve VPC Peerings** dialog is displayed. +5. Click **Create** to send the VPC peering request, and then view the VPC peering information on the **VPC Peering** > **AWS** tab. The status of the newly created VPC peering is **System Checking**. -### Step 2. Approve and configure the VPC peering +6. To view detailed information about your newly created VPC peering, click **...** > **View** in the **Action** column. The **VPC Peering Details** page is displayed. + +
+
+ +1. Open the overview page of the target cluster. + + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + + 2. Click the name of your target cluster to go to its overview page. -Use either of the following two options to approve and configure the VPC peering connection: +2. In the left navigation pane, click **Networking**, and then click **Create VPC Peering**. + +3. Fill in the required information of your existing AWS VPC: + + - Your VPC Region + - AWS Account ID + - VPC ID + - VPC CIDR + + You can get such information from your VPC details page of the [AWS Management Console](https://console.aws.amazon.com/). TiDB Cloud supports creating VPC peerings between VPCs in the same region or from two different regions. + + ![VPC peering](/media/tidb-cloud/vpc-peering/vpc-peering-creating-infos.png) + +4. Click **Create** to send the VPC peering request, and then view the VPC peering information on the **Networking** > **AWS VPC Peering** section. The status of the newly created VPC peering is **System Checking**. + +5. To view detailed information about your newly created VPC peering, click **...** > **View** in the **Action** column. The **AWS VPC Peering Details** page is displayed. + +
+ + +### Step 2. Approve and configure the VPC peering -- [Option 1: Use AWS CLI](#option-1-use-aws-cli) -- [Option 2: Use the AWS dashboard](#option-2-use-the-aws-dashboard) +You can approve and configure the VPC peering connection using AWS CLI or AWS dashboard. -#### Option 1. Use AWS CLI + +
1. Install AWS Command Line Interface (AWS CLI). @@ -141,7 +181,7 @@ Use either of the following two options to approve and configure the VPC peering aws ec2 describe-route-tables --region "$app_region" --filters Name=vpc-id,Values="$app_vpc_id" --query 'RouteTables[*].RouteTableId' --output text | tr "\t" "\n" | while read row do app_route_table_id="$row" - aws ec2 create-route --route-table-id "$app_route_table_id" --destination-cidr-block "$tidbcloud_project_cidr" --vpc-peering-connection-id "$pcx_tidb_to_app_id" + aws ec2 create-route --region "$app_region" --route-table-id "$app_route_table_id" --destination-cidr-block "$tidbcloud_project_cidr" --vpc-peering-connection-id "$pcx_tidb_to_app_id" done ``` @@ -159,19 +199,20 @@ Use either of the following two options to approve and configure the VPC peering After finishing the configuration, the VPC peering has been created. You can [connect to the TiDB cluster](#connect-to-the-tidb-cluster) to verify the result. -#### Option 2. Use the AWS dashboard +
+
You can also use the AWS dashboard to configure the VPC peering connection. -1. Confirm to accept the peer connection request in your AWS console. +1. Confirm to accept the peer connection request in your [AWS Management Console](https://console.aws.amazon.com/). - 1. Sign in to the AWS console and click **Services** on the top menu bar. Enter `VPC` in the search box and go to the VPC service page. + 1. Sign in to the [AWS Management Console](https://console.aws.amazon.com/) and click **Services** on the top menu bar. Enter `VPC` in the search box and go to the VPC service page. ![AWS dashboard](/media/tidb-cloud/vpc-peering/aws-vpc-guide-1.jpg) 2. From the left navigation bar, open the **Peering Connections** page. On the **Create Peering Connection** tab, a peering connection is in the **Pending Acceptance** status. - 3. Confirm the requester owner is TiDB Cloud (`380838443567`). Right-click the peering connection and select **Accept Request** to accept the request in the **Accept VPC peering connection request** dialog. + 3. Confirm that the requester owner and the requester VPC match **TiDB Cloud AWS Account ID** and **TiDB Cloud VPC ID** on the **VPC Peering Details** page of the [TiDB Cloud console](https://tidbcloud.com). Right-click the peering connection and select **Accept Request** to accept the request in the **Accept VPC peering connection request** dialog. ![AWS VPC peering requests](/media/tidb-cloud/vpc-peering/aws-vpc-guide-3.png) @@ -183,7 +224,7 @@ You can also use the AWS dashboard to configure the VPC peering connection. ![Search all route tables related to VPC](/media/tidb-cloud/vpc-peering/aws-vpc-guide-4.png) - 3. Right-click each route table and select **Edit routes**. On the edit page, add a route with a destination to the Project CIDR (by checking the **VPC Peering** configuration page in the TiDB Cloud console) and fill in your peering connection ID in the **Target** column. + 3. Right-click each route table and select **Edit routes**. On the edit page, add a route with a destination to the TiDB Cloud CIDR (by checking the **VPC Peering** configuration page in the TiDB Cloud console) and fill in your peering connection ID in the **Target** column. ![Edit all route tables](/media/tidb-cloud/vpc-peering/aws-vpc-guide-5.png) @@ -201,42 +242,81 @@ You can also use the AWS dashboard to configure the VPC peering connection. Now you have successfully set up the VPC peering connection. Next, [connect to the TiDB cluster via VPC peering](#connect-to-the-tidb-cluster). +
+ + ## Set up VPC peering on Google Cloud +### Step 1. Add VPC peering requests + +You can add VPC peering requests on either the project-level **Network Access** page or the cluster-level **Networking** page in the TiDB Cloud console. + + +
+ 1. Log in to the [TiDB Cloud console](https://tidbcloud.com). 2. Click in the lower-left corner, switch to the target project if you have multiple projects, and then click **Project Settings**. -3. On the **Project Settings** page of your project, click **Network Access** in the left navigation pane, and then click the **VPC Peering** tab. +3. On the **Project Settings** page of your project, click **Network Access** in the left navigation pane, and click the **VPC Peering** > **Google Cloud** tab. The **VPC Peering** configuration is displayed by default. -4. Click **Add**, choose the Google Cloud icon, and then fill in the required information of your existing Google Cloud VPC: +4. In the upper-right corner, click **Create VPC Peering**, select the **TiDB Cloud VPC Region**, and then fill in the required information of your existing Google Cloud VPC: > **Tip:** > - > You can follow instructions next to the **Application Google Cloud Project ID** and **VPC Network Name** fields to find the project ID and VPC network name. + > You can follow instructions next to the **Google Cloud Project ID** and **VPC Network Name** fields to find the project ID and VPC network name. - - Region - - Application Google Cloud Project ID + - Google Cloud Project ID - VPC Network Name - VPC CIDR -5. Click **Initialize**. The **Approve VPC Peerings** dialog is displayed. +5. Click **Create** to send the VPC peering request, and then view the VPC peering information on the **VPC Peering** > **Google Cloud** tab. The status of the newly created VPC peering is **System Checking**. -6. Check the connection information of your TiDB VPC peerings. +6. To view detailed information about your newly created VPC peering, click **...** > **View** in the **Action** column. The **VPC Peering Details** page is displayed. - ![VPC-Peering](/media/tidb-cloud/VPC-Peering3.png) +
+
-7. Execute the following command to finish the setup of VPC peerings: +1. Open the overview page of the target cluster. - {{< copyable "shell-regular" >}} + 1. Log in to the [TiDB Cloud console](https://tidbcloud.com/) and navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. - ```bash - gcloud beta compute networks peerings create --project --network --peer-project --peer-network - ``` + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. - > **Note:** + 2. Click the name of your target cluster to go to its overview page. + +2. In the left navigation pane, click **Networking**, and then click **Create VPC Peering**. + +3. Fill in the required information of your existing Google Cloud VPC: + + > **Tip:** > - > You can name `` as you like. + > You can follow instructions next to the **Google Cloud Project ID** and **VPC Network Name** fields to find the project ID and VPC network name. + + - Google Cloud Project ID + - VPC Network Name + - VPC CIDR + +4. Click **Create** to send the VPC peering request, and then view the VPC peering information on the **Networking** > **Google Cloud VPC Peering** section. The status of the newly created VPC peering is **System Checking**. + +5. To view detailed information about your newly created VPC peering, click **...** > **View** in the **Action** column. The **Google Cloud VPC Peering Details** page is displayed. + +
+ + +### Step 2. Approve the VPC peering + +Execute the following command to finish the setup of VPC peering: + +```bash +gcloud beta compute networks peerings create --project --network --peer-project --peer-network +``` + +> **Note:** +> +> You can name `` as you like. Now you have successfully set up the VPC peering connection. Next, [connect to the TiDB cluster via VPC peering](#connect-to-the-tidb-cluster). @@ -244,10 +324,10 @@ Now you have successfully set up the VPC peering connection. Next, [connect to t 1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click the name of your target cluster to go to its overview page. -2. Click **Connect** in the upper-right corner, and select the **VPC Peering** tab in the connection dialog. +2. Click **Connect** in the upper-right corner, and select **VPC Peering** from the **Connection Type** drop-down list. - You can see the **Status** of the VPC peering is **active**. If **Status** is still **system checking**, wait for about 5 minutes and open the dialog again. + Wait for the VPC peering connection status to change from **system checking** to **active** (approximately 5 minutes). -3. Click **Get Endpoint** and wait for a few minutes. Then the connection command is displayed in the dialog. +3. In the **Connect With** drop-down list, select your preferred connection method. The corresponding connection string is displayed at the bottom of the dialog. -4. Under **Step 2: Connect with a SQL client** in the dialog box, click the tab of your preferred connection method, and then connect to your cluster with the connection string. +4. Connect to your cluster with the connection string. diff --git a/tidb-cloud/setup-self-hosted-kafka-private-link-service.md b/tidb-cloud/setup-self-hosted-kafka-private-link-service.md new file mode 100644 index 0000000000000..7c85c545d95b7 --- /dev/null +++ b/tidb-cloud/setup-self-hosted-kafka-private-link-service.md @@ -0,0 +1,761 @@ +--- +title: Set Up Self-Hosted Kafka Private Link Service in AWS +summary: This document explains how to set up Private Link service for self-hosted Kafka in AWS and how to make it work with TiDB Cloud. +--- + +# Set Up Self-Hosted Kafka Private Link Service in AWS + +This document describes how to set up Private Link service for self-hosted Kafka in AWS, and how to make it work with TiDB Cloud. + +The mechanism works as follows: + +1. The TiDB Cloud VPC connects to the Kafka VPC through private endpoints. +2. Kafka clients need to communicate directly to all Kafka brokers. +3. Each Kafka broker is mapped to a unique port of endpoints within the TiDB Cloud VPC. +4. Leverage the Kafka bootstrap mechanism and AWS resources to achieve the mapping. + +The following diagram shows the mechanism. + +![Connect to AWS Self-Hosted Kafka Private Link Service](/media/tidb-cloud/changefeed/connect-to-aws-self-hosted-kafka-privatelink-service.jpeg) + +The document provides an example of connecting to a Kafka Private Link service deployed across three availability zones (AZ) in AWS. While other configurations are possible based on similar port-mapping principles, this document covers the fundamental setup process of the Kafka Private Link service. For production environments, a more resilient Kafka Private Link service with enhanced operational maintainability and observability is recommended. + +## Prerequisites + +1. Ensure that you have the following authorization to set up a Kafka Private Link service in your own AWS account. + + - Manage EC2 nodes + - Manage VPC + - Manage subnets + - Manage security groups + - Manage load balancer + - Manage endpoint services + - Connect to EC2 nodes to configure Kafka nodes + +2. [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) if you do not have one. + +3. Get the Kafka deployment information from your TiDB Cloud Dedicated cluster. + + 1. In the [TiDB Cloud console](https://tidbcloud.com), navigate to the cluster overview page of the TiDB cluster, and then click **Changefeed** in the left navigation pane. + 2. On the overview page, find the region of the TiDB cluster. Ensure that your Kafka cluster will be deployed to the same region. + 3. Click **Create Changefeed**. + 1. In **Target Type**, select **Kafka**. + 2. In **Connectivity Method**, select **Private Link**. + 4. Note down the information of the TiDB Cloud AWS account in **Reminders before proceeding**. You will use it to authorize TiDB Cloud to create an endpoint for the Kafka Private Link service. + 5. Select **Number of AZs**. In this example, select **3 AZs**. Note down the IDs of the AZs in which you want to deploy your Kafka cluster. If you want to know the relationship between your AZ names and AZ IDs, see [Availability Zone IDs for your AWS resources](https://docs.aws.amazon.com/ram/latest/userguide/working-with-az-ids.html) to find it. + 6. Enter a unique **Kafka Advertised Listener Pattern** for your Kafka Private Link service. + 1. Input a unique random string. It can only include numbers or lowercase letters. You will use it to generate **Kafka Advertised Listener Pattern** later. + 2. Click **Check usage and generate** to check if the random string is unique and generate **Kafka Advertised Listener Pattern** that will be used to assemble the EXTERNAL advertised listener for Kafka brokers. + +Note down all the deployment information. You need to use it to configure your Kafka Private Link service later. + +The following table shows an example of the deployment information. + +| Information | Value | Note | +|--------|-----------------|---------------------------| +| Region | Oregon (`us-west-2`) | N/A | +| Principal of TiDB Cloud AWS Account | `arn:aws:iam:::root` | N/A | +| AZ IDs |
  • `usw2-az1`
  • `usw2-az2`
  • `usw2-az3`
| Align AZ IDs to AZ names in your AWS account.
Example:
  • `usw2-az1` => `us-west-2a`
  • `usw2-az2` => `us-west-2c`
  • `usw2-az3` => `us-west-2b`
| +| Kafka Advertised Listener Pattern | The unique random string: `abc`
Generated pattern for AZs:
  • `usw2-az1` => <broker_id>.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:<port>
  • `usw2-az2` => <broker_id>.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:<port>
  • `usw2-az3` => <broker_id>.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:<port>
| Map AZ names to AZ-specified patterns. Make sure that you configure the right pattern to the broker in a specific AZ later.
  • `us-west-2a` => <broker_id>.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:<port>
  • `us-west-2c` => <broker_id>.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:<port>
  • `us-west-2b` => <broker_id>.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:<port>
| + +## Step 1. Set up a Kafka cluster + +If you need to deploy a new cluster, follow the instructions in [Deploy a new Kafka cluster](#deploy-a-new-kafka-cluster). + +If you need to expose an existing cluster, follow the instructions in [Reconfigure a running Kafka cluster](#reconfigure-a-running-kafka-cluster). + +### Deploy a new Kafka cluster + +#### 1. Set up the Kafka VPC + +The Kafka VPC requires the following: + +- Three private subnets for brokers, one for each AZ. +- One public subnet in any AZ with a bastion node that can connect to the internet and three private subnets, which makes it easy to set up the Kafka cluster. In a production environment, you might have your own bastion node that can connect to the Kafka VPC. + +Before creating subnets, create subnets in AZs based on the mappings of AZ IDs and AZ names. Take the following mapping as an example. + +- `usw2-az1` => `us-west-2a` +- `usw2-az2` => `us-west-2c` +- `usw2-az3` => `us-west-2b` + +Create private subnets in the following AZs: + +- `us-west-2a` +- `us-west-2c` +- `us-west-2b` + +Take the following steps to create the Kafka VPC. + +**1.1. Create the Kafka VPC** + +1. Go to [AWS Console > VPC dashboard](https://console.aws.amazon.com/vpcconsole/home?#vpcs:), and switch to the region in which you want to deploy Kafka. + +2. Click **Create VPC**. Fill in the information on the **VPC settings** page as follows. + + 1. Select **VPC only**. + 2. Enter a tag in **Name tag**, for example, `Kafka VPC`. + 3. Select **IPv4 CIDR manual input**, and enter the IPv4 CIDR, for example, `10.0.0.0/16`. + 4. Use the default values for other options. Click **Create VPC**. + 5. On the VPC detail page, take note of the VPC ID, for example, `vpc-01f50b790fa01dffa`. + +**1.2. Create private subnets in the Kafka VPC** + +1. Go to the [Subnets Listing page](https://console.aws.amazon.com/vpcconsole/home?#subnets:). +2. Click **Create subnet**. +3. Select **VPC ID** (`vpc-01f50b790fa01dffa` in this example) that you noted down before. +4. Add three subnets with the following information. It is recommended that you put the AZ IDs in the subnet names to make it easy to configure the brokers later, because TiDB Cloud requires encoding the AZ IDs in the broker's `advertised.listener` configuration. + + - Subnet1 in `us-west-2a` + - **Subnet name**: `broker-usw2-az1` + - **Availability Zone**: `us-west-2a` + - **IPv4 subnet CIDR block**: `10.0.0.0/18` + + - Subnet2 in `us-west-2c` + - **Subnet name**: `broker-usw2-az2` + - **Availability Zone**: `us-west-2c` + - **IPv4 subnet CIDR block**: `10.0.64.0/18` + + - Subnet3 in `us-west-2b` + - **Subnet name**: `broker-usw2-az3` + - **Availability Zone**: `us-west-2b` + - **IPv4 subnet CIDR block**: `10.0.128.0/18` + +5. Click **Create subnet**. The **Subnets Listing** page is displayed. + +**1.3. Create the public subnet in the Kafka VPC** + +1. Click **Create subnet**. +2. Select **VPC ID** (`vpc-01f50b790fa01dffa` in this example) that you noted down before. +3. Add the public subnet in any AZ with the following information: + + - **Subnet name**: `bastion` + - **IPv4 subnet CIDR block**: `10.0.192.0/18` + +4. Click **Create subnet**. The **Subnets Listing** page is displayed. +5. Configure the bastion subnet to the Public subnet. + + 1. Go to [VPC dashboard > Internet gateways](https://console.aws.amazon.com/vpcconsole/home#igws:). Create an Internet Gateway with the name `kafka-vpc-igw`. + 2. On the **Internet gateways Detail** page, in **Actions**, click **Attach to VPC** to attach the Internet Gateway to the Kafka VPC. + 3. Go to [VPC dashboard > Route tables](https://console.aws.amazon.com/vpcconsole/home#CreateRouteTable:). Create a route table to the Internet Gateway in Kafka VPC and add a new route with the following information: + + - **Name**: `kafka-vpc-igw-route-table` + - **VPC**: `Kafka VPC` + - **Route**: + - **Destination**: `0.0.0.0/0` + - **Target**: `Internet Gateway`, `kafka-vpc-igw` + + 4. Attach the route table to the bastion subnet. On the **Detail** page of the route table, click **Subnet associations > Edit subnet associations** to add the bastion subnet and save changes. + +#### 2. Set up Kafka brokers + +**2.1. Create a bastion node** + +Go to the [EC2 Listing page](https://console.aws.amazon.com/ec2/home#Instances:). Create the bastion node in the bastion subnet. + +- **Name**: `bastion-node` +- **Amazon Machine Image**: `Amazon linux` +- **Instance Type**: `t2.small` +- **Key pair**: `kafka-vpc-key-pair`. Create a new key pair named `kafka-vpc-key-pair`. Download **kafka-vpc-key-pair.pem** to your local for later configuration. +- Network settings + + - **VPC**: `Kafka VPC` + - **Subnet**: `bastion` + - **Auto-assign public IP**: `Enable` + - **Security Group**: create a new security group allow SSH login from anywhere. You can narrow the rule for safety in the production environment. + +**2.2. Create broker nodes** + +Go to the [EC2 Listing page](https://console.aws.amazon.com/ec2/home#Instances:). Create three broker nodes in broker subnets, one for each AZ. + +- Broker 1 in subnet `broker-usw2-az1` + + - **Name**: `broker-node1` + - **Amazon Machine Image**: `Amazon linux` + - **Instance Type**: `t2.large` + - **Key pair**: reuse `kafka-vpc-key-pair` + - Network settings + + - **VPC**: `Kafka VPC` + - **Subnet**: `broker-usw2-az1` + - **Auto-assign public IP**: `Disable` + - **Security Group**: create a new security group to allow all TCP from Kafka VPC. You can narrow the rule for safety in the production environment. + - **Protocol**: `TCP` + - **Port range**: `0 - 65535` + - **Source**: `10.0.0.0/16` + +- Broker 2 in subnet `broker-usw2-az2` + + - **Name**: `broker-node2` + - **Amazon Machine Image**: `Amazon linux` + - **Instance Type**: `t2.large` + - **Key pair**: reuse `kafka-vpc-key-pair` + - Network settings + + - **VPC**: `Kafka VPC` + - **Subnet**: `broker-usw2-az2` + - **Auto-assign public IP**: `Disable` + - **Security Group**: create a new security group to allow all TCP from Kafka VPC. You can narrow the rule for safety in the production environment. + - **Protocol**: `TCP` + - **Port range**: `0 - 65535` + - **Source**: `10.0.0.0/16` + +- Broker 3 in subnet `broker-usw2-az3` + + - **Name**: `broker-node3` + - **Amazon Machine Image**: `Amazon linux` + - **Instance Type**: `t2.large` + - **Key pair**: reuse `kafka-vpc-key-pair` + - Network settings + + - **VPC**: `Kafka VPC` + - **Subnet**: `broker-usw2-az3` + - **Auto-assign public IP**: `Disable` + - **Security Group**: create a new security group to allow all TCP from Kafka VPC. You can narrow the rule for safety in the production environment. + - **Protocol**: `TCP` + - **Port range**: `0 - 65535` + - **Source**: `10.0.0.0/16` + +**2.3. Prepare Kafka runtime binaries** + +1. Go to the detail page of the bastion node. Get the **Public IPv4 address**. Use SSH to log in to the node with the previously downloaded `kafka-vpc-key-pair.pem`. + + ```shell + chmod 400 kafka-vpc-key-pair.pem + ssh -i "kafka-vpc-key-pair.pem" ec2-user@{bastion_public_ip} # replace {bastion_public_ip} with the IP address of your bastion node, for example, 54.186.149.187 + scp -i "kafka-vpc-key-pair.pem" kafka-vpc-key-pair.pem ec2-user@{bastion_public_ip}:~/ + ``` + +2. Download binaries. + + ```shell + # Download Kafka and OpenJDK, and then extract the files. You can choose the binary version based on your preference. + wget https://archive.apache.org/dist/kafka/3.7.1/kafka_2.13-3.7.1.tgz + tar -zxf kafka_2.13-3.7.1.tgz + wget https://download.java.net/java/GA/jdk22.0.2/c9ecb94cd31b495da20a27d4581645e8/9/GPL/openjdk-22.0.2_linux-x64_bin.tar.gz + tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz + ``` + +3. Copy binaries to each broker node. + + ```shell + # Replace {broker-node1-ip} with your broker-node1 IP address + scp -i "kafka-vpc-key-pair.pem" kafka_2.13-3.7.1.tgz ec2-user@{broker-node1-ip}:~/ + ssh -i "kafka-vpc-key-pair.pem" ec2-user@{broker-node1-ip} "tar -zxf kafka_2.13-3.7.1.tgz" + scp -i "kafka-vpc-key-pair.pem" openjdk-22.0.2_linux-x64_bin.tar.gz ec2-user@{broker-node1-ip}:~/ + ssh -i "kafka-vpc-key-pair.pem" ec2-user@{broker-node1-ip} "tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz" + + # Replace {broker-node2-ip} with your broker-node2 IP address + scp -i "kafka-vpc-key-pair.pem" kafka_2.13-3.7.1.tgz ec2-user@{broker-node2-ip}:~/ + ssh -i "kafka-vpc-key-pair.pem" ec2-user@{broker-node2-ip} "tar -zxf kafka_2.13-3.7.1.tgz" + scp -i "kafka-vpc-key-pair.pem" openjdk-22.0.2_linux-x64_bin.tar.gz ec2-user@{broker-node2-ip}:~/ + ssh -i "kafka-vpc-key-pair.pem" ec2-user@{broker-node2-ip} "tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz" + + # Replace {broker-node3-ip} with your broker-node3 IP address + scp -i "kafka-vpc-key-pair.pem" kafka_2.13-3.7.1.tgz ec2-user@{broker-node3-ip}:~/ + ssh -i "kafka-vpc-key-pair.pem" ec2-user@{broker-node3-ip} "tar -zxf kafka_2.13-3.7.1.tgz" + scp -i "kafka-vpc-key-pair.pem" openjdk-22.0.2_linux-x64_bin.tar.gz ec2-user@{broker-node3-ip}:~/ + ssh -i "kafka-vpc-key-pair.pem" ec2-user@{broker-node3-ip} "tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz" + ``` + +**2.4. Set up Kafka nodes on each broker node** + +**2.4.1 Set up a KRaft Kafka cluster with three nodes** + +Each node will act as a broker and controller role. Do the following for each broker: + +1. For the `listeners` item, all three brokers are the same and act as broker and controller roles: + + 1. Configure the same CONTROLLER listener for all **controller** role nodes. If you only want to add the **broker** role nodes, you do not need the CONTROLLER listener in `server.properties`. + 2. Configure two **broker** listeners, `INTERNAL` for internal access and `EXTERNAL` for external access from TiDB Cloud. + +2. For the `advertised.listeners` item, do the following: + + 1. Configure an INTERNAL advertised listener for every broker with the internal IP of the broker node. Advertised internal Kafka clients use this address to visit the broker. + 2. Configure an EXTERNAL advertised listener based on **Kafka Advertised Listener Pattern** you get from TiDB Cloud for each broker node to help TiDB Cloud differentiate between different brokers. Different EXTERNAL advertised listeners help the Kafka client from TiDB Cloud route requests to the right broker. + + - `` differentiates brokers from Kafka Private Link Service access points. Plan a port range for EXTERNAL advertised listeners of all brokers. These ports do not have to be actual ports listened to by brokers. They are ports listened to by the load balancer for Private Link Service that will forward requests to different brokers. + - `AZ ID` in **Kafka Advertised Listener Pattern** indicates where the broker is deployed. TiDB Cloud will route requests to different endpoint DNS names based on the AZ ID. + + It is recommended to configure different broker IDs for different brokers to make it easy for troubleshooting. + +3. The planning values are as follows: + + - **CONTROLLER port**: `29092` + - **INTERNAL port**: `9092` + - **EXTERNAL**: `39092` + - **EXTERNAL advertised listener ports range**: `9093~9095` + +**2.4.2. Create a configuration file** + +Use SSH to log in to every broker node. Create a configuration file `~/config/server.properties` with the following content. + +```properties +# brokers in usw2-az1 + +# broker-node1 ~/config/server.properties +# 1. Replace {broker-node1-ip}, {broker-node2-ip}, {broker-node3-ip} with the actual IP addresses. +# 2. Configure EXTERNAL in "advertised.listeners" based on the "Kafka Advertised Listener Pattern" in the "Prerequisites" section. +# 2.1 The pattern for AZ(ID: usw2-az1) is ".usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:". +# 2.2 So the EXTERNAL can be "b1.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:9093". Replace with "b" prefix plus "node.id" properties, and replace with a unique port (9093) in the port range of the EXTERNAL advertised listener. +# 2.3 If there are more broker role nodes in the same AZ, you can configure them in the same way. +process.roles=broker,controller +node.id=1 +controller.quorum.voters=1@{broker-node1-ip}:29092,2@{broker-node2-ip}:29092,3@{broker-node3-ip}:29092 +listeners=INTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29092,EXTERNAL://0.0.0.0:39092 +inter.broker.listener.name=INTERNAL +advertised.listeners=INTERNAL://{broker-node1-ip}:9092,EXTERNAL://b1.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:9093 +controller.listener.names=CONTROLLER +listener.security.protocol.map=INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL +log.dirs=./data +``` + +```properties +# brokers in usw2-az2 + +# broker-node2 ~/config/server.properties +# 1. Replace {broker-node1-ip}, {broker-node2-ip}, {broker-node3-ip} with the actual IP addresses. +# 2. Configure EXTERNAL in "advertised.listeners" based on the "Kafka Advertised Listener Pattern" in the "Prerequisites" section. +# 2.1 The pattern for AZ(ID: usw2-az2) is ".usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:". +# 2.2 So the EXTERNAL can be "b2.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:9094". Replace with "b" prefix plus "node.id" properties, and replace with a unique port (9094) in the port range of the EXTERNAL advertised listener. +# 2.3 If there are more broker role nodes in the same AZ, you can configure them in the same way. +process.roles=broker,controller +node.id=2 +controller.quorum.voters=1@{broker-node1-ip}:29092,2@{broker-node2-ip}:29092,3@{broker-node3-ip}:29092 +listeners=INTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29092,EXTERNAL://0.0.0.0:39092 +inter.broker.listener.name=INTERNAL +advertised.listeners=INTERNAL://{broker-node2-ip}:9092,EXTERNAL://b2.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:9094 +controller.listener.names=CONTROLLER +listener.security.protocol.map=INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL +log.dirs=./data +``` + +```properties +# brokers in usw2-az3 + +# broker-node3 ~/config/server.properties +# 1. Replace {broker-node1-ip}, {broker-node2-ip}, {broker-node3-ip} with the actual IP addresses. +# 2. Configure EXTERNAL in "advertised.listeners" based on the "Kafka Advertised Listener Pattern" in the "Prerequisites" section. +# 2.1 The pattern for AZ(ID: usw2-az3) is ".usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:". +# 2.2 So the EXTERNAL can be "b3.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:9095". Replace with "b" prefix plus "node.id" properties, and replace with a unique port (9095) in the port range of the EXTERNAL advertised listener. +# 2.3 If there are more broker role nodes in the same AZ, you can configure them in the same way. +process.roles=broker,controller +node.id=3 +controller.quorum.voters=1@{broker-node1-ip}:29092,2@{broker-node2-ip}:29092,3@{broker-node3-ip}:29092 +listeners=INTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29092,EXTERNAL://0.0.0.0:39092 +inter.broker.listener.name=INTERNAL +advertised.listeners=INTERNAL://{broker-node3-ip}:9092,EXTERNAL://b3.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:9095 +controller.listener.names=CONTROLLER +listener.security.protocol.map=INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL +log.dirs=./data +``` + +**2.4.3 Start Kafka brokers** + +Create a script, and then execute it to start the Kafka broker in each broker node. + +```shell +#!/bin/bash + +# Get the directory of the current script +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# Set JAVA_HOME to the Java installation within the script directory +export JAVA_HOME="$SCRIPT_DIR/jdk-22.0.2" +# Define the vars +KAFKA_DIR="$SCRIPT_DIR/kafka_2.13-3.7.1/bin" +KAFKA_STORAGE_CMD=$KAFKA_DIR/kafka-storage.sh +KAFKA_START_CMD=$KAFKA_DIR/kafka-server-start.sh +KAFKA_DATA_DIR=$SCRIPT_DIR/data +KAFKA_LOG_DIR=$SCRIPT_DIR/log +KAFKA_CONFIG_DIR=$SCRIPT_DIR/config + +# Cleanup step, which makes it easy for multiple experiments +# Find all Kafka process IDs +KAFKA_PIDS=$(ps aux | grep 'kafka.Kafka' | grep -v grep | awk '{print $2}') +if [ -z "$KAFKA_PIDS" ]; then + echo "No Kafka processes are running." +else + # Kill each Kafka process + echo "Killing Kafka processes with PIDs: $KAFKA_PIDS" + for PID in $KAFKA_PIDS; do + kill -9 $PID + echo "Killed Kafka process with PID: $PID" + done + echo "All Kafka processes have been killed." +fi + +rm -rf $KAFKA_DATA_DIR +mkdir -p $KAFKA_DATA_DIR +rm -rf $KAFKA_LOG_DIR +mkdir -p $KAFKA_LOG_DIR + +# Magic id: BRl69zcmTFmiPaoaANybiw, you can use your own +$KAFKA_STORAGE_CMD format -t "BRl69zcmTFmiPaoaANybiw" -c "$KAFKA_CONFIG_DIR/server.properties" > $KAFKA_LOG_DIR/server_format.log +LOG_DIR=$KAFKA_LOG_DIR nohup $KAFKA_START_CMD "$KAFKA_CONFIG_DIR/server.properties" & +``` + +**2.5. Test the cluster setting in the bastion node** + +1. Test the Kafka bootstrap. + + ```shell + export JAVA_HOME=/home/ec2-user/jdk-22.0.2 + + # Bootstrap from INTERNAL listener + ./kafka_2.13-3.7.1/bin/kafka-broker-api-versions.sh --bootstrap-server {one_of_broker_ip}:9092 | grep 9092 + # Expected output (the actual order might be different) + {broker-node1-ip}:9092 (id: 1 rack: null) -> ( + {broker-node2-ip}:9092 (id: 2 rack: null) -> ( + {broker-node3-ip}:9092 (id: 3 rack: null) -> ( + + # Bootstrap from EXTERNAL listener + ./kafka_2.13-3.7.1/bin/kafka-broker-api-versions.sh --bootstrap-server {one_of_broker_ip}:39092 + # Expected output for the last 3 lines (the actual order might be different) + # The difference in the output from "bootstrap from INTERNAL listener" is that exceptions or errors might occur because advertised listeners cannot be resolved in Kafka VPC. + # We will make them resolvable in TiDB Cloud side and make it route to the right broker when you create a changefeed connect to this Kafka cluster by Private Link. + b1.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:9093 (id: 1 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + b2.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:9094 (id: 2 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + b3.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:9095 (id: 3 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + ``` + +2. Create a producer script `produce.sh` in the bastion node. + + ```shell + #!/bin/bash + BROKER_LIST=$1 # "{broker_address1},{broker_address2}..." + + # Get the directory of the current script + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + # Set JAVA_HOME to the Java installation within the script directory + export JAVA_HOME="$SCRIPT_DIR/jdk-22.0.2" + # Define the Kafka directory + KAFKA_DIR="$SCRIPT_DIR/kafka_2.13-3.7.1/bin" + TOPIC="test-topic" + + # Create a topic if it does not exist + create_topic() { + echo "Creating topic if it does not exist..." + $KAFKA_DIR/kafka-topics.sh --create --topic $TOPIC --bootstrap-server $BROKER_LIST --if-not-exists --partitions 3 --replication-factor 3 + } + + # Produce messages to the topic + produce_messages() { + echo "Producing messages to the topic..." + for ((chrono=1; chrono <= 10; chrono++)); do + message="Test message "$chrono + echo "Create "$message + echo $message | $KAFKA_DIR/kafka-console-producer.sh --broker-list $BROKER_LIST --topic $TOPIC + done + } + create_topic + produce_messages + ``` + +3. Create a consumer script `consume.sh` in the bastion node. + + ```shell + #!/bin/bash + + BROKER_LIST=$1 # "{broker_address1},{broker_address2}..." + + # Get the directory of the current script + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + # Set JAVA_HOME to the Java installation within the script directory + export JAVA_HOME="$SCRIPT_DIR/jdk-22.0.2" + # Define the Kafka directory + KAFKA_DIR="$SCRIPT_DIR/kafka_2.13-3.7.1/bin" + TOPIC="test-topic" + CONSUMER_GROUP="test-group" + # Consume messages from the topic + consume_messages() { + echo "Consuming messages from the topic..." + $KAFKA_DIR/kafka-console-consumer.sh --bootstrap-server $BROKER_LIST --topic $TOPIC --from-beginning --timeout-ms 5000 --consumer-property group.id=$CONSUMER_GROUP + } + consume_messages + ``` + +4. Execute `produce.sh` and `consume.sh` to verify that the Kafka cluster is running. These scripts will also be reused for later network connection testing. The script will create a topic with `--partitions 3 --replication-factor 3`. Ensure that all these three brokers contain data. Ensure that the script will connect to all three brokers to guarantee that network connection will be tested. + + ```shell + # Test write message. + ./produce.sh {one_of_broker_ip}:9092 + ``` + + ```shell + # Expected output + Creating topic if it does not exist... + + Producing messages to the topic... + Create Test message 1 + >>Create Test message 2 + >>Create Test message 3 + >>Create Test message 4 + >>Create Test message 5 + >>Create Test message 6 + >>Create Test message 7 + >>Create Test message 8 + >>Create Test message 9 + >>Create Test message 10 + ``` + + ```shell + # Test read message + ./consume.sh {one_of_broker_ip}:9092 + ``` + + ```shell + # Expected example output (the actual message order might be different) + Consuming messages from the topic... + Test message 3 + Test message 4 + Test message 5 + Test message 9 + Test message 10 + Test message 6 + Test message 8 + Test message 1 + Test message 2 + Test message 7 + [2024-11-01 08:54:27,547] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) + org.apache.kafka.common.errors.TimeoutException + Processed a total of 10 messages + ``` + +### Reconfigure a running Kafka cluster + +Ensure that your Kafka cluster is deployed in the same region and AZs as the TiDB cluster. If any brokers are in different AZs, move them to the correct ones. + +#### 1. Configure the EXTERNAL listener for brokers + +The following configuration applies to a Kafka KRaft cluster. The ZK mode configuration is similar. + +1. Plan configuration changes. + + 1. Configure an EXTERNAL **listener** for every broker for external access from TiDB Cloud. Select a unique port as the EXTERNAL port, for example, `39092`. + 2. Configure an EXTERNAL **advertised listener** based on **Kafka Advertised Listener Pattern** you get from TiDB Cloud for every broker node to help TiDB Cloud differentiate between different brokers. Different EXTERNAL advertised listeners help Kafka clients from TiDB Cloud route requests to the right broker. + + - `` differentiates brokers from Kafka Private Link Service access points. Plan a port range for EXTERNAL advertised listeners of all brokers, for example, `range from 9093`. These ports do not have to be actual ports listened to by brokers. They are ports listened to by the load balancer for Private Link Service that will forward requests to different brokers. + - `AZ ID` in **Kafka Advertised Listener Pattern** indicates where the broker is deployed. TiDB Cloud will route requests to different endpoint DNS names based on the AZ ID. + + It is recommended to configure different broker IDs for different brokers to make it easy for troubleshooting. + +2. Use SSH to log in to each broker node. Modify the configuration file of each broker with the following content: + + ```properties + # brokers in usw2-az1 + + # Add EXTERNAL listener + listeners=INTERNAL:...,EXTERNAL://0.0.0.0:39092 + + # Add EXTERNAL advertised listeners based on the "Kafka Advertised Listener Pattern" in "Prerequisites" section + # 1. The pattern for AZ(ID: usw2-az1) is ".usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:" + # 2. So the EXTERNAL can be "b1.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:9093", replace with "b" prefix plus "node.id" properties, replace with a unique port(9093) in EXTERNAL advertised listener ports range + advertised.listeners=...,EXTERNAL://b1.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:9093 + + # Configure EXTERNAL map + listener.security.protocol.map=...,EXTERNAL:PLAINTEXT + ``` + + ```properties + # brokers in usw2-az2 + + # Add EXTERNAL listener + listeners=INTERNAL:...,EXTERNAL://0.0.0.0:39092 + + # Add EXTERNAL advertised listeners based on the "Kafka Advertised Listener Pattern" in "Prerequisites" section + # 1. The pattern for AZ(ID: usw2-az2) is ".usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:" + # 2. So the EXTERNAL can be "b2.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:9094". Replace with "b" prefix plus "node.id" properties, and replace with a unique port(9094) in EXTERNAL advertised listener ports range. + advertised.listeners=...,EXTERNAL://b2.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:9094 + + # Configure EXTERNAL map + listener.security.protocol.map=...,EXTERNAL:PLAINTEXT + ``` + + ```properties + # brokers in usw2-az3 + + # Add EXTERNAL listener + listeners=INTERNAL:...,EXTERNAL://0.0.0.0:39092 + + # Add EXTERNAL advertised listeners based on the "Kafka Advertised Listener Pattern" in "Prerequisites" section + # 1. The pattern for AZ(ID: usw2-az3) is ".usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:" + # 2. So the EXTERNAL can be "b2.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:9095". Replace with "b" prefix plus "node.id" properties, and replace with a unique port(9095) in EXTERNAL advertised listener ports range. + advertised.listeners=...,EXTERNAL://b3.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:9095 + + # Configure EXTERNAL map + listener.security.protocol.map=...,EXTERNAL:PLAINTEXT + ``` + +3. After you reconfigure all the brokers, restart your Kafka brokers one by one. + +#### 2. Test EXTERNAL listener settings in your internal network + +You can download the Kafka and OpenJDK in you Kafka client node. + +```shell +# Download Kafka and OpenJDK, and then extract the files. You can choose the binary version based on your preference. +wget https://archive.apache.org/dist/kafka/3.7.1/kafka_2.13-3.7.1.tgz +tar -zxf kafka_2.13-3.7.1.tgz +wget https://download.java.net/java/GA/jdk22.0.2/c9ecb94cd31b495da20a27d4581645e8/9/GPL/openjdk-22.0.2_linux-x64_bin.tar.gz +tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz +``` + +Execute the following script to test if the bootstrap works as expected. + +```shell +export JAVA_HOME=/home/ec2-user/jdk-22.0.2 + +# Bootstrap from the EXTERNAL listener +./kafka_2.13-3.7.1/bin/kafka-broker-api-versions.sh --bootstrap-server {one_of_broker_ip}:39092 + +# Expected output for the last 3 lines (the actual order might be different) +# There will be some exceptions or errors because advertised listeners cannot be resolved in your Kafka network. +# We will make them resolvable in TiDB Cloud side and make it route to the right broker when you create a changefeed connect to this Kafka cluster by Private Link. +b1.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:9093 (id: 1 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException +b2.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:9094 (id: 2 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException +b3.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:9095 (id: 3 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException +``` + +## Step 2. Expose the Kafka cluster as Private Link Service + +### 1. Set up the load balancer + +Create a network load balancer with four target groups with different ports. One target group is for bootstrap, and the others will map to different brokers. + +1. bootstrap target group => 9092 => broker-node1:39092,broker-node2:39092,broker-node3:39092 +2. broker target group 1 => 9093 => broker-node1:39092 +3. broker target group 2 => 9094 => broker-node2:39092 +4. broker target group 3 => 9095 => broker-node3:39092 + +If you have more broker role nodes, you need to add more mappings. Ensure that you have at least one node in the bootstrap target group. It is recommended to add three nodes, one for each AZ for resilience. + +Do the following to set up the load balancer: + +1. Go to [Target groups](https://console.aws.amazon.com/ec2/home#CreateTargetGroup:) to create four target groups. + + - Bootstrap target group + + - **Target type**: `Instances` + - **Target group name**: `bootstrap-target-group` + - **Protocol**: `TCP` + - **Port**: `9092` + - **IP address type**: `IPv4` + - **VPC**: `Kafka VPC` + - **Health check protocol**: `TCP` + - **Register targets**: `broker-node1:39092`, `broker-node2:39092`, `broker-node3:39092` + + - Broker target group 1 + + - **Target type**: `Instances` + - **Target group name**: `broker-target-group-1` + - **Protocol**: `TCP` + - **Port**: `9093` + - **IP address type**: `IPv4` + - **VPC**: `Kafka VPC` + - **Health check protocol**: `TCP` + - **Register targets**: `broker-node1:39092` + + - Broker target group 2 + + - **Target type**: `Instances` + - **Target group name**: `broker-target-group-2` + - **Protocol**: `TCP` + - **Port**: `9094` + - **IP address type**: `IPv4` + - **VPC**: `Kafka VPC` + - **Health check protocol**: `TCP` + - **Register targets**: `broker-node2:39092` + + - Broker target group 3 + + - **Target type**: `Instances` + - **Target group name**: `broker-target-group-3` + - **Protocol**: `TCP` + - **Port**: `9095` + - **IP address type**: `IPv4` + - **VPC**: `Kafka VPC` + - **Health check protocol**: `TCP` + - **Register targets**: `broker-node3:39092` + +2. Go to [Load balancers](https://console.aws.amazon.com/ec2/home#LoadBalancers:) to create a network load balancer. + + - **Load balancer name**: `kafka-lb` + - **Schema**: `Internal` + - **Load balancer IP address type**: `IPv4` + - **VPC**: `Kafka VPC` + - **Availability Zones**: + - `usw2-az1` with `broker-usw2-az1 subnet` + - `usw2-az2` with `broker-usw2-az2 subnet` + - `usw2-az3` with `broker-usw2-az3 subnet` + - **Security groups**: create a new security group with the following rules. + - Inbound rule allows all TCP from Kafka VPC: Type - `All TCP`; Source - `Anywhere-IPv4` + - Outbound rule allows all TCP to Kafka VPC: Type - `All TCP`; Destination - `Anywhere-IPv4` + - Listeners and routing: + - Protocol: `TCP`; Port: `9092`; Forward to: `bootstrap-target-group` + - Protocol: `TCP`; Port: `9093`; Forward to: `broker-target-group-1` + - Protocol: `TCP`; Port: `9094`; Forward to: `broker-target-group-2` + - Protocol: `TCP`; Port: `9095`; Forward to: `broker-target-group-3` + +3. Test the load balancer in the bastion node. This example only tests the Kafka bootstrap. Because the load balancer is listening on the Kafka EXTERNAL listener, the addresses of EXTERNAL advertised listeners can not be resolved in the bastion node. Note down the `kafka-lb` DNS name from the load balancer detail page, for example `kafka-lb-77405fa57191adcb.elb.us-west-2.amazonaws.com`. Execute the script in the bastion node. + + ```shell + # Replace {lb_dns_name} to your actual value + export JAVA_HOME=/home/ec2-user/jdk-22.0.2 + ./kafka_2.13-3.7.1/bin/kafka-broker-api-versions.sh --bootstrap-server {lb_dns_name}:9092 + + # Expected output for the last 3 lines (the actual order might be different) + b1.usw2-az1.abc.us-west-2.aws.3199015.tidbcloud.com:9093 (id: 1 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + b2.usw2-az2.abc.us-west-2.aws.3199015.tidbcloud.com:9094 (id: 2 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + b3.usw2-az3.abc.us-west-2.aws.3199015.tidbcloud.com:9095 (id: 3 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + + # You can also try bootstrap in other ports 9093/9094/9095. It will succeed probabilistically because NLB in AWS resolves LB DNS to the IP address of any availability zone and disables cross-zone load balancing by default. + # If you enable cross-zone load balancing in LB, it will succeed. However, it is unnecessary and might cause additional cross-AZ traffic. + ``` + +### 2. Set up Private Link Service + +1. Go to [Endpoint service](https://console.aws.amazon.com/vpcconsole/home#EndpointServices:). Click **Create endpoint service** to create a Private Link service for the Kafka load balancer. + + - **Name**: `kafka-pl-service` + - **Load balancer type**: `Network` + - **Load balancers**: `kafka-lb` + - **Included Availability Zones**: `usw2-az1`,`usw2-az2`, `usw2-az3` + - **Require acceptance for endpoint**: `Acceptance required` + - **Enable private DNS name**: `No` + +2. Note down the **Service name**. You need to provide it to TiDB Cloud, for example `com.amazonaws.vpce.us-west-2.vpce-svc-0f49e37e1f022cd45`. + +3. On the detail page of the kafka-pl-service, click the **Allow principals** tab, and allow the AWS account of TiDB Cloud to create the endpoint. You can get the AWS account of TiDB Cloud in [Prerequisites](#prerequisites), for example, `arn:aws:iam:::root`. + +## Step 3. Connect from TiDB Cloud + +1. Return to the [TiDB Cloud console](https://tidbcloud.com) to create a changefeed for the cluster to connect to the Kafka cluster by **Private Link**. For more information, see [Sink to Apache Kafka](/tidb-cloud/changefeed-sink-to-apache-kafka.md). + +2. When you proceed to **Configure the changefeed target > Connectivity Method > Private Link**, fill in the following fields with corresponding values and other fields as needed. + + - **Kafka Type**: `3 AZs`. Ensure that your Kafka cluster is deployed in the same three AZs. + - **Kafka Advertised Listener Pattern**: `abc`. It is the same as the unique random string you use to generate **Kafka Advertised Listener Pattern** in [Prerequisites](#prerequisites). + - **Endpoint Service Name**: the Kafka service name. + - **Bootstrap Ports**: `9092`. A single port is sufficient because you configure a dedicated bootstrap target group behind it. + +3. Proceed with the steps in [Sink to Apache Kafka](/tidb-cloud/changefeed-sink-to-apache-kafka.md). + +Now you have successfully finished the task. + +## FAQ + +### How to connect to the same Kafka Private Link service from two different TiDB Cloud projects? + +If you have already followed this document to successfully set up the connection from the first project, you can connect to the same Kafka Private Link service from the second project as follows: + +1. Follow instructions from the beginning of this document. + +2. When you proceed to [Step 1. Set up a Kafka cluster](#step-1-set-up-a-kafka-cluster), follow [Reconfigure a running Kafka cluster](#reconfigure-a-running-kafka-cluster) to create another group of EXTERNAL listeners and advertised listeners. You can name it as **EXTERNAL2**. Note that the port range of **EXTERNAL2** cannot overlap with the **EXTERNAL**. + +3. After reconfiguring brokers, add another target group in the load balancer, including the bootstrap and broker target groups. + +4. Configure the TiDB Cloud connection with the following information: + + - New Bootstrap port + - New Kafka Advertised Listener Group + - The same Endpoint Service diff --git a/tidb-cloud/setup-self-hosted-kafka-private-service-connect.md b/tidb-cloud/setup-self-hosted-kafka-private-service-connect.md new file mode 100644 index 0000000000000..7a8e77ea63c45 --- /dev/null +++ b/tidb-cloud/setup-self-hosted-kafka-private-service-connect.md @@ -0,0 +1,704 @@ +--- +title: Set Up Self-Hosted Kafka Private Service Connect in Google Cloud +summary: This document explains how to set up Private Service Connect for self-hosted Kafka in Google Cloud and how to make it work with TiDB Cloud. +--- + +# Set Up Self-Hosted Kafka Private Service Connect in Google Cloud + +This document explains how to set up Private Service Connect for self-hosted Kafka in Google Cloud, and how to make it work with TiDB Cloud. + +The mechanism works as follows: + +1. The TiDB Cloud VPC connects to the Kafka VPC through private endpoints. +2. Kafka clients need to communicate directly to all Kafka brokers. +3. Each Kafka broker is mapped to a unique port within the TiDB Cloud VPC. +4. Leverage the Kafka bootstrap mechanism and Google Cloud resources to achieve the mapping. + +There are two ways to set up Private Service Connect for self-hosted Kafka in Google Cloud: + +- Use the Private Service Connect (PSC) port mapping mechanism. This method requires static port-broker mapping configuration. You need to reconfigure the existing Kafka cluster to add a group of EXTERNAL listeners and advertised listeners. See [Set up self-hosted Kafka Private Service Connect service by PSC port mapping](#set-up-self-hosted-kafka-private-service-connect-service-by-psc-port-mapping). + +- Use [Kafka-proxy](https://github.com/grepplabs/kafka-proxy). This method introduces an extra running process as the proxy between Kafka clients and Kafka brokers. The proxy dynamically configures the port-broker mapping and forwards requests. You do not need to reconfigure the existing Kafka cluster. See [Set up self-hosted Kafka Private Service Connect by Kafka-proxy](#set-up-self-hosted-kafka-private-service-connect-by-kafka-proxy). + +The document provides an example of connecting to a Kafka Private Service Connect service deployed across three availability zones (AZ) in Google Cloud. While other configurations are possible based on similar port-mapping principles, this document covers the fundamental setup process of the Kafka Private Service Connect service. For production environments, a more resilient Kafka Private Service Connect service with enhanced operational maintainability and observability is recommended. + +## Prerequisites + +1. Ensure that you have the following authorization to set up Kafka Private Service Connect in your own Google Cloud account. + + - Manage VM nodes + - Manage VPC + - Manage subnets + - Manage load balancer + - Manage Private Service Connect + - Connect to VM nodes to configure Kafka nodes + +2. [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) if you do not have one. + +3. Get the Kafka deployment information from your TiDB Cloud Dedicated cluster. + + 1. In the [TiDB Cloud console](https://tidbcloud.com), navigate to the cluster overview page of the TiDB cluster, and then click **Changefeed** in the left navigation pane. + 2. On the overview page, find the region of the TiDB cluster. Ensure that your Kafka cluster will be deployed to the same region. + 3. Click **Create Changefeed**. + 1. In **Target Type**, select **Kafka**. + 2. In **Connectivity Method**, select **Private Service Connect**. + 4. Note down the Google Cloud project in **Reminders before proceeding**. You will use it to authorize the auto-accept endpoint creation request from TiDB Cloud. + 5. Note down the **Zones of TiDB Cluster**. You will deploy your TiDB cluster in these zones. It is recommended that you deploy Kafka in these zones to reduce cross-zone traffic. + 6. Pick a unique **Kafka Advertised Listener Pattern** for your Kafka Private Service Connect service. + 1. Input a unique random string. It can only include numbers or lowercase letters. You will use it to generate **Kafka Advertised Listener Pattern** later. + 2. Click **Check usage and generate** to check if the random string is unique and generate **Kafka Advertised Listener Pattern** that will be used to assemble the EXTERNAL advertised listener for Kafka brokers, or configure Kafka-proxy. + +Note down all the deployment information. You need to use it to configure your Kafka Private Service Connect service later. + +The following table shows an example of the deployment information. + +| Information | Value | +|------------------------------------|---------------------------------| +| Region | Oregon (`us-west1`) | +| Google Cloud project of TiDB Cloud | `tidbcloud-prod-000` | +| Zones |
  • `us-west1-a`
  • `us-west1-b`
  • `us-west1-c`
| +| Kafka Advertised Listener Pattern | The unique random string: `abc`
Generated pattern: <broker_id>.abc.us-west1.gcp.3199745.tidbcloud.com:<port> | + +## Set up self-hosted Kafka Private Service Connect service by PSC port mapping + +Expose each Kafka broker to TiDB Cloud VPC with a unique port by using the PSC port mapping mechanism. The following diagram shows how it works. + +![Connect to Google Cloud self-hosted Kafka Private Service Connect by port mapping](/media/tidb-cloud/changefeed/connect-to-google-cloud-self-hosted-kafka-private-service-connect-by-portmapping.jpeg) + +### Step 1. Set up the Kafka cluster + +If you need to deploy a new cluster, follow the instructions in [Deploy a new Kafka cluster](#deploy-a-new-kafka-cluster). + +If you need to expose an existing cluster, follow the instructions in [Reconfigure a running Kafka cluster](#reconfigure-a-running-kafka-cluster). + +#### Deploy a new Kafka cluster + +**1. Set up the Kafka VPC** + +You need to create two subnets for Kafka VPC, one for Kafka brokers, and the other for the bastion node to make it easy to configure the Kafka cluster. + +Go to the [Google Cloud console](https://cloud.google.com/cloud-console), and navigate to the [VPC networks](https://console.cloud.google.com/networking/networks/list) page to create the Kafka VPC with the following attributes: + +- **Name**: `kafka-vpc` +- Subnets + - **Name**: `bastion-subnet`; **Region**: `us-west1`; **IPv4 range**: `10.0.0.0/18` + - **Name**: `brokers-subnet`; **Region**: `us-west1`; **IPv4 range**: `10.64.0.0/18` +- Firewall rules + - `kafka-vpc-allow-custom` + - `kafka-vpc-allow-ssh` + +**2. Provisioning VMs** + +Go to the [VM instances](https://console.cloud.google.com/compute/instances) page to provision VMs: + +1. Bastion node + + - **Name**: `bastion-node` + - **Region**: `us-west1` + - **Zone**: `Any` + - **Machine Type**: `e2-medium` + - **Image**: `Debian GNU/Linux 12` + - **Network**: `kafka-vpc` + - **Subnetwork**: `bastion-subnet` + - **External IPv4 address**: `Ephemeral` + +2. Broker node 1 + + - **Name**: `broker-node1` + - **Region**: `us-west1` + - **Zone**: `us-west1-a` + - **Machine Type**: `e2-medium` + - **Image**: `Debian GNU/Linux 12` + - **Network**: `kafka-vpc` + - **Subnetwork**: `brokers-subnet` + - **External IPv4 address**: `None` + +3. Broker node 2 + + - **Name**: `broker-node2` + - **Region**: `us-west1` + - **Zone**: `us-west1-b` + - **Machine Type**: `e2-medium` + - **Image**: `Debian GNU/Linux 12` + - **Network**: `kafka-vpc` + - **Subnetwork**: `brokers-subnet` + - **External IPv4 address**: `None` + +4. Broker node 3 + + - **Name**: `broker-node3` + - **Region**: `us-west1` + - **Zone**: `us-west1-c` + - **Machine Type**: `e2-medium` + - **Image**: `Debian GNU/Linux 12` + - **Network**: `kafka-vpc` + - **Subnetwork**: `brokers-subnet` + - **External IPv4 address**: `None` + +**3. Prepare Kafka runtime binaries** + +1. Go to the detail page of the bastion node. Click **SSH** to log in to the bastion node. Download binaries. + + ```shell + # Download Kafka and OpenJDK, and then extract the files. You can choose the binary version based on your preference. + wget https://archive.apache.org/dist/kafka/3.7.1/kafka_2.13-3.7.1.tgz + tar -zxf kafka_2.13-3.7.1.tgz + wget https://download.java.net/java/GA/jdk22.0.2/c9ecb94cd31b495da20a27d4581645e8/9/GPL/openjdk-22.0.2_linux-x64_bin.tar.gz + tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz + ``` + +2. Copy binaries to each broker node. + + ```shell + # Run this command to authorize gcloud to access the Cloud Platform with Google user credentials + # Follow the instruction in output to finish the login + gcloud auth login + + # Copy binaries to broker nodes + gcloud compute scp kafka_2.13-3.7.1.tgz openjdk-22.0.2_linux-x64_bin.tar.gz broker-node1:~ --zone=us-west1-a + gcloud compute ssh broker-node1 --zone=us-west1-a --command="tar -zxf kafka_2.13-3.7.1.tgz && tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz" + gcloud compute scp kafka_2.13-3.7.1.tgz openjdk-22.0.2_linux-x64_bin.tar.gz broker-node2:~ --zone=us-west1-b + gcloud compute ssh broker-node2 --zone=us-west1-b --command="tar -zxf kafka_2.13-3.7.1.tgz && tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz" + gcloud compute scp kafka_2.13-3.7.1.tgz openjdk-22.0.2_linux-x64_bin.tar.gz broker-node3:~ --zone=us-west1-c + gcloud compute ssh broker-node3 --zone=us-west1-c --command="tar -zxf kafka_2.13-3.7.1.tgz && tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz" + ``` + +**4. Configure Kafka brokers** + +1. Set up a KRaft Kafka cluster with three nodes. Each node acts as a broker and controller roles. For every broker: + + 1. For `listeners`, all three brokers are the same and act as brokers and controller roles: + 1. Configure the same CONTROLLER listener for all **controller** role nodes. If you only want to add the **broker** role nodes, you do not need the CONTROLLER listener in `server.properties`. + 2. Configure two **broker** listeners. INTERNAL for internal access; EXTERNAL for external access from TiDB Cloud. + + 2. For `advertised.listeners`, do the following: + 1. Configure an INTERNAL advertised listener for each broker using the internal IP address of the broker node, which allows internal Kafka clients to connect to the broker via the advertised address. + 2. Configure an EXTERNAL advertised listener based on **Kafka Advertised Listener Pattern** you get from TiDB Cloud for every broker node to help TiDB Cloud differentiate between different brokers. Different EXTERNAL advertised listeners help Kafka clients from TiDB Cloud side route requests to the right broker. + - `` differentiates brokers from Kafka Private Service Connect access points. Plan a port range for EXTERNAL advertised listeners of all brokers. These ports do not have to be actual ports listened to by brokers. They are ports listened to by the load balancer for Private Service Connect that will forward requests to different brokers. + - It is recommended to configure different broker IDs for different brokers to make it easy for troubleshooting. + + 3. The planning values: + - CONTROLLER port: `29092` + - INTERNAL port: `9092` + - EXTERNAL: `39092` + - EXTERNAL advertised listener ports range: `9093~9095` + +2. Use SSH to log in to every broker node. Create a configuration file `~/config/server.properties` with the following content for each broker node respectively. + + ```properties + # broker-node1 ~/config/server.properties + # 1. Replace {broker-node1-ip}, {broker-node2-ip}, {broker-node3-ip} with the actual IP addresses. + # 2. Configure EXTERNAL in "advertised.listeners" based on the "Kafka Advertised Listener Pattern" in the "Prerequisites" section. + # 2.1 The pattern is ".abc.us-west1.gcp.3199745.tidbcloud.com:". + # 2.2 So the EXTERNAL can be "b1.abc.us-west1.gcp.3199745.tidbcloud.com:9093". Replace with "b" prefix plus "node.id" properties, and replace with a unique port (9093) in the port range of the EXTERNAL advertised listener. + process.roles=broker,controller + node.id=1 + controller.quorum.voters=1@{broker-node1-ip}:29092,2@{broker-node2-ip}:29092,3@{broker-node3-ip}:29092 + listeners=INTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29092,EXTERNAL://0.0.0.0:39092 + inter.broker.listener.name=INTERNAL + advertised.listeners=INTERNAL://{broker-node1-ip}:9092,EXTERNAL://b1.abc.us-west1.gcp.3199745.tidbcloud.com:9093 + controller.listener.names=CONTROLLER + listener.security.protocol.map=INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL + log.dirs=./data + ``` + + ```properties + # broker-node2 ~/config/server.properties + # 1. Replace {broker-node1-ip}, {broker-node2-ip}, {broker-node3-ip} with the actual IP addresses. + # 2. Configure EXTERNAL in "advertised.listeners" based on the "Kafka Advertised Listener Pattern" in the "Prerequisites" section. + # 2.1 The pattern is ".abc.us-west1.gcp.3199745.tidbcloud.com:". + # 2.2 So the EXTERNAL can be "b2.abc.us-west1.gcp.3199745.tidbcloud.com:9094". Replace with "b" prefix plus "node.id" properties, and replace with a unique port (9094) in the port range of the EXTERNAL advertised listener. + process.roles=broker,controller + node.id=2 + controller.quorum.voters=1@{broker-node1-ip}:29092,2@{broker-node2-ip}:29092,3@{broker-node3-ip}:29092 + listeners=INTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29092,EXTERNAL://0.0.0.0:39092 + inter.broker.listener.name=INTERNAL + advertised.listeners=INTERNAL://{broker-node2-ip}:9092,EXTERNAL://b2.abc.us-west1.gcp.3199745.tidbcloud.com:9094 + controller.listener.names=CONTROLLER + listener.security.protocol.map=INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL + log.dirs=./data + ``` + + ```properties + # broker-node3 ~/config/server.properties + # 1. Replace {broker-node1-ip}, {broker-node2-ip}, {broker-node3-ip} with the actual IP addresses. + # 2. Configure EXTERNAL in "advertised.listeners" based on the "Kafka Advertised Listener Pattern" in the "Prerequisites" section. + # 2.1 The pattern is ".abc.us-west1.gcp.3199745.tidbcloud.com:". + # 2.2 So the EXTERNAL can be "b3.abc.us-west1.gcp.3199745.tidbcloud.com:9095". Replace with "b" prefix plus "node.id" properties, and replace with a unique port (9095) in the port range of the EXTERNAL advertised listener. + process.roles=broker,controller + node.id=3 + controller.quorum.voters=1@{broker-node1-ip}:29092,2@{broker-node2-ip}:29092,3@{broker-node3-ip}:29092 + listeners=INTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:29092,EXTERNAL://0.0.0.0:39092 + inter.broker.listener.name=INTERNAL + advertised.listeners=INTERNAL://{broker-node3-ip}:9092,EXTERNAL://b3.abc.us-west1.gcp.3199745.tidbcloud.com:9095 + controller.listener.names=CONTROLLER + listener.security.protocol.map=INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT,EXTERNAL:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL + log.dirs=./data + ``` + +3. Create a script, and then execute it to start the Kafka broker in each broker node. + + ```shell + #!/bin/bash + + # Get the directory of the current script + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + # Set JAVA_HOME to the Java installation within the script directory + export JAVA_HOME="$SCRIPT_DIR/jdk-22.0.2" + # Define the vars + KAFKA_DIR="$SCRIPT_DIR/kafka_2.13-3.7.1/bin" + KAFKA_STORAGE_CMD=$KAFKA_DIR/kafka-storage.sh + KAFKA_START_CMD=$KAFKA_DIR/kafka-server-start.sh + KAFKA_DATA_DIR=$SCRIPT_DIR/data + KAFKA_LOG_DIR=$SCRIPT_DIR/log + KAFKA_CONFIG_DIR=$SCRIPT_DIR/config + + # Cleanup step, which makes it easy for multiple experiments + # Find all Kafka process IDs + KAFKA_PIDS=$(ps aux | grep 'kafka.Kafka' | grep -v grep | awk '{print $2}') + if [ -z "$KAFKA_PIDS" ]; then + echo "No Kafka processes are running." + else + # Kill each Kafka process + echo "Killing Kafka processes with PIDs: $KAFKA_PIDS" + for PID in $KAFKA_PIDS; do + kill -9 $PID + echo "Killed Kafka process with PID: $PID" + done + echo "All Kafka processes have been killed." + fi + + rm -rf $KAFKA_DATA_DIR + mkdir -p $KAFKA_DATA_DIR + rm -rf $KAFKA_LOG_DIR + mkdir -p $KAFKA_LOG_DIR + + # Magic id: BRl69zcmTFmiPaoaANybiw. You can use your own magic ID. + $KAFKA_STORAGE_CMD format -t "BRl69zcmTFmiPaoaANybiw" -c "$KAFKA_CONFIG_DIR/server.properties" > $KAFKA_LOG_DIR/server_format.log + LOG_DIR=$KAFKA_LOG_DIR nohup $KAFKA_START_CMD "$KAFKA_CONFIG_DIR/server.properties" & + ``` + +**5. Test the Kafka cluster in the bastion node** + +1. Test the Kafka bootstrap. + + ```shell + export JAVA_HOME=~/jdk-22.0.2 + + # Bootstrap from INTERNAL listener + ./kafka_2.13-3.7.1/bin/kafka-broker-api-versions.sh --bootstrap-server {one_of_broker_ip}:9092 | grep 9092 + # Expected output (the actual order might be different) + {broker-node1-ip}:9092 (id: 1 rack: null) -> ( + {broker-node2-ip}:9092 (id: 2 rack: null) -> ( + {broker-node3-ip}:9092 (id: 3 rack: null) -> ( + + # Bootstrap from EXTERNAL listener + ./kafka_2.13-3.7.1/bin/kafka-broker-api-versions.sh --bootstrap-server {one_of_broker_ip}:39092 + # Expected output for the last 3 lines (the actual order might be different) + # The difference in the output from "bootstrap from INTERNAL listener" is that exceptions or errors might occur because advertised listeners cannot be resolved in Kafka VPC. + # We will make them resolvable in TiDB Cloud side and make it route to the right broker when you create a changefeed connect to this Kafka cluster by Private Service Connect. + b1.abc.us-west1.gcp.3199745.tidbcloud.com:9093 (id: 1 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + b2.abc.us-west1.gcp.3199745.tidbcloud.com:9094 (id: 2 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + b3.abc.us-west1.gcp.3199745.tidbcloud.com:9095 (id: 3 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + ``` + +2. Create a producer script `produce.sh` in the bastion node. + + ```shell + #!/bin/bash + BROKER_LIST=$1 # "{broker_address1},{broker_address2}..." + + # Get the directory of the current script + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + # Set JAVA_HOME to the Java installation within the script directory + export JAVA_HOME="$SCRIPT_DIR/jdk-22.0.2" + # Define the Kafka directory + KAFKA_DIR="$SCRIPT_DIR/kafka_2.13-3.7.1/bin" + TOPIC="test-topic" + + # Create a topic if it does not exist + create_topic() { + echo "Creating topic if it does not exist..." + $KAFKA_DIR/kafka-topics.sh --create --topic $TOPIC --bootstrap-server $BROKER_LIST --if-not-exists --partitions 3 --replication-factor 3 + } + + # Produce messages to the topic + produce_messages() { + echo "Producing messages to the topic..." + for ((chrono=1; chrono <= 10; chrono++)); do + message="Test message "$chrono + echo "Create "$message + echo $message | $KAFKA_DIR/kafka-console-producer.sh --broker-list $BROKER_LIST --topic $TOPIC + done + } + create_topic + produce_messages + ``` + +3. Create a consumer script `consume.sh` in the bastion node. + + ```shell + #!/bin/bash + + BROKER_LIST=$1 # "{broker_address1},{broker_address2}..." + + # Get the directory of the current script + SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + # Set JAVA_HOME to the Java installation within the script directory + export JAVA_HOME="$SCRIPT_DIR/jdk-22.0.2" + # Define the Kafka directory + KAFKA_DIR="$SCRIPT_DIR/kafka_2.13-3.7.1/bin" + TOPIC="test-topic" + CONSUMER_GROUP="test-group" + # Consume messages from the topic + consume_messages() { + echo "Consuming messages from the topic..." + $KAFKA_DIR/kafka-console-consumer.sh --bootstrap-server $BROKER_LIST --topic $TOPIC --from-beginning --timeout-ms 5000 --consumer-property group.id=$CONSUMER_GROUP + } + consume_messages + ``` + +4. Execute `produce.sh` and `consume.sh` to verify that the Kafka cluster is running. These scripts will also be reused for later network connection testing. The script will create a topic with `--partitions 3 --replication-factor 3`. Ensure that all three brokers contain data. Ensure that the scripts will connect to all three brokers to guarantee that network connection will be tested. + + ```shell + # Test write message. + ./produce.sh {one_of_broker_ip}:9092 + ``` + + ```text + # Expected output + Creating topic if it does not exist... + + Producing messages to the topic... + Create Test message 1 + >>Create Test message 2 + >>Create Test message 3 + >>Create Test message 4 + >>Create Test message 5 + >>Create Test message 6 + >>Create Test message 7 + >>Create Test message 8 + >>Create Test message 9 + >>Create Test message 10 + ``` + + ```shell + # Test read message + ./consume.sh {one_of_broker_ip}:9092 + ``` + + ```text + # Expected example output (the actual message order might be different) + Consuming messages from the topic... + Test message 3 + Test message 4 + Test message 5 + Test message 9 + Test message 10 + Test message 6 + Test message 8 + Test message 1 + Test message 2 + Test message 7 + [2024-11-01 08:54:27,547] ERROR Error processing message, terminating consumer process: (kafka.tools.ConsoleConsumer$) + org.apache.kafka.common.errors.TimeoutException + Processed a total of 10 messages + ``` + +#### Reconfigure a running Kafka cluster + +Ensure that your Kafka cluster is deployed in the same region as the TiDB cluster. It is recommended that the zones are also in the same region to reduce cross-zone traffic. + +**1. Configure the EXTERNAL listener for brokers** + +The following configuration applies to a Kafka KRaft cluster. The ZK mode configuration is similar. + +1. Plan configuration changes. + + 1. Configure an EXTERNAL **listener** for every broker for external access from TiDB Cloud. Select a unique port as the EXTERNAL port, for example, `39092`. + 2. Configure an EXTERNAL **advertised listener** based on **Kafka Advertised Listener Pattern** you get from TiDB Cloud for every broker node to help TiDB Cloud differentiate between different brokers. Different EXTERNAL advertised listeners help Kafka clients from TiDB Cloud side route requests to the right broker. + - `` differentiates brokers from Kafka Private Service Connect access points. Plan a port range for EXTERNAL advertised listeners of all brokers, for example, `range from 9093`. These ports do not have to be actual ports listened to by brokers. They are ports listened to by the load balancer for Private Service Connect that will forward requests to different brokers. + - It is recommended to configure different broker IDs for different brokers to make it easy for troubleshooting. + +2. Use SSH to log in to each broker node. Modify the configuration file of each broker with the following content: + + ```properties + # Add EXTERNAL listener + listeners=INTERNAL:...,EXTERNAL://0.0.0.0:39092 + + # Add EXTERNAL advertised listeners based on the "Kafka Advertised Listener Pattern" in the "Prerequisites" section + # 1. The pattern is ".abc.us-west1.gcp.3199745.tidbcloud.com:". + # 2. So the EXTERNAL can be "bx.abc.us-west1.gcp.3199745.tidbcloud.com:xxxx". Replace with "b" prefix plus "node.id" properties, and replace with a unique port in the port range of the EXTERNAL advertised listener. + # For example + advertised.listeners=...,EXTERNAL://b1.abc.us-west1.gcp.3199745.tidbcloud.com:9093 + + # Configure EXTERNAL map + listener.security.protocol.map=...,EXTERNAL:PLAINTEXT + ``` + +3. After you reconfigure all the brokers, restart your Kafka brokers one by one. + +**2. Test EXTERNAL listener settings in your internal network** + +You can download Kafka and OpenJDK in your Kafka client node. + +```shell +# Download Kafka and OpenJDK, and then extract the files. You can choose the binary version based on your preference. +wget https://archive.apache.org/dist/kafka/3.7.1/kafka_2.13-3.7.1.tgz +tar -zxf kafka_2.13-3.7.1.tgz +wget https://download.java.net/java/GA/jdk22.0.2/c9ecb94cd31b495da20a27d4581645e8/9/GPL/openjdk-22.0.2_linux-x64_bin.tar.gz +tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz +``` + +Execute the following script to test if the bootstrap works as expected. + +```shell +export JAVA_HOME=~/jdk-22.0.2 + +# Bootstrap from the EXTERNAL listener +./kafka_2.13-3.7.1/bin/kafka-broker-api-versions.sh --bootstrap-server {one_of_broker_ip}:39092 + +# Expected output for the last 3 lines (the actual order might be different) +# There will be some exceptions or errors because advertised listeners cannot be resolved in your Kafka network. +# We will make them resolvable in TiDB Cloud side and make it route to the right broker when you create a changefeed connect to this Kafka cluster by Private Service Connect. +b1.abc.us-west1.gcp.3199745.tidbcloud.com:9093 (id: 1 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException +b2.abc.us-west1.gcp.3199745.tidbcloud.com:9094 (id: 2 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException +b3.abc.us-west1.gcp.3199745.tidbcloud.com:9095 (id: 3 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException +``` + +### Step 2. Expose the Kafka cluster as Private Service Connect + +1. Go to the [Network endpoint group](https://console.cloud.google.com/compute/networkendpointgroups/list) page. Create a network endpoint group as follows: + + - **Name**: `kafka-neg` + - **Network endpoint group type**: `Port Mapping NEG(Regional)` + - **Region**: `us-west1` + - **Network**: `kafka-vpc` + - **Subnet**: `brokers-subnet` + +2. Go to the detail page of the network endpoint group, and add the network endpoints to configure the port mapping to broker nodes. + + 1. Network endpoint 1 + - **Instance**: `broker-node1` + - **VM Port**: `39092` + - **Client Port**: `9093` + 2. Network endpoint 2 + - **Instance**: `broker-node2` + - **VM Port**: `39092` + - **Client Port**: `9094` + 3. Network endpoint 3 + - **Instance**: `broker-node3` + - **VM Port**: `39092` + - **Client Port**: `9095` + +3. Go to the [Load balancing](https://console.cloud.google.com/net-services/loadbalancing/list/loadBalancers) page. Create a load balancer as follows: + + - **Type of load balancer**: `Network Load Balancer` + - **Proxy or Passthrough**: `Passthrough` + - **Public facing or internal**: `Internal` + - **Load Balancer name**: `kafka-lb` + - **Region**: `us-west1` + - **Network**: `kafka-vpc` + - Backend configuration + - **Backend type**: `Port mapping network endpoint group` + - **Protocol**: `TCP` + - **Port mapping network endpoint group**: `kafka-neg` + - Frontend configuration + - **Subnetwork**: `brokers-subnet` + - **Ports**: `All` + +4. Go to [**Private Service Connect** > **PUBLISH SERVICE**](https://console.cloud.google.com/net-services/psc/list/producers). + + - **Load Balancer Type**: `Internal passthrough Network Load Balancer` + - **Internal load balancer**: `kafka-lb` + - **Service name**: `kafka-psc` + - **Subnets**: `RESERVE NEW SUBNET` + - **Name**: `psc-subnet` + - **VPC Network**: `kafka-vpc` + - **Region**: `us-west1` + - **IPv4 range**: `10.128.0.0/18` + - **Accepted projects**: the Google Cloud project of TiDB Cloud you got in [Prerequisites](#prerequisites), for example, `tidbcloud-prod-000`. + +5. Navigate to the detail page of the `kafka-psc`. Note down the **Service attachment**, for example, `projects/tidbcloud-dp-stg-000/regions/us-west1/serviceAttachments/kafka-psc`. You will use it in TiDB Cloud to connect to this PSC. + +6. Go to the detail page of the VPC network `kafka-vpc`. Add a firewall rule to allow PSC traffic to all brokers. + + - **Name**: `allow-psc-traffic` + - **Direction of traffic**: `Ingress` + - **Action on match**: `Allow` + - **Targets**: `All instances in the network` + - **Source filter**: `IPv4 ranges` + - **Source IPv4 ranges**: `10.128.0.0/18`. The range of psc-subnet. + - **Protocols and ports**: Allow all + +### Step 3. Connect from TiDB Cloud + +1. Go back to the [TiDB Cloud console](https://tidbcloud.com) to create a changefeed for the cluster to connect to the Kafka cluster by **Private Service Connect**. For more information, see [Sink to Apache Kafka](/tidb-cloud/changefeed-sink-to-apache-kafka.md). + +2. When you proceed to **Configure the changefeed target > Connectivity Method > Private Service Connect**, fill in the following fields with corresponding values and other fields as needed. + + - **Kafka Advertised Listener Pattern**: `abc`. It is the same as the unique random string you use to generate **Kafka Advertised Listener Pattern** in [Prerequisites](#prerequisites). + - **Service Attachment**: the Kafka service attachment of PSC, for example, `projects/tidbcloud-dp-stg-000/regions/us-west1/serviceAttachments/kafka-psc`. + - **Bootstrap Ports**: `9092,9093,9094` + +3. Proceed with the steps in [Sink to Apache Kafka](/tidb-cloud/changefeed-sink-to-apache-kafka.md). + +## Set up self-hosted Kafka Private Service Connect by Kafka-proxy + +Expose each Kafka broker to TiDB Cloud VPC with a unique port by using the Kafka-proxy dynamic port mapping mechanism. The following diagram shows how it works. + +![Connect to Google Cloud self-hosted Kafka Private Service Connect by Kafka proxy](/media/tidb-cloud/changefeed/connect-to-google-cloud-self-hosted-kafka-private-service-connect-by-kafka-proxy.jpeg) + +### Step 1. Set up Kafka-proxy + +Assume that you already have a Kafka cluster running in the same region as the TiDB cluster. You can connect to the Kafka cluster from your VPC network. The Kafka cluster can be self-hosted or provided by third-party providers, such as Confluent. + +1. Go to the [Instance groups](https://console.cloud.google.com/compute/instanceGroups/list) page, and create an instance group for Kafka-proxy. + + - **Name**: `kafka-proxy-ig` + - Instance template: + - **Name**: `kafka-proxy-tpl` + - **Location**: `Regional` + - **Region**: `us-west1` + - **Machine type**: `e2-medium`. You can choose your own machine type based on your workload. + - **Network**: your VPC network that can connect to the Kafka cluster. + - **Subnetwork**: your subnet that can connect to the Kafka cluster. + - **External IPv4 address**: `Ephemeral`. Enable Internet access to make it easy to configure Kafka-proxy. You can select **None** in your production environment and log in to the node in your way. + - **Location**: `Single zone` + - **Region**: `us-west1` + - **Zone**: choose one of your broker's zones. + - **Autoscaling mode**: `Off` + - **Minimum number of instances**: `1` + - **Maximum number of instances**: `1`. Kafka-proxy does not support the cluster mode, so only one instance can be deployed. Each Kafka-proxy randomly maps local ports to the ports of the broker, resulting in different mappings across proxies. Deploying multiple Kafka-proxies behind a load balancer can cause issues. If a Kafka client connects to one proxy and then accesses a broker through another, the request might be routed to the wrong broker. + +2. Go to the detail page of the node in kafka-proxy-ig. Click **SSH** to log in to the node. Download the binaries: + + ```shell + # You can choose another version + wget https://github.com/grepplabs/kafka-proxy/releases/download/v0.3.11/kafka-proxy-v0.3.11-linux-amd64.tar.gz + tar -zxf kafka-proxy-v0.3.11-linux-amd64.tar.gz + ``` + +3. Run Kafka-proxy and connect to Kafka brokers. + + ```shell + # There are three kinds of parameters that need to feed to the Kafka-proxy + # 1. --bootstrap-server-mapping defines the bootstrap mapping. Suggest that you configure three mappings, one for each zone for resilience. + # a) Kafka broker address; + # b) Local address for the broker in Kafka-proxy; + # c) Advertised listener for the broker if Kafka clients bootstrap from Kafka-proxy + # 2. --dynamic-sequential-min-port defines the start port of the random mapping for other brokers + # 3. --dynamic-advertised-listener defines advertised listener address for other brokers based on the pattern obtained from the "Prerequisites" section + # a) The pattern: .abc.us-west1.gcp.3199745.tidbcloud.com: + # b) Make sure to replace with a fixed lowercase string, for example, "brokers". You can use your own string. This step will help TiDB Cloud route requests properly. + # c) Remove ":" + # d) The advertised listener address would be: brokers.abc.us-west1.gcp.3199745.tidbcloud.com + ./kafka-proxy server \ + --bootstrap-server-mapping "{address_of_broker1},0.0.0.0:9092,b1.abc.us-west1.gcp.3199745.tidbcloud.com:9092" \ + --bootstrap-server-mapping "{address_of_broker2},0.0.0.0:9093,b2.abc.us-west1.gcp.3199745.tidbcloud.com:9093" \ + --bootstrap-server-mapping "{address_of_broker3},0.0.0.0:9094,b3.abc.us-west1.gcp.3199745.tidbcloud.com:9094" \ + --dynamic-sequential-min-port=9095 \ + --dynamic-advertised-listener=brokers.abc.us-west1.gcp.3199745.tidbcloud.com > ./kafka_proxy.log 2>&1 & + ``` + +4. Test bootstrap in the Kafka-proxy node. + + ```shell + # Download Kafka and OpenJDK, and then extract the files. You can choose the binary version based on your preference. + wget https://archive.apache.org/dist/kafka/3.7.1/kafka_2.13-3.7.1.tgz + tar -zxf kafka_2.13-3.7.1.tgz + wget https://download.java.net/java/GA/jdk22.0.2/c9ecb94cd31b495da20a27d4581645e8/9/GPL/openjdk-22.0.2_linux-x64_bin.tar.gz + tar -zxf openjdk-22.0.2_linux-x64_bin.tar.gz + + export JAVA_HOME=~/jdk-22.0.2 + + ./kafka_2.13-3.7.1/bin/kafka-broker-api-versions.sh --bootstrap-server 0.0.0.0:9092 + # Expected output of the last few lines (the actual order might be different) + # There might be exceptions or errors because advertised listeners cannot be resolved in your network. + # We will make them resolvable in TiDB Cloud side and make it route to the right broker when you create a changefeed connect to this Kafka cluster by Private Service Connect. + b1.abc.us-west1.gcp.3199745.tidbcloud.com:9092 (id: 1 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + b2.abc.us-west1.gcp.3199745.tidbcloud.com:9093 (id: 2 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + b3.abc.us-west1.gcp.3199745.tidbcloud.com:9094 (id: 3 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + brokers.abc.us-west1.gcp.3199745.tidbcloud.com:9095 (id: 4 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + brokers.abc.us-west1.gcp.3199745.tidbcloud.com:9096 (id: 5 rack: null) -> ERROR: org.apache.kafka.common.errors.DisconnectException + ... + ``` + +### Step 2. Expose Kafka-proxy as Private Service Connect Service + +1. Go to the [Load balancing](https://console.cloud.google.com/net-services/loadbalancing/list/loadBalancers) page, and create a load balancer. + + - **Type of load balancer**: `Network Load Balancer` + - **Proxy or Passthrough**: `Passthrough` + - **Public facing or internal**: `Internal` + - **Load Balancer name**: `kafka-proxy-lb` + - **Region**: `us-west1` + - **Network**: your network + - Backend configuration + - **Backend type**: `Instance group` + - **Protocol**: `TCP` + - **Instance group**: `kafka-proxy-ig` + - Frontend configuration + - **Subnetwork**: your subnet + - **Ports**: `All` + - Heath check: + - **Name**: `kafka-proxy-hc` + - **Scope**: `Regional` + - **Protocol**: `TCP` + - **Port**: `9092`. You can select one of the bootstrap ports in Kafka-proxy. + +2. Go to [**Private Service Connect** > **PUBLISH SERVICE**](https://console.cloud.google.com/net-services/psc/list/producers). + + - **Load Balancer Type**: `Internal passthrough Network Load Balancer` + - **Internal load balancer**: `kafka-proxy-lb` + - **Service name**: `kafka-proxy-psc` + - **Subnets**: `RESERVE NEW SUBNET` + - **Name**: `proxy-psc-subnet` + - **VPC Network**: your network + - **Region**: `us-west1` + - **IPv4 range**: set the CIDR based on your network planning + - **Accepted projects**: the Google Cloud project of TiDB Cloud you get in [Prerequisites](#prerequisites), for example, `tidbcloud-prod-000`. + +3. Navigate to the detail page of the **kafka-proxy-psc**. Note down the `Service attachment`, for example, `projects/tidbcloud-dp-stg-000/regions/us-west1/serviceAttachments/kafka-proxy-psc`, which will be used in TiDB Cloud to connect to this PSC. + +4. Go to the detail page of your VPC network. Add a firewall rule to allow the PSC traffic for all brokers. + + - **Name**: `allow-proxy-psc-traffic` + - **Direction of traffic**: `Ingress` + - **Action on match**: `Allow` + - **Targets**: All instances in the network + - **Source filter**: `IPv4 ranges` + - **Source IPv4 ranges**: the CIDR of proxy-psc-subnet + - **Protocols and ports**: Allow all + +### Step 3. Connect from TiDB Cloud + +1. Return to the [TiDB Cloud console](https://tidbcloud.com) and create a changefeed for the cluster to connect to the Kafka cluster by **Private Service Connect**. For more information, see [Sink to Apache Kafka](/tidb-cloud/changefeed-sink-to-apache-kafka.md). + +2. After you proceed to the **Configure the changefeed target** > **Connectivity Method** > **Private Service Connect**, fill in the following fields with corresponding values and other fields as needed. + + - **Kafka Advertised Listener Pattern**: `abc`. The same as the unique random string you use to generate **Kafka Advertised Listener Pattern** in [Prerequisites](#prerequisites). + - **Service Attachment**: the kafka-proxy service attachment of PSC, for example, `projects/tidbcloud-dp-stg-000/regions/us-west1/serviceAttachments/kafka-proxy-psc`. + - **Bootstrap Ports**: `9092,9093,9094` + +3. Continue to follow the guideline in [Sink to Apache Kafka](/tidb-cloud/changefeed-sink-to-apache-kafka.md). + +## FAQ + +### How to connect to the same Kafka Private Service Connect service from two different TiDB Cloud projects? + +If you have already followed the steps in this document and successfully set up the connection from the first project, and you want to set up a second connection from the second project, you can connect to the same Kafka Private Service Connect service from two different TiDB Cloud projects as follows: + +- If you set up Kafka PSC by PSC port mapping, do the following: + + 1. Follow instructions from the beginning of this document. When you proceed to [Step 1. Set up Kafka Cluster](#step-1-set-up-the-kafka-cluster), follow the [Reconfigure a running Kafka cluster](#reconfigure-a-running-kafka-cluster) section to create another group of EXTERNAL listeners and advertised listeners. You can name it as `EXTERNAL2`. Note that the port range of `EXTERNAL2` cannot overlap with the EXTERNAL. + + 2. After reconfiguring the brokers, add another group of Network endpoints to the Network endpoint group, which maps the ports range to the `EXTERNAL2` listener. + + 3. Configure the TiDB Cloud connection with the following input to create the new changefeed: + + - New Bootstrap ports + - New Kafka Advertised Listener Pattern + - The same Service Attachment + +- If you [set up self-hosted Kafka Private Service Connect by Kafka-proxy](#set-up-self-hosted-kafka-private-service-connect-by-kafka-proxy), create a new Kafka-proxy PSC from the beginning with a new Kafka Advertised Listener Pattern. diff --git a/tidb-cloud/size-your-cluster.md b/tidb-cloud/size-your-cluster.md index 26a2ac7e3ed90..43fb92bce9261 100644 --- a/tidb-cloud/size-your-cluster.md +++ b/tidb-cloud/size-your-cluster.md @@ -5,17 +5,17 @@ summary: Learn how to determine the size of your TiDB Cloud cluster. # Determine Your TiDB Size -This document describes how to determine the size of a TiDB Dedicated cluster. +This document describes how to determine the size of a TiDB Cloud Dedicated cluster. > **Note:** > -> You cannot change the size of a [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster. +> You cannot change the size of a [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) cluster. ## Size TiDB TiDB is for computing only and does not store data. It is horizontally scalable. -You can configure node number, vCPU, and RAM for TiDB. +You can configure node count, vCPU, and RAM for TiDB. To learn performance test results of different cluster scales, see [TiDB Cloud Performance Reference](/tidb-cloud/tidb-cloud-performance-reference.md). @@ -23,19 +23,24 @@ To learn performance test results of different cluster scales, see [TiDB Cloud P The supported vCPU and RAM sizes include the following: -- 4 vCPU, 16 GiB -- 8 vCPU, 16 GiB -- 16 vCPU, 32 GiB +| Standard size | High memory size | +|:---------:|:----------------:| +| 4 vCPU, 16 GiB | N/A | +| 8 vCPU, 16 GiB | 8 vCPU, 32 GiB | +| 16 vCPU, 32 GiB | 16 vCPU, 64 GiB | +| 32 vCPU, 64 GiB | 32 vCPU, 128 GiB | > **Note:** > +> To use the **32 vCPU, 128 GiB** size of TiDB, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). +> > If the vCPU and RAM size of TiDB is set as **4 vCPU, 16 GiB**, note the following restrictions: > -> - The node number of TiDB can only be set to 1 or 2, and the node number of TiKV is fixed to 3. +> - The node count of TiDB can only be set to 1 or 2, and the node count of TiKV is fixed to 3. > - 4 vCPU TiDB can only be used with 4 vCPU TiKV. > - TiFlash is unavailable. -### TiDB node number +### TiDB node count For high availability, it is recommended that you configure at least two TiDB nodes for each TiDB Cloud cluster. @@ -60,13 +65,13 @@ If the number of TiDB nodes is less than 8, the performance deviation coefficien When planning your cluster size, you can estimate the number of TiDB nodes according to your workload type, your overall expected performance (QPS), and the performance of a single TiDB node corresponding to the workload type using the following formula: -`node num = ceil(overall expected performance ÷ performance per node * (1 - performance deviation coefficient))` +`node count = ceil(overall expected performance ÷ performance per node * (1 - performance deviation coefficient))` -In the formula, you need to calculate `node num = ceil(overall expected performance ÷ performance per node)` first to get a rough node number, and then use the corresponding performance deviation coefficient to get the final result of the node number. +In the formula, you need to calculate `node count = ceil(overall expected performance ÷ performance per node)` first to get a rough node count, and then use the corresponding performance deviation coefficient to get the final result of the node count. For example, your overall expected performance is 110,000 QPS under a mixed workload, your P95 latency is about 100 ms, and you want to use 8 vCPU, 16 GiB TiDB nodes. Then, you can get the estimated TiDB performance of an 8 vCPU, 16 GiB TiDB node from the preceding table (which is `15,500`), and calculate a rough number of TiDB nodes as follows: -`node num = ceil(110,000 ÷ 15,500) = 8` +`node count = ceil(110,000 ÷ 15,500) = 8` As the performance deviation coefficient of 8 nodes is about 5%, the estimated TiDB performance is `8 * 15,500 * (1 - 5%) = 117,800`, which can meet your expected performance of 110,000 QPS. @@ -76,7 +81,7 @@ Therefore, 8 TiDB nodes (8 vCPU, 16 GiB) are recommended for you. TiKV is responsible for storing data. It is horizontally scalable. -You can configure node number, vCPU and RAM, and storage for TiKV. +You can configure node count, vCPU and RAM, and storage for TiKV. To learn performance test results of different cluster scales, see [TiDB Cloud Performance Reference](/tidb-cloud/tidb-cloud-performance-reference.md). @@ -84,20 +89,22 @@ To learn performance test results of different cluster scales, see [TiDB Cloud P The supported vCPU and RAM sizes include the following: -- 4 vCPU, 16 GiB -- 8 vCPU, 32 GiB -- 8 vCPU, 64 GiB -- 16 vCPU, 64 GiB +| Standard size | High memory size | +|:---------:|:----------------:| +| 4 vCPU, 16 GiB | N/A | +| 8 vCPU, 32 GiB | 8 vCPU, 64 GiB | +| 16 vCPU, 64 GiB | Coming soon | +| 32 vCPU, 128 GiB | N/A | > **Note:** > > If the vCPU and RAM size of TiKV is set as **4 vCPU, 16 GiB**, note the following restrictions: > -> - The node number of TiDB can only be set to 1 or 2, and the node number of TiKV is fixed to 3. +> - The node count of TiDB can only be set to 1 or 2, and the node count of TiKV is fixed to 3. > - 4 vCPU TiKV can only be used with 4 vCPU TiDB. > - TiFlash is unavailable. -### TiKV node number +### TiKV node count The number of TiKV nodes should be **at least 1 set (3 nodes in 3 different Available Zones)**. @@ -107,13 +114,13 @@ TiDB Cloud deploys TiKV nodes evenly to all availability zones (at least 3) in t > > When you scale your TiDB cluster, nodes in the 3 availability zones are increased or decreased at the same time. For how to scale in or scale out a TiDB cluster based on your needs, see [Scale Your TiDB Cluster](/tidb-cloud/scale-tidb-cluster.md). -Although TiKV is mainly used for data storage, the performance of the TiKV node also varies depending on different workloads. Therefore, when planning the number of TiKV nodes, you need to estimate it according to both your [**data volume**](#estimate-tikv-node-number-according-to-data-volume) and [expected performance](#estimate-tikv-node-number-according-to-expected-performance), and then take the larger of the two estimates as the recommended node number. +Although TiKV is mainly used for data storage, the performance of the TiKV node also varies depending on different workloads. Therefore, when planning the number of TiKV nodes, you need to estimate it according to both your [**data volume**](#estimate-tikv-node-count-according-to-data-volume) and [expected performance](#estimate-tikv-node-count-according-to-expected-performance), and then take the larger of the two estimates as the recommended node count. -#### Estimate TiKV node number according to data volume +#### Estimate TiKV node count according to data volume You can calculate a recommended number of TiKV nodes according to your data volume as follows: -`node num = ceil(size of your data * TiKV compression ratio * the number of replicas ÷ TiKV storage usage ratio ÷ one TiKV capacity ÷ 3) * 3` +`node count = ceil(size of your data * TiKV compression ratio * the number of replicas ÷ TiKV storage usage ratio ÷ one TiKV capacity ÷ 3) * 3` Generally, it is recommended to keep the usage ratio of TiKV storage below 80%. The number of replicas in TiDB Cloud is 3 by default. The maximum storage capacity of an 8 vCPU, 64 GiB TiKV node is 4096 GiB. @@ -121,9 +128,9 @@ Based on historical data, the average TiKV compression ratio is around 40%. Suppose that the size of your MySQL dump files is 20 TB and the TiKV compression ratio is 40%. Then, you can calculate a recommended number of TiKV nodes according to your data volume as follows: -`node num = ceil(20 TB * 40% * 3 ÷ 0.8 ÷ 4096 GiB ÷ 3) * 3 = 9` +`node count = ceil(20 TB * 40% * 3 ÷ 0.8 ÷ 4096 GiB ÷ 3) * 3 = 9` -#### Estimate TiKV node number according to expected performance +#### Estimate TiKV node count according to expected performance Similarly as TiDB performance, TiKV performance increases linearly with the number of TiKV nodes. However, when the number of TiKV nodes exceeds 8, the performance increment becomes slightly less than linearly proportional. For each additional 8 nodes, the performance deviation coefficient is about 5%. @@ -146,39 +153,67 @@ If the number of TiKV nodes is less than 8, the performance deviation coefficien When planning your cluster size, you can estimate the number of TiKV nodes according to your workload type, your overall expected performance (QPS), and the performance of a single TiKV node corresponding to the workload type using the following formula: -`node num = ceil(overall expected performance ÷ performance per node * (1 - performance deviation coefficient))` +`node count = ceil(overall expected performance ÷ performance per node * (1 - performance deviation coefficient))` -In the formula, you need to calculate `node num = ceil(overall expected performance ÷ performance per node)` first to get a rough node number, and then use the corresponding performance deviation coefficient to get the final result of the node number. +In the formula, you need to calculate `node count = ceil(overall expected performance ÷ performance per node)` first to get a rough node count, and then use the corresponding performance deviation coefficient to get the final result of the node count. For example, your overall expected performance is 110,000 QPS under a mixed workload, your P95 latency is about 100 ms, and you want to use 8 vCPU, 32 GiB TiKV nodes. Then, you can get the estimated TiKV performance of an 8 vCPU, 32 GiB TiKV node from the preceding table (which is `17,800`), and calculate a rough number of TiKV nodes as follows: -`node num = ceil(110,000 / 17,800 ) = 7` +`node count = ceil(110,000 / 17,800 ) = 7` As 7 is less than 8, the performance deviation coefficient of 7 nodes is 0. The estimated TiKV performance is `7 * 17,800 * (1 - 0) = 124,600`, which can meet your expected performance of 110,000 QPS. Therefore, 7 TiKV nodes (8 vCPU, 32 GiB) are recommended for you according to your expected performance. -Next, you can compare the TiKV node number calculated according to data volume with the number calculated according to your expected performance, and take the larger one as a recommended number of your TiKV nodes. +Next, you can compare the TiKV node count calculated according to data volume with the number calculated according to your expected performance, and take the larger one as a recommended number of your TiKV nodes. -### TiKV node storage +### TiKV node storage size -The supported node storage of different TiKV vCPUs is as follows: +The supported node storage sizes of different TiKV vCPUs are as follows: | TiKV vCPU | Min node storage | Max node storage | Default node storage | |:---------:|:----------------:|:----------------:|:--------------------:| | 4 vCPU | 200 GiB | 2048 GiB | 500 GiB | | 8 vCPU | 200 GiB | 4096 GiB | 500 GiB | -| 16 vCPU | 200 GiB | 6144 GiB | 500 GiB | +| 16 vCPU | 200 GiB | 4096 GiB | 500 GiB | +| 32 vCPU | 200 GiB | 4096 GiB | 500 GiB | > **Note:** > -> You cannot decrease the TiKV node storage after the cluster creation. +> You cannot decrease the TiKV node storage size after the cluster creation. + +### TiKV node storage types + +TiDB Cloud provides the following TiKV storage types for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters hosted on AWS: + +- [Basic storage](#basic-storage) +- [Standard storage](#standard-storage) +- [Performance and Plus storage](#performance-and-plus-storage) + +#### Basic storage + +The Basic storage is a general-purpose storage type that provides lower performance than the Standard storage. + +The Basic storage type is applied automatically to the following clusters hosted on AWS: + +- Existing clusters that are created before April 1, 2025. +- New clusters that are created with TiDB versions earlier than v7.5.5, v8.1.2, or v8.5.0. + +#### Standard storage + +The Standard storage is ideal for most workloads, providing a balance between performance and cost efficiency. Compared with the Basic storage, it offers better performance by reserving ample disk resources for Raft logs. This reduces the impact of Raft I/O on data disk I/O, improving read and write performance for TiKV. + +The Standard storage type is applied automatically to new clusters hosted on AWS and created with TiDB versions v7.5.5, v8.1.2, v8.5.0, or later. + +#### Performance and Plus storage + +The Performance and Plus storage provide higher performance and stability, with pricing that reflects these enhanced capabilities. Currently, these two storage types are only available upon request for clusters deployed on AWS. To request the Performance or Plus storage, click **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com) and click **Request Support**. Then, fill in "Apply for TiKV storage type" in the **Description** field and click **Submit**. ## Size TiFlash TiFlash synchronizes data from TiKV in real time and supports real-time analytics workloads right out of the box. It is horizontally scalable. -You can configure node number, vCPU and RAM, and storage for TiFlash. +You can configure node count, vCPU and RAM, and storage for TiFlash. ### TiFlash vCPU and RAM @@ -186,10 +221,12 @@ The supported vCPU and RAM sizes include the following: - 8 vCPU, 64 GiB - 16 vCPU, 128 GiB +- 32 vCPU, 128 GiB +- 32 vCPU, 256 GiB Note that TiFlash is unavailable when the vCPU and RAM size of TiDB or TiKV is set as **4 vCPU, 16 GiB**. -### TiFlash node number +### TiFlash node count TiDB Cloud deploys TiFlash nodes evenly to different availability zones in a region. It is recommended that you configure at least two TiFlash nodes in each TiDB Cloud cluster and create at least two replicas of the data for high availability in your production environment. @@ -207,9 +244,25 @@ The supported node storage of different TiFlash vCPUs is as follows: | TiFlash vCPU | Min node storage | Max node storage | Default node storage | |:---------:|:----------------:|:----------------:|:--------------------:| -| 8 vCPU | 200 GiB | 2048 GiB | 500 GiB | -| 16 vCPU | 200 GiB | 2048 GiB | 500 GiB | +| 8 vCPU | 200 GiB | 4096 GiB | 500 GiB | +| 16 vCPU | 200 GiB | 4096 GiB | 500 GiB | +| 32 vCPU | 200 GiB | 8192 GiB | 500 GiB | > **Note:** > > You cannot decrease the TiFlash node storage after the cluster creation. + +### TiFlash node storage types + +TiDB Cloud provides the following TiFlash storage types for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters hosted on AWS: + +- [Basic storage](#basic-storage-1) +- [Plus storage](#plus-storage) + +#### Basic storage + +The Basic storage is ideal for most workloads, providing a balance between performance and cost efficiency. + +#### Plus storage + +The Plus storage provides higher performance and stability, with pricing that reflects these enhanced capabilities. Currently, this storage type is only available upon request for clusters deployed on AWS. To request it, click **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com) and click **Request Support**. Then, fill in "Apply for TiFlash storage type" in the **Description** field and click **Submit**. \ No newline at end of file diff --git a/tidb-cloud/sql-concepts.md b/tidb-cloud/sql-concepts.md new file mode 100644 index 0000000000000..3be8502a02b37 --- /dev/null +++ b/tidb-cloud/sql-concepts.md @@ -0,0 +1,94 @@ +--- +title: SQL +summary: Learn about SQL concepts for TiDB. +--- + +# SQL + +TiDB is highly compatible with the MySQL protocol and the common features and syntax of MySQL 5.7 and MySQL 8.0. The ecosystem tools for MySQL (PHPMyAdmin, Navicat, MySQL Workbench, DBeaver and [more](https://docs.pingcap.com/tidb/v7.2/dev-guide-third-party-support#gui)) and the MySQL client can be used for TiDB. + +However, some features of MySQL are not supported in TiDB. This could be because there is now a better way to solve the problem (such as the use of JSON instead of XML functions) or a lack of current demand versus effort required (such as stored procedures and functions). Additionally, some features might be difficult to implement in a distributed system. For more information, see [MySQL Compatibility](/mysql-compatibility.md). + +## SQL statements + +A SQL statement is a command or instruction in SQL (Structured Query Language) composed of identifiers, parameters, variables, data types, and reserved SQL keywords. It directs the database to perform specific actions, such as retrieving, modifying, or managing data and database structures. + +TiDB uses SQL statements that aim to follow ISO/IEC SQL standards, with extensions for MySQL and TiDB-specific statements where necessary. + +SQL is divided into the following 4 types according to their functions: + +- DDL (Data Definition Language): It is used to define database objects, including databases, tables, views, and indexes. For DDL statements in TiDB, see [Schema management / Data definition statements (DDL)](/sql-statements/sql-statement-overview.md#schema-management--data-definition-statements-ddl). + +- DML (Data Manipulation Language): It is used to manipulate application-related records. For DML statements in TiDB, see [Data manipulation statements (DML)](/sql-statements/sql-statement-overview.md#data-manipulation-statements-dml). + +- DQL (Data Query Language): It is used to query the records after conditional filtering. + +- DCL (Data Control Language): It is used to define access privileges and security levels. + +To get an overview of SQL statements in TiDB, see [SQL Statement Overview](/sql-statements/sql-statement-overview.md). + +## SQL mode + +TiDB servers operate in different SQL modes and apply these modes differently for different clients. SQL mode defines the SQL syntax that TiDB supports and the type of data validation check to perform. + +For more information, see [SQL Mode](/sql-mode.md). + +## Row ID generation attributes + +TiDB provides three SQL attributes to optimize row ID generation and data distribution. + +- AUTO_INCREMENT + +- AUTO_RANDOM + +- SHARD_ROW_ID_BITS + +### AUTO_INCREMENT + +`AUTO_INCREMENT` is a column attribute that is used to automatically fill in default column values. When the `INSERT` statement does not specify values for the `AUTO_INCREMENT` column, the system automatically assigns values to this column. + +For performance reasons, `AUTO_INCREMENT` numbers are allocated in a batch of values (30 thousand by default) to each TiDB server. This means that while `AUTO_INCREMENT` numbers are guaranteed to be unique, values assigned to an `INSERT` statement will only be monotonic on a per TiDB server basis. + +If you want the `AUTO_INCREMENT` numbers to be monotonic on all TiDB servers and your TiDB version is v6.5.0 or later, it is recommended to enable the [MySQL compatibility mode](/auto-increment.md#mysql-compatibility-mode). + +For more information, see [AUTO_INCREMENT](/auto-increment.md). + +### AUTO_RANDOM + +`AUTO_RANDOM` is a column attribute that is used to automatically assign values to a `BIGINT` column. Values assigned automatically are random and unique. Since the value of `AUTO_RANDOM` is random and unique, `AUTO_RANDOM` is often used in place of [`AUTO_INCREMENT`](/auto-increment.md) to avoid write hotspots in a single storage node caused by TiDB assigning consecutive IDs. + +Since the value of `AUTO_RANDOM` is random and unique, `AUTO_RANDOM` is often used in place of [`AUTO_INCREMENT`](/auto-increment.md) to avoid write hotspots in a single storage node caused by TiDB assigning consecutive IDs. If the current `AUTO_INCREMENT` column is a primary key and the type is `BIGINT`, you can execute the `ALTER TABLE t MODIFY COLUMN id BIGINT AUTO_RANDOM(5);` statement to switch from `AUTO_INCREMENT` to `AUTO_RANDOM`. + +For more information, see [AUTO_RANDOM](/auto-random.md). + +### SHARD_ROW_ID_BITS + +For the tables with a non-clustered primary key or no primary key, TiDB uses an implicit auto-increment row ID. When a large number of `INSERT` operations are performed, the data is written into a single Region, causing a write hot spot. + +To mitigate the hot spot issue, you can configure [`SHARD_ROW_ID_BITS`](/shard-row-id-bits.md). The row IDs are scattered, and the data are written into multiple different Regions. + +## Keywords + +Keywords are words that have special meanings in SQL statements, such as `SELECT`, `UPDATE`, and `DELETE`. + +- Some of them can be used as identifiers directly, which are called non-reserved keywords. + +- Some of them require special treatment before being used as identifiers, which are called reserved keywords. + +However, some non-reserved keywords might still require special treatment. It is recommended that you treat them as reserved keywords. + +For more information, see [Keywords](/keywords.md). + +## User-defined variables + +TiDB lets you set and read the user-defined variables. The format of the user-defined variables is `@var_name`. The characters that compose `var_name` can be any characters that can compose an identifier, including the numbers `0-9`, the letters `a-zA-Z`, the underscore `_`, the dollar sign `$`, and the UTF-8 characters. In addition, it also includes the English period `.`. The user-defined variables are case-insensitive. + +The user-defined variables are session-specific, which means a user variable defined by one client connection cannot be seen or used by other client connections. + +For more information, see [User-Defined Variables](/user-defined-variables.md). + +## Metadata lock + +In TiDB, a metadata lock is a mechanism introduced to manage changes to table metadata during online schema changes. When a transaction begins, it locks onto a snapshot of the current metadata. If the metadata changes during the transaction, TiDB throws an "Information schema is changed" error, preventing the transaction from committing. The metadata lock coordinates Data Manipulation Language (DML) and Data Definition Language (DDL) operations by prioritizing DMLs, ensuring that in-progress DML transactions with outdated metadata commit before applying new DDL changes, thus minimizing errors and maintaining data consistency. + +For more information, see [Metadata Lock](/metadata-lock.md). \ No newline at end of file diff --git a/tidb-cloud/sql-proxy-account.md b/tidb-cloud/sql-proxy-account.md new file mode 100644 index 0000000000000..6b2314d630aba --- /dev/null +++ b/tidb-cloud/sql-proxy-account.md @@ -0,0 +1,90 @@ +--- +title: SQL Proxy Account +summary: Learn about the SQL proxy account in TiDB Cloud. +--- + +# SQL Proxy Account + +A SQL proxy account is a SQL user account that is automatically created by TiDB Cloud to access the database via [SQL Editor](/tidb-cloud/explore-data-with-chat2query.md) or [Data Service](https://docs.pingcap.com/tidbcloud/api/v1beta1/dataservice) on behalf of a TiDB Cloud user. For example, `testuser@pingcap.com` is a TiDB Cloud user account, while `3jhEcSimm7keKP8.testuser._41mqK6H4` is its corresponding SQL proxy account. + +SQL proxy accounts provide a secure, token-based authentication mechanism for accessing the database in TiDB Cloud. By eliminating the need for traditional username and password credentials, SQL proxy accounts enhance security and simplify access management. + +The key benefits of SQL proxy accounts are as follows: + +- Enhanced security: mitigates risks associated with static credentials by using JWT tokens. +- Streamlined access: restricts access specifically to the SQL Editor and Data Service, ensuring precise control. +- Ease of management: simplifies authentication for developers and administrators working with TiDB Cloud. + +## Identify the SQL proxy account + +If you want to identify whether a specific SQL account is a SQL proxy account, take the following steps: + +1. Examine the `mysql.user` table: + + ```sql + USE mysql; + SELECT user FROM user WHERE plugin = 'tidb_auth_token'; + ``` + +2. Check grants for the SQL account. If roles like `role_admin`, `role_readonly`, or `role_readwrite` are listed, then it is a SQL proxy account. + + ```sql + SHOW GRANTS for 'username'; + ``` + +## How the SQL proxy account is created + +The SQL proxy account is automatically created during TiDB Cloud cluster initialization for the TiDB Cloud user who is granted a role with permissions in the cluster. + +## How the SQL proxy account is deleted + +When a user is removed from [an organization](/tidb-cloud/manage-user-access.md#remove-an-organization-member) or [a project](/tidb-cloud/manage-user-access.md#remove-a-project-member), or their role changes to one that does not have access to the cluster, the SQL proxy account is automatically deleted. + +Note that if a SQL proxy account is manually deleted, it will be automatically recreated when the user log in to the TiDB Cloud console next time. + +## SQL proxy account username + +In some cases, the SQL proxy account username is exactly the same as the TiDB Cloud username, but in other cases it is not exactly the same. The SQL proxy account username is determined by the length of the TiDB Cloud user's email address. The rules are as follows: + +| Environment | Email length | Username format | +| ----------- | ------------ | --------------- | +| TiDB Cloud Dedicated | <= 32 characters | Full email address | +| TiDB Cloud Dedicated | > 32 characters | `prefix($email, 23)_prefix(base58(sha1($email)), 8)` | +| TiDB Cloud Serverless | <= 15 characters | `serverless_unique_prefix + "." + email` | +| TiDB Cloud Serverless | > 15 characters | `serverless_unique_prefix + "." + prefix($email, 6)_prefix(base58(sha1($email)), 8)` | + +Examples: + +| Environment | Email address | SQL proxy account username | +| ----------- | ----- | -------- | +| TiDB Cloud Dedicated | `user@pingcap.com` | `user@pingcap.com` | +| TiDB Cloud Dedicated | `longemailaddressexample@pingcap.com` | `longemailaddressexample_48k1jwL9` | +| TiDB Cloud Serverless | `u1@pingcap.com` | `{user_name_prefix}.u1@pingcap.com` | +| TiDB Cloud Serverless | `longemailaddressexample@pingcap.com` | `{user_name_prefix}.longem_48k1jwL9`| + +> **Note:** +> +> In the preceding table, `{user_name_prefix}` is a unique prefix generated by TiDB Cloud to distinguish TiDB Cloud Serverless clusters. For details, see the [user name prefix](/tidb-cloud/select-cluster-tier.md#user-name-prefix) of TiDB Cloud Serverless clusters. + +## SQL proxy account password + +Since SQL proxy accounts are JWT token-based, it is not necessary to manage passwords for these accounts. The security token is automatically managed by the system. + +## SQL proxy account roles + +The SQL proxy account's role depends on the TiDB Cloud user's IAM role: + +- Organization level: + - Organization Owner: role_admin + - Organization Billing Manager: No proxy account + - Organization Viewer: No proxy account + - Organization Console Audit Manager: No proxy account + +- Project level: + - Project Owner: role_admin + - Project Data Access Read-Write: role_readwrite + - Project Data Access Read-Only: role_readonly + +## SQL proxy account access control + +SQL proxy accounts are JWT token-based and only accessible to the Data Service and SQL Editor. It is impossible to access the TiDB Cloud cluster using a SQL proxy account with a username and password. diff --git a/tidb-cloud/terraform-get-tidbcloud-provider.md b/tidb-cloud/terraform-get-tidbcloud-provider.md index feebb0ae99d27..1b2637dfc2a00 100644 --- a/tidb-cloud/terraform-get-tidbcloud-provider.md +++ b/tidb-cloud/terraform-get-tidbcloud-provider.md @@ -48,7 +48,7 @@ For detailed steps, see [TiDB Cloud API documentation](https://docs.pingcap.com/ required_providers { tidbcloud = { source = "tidbcloud/tidbcloud" - version = "~> 0.1.0" + version = "~> 0.3.0" } } required_version = ">= 1.0.0" @@ -103,12 +103,28 @@ provider "tidbcloud" { `public_key` and `private_key` are the API key's public key and private key. You can also pass them through the environment variables: ``` -export TIDBCLOUD_PUBLIC_KEY = ${public_key} -export TIDBCLOUD_PRIVATE_KEY = ${private_key} +export TIDBCLOUD_PUBLIC_KEY=${public_key} +export TIDBCLOUD_PRIVATE_KEY=${private_key} ``` Now, you can use the TiDB Cloud Terraform Provider. +## Step 5. Configure TiDB Cloud Terraform Provider with sync configuration + +Terraform provider (>= 0.3.0) supports an optional parameter `sync`. + +By setting `sync` to `true`, you can create, update, or delete resources synchronously. Here is an example: + +``` +provider "tidbcloud" { + public_key = "your_public_key" + private_key = "your_private_key" + sync = true +} +``` + +Setting `sync` to `true` is recommended, but note that `sync` currently only works with the cluster resource. If you need synchronous operations for other resources, [contact TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). + ## Next step -Get started by managing a cluster with the [cluster resource](/tidb-cloud/terraform-use-cluster-resource.md). \ No newline at end of file +Get started by managing a cluster with the [cluster resource](/tidb-cloud/terraform-use-cluster-resource.md). diff --git a/tidb-cloud/terraform-use-backup-resource.md b/tidb-cloud/terraform-use-backup-resource.md index f882f8f2ea20e..e8f843e7853d0 100644 --- a/tidb-cloud/terraform-use-backup-resource.md +++ b/tidb-cloud/terraform-use-backup-resource.md @@ -9,13 +9,13 @@ You can learn how to create a backup of a TiDB Cloud cluster with the `tidbcloud The features of the `tidbcloud_backup` resource include the following: -- Create backups for TiDB Dedicated clusters. -- Delete backups for TiDB Dedicated clusters. +- Create backups for TiDB Cloud Dedicated clusters. +- Delete backups for TiDB Cloud Dedicated clusters. ## Prerequisites - [Get TiDB Cloud Terraform Provider](/tidb-cloud/terraform-get-tidbcloud-provider.md). -- The backup and restore feature is unavailable to TiDB Serverless clusters. To use backup resources, make sure that you have created a TiDB Dedicated cluster. +- The backup and restore feature is unavailable to TiDB Cloud Serverless clusters. To use backup resources, make sure that you have created a TiDB Cloud Dedicated cluster. ## Create a backup with the backup resource diff --git a/tidb-cloud/terraform-use-cluster-resource.md b/tidb-cloud/terraform-use-cluster-resource.md index 1f223d17ce471..66e3b06aae03c 100644 --- a/tidb-cloud/terraform-use-cluster-resource.md +++ b/tidb-cloud/terraform-use-cluster-resource.md @@ -11,9 +11,9 @@ In addition, you will also learn how to get the necessary information with the ` The features of the `tidbcloud_cluster` resource include the following: -- Create TiDB Serverless and TiDB Dedicated clusters. -- Modify TiDB Dedicated clusters. -- Delete TiDB Serverless and TiDB Dedicated clusters. +- Create TiDB Cloud Serverless and TiDB Cloud Dedicated clusters. +- Modify TiDB Cloud Dedicated clusters. +- Delete TiDB Cloud Serverless and TiDB Cloud Dedicated clusters. ## Prerequisites @@ -39,6 +39,7 @@ To view the information of all available projects, you can use the `tidbcloud_pr provider "tidbcloud" { public_key = "your_public_key" private_key = "your_private_key" + sync = true } data "tidbcloud_projects" "example_project" { @@ -137,6 +138,7 @@ To get the cluster specification information, you can use the `tidbcloud_cluster provider "tidbcloud" { public_key = "your_public_key" private_key = "your_private_key" + sync = true } data "tidbcloud_cluster_specs" "example_cluster_spec" { } @@ -267,11 +269,11 @@ In the results: > **Note:** > -> Before you begin, make sure that you have set a Project CIDR in the TiDB Cloud console. For more information, see [Set a Project CIDR](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-project-cidr). +> Before you begin, make sure that you have set a CIDR in the TiDB Cloud console. For more information, see [Set a CIDR](/tidb-cloud/set-up-vpc-peering-connections.md#prerequisite-set-a-cidr-for-a-region). You can create a cluster using the `tidbcloud_cluster` resource. -The following example shows how to create a TiDB Dedicated cluster. +The following example shows how to create a TiDB Cloud Dedicated cluster. 1. Create a directory for the cluster and enter it. @@ -289,6 +291,7 @@ The following example shows how to create a TiDB Dedicated cluster. provider "tidbcloud" { public_key = "your_public_key" private_key = "your_private_key" + sync = true } resource "tidbcloud_cluster" "example_cluster" { @@ -469,9 +472,9 @@ The following example shows how to create a TiDB Dedicated cluster. When the status is `AVAILABLE`, it indicates that your TiDB cluster is created and ready for use. -## Modify a TiDB Dedicated cluster +## Modify a TiDB Cloud Dedicated cluster -For a TiDB Dedicated cluster, you can use Terraform to manage cluster resources as follows: +For a TiDB Cloud Dedicated cluster, you can use Terraform to manage cluster resources as follows: - Add a TiFlash component to the cluster. - Scale the cluster. @@ -827,7 +830,7 @@ You can pause a cluster when its status is `AVAILABLE` or resume a cluster when 6. Wait for a moment, then use the `terraform refersh` command to update the state. The status will be changed to `AVAILABLE` finally. -Now, you have created and managed a TiDB Dedicated cluster with Terraform. Next, you can try creating a backup of the cluster by our [backup resource](/tidb-cloud/terraform-use-backup-resource.md). +Now, you have created and managed a TiDB Cloud Dedicated cluster with Terraform. Next, you can try creating a backup of the cluster by our [backup resource](/tidb-cloud/terraform-use-backup-resource.md). ## Import a cluster diff --git a/tidb-cloud/terraform-use-import-resource.md b/tidb-cloud/terraform-use-import-resource.md index 333b805ef5c29..3b75f6d3d92d7 100644 --- a/tidb-cloud/terraform-use-import-resource.md +++ b/tidb-cloud/terraform-use-import-resource.md @@ -9,14 +9,14 @@ You can learn how to import data to a TiDB Cloud cluster with the `tidbcloud_imp The features of the `tidbcloud_import` resource include the following: -- Create import tasks for TiDB Serverless and TiDB Dedicated clusters. +- Create import tasks for TiDB Cloud Serverless and TiDB Cloud Dedicated clusters. - Import data either from local disks or from Amazon S3 buckets. - Cancel ongoing import tasks. ## Prerequisites - [Get TiDB Cloud Terraform Provider](/tidb-cloud/terraform-get-tidbcloud-provider.md). -- [Create a TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) or [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). +- [Create a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) or [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). ## Create and run an import task @@ -26,7 +26,7 @@ You can manage either a local import task or an Amazon S3 import task using the > **Note:** > -> Importing local files is supported only for TiDB Serverless clusters, not for TiDB Dedicated clusters. +> Importing local files is supported only for TiDB Cloud Serverless clusters, not for TiDB Cloud Dedicated clusters. 1. Create a CSV file for import. For example: diff --git a/tidb-cloud/terraform-use-restore-resource.md b/tidb-cloud/terraform-use-restore-resource.md index 9afd88ab46e2f..2793dc7e5ad6b 100644 --- a/tidb-cloud/terraform-use-restore-resource.md +++ b/tidb-cloud/terraform-use-restore-resource.md @@ -9,12 +9,12 @@ You can learn how to manage a restore task with the `tidbcloud_restore` resource The features of the `tidbcloud_restore` resource include the following: -- Create restore tasks for TiDB Dedicated clusters according to your backup. +- Create restore tasks for TiDB Cloud Dedicated clusters according to your backup. ## Prerequisites - [Get TiDB Cloud Terraform Provider](/tidb-cloud/terraform-get-tidbcloud-provider.md). -- The backup and restore feature is unavailable for TiDB Serverless clusters. To use restore resources, make sure that you have created a TiDB Dedicated cluster. +- The backup and restore feature is unavailable for TiDB Cloud Serverless clusters. To use restore resources, make sure that you have created a TiDB Cloud Dedicated cluster. ## Create a restore task diff --git a/tidb-cloud/third-party-monitoring-integrations.md b/tidb-cloud/third-party-monitoring-integrations.md index d0cd59441a9cf..484c6db9b75c7 100644 --- a/tidb-cloud/third-party-monitoring-integrations.md +++ b/tidb-cloud/third-party-monitoring-integrations.md @@ -21,7 +21,7 @@ The available third-party integrations are displayed. ## Limitation -- For [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters, third-party metrics integrations are not supported. +- For [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters, third-party metrics integrations are not supported. - Third-party metrics integrations are not available when the cluster status is **CREATING**, **RESTORING**, **PAUSED**, or **RESUMING**. @@ -35,7 +35,7 @@ For the detailed integration steps and a list of metrics that Datadog tracks, re ### Prometheus and Grafana integration (beta) -With the Prometheus and Grafana integration, you can get a scrape_config file for Prometheus from TiDB Cloud and use the content from the file to configure Prometheus. You can view these metrics in your Grafana dashboards. +With the Prometheus and Grafana integration, you can get a `scrape_config` file for Prometheus from TiDB Cloud and use the content from the file to configure Prometheus. You can view these metrics in your Grafana dashboards. For the detailed integration steps and a list of metrics that Prometheus tracks, see [Integrate TiDB Cloud with Prometheus and Grafana](/tidb-cloud/monitor-prometheus-and-grafana-integration.md). diff --git a/tidb-cloud/ticloud-ai.md b/tidb-cloud/ticloud-ai.md new file mode 100644 index 0000000000000..7f2a6b705c498 --- /dev/null +++ b/tidb-cloud/ticloud-ai.md @@ -0,0 +1,47 @@ +--- +title: ticloud ai +summary: The reference of `ticloud ai`. +--- + +# ticloud ai + +Chat with TiDB Bot: + +```shell +ticloud ai [flags] +``` + +## Examples + +Chat with TiDB Bot in interactive mode: + +```shell +ticloud ai +``` + +Chat with TiDB Bot in non-interactive mode: + +```shell +ticloud ai -q "How to create a cluster?" +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|--------------------|-----------------------------------|----------|------------------------------------------------------| +| -q, --query string | Specifies your query to TiDB Bot. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|--------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-auth-login.md b/tidb-cloud/ticloud-auth-login.md new file mode 100644 index 0000000000000..3ed324f317c67 --- /dev/null +++ b/tidb-cloud/ticloud-auth-login.md @@ -0,0 +1,47 @@ +--- +title: ticloud auth login +summary: The reference of `ticloud auth login`. +--- + +# ticloud auth login + +Authenticate with TiDB Cloud: + +```shell +ticloud auth login [flags] +``` + +## Examples + +To log into TiDB Cloud: + +```shell +ticloud auth login +``` + +To log into TiDB Cloud with insecure storage: + +```shell +ticloud auth login --insecure-storage +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|--------------------|---------------------------------------------------------------------------|----------|------------------------------------------------------| +| --insecure-storage | Saves authentication credentials in plain text instead of credential store. | No | Works in both non-interactive and interactive modes. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|--------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-update.md b/tidb-cloud/ticloud-auth-logout.md similarity index 53% rename from tidb-cloud/ticloud-update.md rename to tidb-cloud/ticloud-auth-logout.md index 0a2c85dd3eca7..39bfa978c3c51 100644 --- a/tidb-cloud/ticloud-update.md +++ b/tidb-cloud/ticloud-auth-logout.md @@ -1,36 +1,31 @@ --- -title: ticloud update -summary: The reference of `ticloud update`. +title: ticloud auth logout +summary: The reference of `ticloud auth logout`. --- -# ticloud update +# ticloud auth logout -Update the TiDB Cloud CLI to the latest version: +Log out of TiDB Cloud: ```shell -ticloud update [flags] +ticloud auth logout [flags] ``` ## Examples -Update the TiDB Cloud CLI to the latest version: +To log out of TiDB Cloud: ```shell -ticloud update +ticloud auth logout ``` -## Flags - -| Flag | Description | -|------------|--------------------------| - | -h, --help | Help information for this command | - ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| +| Flag | Description | Required | Note | +|----------------------|--------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-auth-whoami.md b/tidb-cloud/ticloud-auth-whoami.md new file mode 100644 index 0000000000000..9d89791916283 --- /dev/null +++ b/tidb-cloud/ticloud-auth-whoami.md @@ -0,0 +1,40 @@ +--- +title: ticloud auth whoami +summary: The reference of `ticloud auth whoami`. +--- + +# ticloud auth whoami + +Display information about the current user: + +```shell +ticloud auth whoami [flags] +``` + +## Examples + +To display information about the current user: + +```shell +ticloud auth whoami +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|------------|------------------------------------------|----------|------------------------------------------------------| +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-branch-connect-info.md b/tidb-cloud/ticloud-branch-connect-info.md deleted file mode 100644 index f2aac9ad26e80..0000000000000 --- a/tidb-cloud/ticloud-branch-connect-info.md +++ /dev/null @@ -1,49 +0,0 @@ ---- -title: ticloud branch connect-info -summary: The reference of `ticloud branch connect-info`. ---- - -# ticloud branch connect-info - -Get the connection string of a branch: - -```shell -ticloud branch connect-info [flags] -``` - -## Examples - -Get the connection string of a branch in interactive mode: - -```shell -ticloud branch connect-info -``` - -Get the connection string of a branch in non-interactive mode: - -```shell -ticloud branch connect-info --branch-id --cluster-id --client --operating-system -``` - -## Flags - -In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. - -| Flag | Description | Required | Note | -|---------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| -| -c, --cluster-id string | The ID of the cluster, in which the branch is created | Yes | Only works in non-interactive mode. | -| -b, --branch-id string | The ID of the branch | Yes | Only works in non-interactive mode. | -| --client string | The desired client used for the connection. Supported clients include `general`, `mysql_cli`, `mycli`, `libmysqlclient`, `python_mysqlclient`, `pymysql`, `mysql_connector_python`, `mysql_connector_java`, `go_mysql_driver`, `node_mysql2`, `ruby_mysql2`, `php_mysqli`, `rust_mysql`, `mybatis`, `hibernate`, `spring_boot`, `gorm`, `prisma`, `sequelize_mysql2`, `django_tidb`, `sqlalchemy_mysqlclient`, and `active_record`. | Yes | Only works in non-interactive mode. | -| --operating-system string | The operating system name. Supported operating systems include `macOS`, `Windows`, `Ubuntu`, `CentOS`, `RedHat`, `Fedora`, `Debian`, `Arch`, `OpenSUSE`, `Alpine`, and `Others`. | Yes | Only works in non-interactive mode. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | - -## Inherited flags - -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------------------|----------|-------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | - -## Feedback - -If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-branch-create.md b/tidb-cloud/ticloud-branch-create.md index d73a7dc20853b..5caca96a9f52d 100644 --- a/tidb-cloud/ticloud-branch-create.md +++ b/tidb-cloud/ticloud-branch-create.md @@ -1,50 +1,55 @@ --- -title: ticloud branch create -summary: The reference of `ticloud branch create`. +title: ticloud serverless branch create +summary: The reference of `ticloud serverless branch create`. --- -# ticloud branch create +# ticloud serverless branch create -Create a branch for a cluster: +Create a [branch](/tidb-cloud/branch-overview.md) for a TiDB Cloud Serverless cluster: ```shell -ticloud branch create [flags] +ticloud serverless branch create [flags] ``` -> **Note:** -> -> Currently, you can only create branches for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster. - ## Examples -Create a branch in interactive mode: +Create a branch for a TiDB Cloud Serverless cluster in interactive mode: + +```shell +ticloud serverless branch create +``` + +Create a branch for a TiDB Cloud Serverless cluster in non-interactive mode: ```shell -ticloud branch create +ticloud serverless branch create --cluster-id --display-name ``` -Create a branch in non-interactive mode: +Create a branch from another branch with a specified timestamp in non-interactive mode: ```shell -ticloud branch create --cluster-id --branch-name +ticloud serverless branch create --cluster-id --display-name --parent-id --parent-timestamp ``` ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|-------------------------|------------------------------------------------------------|----------|-----------------------------------------------------| -| -c, --cluster-id string | The ID of the cluster, in which the branch will be created | Yes | Only works in non-interactive mode. | -| --branch-name string | The name of the branch to be created | Yes | Only works in non-interactive mode. | -| -h, --help | Get help information for this command | No | Works in both non-interactive and interactive modes | +| Flag | Description | Required | Note | +|---------------------------|-----------------------------------------------------------------------------------------------------------|----------|-----------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster, in which the branch will be created. | Yes | Only works in non-interactive mode. | +| -n, --display-name string | Specifies the name of the branch to be created. | Yes | Only works in non-interactive mode. | +| --parent-id string | Specifies the ID of the branch parent. The default value is the cluster ID. | No | Only works in non-interactive mode. | +| --parent-timestamp string | Specifies the timestamp of the parent branch in RFC3339 format, such as `2024-01-01T00:00:00Z`. The default value is the current time. | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|------------------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-branch-delete.md b/tidb-cloud/ticloud-branch-delete.md index 678d7e64bf770..b11209b7f9293 100644 --- a/tidb-cloud/ticloud-branch-delete.md +++ b/tidb-cloud/ticloud-branch-delete.md @@ -1,31 +1,31 @@ --- -title: ticloud branch delete -summary: The reference of `ticloud branch delete`. +title: ticloud serverless branch delete +summary: The reference of `ticloud serverless branch delete`. --- -# ticloud branch delete +# ticloud serverless branch delete -Delete a branch from your cluster: +Delete a branch from your TiDB Cloud Serverless cluster: ```shell -ticloud branch delete [flags] +ticloud serverless branch delete [flags] ``` Or use the following alias command: ```shell -ticloud branch rm [flags] +ticloud serverless branch rm [flags] ``` ## Examples -Delete a branch in interactive mode: +Delete a TiDB Cloud Serverless branch in interactive mode: ```shell -ticloud branch delete +ticloud serverless branch delete ``` -Delete a branch in non-interactive mode: +Delete a TiDB Cloud Serverless branch in non-interactive mode: ```shell ticloud branch delete --branch-id --cluster-id @@ -37,17 +37,18 @@ In non-interactive mode, you need to manually enter the required flags. In inter | Flag | Description | Required | Note | |-------------------------|--------------------------------------------|----------|------------------------------------------------------| -| -b, --branch-id string | The ID of the branch to be deleted | Yes | Only works in non-interactive mode. | -| --force | Deletes a branch without confirmation | No | Works in both non-interactive and interactive modes. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| -c, --cluster-id string | The cluster ID of the branch to be deleted | Yes | Only works in non-interactive mode. | +| -b, --branch-id string | Specifies the ID of the branch to be deleted. | Yes | Only works in non-interactive mode. | +| --force | Deletes a branch without confirmation. | No | Works in both non-interactive and interactive modes. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|--------------------------------------------------------------------------------------------|----------|-------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | The active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|--------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-branch-describe.md b/tidb-cloud/ticloud-branch-describe.md index c5d8299a42c86..9074460c63fac 100644 --- a/tidb-cloud/ticloud-branch-describe.md +++ b/tidb-cloud/ticloud-branch-describe.md @@ -1,34 +1,34 @@ --- -title: ticloud branch describe -summary: The reference of `ticloud branch describe`. +title: ticloud serverless branch describe +summary: The reference of `ticloud serverless branch describe`. --- -# ticloud branch describe +# ticloud serverless branch describe Get information about a branch (such as the endpoints, [user name prefix](/tidb-cloud/select-cluster-tier.md#user-name-prefix), and usages): ```shell -ticloud branch describe [flags] +ticloud serverless branch describe [flags] ``` Or use the following alias command: ```shell -ticloud branch get [flags] +ticloud serverless branch get [flags] ``` ## Examples -Get the branch information in interactive mode: +Get branch information of a TiDB Cloud Serverless cluster in interactive mode: ```shell -ticloud branch describe +ticloud serverless branch describe ``` -Get the branch information in non-interactive mode: +Get branch information of a TiDB Cloud Serverless cluster in non-interactive mode: ```shell -ticloud branch describe --branch-id --cluster-id +ticloud serverless branch describe --branch-id --cluster-id ``` ## Flags @@ -37,16 +37,17 @@ In non-interactive mode, you need to manually enter the required flags. In inter | Flag | Description | Required | Note | |-------------------------|-----------------------------------|----------|------------------------------------------------------| -| -b, --branch-id string | The ID of the branch | Yes | Only works in non-interactive mode. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| -c, --cluster-id string | The cluster ID of the branch | Yes | Only works in non-interactive mode. | +| -b, --branch-id string | Specifies the ID of the branch. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|------------------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-branch-list.md b/tidb-cloud/ticloud-branch-list.md index a3cacf5d665d5..805ffa5a7d574 100644 --- a/tidb-cloud/ticloud-branch-list.md +++ b/tidb-cloud/ticloud-branch-list.md @@ -1,65 +1,59 @@ --- -title: ticloud branch list -summary: The reference of `ticloud branch list`. +title: ticloud serverless branch list +summary: The reference of `ticloud serverless branch list`. --- -# ticloud branch list +# ticloud serverless branch list -List all branches for a cluster: +List all branches for a TiDB Cloud Serverless cluster: ```shell -ticloud branch list [flags] +ticloud serverless branch list [flags] ``` Or use the following alias command: ```shell -ticloud branch ls [flags] +ticloud serverless branch ls [flags] ``` ## Examples -List all branches for a cluster (interactive mode): +List all branches for a TiDB Cloud Serverless cluster in interactive mode: ```shell -ticloud branch list +ticloud serverless branch list ``` -List all branches for a specified cluster (non-interactive mode): +List all branches for a specific TiDB Cloud Serverless cluster in non-interactive mode: ```shell -ticloud branch list +ticloud serverless branch list -c ``` -List all branches for a specified cluster in the JSON format: +List all branches for a specific TiDB Cloud Serverless cluster in the JSON format: ```shell -ticloud branch list -o json +ticloud serverless branch list -o json ``` -## Arguments - -The `branch list` command has the following arguments: - -| Argument Index | Description | Required | Note | -|----------------|-----------------------------------------------------|----------|---------------------------------------| -| `` | The cluster ID of the branches which will be listed | Yes | Only works in non-interactive mode. | - ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|---------------------|--------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| -o, --output string | Output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|-------------------------|--------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| -o, --output string | Specifies the output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|------------------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-branch-shell.md b/tidb-cloud/ticloud-branch-shell.md new file mode 100644 index 0000000000000..835cd99f440fe --- /dev/null +++ b/tidb-cloud/ticloud-branch-shell.md @@ -0,0 +1,63 @@ +--- +title: ticloud serverless branch shell +summary: The reference of `ticloud serverless branch shell`. +aliases: ['/tidbcloud/ticloud-connect'] +--- + +# ticloud serverless branch shell + +Connect to a branch of a TiDB Cloud Serverless cluster: + +```shell +ticloud serverless branch shell [flags] +``` + +## Examples + +Connect to a TiDB Cloud Serverless branch in interactive mode: + +```shell +ticloud serverless branch shell +``` + +Connect to a TiDB Cloud Serverless branch with the default user in non-interactive mode: + +```shell +ticloud serverless branch shell -c -b +``` + +Connect to a TiDB Cloud Serverless branch with the default user and password in non-interactive mode: + +```shell +ticloud serverless branch shell -c -b --password +``` + +Connect to a TiDB Cloud Serverless branch with a specific user and password in non-interactive mode: + +```shell +ticloud serverless branch shell -c -b -u --password +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|-----------------------------------|----------|------------------------------------------------------| +| -b, --branch-id string | Specifies the ID of the branch. | Yes | Only works in non-interactive mode. | +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| --password | Specifies the password of the user. | No | Only works in non-interactive mode. | +| -u, --user string | Specifies the user for login. | No | Only works in non-interactive mode. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. \ No newline at end of file diff --git a/tidb-cloud/ticloud-cluster-connect-info.md b/tidb-cloud/ticloud-cluster-connect-info.md deleted file mode 100644 index 0e2c4ec1e28e1..0000000000000 --- a/tidb-cloud/ticloud-cluster-connect-info.md +++ /dev/null @@ -1,53 +0,0 @@ ---- -title: ticloud cluster connect-info -summary: The reference of `ticloud cluster connect-info`. ---- - -# ticloud cluster connect-info - -Get the connection string of a cluster: - -```shell -ticloud cluster connect-info [flags] -``` - -> **Note:** -> -> Currently, this command only supports getting the connection string of a [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster. - -## Examples - -Get the connection string of a cluster in interactive mode: - -```shell -ticloud cluster connect-info -``` - -Get the connection string of a cluster in non-interactive mode: - -```shell -ticloud cluster connect-info --project-id --cluster-id --client --operating-system -``` - -## Flags - -In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. - -| Flag | Description | Required | Note | -|----------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| -| -p, --project-id string | The ID of the project, in which the cluster is created | Yes | Only works in non-interactive mode. | -| -c, --cluster-id string | The ID of the cluster | Yes | Only works in non-interactive mode. | -| --client string | The desired client used for the connection. Supported clients include `general`, `mysql_cli`, `mycli`, `libmysqlclient`, `python_mysqlclient`, `pymysql`, `mysql_connector_python`, `mysql_connector_java`, `go_mysql_driver`, `node_mysql2`, `ruby_mysql2`, `php_mysqli`, `rust_mysql`, `mybatis`, `hibernate`, `spring_boot`, `gorm`, `prisma`, `sequelize_mysql2`, `django_tidb`, `sqlalchemy_mysqlclient`, and `active_record`. | Yes | Only works in non-interactive mode. | -| --operating-system string | The operating system name. Supported operating systems include `macOS`, `Windows`, `Ubuntu`, `CentOS`, `RedHat`, `Fedora`, `Debian`, `Arch`, `OpenSUSE`, `Alpine`, and `Others`. | Yes | Only works in non-interactive mode. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | - -## Inherited flags - -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------------------|----------|-------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | - -## Feedback - -If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-cluster-create.md b/tidb-cloud/ticloud-cluster-create.md index f9dbd9ba85b10..d295836efe330 100644 --- a/tidb-cloud/ticloud-cluster-create.md +++ b/tidb-cloud/ticloud-cluster-create.md @@ -1,54 +1,57 @@ --- -title: ticloud cluster create -summary: The reference of `ticloud cluster create`. +title: ticloud serverless create +summary: The reference of `ticloud serverless create`. --- -# ticloud cluster create +# ticloud serverless create -Create a cluster: +Create a TiDB Cloud Serverless cluster: ```shell -ticloud cluster create [flags] +ticloud serverless create [flags] ``` -> **Note:** -> -> Currently, you can only create a [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster using the preceding command. - ## Examples -Create a cluster in interactive mode: +Create a TiDB Cloud Serverless cluster in interactive mode: ```shell -ticloud cluster create +ticloud serverless create ``` -Create a cluster in non-interactive mode: +Create a TiDB Cloud Serverless cluster in non-interactive mode: ```shell -ticloud cluster create --project-id --cluster-name --cloud-provider --region --root-password --cluster-type +ticloud serverless create --display-name --region ``` +Create a TiDB Cloud Serverless cluster with a spending limit in non-interactive mode: + +```shell +ticloud serverless create --display-name --region --spending-limit-monthly +``` + ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|-------------------------|-------------------------------------------------------------|----------|-----------------------------------| -| --cloud-provider string | Cloud provider (Currently, only `AWS` is supported) | Yes | Only works in non-interactive mode. | -| --cluster-name string | Name of the cluster to be created | Yes | Only works in non-interactive mode. | -| --cluster-type string | Cluster type (Currently, only `SERVERLESS` is supported) | Yes | Only works in non-interactive mode. | -| -h, --help | Get help information for this command | No | Works in both non-interactive and interactive modes | -| -p, --project-id string | The ID of the project, in which the cluster will be created | Yes | Only works in non-interactive mode. | -| -r, --region string | Cloud region | Yes | Only works in non-interactive mode. | -| --root-password string | The root password of the cluster | Yes | Only works in non-interactive mode. | +| Flag | Description | Required | Note | +|------------------------------|----------------------------------------------------------------------------------------------------------------|----------|-----------------------------------------------------| +| -n --display-name string | Specifies the name of the cluster to be created. | Yes | Only works in non-interactive mode. | +| --spending-limit-monthly int | Specifies the maximum monthly spending limit in USD cents. | No | Only works in non-interactive mode. | +| -p, --project-id string | Specifies the ID of the project, in which the cluster will be created. The default value is `default project`. | No | Only works in non-interactive mode. | +| -r, --region string | Specifies the name of cloud region. You can use "ticloud serverless region" to see all regions. | Yes | Only works in non-interactive mode. | +| --disable-public-endpoint | Disables the public endpoint. | No | Only works in non-interactive mode. | +| --encryption | Enables enhanced encryption at rest. | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-cluster-delete.md b/tidb-cloud/ticloud-cluster-delete.md index 080e92df19a11..3bbb3342cdd77 100644 --- a/tidb-cloud/ticloud-cluster-delete.md +++ b/tidb-cloud/ticloud-cluster-delete.md @@ -1,53 +1,53 @@ --- -title: ticloud cluster delete -summary: The reference of `ticloud cluster delete`. +title: ticloud serverless cluster delete +summary: The reference of `ticloud serverless delete`. --- -# ticloud cluster delete +# ticloud serverless delete -Delete a cluster from your project: +Delete a TiDB Cloud Serverless cluster from your project: ```shell -ticloud cluster delete [flags] +ticloud serverless delete [flags] ``` Or use the following alias command: ```shell -ticloud cluster rm [flags] +ticloud serverless rm [flags] ``` ## Examples -Delete a cluster in interactive mode: +Delete a TiDB Cloud Serverless cluster in interactive mode: ```shell -ticloud cluster delete +ticloud serverless delete ``` -Delete a cluster in non-interactive mode: +Delete a TiDB Cloud Serverless cluster in non-interactive mode: ```shell -ticloud cluster delete --project-id --cluster-id +ticloud serverless delete --cluster-id ``` ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|-------------------------|---------------------------------------------|----------|-----------------------------------------------------| -| -c, --cluster-id string | The ID of the cluster to be deleted | Yes | Only works in non-interactive mode. | -| --force | Deletes a cluster without confirmation | No | Works in both non-interactive and interactive modes. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| -p, --project-id string | The project ID of the cluster to be deleted | Yes | Only works in non-interactive mode. | +| Flag | Description | Required | Note | +|-------------------------|----------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster to be deleted. | Yes | Only works in non-interactive mode. | +| --force | Deletes a cluster without confirmation. | No | Works in both non-interactive and interactive modes. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| +| Flag | Description | Required | Note | +|----------------------|--------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | The active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-cluster-describe.md b/tidb-cloud/ticloud-cluster-describe.md index 2215ecccb1264..7f23d0efdba06 100644 --- a/tidb-cloud/ticloud-cluster-describe.md +++ b/tidb-cloud/ticloud-cluster-describe.md @@ -1,52 +1,52 @@ --- -title: ticloud cluster describe -summary: The reference of `ticloud cluster describe`. +title: ticloud serverless cluster describe +summary: The reference of `ticloud serverless describe`. --- -# ticloud cluster describe +# ticloud serverless describe -Get information about a cluster (such as the cloud provider, cluster type, cluster configurations, and cluster status): +Get information about a TiDB Cloud Serverless cluster (such as the cluster configurations and cluster status): ```shell -ticloud cluster describe [flags] +ticloud serverless describe [flags] ``` Or use the following alias command: ```shell -ticloud cluster get [flags] +ticloud serverless get [flags] ``` ## Examples -Get the cluster information in interactive mode: +Get information about a TiDB Cloud Serverless cluster in interactive mode: ```shell -ticloud cluster describe +ticloud serverless describe ``` -Get the cluster information in non-interactive mode: +Get information about a TiDB Cloud Serverless cluster in non-interactive mode: ```shell -ticloud cluster describe --project-id --cluster-id +ticloud serverless describe --cluster-id ``` ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|-------------------------|-------------------------------|----------|-----------------------------------| -| -c, --cluster-id string | The ID of the cluster | Yes | Only works in non-interactive mode. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| -p, --project-id string | The project ID of the cluster | Yes | Only works in non-interactive mode. | +| Flag | Description | Required | Note | +|-------------------------|-----------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-cluster-list.md b/tidb-cloud/ticloud-cluster-list.md index a8d05cea36a75..50092432c309c 100644 --- a/tidb-cloud/ticloud-cluster-list.md +++ b/tidb-cloud/ticloud-cluster-list.md @@ -1,57 +1,59 @@ --- -title: ticloud cluster list -summary: The reference of `ticloud cluster list`. +title: ticloud serverless cluster list +summary: The reference of `ticloud serverless list`. --- -# ticloud cluster list +# ticloud serverless list -List all clusters in a project: +List all TiDB Cloud Serverless clusters in a project: ```shell -ticloud cluster list [flags] +ticloud serverless list [flags] ``` Or use the following alias command: ```shell -ticloud cluster ls [flags] +ticloud serverless ls [flags] ``` ## Examples -List all clusters in a project (interactive mode): +List all TiDB Cloud Serverless clusters in interactive mode: ```shell -ticloud cluster list +ticloud serverless list ``` -List all clusters in a specified project (non-interactive mode): +List all TiDB Cloud Serverless clusters in a specified project in non-interactive mode: ```shell -ticloud cluster list +ticloud serverless list -p ``` -List all clusters in a specified project in the JSON format: +List all TiDB Cloud Serverless clusters in a specified project with the JSON format in non-interactive mode: ```shell -ticloud cluster list -o json +ticloud serverless list -p -o json ``` ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|---------------------|--------------------------------------------------------------------------------------------------------|----------|-----------------------------------------------------| -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| -o, --output string | Output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|-------------------------|--------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -p, --project-id string | Specifies the ID of the project. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| -o, --output string | Specifies the output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-completion.md b/tidb-cloud/ticloud-completion.md new file mode 100644 index 0000000000000..2c563df51d004 --- /dev/null +++ b/tidb-cloud/ticloud-completion.md @@ -0,0 +1,46 @@ +--- +title: ticloud completion +summary: The reference of `ticloud completion`. +--- + +# ticloud completion + +Generate the autocompletion script for the specified shell of TiDB Cloud CLI: + +```shell +ticloud completion [command] +``` + +## Examples + +Generate the autocompletion script for bash: + +```shell +ticloud completion bash +``` + +Generate the autocompletion script for zsh: + +```shell +ticloud completion zsh +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|---------------------------------------------------------------|----------|------------------------------------------------------| +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. \ No newline at end of file diff --git a/tidb-cloud/ticloud-config-create.md b/tidb-cloud/ticloud-config-create.md index c5d2f566984ce..74d7a64720323 100644 --- a/tidb-cloud/ticloud-config-create.md +++ b/tidb-cloud/ticloud-config-create.md @@ -35,10 +35,10 @@ In non-interactive mode, you need to manually enter the required flags. In inter | Flag | Description | Required | Note | |-----------------------|-----------------------------------------------|----------|-----------------------------------| -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| --private-key string | The private key of the TiDB Cloud API | Yes | Only works in non-interactive mode. | -| --profile-name string | The name of the profile, which must not contain `.` | Yes | Only works in non-interactive mode. | -| --public-key string | The public key of the TiDB Cloud API | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| --private-key string | Specifies the private key of the TiDB Cloud API. | Yes | Only works in non-interactive mode. | +| --profile-name string | Specifies the name of the profile (which must not contain `.`). | Yes | Only works in non-interactive mode. | +| --public-key string | Specifies the public key of the TiDB Cloud API. | Yes | Only works in non-interactive mode. | ## Inherited flags @@ -46,6 +46,7 @@ In non-interactive mode, you need to manually enter the required flags. In inter |----------------------|----------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | | -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-config-delete.md b/tidb-cloud/ticloud-config-delete.md index 6e039e49c0075..e9e35fb5759ba 100644 --- a/tidb-cloud/ticloud-config-delete.md +++ b/tidb-cloud/ticloud-config-delete.md @@ -29,8 +29,8 @@ ticloud config delete | Flag | Description | |------------|---------------------------------------| -| --force | Deletes a profile without confirmation | -| -h, --help | Help information for this command | +| --force | Deletes a profile without confirmation. | +| -h, --help | Shows help information for this command. | ## Inherited flags @@ -38,6 +38,7 @@ ticloud config delete |----------------------|-----------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | | -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-config-describe.md b/tidb-cloud/ticloud-config-describe.md index acdf34e102a6b..802421ce671a9 100644 --- a/tidb-cloud/ticloud-config-describe.md +++ b/tidb-cloud/ticloud-config-describe.md @@ -29,7 +29,7 @@ ticloud config describe | Flag | Description | |------------|--------------------------| -| -h, --help | Help information for this command | +| -h, --help | Shows help information for this command. | ## Inherited flags @@ -37,6 +37,7 @@ ticloud config describe |----------------------|-----------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | | -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-config-edit.md b/tidb-cloud/ticloud-config-edit.md index 6fb82659297b8..be5a2c24962ac 100644 --- a/tidb-cloud/ticloud-config-edit.md +++ b/tidb-cloud/ticloud-config-edit.md @@ -15,7 +15,7 @@ If you are using Windows, after you execute the preceding command, the path of t > **Note:** > -> To avoid format errors and execution failures, it is NOT recommended to manually edit the configuration file. Instead, you can use [`ticloud config create`](/tidb-cloud/ticloud-config-create.md), [`ticloud config delete`](/tidb-cloud/ticloud-config-delete.md), or [`ticloud config set`](/tidb-cloud/ticloud-config-set.md) to modify the confiturations. +> To avoid format errors and execution failures, it is NOT recommended to manually edit the configuration file. Instead, you can use [`ticloud config create`](/tidb-cloud/ticloud-config-create.md), [`ticloud config delete`](/tidb-cloud/ticloud-config-delete.md), or [`ticloud config set`](/tidb-cloud/ticloud-config-set.md) to modify the configurations. ## Examples @@ -29,7 +29,7 @@ ticloud config edit | Flag | Description | |------------|--------------------------| - | -h, --help | Help information for this command | + | -h, --help | Shows help information for this command. | ## Inherited flags @@ -37,6 +37,7 @@ ticloud config edit |----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | | -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-config-list.md b/tidb-cloud/ticloud-config-list.md index 766f7c8e99f2d..549005ed9fcea 100644 --- a/tidb-cloud/ticloud-config-list.md +++ b/tidb-cloud/ticloud-config-list.md @@ -29,7 +29,7 @@ ticloud config list | Flag | Description | |------------|--------------------------| -| -h, --help | Help information for this command | +| -h, --help | Shows help information for this command. | ## Inherited flags @@ -37,6 +37,7 @@ ticloud config list |----------------------|-----------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | | -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-config-set.md b/tidb-cloud/ticloud-config-set.md index 8059c79e9fcc7..bd8d5f8227221 100644 --- a/tidb-cloud/ticloud-config-set.md +++ b/tidb-cloud/ticloud-config-set.md @@ -15,11 +15,11 @@ The properties that can be configured include `public-key`, `private-key`, and ` | Properties | Description | Required | |-------------|--------------------------------------------------------------------|----------| -| public-key | The public key of the TiDB Cloud API | Yes | -| private-key | The private key of the TiDB Cloud API | Yes | -| api-url | The base API URL of TiDB Cloud (`https://api.tidbcloud.com` by default) | No | +| public-key | Specifies the public key of the TiDB Cloud API. | Yes | +| private-key | Specifies the private key of the TiDB Cloud API. | Yes | +| api-url | Specifies the base API URL of TiDB Cloud (`https://api.tidbcloud.com` by default). | No | -> **Notes:** +> **Note:** > > If you want to configure properties for a specific user profile, you can add the `-P` flag and specify the target user profile name in the command. @@ -51,7 +51,7 @@ ticloud config set api-url https://api.tidbcloud.com | Flag | Description | |------------|--------------------------| -| -h, --help | Help information for this command | +| -h, --help | Shows help information for this command. | ## Inherited flags @@ -59,6 +59,7 @@ ticloud config set api-url https://api.tidbcloud.com |----------------------|-----------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | | -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-config-use.md b/tidb-cloud/ticloud-config-use.md index d73b9c76860f1..197735abface4 100644 --- a/tidb-cloud/ticloud-config-use.md +++ b/tidb-cloud/ticloud-config-use.md @@ -23,7 +23,7 @@ ticloud config use test | Flag | Description | |------------|--------------------------| -| -h, --help | Help information for this command | +| -h, --help | Shows help information for this command. | ## Inherited flags @@ -31,6 +31,7 @@ ticloud config use test |----------------------|-----------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | | -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-connect.md b/tidb-cloud/ticloud-connect.md deleted file mode 100644 index 93a631f3f7eb7..0000000000000 --- a/tidb-cloud/ticloud-connect.md +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: ticloud connect -summary: The reference of `ticloud connect`. ---- - -# ticloud connect - -Connect to a TiDB Cloud cluster or branch: - -```shell -ticloud connect [flags] -``` - -> **Note:** -> -> - If you are prompted about whether to use the default user, you can choose `Y` to use the default root user or choose `n` to specify another user. For [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters, the name of the default root user has a [prefix](/tidb-cloud/select-cluster-tier.md#user-name-prefix) such as `3pTAoNNegb47Uc8`. -> - The connection forces the [ANSI SQL mode](https://dev.mysql.com/doc/refman/8.0/en/sql-mode.html#sqlmode_ansi) for the session. To exit the session, enter `\q`. - -## Examples - -Connect to a TiDB Cloud cluster or branch in interactive mode: - -```shell -ticloud connect -``` - -Use the default user to connect to a TiDB Cloud cluster or branch in non-interactive mode: - -```shell -ticloud connect -p -c -ticloud connect -p -c -b -``` - -Use the default user to connect to the TiDB Cloud cluster or branch with password in non-interactive mode: - -```shell -ticloud connect -p -c --password -ticloud connect -p -c -b --password -``` - -Use a specific user to connect to the TiDB Cloud cluster or branch in non-interactive mode: - -```shell -ticloud connect -p -c -u -ticloud connect -p -c -b -u -``` - -## Flags - -In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. - -| Flag | Description | Required | Note | -|-------------------------|-----------------------------------|----------|------------------------------------------------------| -| -c, --cluster-id string | Cluster ID | Yes | Only works in non-interactive mode. | -| -b, --branch-id string | Branch ID | No | Only works in non-interactive mode. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| --password | The password of the user | No | Only works in non-interactive mode. | -| -p, --project-id string | Project ID | Yes | Only works in non-interactive mode. | -| -u, --user string | A specific user for login | No | Only works in non-interactive mode. | - -## Inherited flags - -| Flag | Description | Required | Note | -|----------------------|------------------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | - -## Feedback - -If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-help.md b/tidb-cloud/ticloud-help.md new file mode 100644 index 0000000000000..11aa4732d3e6d --- /dev/null +++ b/tidb-cloud/ticloud-help.md @@ -0,0 +1,46 @@ +--- +title: ticloud help +summary: The reference of `ticloud help`. +--- + +# ticloud help + +Get help information for any command in TiDB Cloud CLI: + +```shell +ticloud help [command] [flags] +``` + +## Examples + +To get help for the `auth` command: + +```shell +ticloud help auth +``` + +To get help for the `serverless create` command: + +```shell +ticloud help serverless create +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|---------------------------------------------------------------|----------|------------------------------------------------------| +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. \ No newline at end of file diff --git a/tidb-cloud/ticloud-import-cancel.md b/tidb-cloud/ticloud-import-cancel.md index bb42e358db4c7..8ce763008620d 100644 --- a/tidb-cloud/ticloud-import-cancel.md +++ b/tidb-cloud/ticloud-import-cancel.md @@ -1,14 +1,14 @@ --- -title: ticloud import cancel -summary: The reference of `ticloud import cancel`. +title: ticloud serverless import cancel +summary: The reference of `ticloud serverless import cancel`. --- -# ticloud import cancel +# ticloud serverless import cancel Cancel a data import task: ```shell -ticloud import cancel [flags] +ticloud serverless import cancel [flags] ``` ## Examples @@ -16,26 +16,25 @@ ticloud import cancel [flags] Cancel an import task in interactive mode: ```shell -ticloud import cancel +ticloud serverless import cancel ``` Cancel an import task in non-interactive mode: ```shell -ticloud import cancel --project-id --cluster-id --import-id +ticloud serverless import cancel --cluster-id --import-id ``` ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|-------------------------|---------------------------------------|----------|-----------------------------------------------------| -| -c, --cluster-id string | Cluster ID | Yes | Only works in non-interactive mode. | -| --force | Deletes a profile without confirmation | No | Works in both non-interactive and interactive modes. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| --import-id string | The ID of the import task | Yes | Only works in non-interactive mode. | -| -p, --project-id string | Project ID | Yes | Only works in non-interactive mode. | +| Flag | Description | Required | Note | +|-------------------------|----------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| --force | Cancels an import task without confirmation. | No | Works in both non-interactive and interactive modes. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| --import-id string | Specifies the ID of the import task. | Yes | Only works in non-interactive mode. | ## Inherited flags @@ -43,6 +42,7 @@ In non-interactive mode, you need to manually enter the required flags. In inter |----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| | --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | | -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-import-describe.md b/tidb-cloud/ticloud-import-describe.md index c01062e039c2f..bcbf64fcdb025 100644 --- a/tidb-cloud/ticloud-import-describe.md +++ b/tidb-cloud/ticloud-import-describe.md @@ -1,20 +1,20 @@ --- -title: ticloud import describe -summary: The reference of `ticloud import describe`. +title: ticloud serverless import describe +summary: The reference of `ticloud serverless import describe`. --- -# ticloud import describe +# ticloud serverless import describe -Get the import details of a data import task: +Describe a data import task: ```shell -ticloud import describe [flags] +ticloud serverless import describe [flags] ``` Or use the following alias command: ```shell -ticloud import get [flags] +ticloud serverless import get [flags] ``` ## Examples @@ -22,32 +22,32 @@ ticloud import get [flags] Describe an import task in interactive mode: ```shell -ticloud import describe +ticloud serverless import describe ``` Describe an import task in non-interactive mode: ```shell -ticloud import describe --project-id --cluster-id --import-id +ticloud serverless import describe --cluster-id --import-id ``` ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|-------------------------|--------------------------|----------|-----------------------------------| -| -c, --cluster-id string | Cluster ID | Yes | Only works in non-interactive mode. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| --import-id string | The ID of the import task | Yes | Only works in non-interactive mode. | -| -p, --project-id string | Project ID | Yes | Only works in non-interactive mode. | +| Flag | Description | Required | Note | +|-------------------------|-----------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| --import-id string | Specifies the ID of the import task. | Yes | Only works in non-interactive mode. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-import-list.md b/tidb-cloud/ticloud-import-list.md index d02d5c26b6798..55cd733c718d8 100644 --- a/tidb-cloud/ticloud-import-list.md +++ b/tidb-cloud/ticloud-import-list.md @@ -1,20 +1,20 @@ --- -title: ticloud import list -summary: The reference of `ticloud import list`. +title: ticloud serverless import list +summary: The reference of `ticloud serverless import list`. --- -# ticloud import list +# ticloud serverless import list List data import tasks: ```shell -ticloud import list [flags] +ticloud serverless import list [flags] ``` Or use the following alias command: ```shell -ticloud import ls [flags] +ticloud serverless import ls [flags] ``` ## Examples @@ -22,38 +22,38 @@ ticloud import ls [flags] List import tasks in interactive mode: ```shell -ticloud import list +ticloud serverless import list ``` List import tasks in non-interactive mode: ```shell -ticloud import list --project-id --cluster-id +ticloud serverless import list --cluster-id ``` List import tasks for a specified cluster in the JSON format: ```shell -ticloud import list --project-id --cluster-id --output json +ticloud serverless import list --cluster-id --output json ``` ## Flags In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|-------------------------|--------------------------------------------------------------------------------------------------------|----------|-----------------------------------------------------| -| -c, --cluster-id string | Cluster ID | Yes | Only works in non-interactive mode. | -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| -o, --output string | Output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | -| -p, --project-id string | Project ID | Yes | Only works in non-interactive mode. | +| Flag | Description | Required | Note | +|-------------------------|--------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| -o, --output string | Specifies the output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-import-start-local.md b/tidb-cloud/ticloud-import-start-local.md deleted file mode 100644 index c3576a4f901ae..0000000000000 --- a/tidb-cloud/ticloud-import-start-local.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -title: ticloud import start local -summary: The reference of `ticloud import start local`. ---- - -# ticloud import start local - -Import a local file to a [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster: - -```shell -ticloud import start local [flags] -``` - -> **Note:** -> -> Currently, you can only import one CSV file for one import task. - -## Examples - -Start an import task in interactive mode: - -```shell -ticloud import start local -``` - -Start an import task in non-interactive mode: - -```shell -ticloud import start local --project-id --cluster-id --data-format --target-database --target-table -``` - -Start an import task with a custom CSV format: - -```shell -ticloud import start local --project-id --cluster-id --data-format CSV --target-database --target-table --separator \" --delimiter \' --backslash-escape=false --trim-last-separator=true -``` - -## Flags - -In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. - -| Flag | Description | Required | Note | -|---|---|---|---| -| --backslash-escape | Whether to parse backslashes inside fields as escape characters for CSV files. The default value is `true`. | No | Only works in non-interactive mode when `--data-format CSV` is specified. | -| -c, --cluster-id string | Specifies the cluster ID. | Yes | Only works in non-interactive mode. | -| --data-format string | Specifies the data format. Currently, only `CSV` is supported. | Yes | Only works in non-interactive mode. | -| --delimiter string | Specifies the delimiter used for quoting for CSV files. The default value is `"`. | No | Only works in non-interactive mode when `--data-format CSV` is specified. | -| -h, --help | Displays help information for this command. | No | Works in both non-interactive and interactive modes. | -| -p, --project-id string | Specifies the project ID. | Yes | Only works in non-interactive mode. | -| --separator string | Specifies the field separator for CSV files. The default value is `,`. | No | Only works in non-interactive mode when `--data-format CSV` is specified. | -| --target-database string | Specifies the target database to import data to. | Yes | Only works in non-interactive mode. | -| --target-table string | Specifies the target table to import data to. | Yes | Only works in non-interactive mode. | -| --trim-last-separator | Whether to treat separators as line terminators and trim all trailing separators for CSV files. The default value is `false`. | No | Only works in non-interactive mode when `--data-format CSV` is specified. | - -## Inherited flags - -| Flag | Description | Required | Note | -|---|---|---|---| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | - -## Feedback - -If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-import-start-mysql.md b/tidb-cloud/ticloud-import-start-mysql.md deleted file mode 100644 index 3c67cc007d95a..0000000000000 --- a/tidb-cloud/ticloud-import-start-mysql.md +++ /dev/null @@ -1,79 +0,0 @@ ---- -title: ticloud import start mysql -summary: The reference of `ticloud import start mysql`. ---- - -# ticloud import start mysql - -Import a table from a MySQL-compatible database to a [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster: - -```shell -ticloud import start mysql [flags] -``` - -> **Note:** -> -> - Before running this command, make sure that you have installed the MySQL command-line tool first. For more details, see [Installation](/tidb-cloud/get-started-with-cli.md#installation). -> - If the target table already exists in the target database, to use this command for table import, make sure that the target table name is the same as the source table name and add the `skip-create-table` flag in the command. -> - If the target table does not exist in the target database, executing this command automatically creates a table with the same name as the source table in the target database. - -## Examples - -- Start an import task in interactive mode: - - ```shell - ticloud import start mysql - ``` - -- Start an import task in non-interactive mode (using the TiDB Serverless cluster default user `.root`): - - ```shell - ticloud import start mysql --project-id --cluster-id --source-host --source-port --source-user --source-password --source-database --source-table --target-database --target-password - ``` - -- Start an import task in non-interactive mode (using a specific user): - - ```shell - ticloud import start mysql --project-id --cluster-id --source-host --source-port --source-user --source-password --source-database --source-table --target-database --target-password --target-user - ``` - -- Start an import task that skips creating the target table if it already exists in the target database: - - ```shell - ticloud import start mysql --project-id --cluster-id --source-host --source-port --source-user --source-password --source-database --source-table --target-database --target-password --skip-create-table - ``` - -> **Note:** -> -> MySQL 8.0 uses `utf8mb4_0900_ai_ci` as the default collation, which is currently not supported by TiDB. If your source table uses the `utf8mb4_0900_ai_ci` collation, before the import, you need to either alter the source table collation to a [supported collation of TiDB](/character-set-and-collation.md#character-sets-and-collations-supported-by-tidb) or manually create the target table in TiDB. - -## Flags - -In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. - -| Flag | Description | Required | Note | -|---|---|---|---| -| -c, --cluster-id string | Specifies the cluster ID. | Yes | Only works in non-interactive mode. | -| -h, --help | Displays help information for this command. | No | Works in both non-interactive and interactive modes. | -| -p, --project-id string | Specifies the project ID. | Yes | Only works in non-interactive mode. | -| --skip-create-table | Skips creating the target table if it already exists in the target database. | No | Only works in non-interactive mode. | -| --source-database string | The name of the source MySQL database. | Yes | Only works in non-interactive mode. | -| --source-host string | The host of the source MySQL instance. | Yes | Only works in non-interactive mode. | -| --source-password string | The password for the source MySQL instance. | Yes | Only works in non-interactive mode. | -| --source-port int | The port of the source MySQL instance. | Yes | Only works in non-interactive mode. | -| --source-table string | The source table name in the source MySQL database. | Yes | Only works in non-interactive mode. | -| --source-user string | The user to log in to the source MySQL instance. | Yes | Only works in non-interactive mode. | -| --target-database string | The target database name in the TiDB Serverless cluster. | Yes | Only works in non-interactive mode. | -| --target-password string | The password for the target TiDB Serverless cluster. | Yes | Only works in non-interactive mode. | -| --target-user string | The user to log in to the target TiDB Serverless cluster. | No | Only works in non-interactive mode. | - -## Inherited flags - -| Flag | Description | Required | Note | -|---|---|---|---| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | - -## Feedback - -If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-import-start-s3.md b/tidb-cloud/ticloud-import-start-s3.md deleted file mode 100644 index d42f84def8128..0000000000000 --- a/tidb-cloud/ticloud-import-start-s3.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -title: ticloud import start s3 -summary: The reference of `ticloud import start s3`. ---- - -# ticloud import start s3 - -Import files from Amazon S3 into TiDB Cloud: - -```shell -ticloud import start s3 [flags] -``` - -> **Note:** -> -> Before importing files from Amazon S3 into TiDB Cloud, you need to configure the Amazon S3 bucket access for TiDB Cloud and get the Role ARN. For more information, see [Configure Amazon S3 access](/tidb-cloud/config-s3-and-gcs-access.md#configure-amazon-s3-access). - -## Examples - -Start an import task in interactive mode: - -```shell -ticloud import start s3 -``` - -Start an import task in non-interactive mode: - -```shell -ticloud import start s3 --project-id --cluster-id --aws-role-arn --data-format --source-url -``` - -Start an import task with a custom CSV format: - -```shell -ticloud import start s3 --project-id --cluster-id --aws-role-arn --data-format CSV --source-url --separator \" --delimiter \' --backslash-escape=false --trim-last-separator=true -``` - -## Flags - -In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. - -| Flag | Description | Required | Note | -|---|---|---|---| -| --aws-role-arn string | Specifies the AWS role ARN that is used to access the Amazon S3 data source. | Yes | Only works in non-interactive mode. | -| --backslash-escape | Whether to parse backslashes inside fields as escape characters for CSV files. The default value is `true`. | No | Only works in non-interactive mode when `--data-format CSV` is specified. | -| -c, --cluster-id string | Specifies the cluster ID. | Yes | Only works in non-interactive mode. | -| --data-format string | Specifies the data format. Valid values are `CSV`, `SqlFile`, `Parquet`, or `AuroraSnapshot`. | Yes | Only works in non-interactive mode. | -| --delimiter string | Specifies the delimiter used for quoting for CSV files. The default value is `"`. | No | Only works in non-interactive mode when `--data-format CSV` is specified. | -| -h, --help | Displays help information for this command. | No | Works in both non-interactive and interactive modes. | -| -p, --project-id string | Specifies the project ID. | Yes | Only works in non-interactive mode. | -| --separator string | Specifies the field separator for CSV files. The default value is `,`. | No | Only works in non-interactive mode when `--data-format CSV` is specified. | -| --source-url string | The S3 path where the source data files are stored. | Yes | Only works in non-interactive mode. | -| --trim-last-separator | Whether to treat separators as line terminators and trim all trailing separators for CSV files. The default value is `false`. | No | Only works in non-interactive mode when `--data-format CSV` is specified. | - -## Inherited flags - -| Flag | Description | Required | Note | -|---|---|---|---| -| --no-color | Disables color in output | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | - -## Feedback - -If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-import-start.md b/tidb-cloud/ticloud-import-start.md new file mode 100644 index 0000000000000..778baf3283b01 --- /dev/null +++ b/tidb-cloud/ticloud-import-start.md @@ -0,0 +1,109 @@ +--- +title: ticloud serverless import start +summary: The reference of `ticloud serverless import start`. +aliases: ['/tidbcloud/ticloud-import-start-local','/tidbcloud/ticloud-import-start-mysql','/tidbcloud/ticloud-import-start-s3'] +--- + +# ticloud serverless import start + +Start a data import task: + +```shell +ticloud serverless import start [flags] +``` + +Or use the following alias command: + +```shell +ticloud serverless import create [flags] +``` + +> **Note:** +> +> Currently, you can only import one CSV file for one local import task. + +## Examples + +Start an import task in interactive mode: + +```shell +ticloud serverless import start +``` + +Start a local import task in non-interactive mode: + +```shell +ticloud serverless import start --local.file-path --cluster-id --file-type --local.target-database --local.target-table +``` + +Start a local import task with custom upload concurrency: + +```shell +ticloud serverless import start --local.file-path --cluster-id --file-type --local.target-database --local.target-table --local.concurrency 10 +``` + +Start a local import task with custom CSV format: + +```shell +ticloud serverless import start --local.file-path --cluster-id --file-type CSV --local.target-database --local.target-table --csv.separator \" --csv.delimiter \' --csv.backslash-escape=false --csv.trim-last-separator=true +``` + +Start an S3 import task in non-interactive mode: + +```shell +ticloud serverless import start --source-type S3 --s3.uri --cluster-id --file-type --s3.role-arn +``` + +Start a GCS import task in non-interactive mode: + +```shell +ticloud serverless import start --source-type GCS --gcs.uri --cluster-id --file-type --gcs.service-account-key +``` + +Start an Azure Blob import task in non-interactive mode: + +```shell +ticloud serverless import start --source-type AZURE_BLOB --azblob.uri --cluster-id --file-type --azblob.sas-token +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|----------------------------------|----------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| --azblob.sas-token string | Specifies the SAS token of Azure Blob. | No | Only works in non-interactive mode. | +| --azblob.uri string | Specifies the Azure Blob URI in `azure://.blob.core.windows.net//` format. | No | Only works in non-interactive mode. | +| --gcs.service-account-key string | Specifies the base64 encoded service account key of GCS. | No | Only works in non-interactive mode. | +| --gcs.uri string | Specifies the GCS URI in `gcs:///` format. Required when source type is GCS. | Yes | Only works in non-interactive mode. | +| --s3.access-key-id string | Specifies the access key ID of Amazon S3. You only need to set one of the `s3.role-arn` and [`s3.access-key-id`, `s3.secret-access-key`]. | No | Only works in non-interactive mode. | +| --s3.role-arn string | Specifies the role ARN of Amazon S3. You only need to set one of the `s3.role-arn` and [`s3.access-key-id`, `s3.secret-access-key`]. | No | Only works in non-interactive mode. | +| --s3.secret-access-key string | Specifies the secret access key of Amazon S3. You only need to set one of the `s3.role-arn` and [`s3.access-key-id`, `s3.secret-access-key`]. | No | Only works in non-interactive mode. | +| --s3.uri string | Specifies the S3 URI in `s3:///` format. Required when source type is S3. | Yes | Only works in non-interactive mode. | +| --source-type string | Specifies the import source type, one of [`"LOCAL"` `"S3"` `"GCS"` `"AZURE_BLOB"`]. The default value is `"LOCAL"`. | No | Only works in non-interactive mode. | +| -c, --cluster-id string | Specifies the cluster ID. | Yes | Only works in non-interactive mode. | +| --local.concurrency int | Specifies the concurrency for uploading files. The default value is `5`. | No | Only works in non-interactive mode. | +| --local.file-path string | Specifies the path of the local file to be imported. | No | Only works in non-interactive mode. | +| --local.target-database string | Specifies the target database to which the data is imported. | No | Only works in non-interactive mode. | +| --local.target-table string | Specifies the target table to which the data is imported. | No | Only works in non-interactive mode. | +| --file-type string | Specifies the import file type, one of ["CSV" "SQL" "AURORA_SNAPSHOT" "PARQUET"]. | Yes | Only works in non-interactive mode. | +| --csv.backslash-escape | Specifies whether to parse backslash inside fields as escape characters in a CSV file. The default value is `true`. | No | Only works in non-interactive mode. | +| --csv.delimiter string | Specifies the delimiter used for quoting a CSV file. The default value is `\`. | No | Only works in non-interactive mode. | +| --csv.separator string | Specifies the field separator in a CSV file. The default value is `,`. | No | Only works in non-interactive mode. | +| --csv.skip-header | Specifies whether the CSV file contains a header line. | No | Only works in non-interactive mode. | +| --csv.trim-last-separator | Specifies whether to treat the separator as the line terminator and trim all trailing separators in a CSV file. | No | Only works in non-interactive mode. | +| --csv.not-null | Specifies whether a CSV file can contain any NULL values. | No | Only works in non-interactive mode. | +| --csv.null-value string | Specifies the representation of NULL values in the CSV file. (default "\\N") | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-project-list.md b/tidb-cloud/ticloud-project-list.md index eee14b32cd698..da1fba930bcd9 100644 --- a/tidb-cloud/ticloud-project-list.md +++ b/tidb-cloud/ticloud-project-list.md @@ -35,17 +35,18 @@ ticloud project list -o json In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. -| Flag | Description | Required | Note | -|---------------------|--------------------------------------------------------------------------------------------------------|----------|-----------------------------------------------------| -| -h, --help | Help information for this command | No | Works in both non-interactive and interactive modes. | -| -o, --output string | Output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|---------------------|--------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| -o, --output string | Specifies the output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | ## Inherited flags -| Flag | Description | Required | Note | -|----------------------|-------------------------------------------------------------------------------------------|----------|--------------------------------------------------------------------------------------------------------------------------| -| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | -| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | ## Feedback diff --git a/tidb-cloud/ticloud-serverless-authorized-network-create.md b/tidb-cloud/ticloud-serverless-authorized-network-create.md new file mode 100644 index 0000000000000..1bcbc3c9edd88 --- /dev/null +++ b/tidb-cloud/ticloud-serverless-authorized-network-create.md @@ -0,0 +1,50 @@ +--- +title: ticloud serverless authorized-network create +summary: The reference of `ticloud serverless authorized-network create`. +--- + +# ticloud serverless authorized-network create + +Create an authorized network: + +```shell +ticloud serverless authorized-network create [flags] +``` + +## Examples + +Create an authorized network in interactive mode: + +```shell +ticloud serverless authorized-network create +``` + +Create an authorized network in non-interactive mode: + +```shell +ticloud serverless authorized-network create -c --display-name --start-ip-address --end-ip-address +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| --display-name string | Specifies the name of the authorized network. | Yes | Only works in non-interactive mode. | +| --start-ip-address string | Specifies the start IP address of the authorized network. | Yes | Only works in non-interactive mode. | +| --end-ip-address string | Specifies the end IP address of the authorized network. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-authorized-network-delete.md b/tidb-cloud/ticloud-serverless-authorized-network-delete.md new file mode 100644 index 0000000000000..e5113d5e9f5fd --- /dev/null +++ b/tidb-cloud/ticloud-serverless-authorized-network-delete.md @@ -0,0 +1,50 @@ +--- +title: ticloud serverless authorized-network delete +summary: The reference of `ticloud serverless authorized-network delete`. +--- + +# ticloud serverless authorized-network delete + +Delete an authorized network: + +```shell +ticloud serverless authorized-network delete [flags] +``` + +## Examples + +Delete an authorized network in interactive mode: + +```shell +ticloud serverless authorized-network delete +``` + +Delete an authorized network in non-interactive mode: + +```shell +ticloud serverless authorized-network delete -c --start-ip-address --end-ip-address +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| --force string | Delete an authorized network without confirmation. | No | Only works in non-interactive mode. | +| --start-ip-address string | Specifies the start IP address of the authorized network. | Yes | Only works in non-interactive mode. | +| --end-ip-address string | Specifies the end IP address of the authorized network. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-authorized-network-list.md b/tidb-cloud/ticloud-serverless-authorized-network-list.md new file mode 100644 index 0000000000000..3ce3c5fe2424a --- /dev/null +++ b/tidb-cloud/ticloud-serverless-authorized-network-list.md @@ -0,0 +1,48 @@ +--- +title: ticloud serverless authorized-network list +summary: The reference of `ticloud serverless authorized-network list`. +--- + +# ticloud serverless authorized-network list + +List all authorized networks: + +```shell +ticloud serverless authorized-network list [flags] +``` + +## Examples + +List all authorized networks in interactive mode: + +```shell +ticloud serverless authorized-network list +``` + +List all authorized networks in non-interactive mode: + +```shell +ticloud serverless authorized-network list -c +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| --output string | Specifies the output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-authorized-network-update.md b/tidb-cloud/ticloud-serverless-authorized-network-update.md new file mode 100644 index 0000000000000..7768ef4651f67 --- /dev/null +++ b/tidb-cloud/ticloud-serverless-authorized-network-update.md @@ -0,0 +1,52 @@ +--- +title: ticloud serverless authorized-network update +summary: The reference of `ticloud serverless authorized-network update`. +--- + +# ticloud serverless authorized-network update + +Update an authorized network: + +```shell +ticloud serverless authorized-network update [flags] +``` + +## Examples + +Update an authorized network in interactive mode: + +```shell +ticloud serverless authorized-network update +``` + +Update an authorized network in non-interactive mode: + +```shell +ticloud serverless authorized-network update -c --start-ip-address --end-ip-address --new-start-ip-address --new-end-ip-address +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| --start-ip-address string | Specifies the starting IP address of the authorized network. | Yes | Only works in non-interactive mode. | +| --end-ip-address string | Specifies the ending IP address of the authorized network. | Yes | Only works in non-interactive mode. | +| --new-display-name string | Specifies the new name of the authorized network. | No | Only works in non-interactive mode. | +| --new-start-ip-address string | Specifies the new start IP address of the authorized network. | No | Only works in non-interactive mode. | +| --new-end-ip-address string | Specifies the new end IP address of the authorized network. | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-export-cancel.md b/tidb-cloud/ticloud-serverless-export-cancel.md new file mode 100644 index 0000000000000..777ec6686b79a --- /dev/null +++ b/tidb-cloud/ticloud-serverless-export-cancel.md @@ -0,0 +1,49 @@ +--- +title: ticloud serverless export cancel +summary: The reference of `ticloud serverless export cancel`. +--- + +# ticloud serverless export cancel + +Cancel a data export task: + +```shell +ticloud serverless export cancel [flags] +``` + +## Examples + +Cancel an export task in interactive mode: + +```shell +ticloud serverless export cancel +``` + +Cancel an export task in non-interactive mode: + +```shell +ticloud serverless export cancel -c -e +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|----------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -e, --export-id string | Specifies the ID of the export task. | Yes | Only works in non-interactive mode. | +| --force | Cancels an export task without confirmation | No | Works in both non-interactive and interactive modes. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-export-create.md b/tidb-cloud/ticloud-serverless-export-create.md new file mode 100644 index 0000000000000..4fd835e827c4a --- /dev/null +++ b/tidb-cloud/ticloud-serverless-export-create.md @@ -0,0 +1,106 @@ +--- +title: ticloud serverless export create +summary: The reference of `ticloud serverless export create`. +--- + +# ticloud serverless export create + +Export data from a TiDB Cloud Serverless cluster: + +```shell +ticloud serverless export create [flags] +``` + +## Examples + +Export data from a TiDB Cloud Serverless cluster in interactive mode: + +```shell +ticloud serverless export create +``` + +Export data from a TiDB Cloud Serverless cluster to a local file in non-interactive mode: + +```shell +ticloud serverless export create -c --filter +``` + +Export data from a TiDB Cloud Serverless cluster to Amazon S3 in non-interactive mode: + +```shell +ticloud serverless export create -c --s3.uri --s3.access-key-id --s3.secret-access-key --filter +``` + +Export data from a TiDB Cloud Serverless cluster to Google Cloud Storage in non-interactive mode: + +```shell +ticloud serverless export create -c --gcs.uri --gcs.service-account-key --filter +``` + +Export data from a TiDB Cloud Serverless cluster to Azure Blob Storage in non-interactive mode: + +```shell +ticloud serverless export create -c --azblob.uri --azblob.sas-token --filter +``` + +Export data from a TiDB Cloud Serverless cluster to Alibaba Cloud OSS in non-interactive mode: + +```shell +ticloud serverless export create -c --oss.uri --oss.access-key-id --oss.access-key-secret --filter +``` + +Export data to a Parquet file and compress it with `SNAPPY` in non-interactive mode: + +```shell +ticloud serverless export create -c --file-type parquet --parquet.compression SNAPPY --filter +``` + +Export data with SQL statements in non-interactive mode: + +```shell +ticloud serverless export create -c --sql 'select * from database.table' +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|----------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster, from which you want to export data. | Yes | Only works in non-interactive mode. | +| --file-type string | Specifies the export file type. One of ["SQL" "CSV" "PARQUET"]. (default "CSV") | No | Only works in non-interactive mode. | +| --target-type string | Specifies the export target. One of [`"LOCAL"` `"S3"` `"GCS"` `"AZURE_BLOB"` `"OSS"`]. The default value is `"LOCAL"`. | No | Only works in non-interactive mode. | +| --s3.uri string | Specifies the S3 URI in `s3:///` format. Required when the target type is S3. | No | Only works in non-interactive mode. | +| --s3.access-key-id string | Specifies the access key ID of Amazon S3. You only need to set one of the s3.role-arn and [s3.access-key-id, s3.secret-access-key]. | NO | Only works in non-interactive mode. | +| --s3.secret-access-key string | Specifies the secret access key of Amazon S3. You only need to set one of the s3.role-arn and [s3.access-key-id, s3.secret-access-key]. | No | Only works in non-interactive mode. | +| --s3.role-arn string | Specifies the role ARN of Amazon S3. You only need to set one of the s3.role-arn and [s3.access-key-id, s3.secret-access-key]. | No | Only works in non-interactive mode. | +| --gcs.uri string | Specifies the GCS URI in `gcs:///` format. Required when the target type is GCS. | No | Only works in non-interactive mode. | +| --gcs.service-account-key string | Specifies the base64 encoded service account key of GCS. | No | Only works in non-interactive mode. | +| --azblob.uri string | Specifies the Azure Blob URI in `azure://.blob.core.windows.net//` format. Required when the target type is AZURE_BLOB. | No | Only works in non-interactive mode. | +| --azblob.sas-token string | Specifies the SAS token of Azure Blob. | No | Only works in non-interactive mode. | +| --oss.uri string | Specifies the Alibaba Cloud OSS URI in `oss:///` format. Required when the export `target-type` is `"OSS"`. | No | Only works in non-interactive mode. | +| --oss.access-key-id string | Specifies the AccessKey ID to access Alibaba Cloud OSS. | No | Only works in non-interactive mode. | +| --oss.access-key-secret string | Specifies the AccessKey secret to access Alibaba Cloud OSS. | No | Only works in non-interactive mode. | +| --csv.delimiter string | Specifies the delimiter of string type variables in CSV files. (default "\"") | No | Only works in non-interactive mode. | +| --csv.null-value string | Specifies the representation of null values in CSV files. (default "\\N") | No | Only works in non-interactive mode. | +| --csv.separator string | Specifies the separator of each value in CSV files. (default ",") | No | Only works in non-interactive mode. | +| --csv.skip-header | Exports CSV files of the tables without header. | No | Only works in non-interactive mode. | +| --parquet.compression string | Specifies the Parquet compression algorithm. One of [`"GZIP"` `"SNAPPY"` `"ZSTD"` `"NONE"`]. The default value is `"ZSTD"`. | No | Only works in non-interactive mode. | +| --filter strings | Specifies the exported tables with table filter patterns. Do not use it with --sql. For more information, see [Table Filter](/table-filter.md). | No | Only works in non-interactive mode. | +| --sql string | Filters the exported data with the `SQL SELECT` statement. | No | Only works in non-interactive mode. | +| --where string | Filters the exported tables with the `WHERE` condition. Do not use it with --sql. | No | Only works in non-interactive mode. | +| --compression string | Specifies the compression algorithm of the export file. The supported algorithms include `GZIP`, `SNAPPY`, `ZSTD`, and `NONE`. The default value is `GZIP`. | No | Only works in non-interactive mode. | +| --force | Creates the export task without confirmation. You need to confirm when you want to export the whole cluster in non-interactive mode. | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-export-describe.md b/tidb-cloud/ticloud-serverless-export-describe.md new file mode 100644 index 0000000000000..ea9fd4ca7311c --- /dev/null +++ b/tidb-cloud/ticloud-serverless-export-describe.md @@ -0,0 +1,54 @@ +--- +title: ticloud serverless export describe +summary: The reference of `ticloud serverless export describe`. +--- + +# ticloud serverless export describe + +Get the export information of a TiDB Cloud Serverless cluster: + +```shell +ticloud serverless export describe [flags] +``` + +Or use the following alias command: + +```shell +ticloud serverless export get [flags] +``` + +## Examples + +Get the export information in interactive mode: + +```shell +ticloud serverless export describe +``` + +Get the export information in non-interactive mode: + +```shell +ticloud serverless export describe -c -e +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|----------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -e, --export-id string | Specifies the ID of the export task. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-export-download.md b/tidb-cloud/ticloud-serverless-export-download.md new file mode 100644 index 0000000000000..bb1ee9b9dc5de --- /dev/null +++ b/tidb-cloud/ticloud-serverless-export-download.md @@ -0,0 +1,51 @@ +--- +title: ticloud serverless export download +summary: The reference of `ticloud serverless export download`. +--- + +# ticloud serverless export download + +Download the exported data from a TiDB Cloud Serverless cluster to your local storage: + +```shell +ticloud serverless export download [flags] +``` + +## Examples + +Download the exported data in interactive mode: + +```shell +ticloud serverless export download +``` + +Download the exported data in non-interactive mode: + +```shell +ticloud serverless export download -c -e +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|-----------------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -e, --export-id string | Specifies the ID of the export task. | Yes | Only works in non-interactive mode. | +| --output-path string | Specifies the destination path for saving the downloaded data. If not specified, the data is downloaded to the current directory. | No | Only works in non-interactive mode. | +| --concurrency int | Specifies the download concurrency. The default value is `3`. | No | Works in both non-interactive and interactive modes. | +| --force | Downloads the exported data without confirmation. | No | Works in both non-interactive and interactive modes. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-export-list.md b/tidb-cloud/ticloud-serverless-export-list.md new file mode 100644 index 0000000000000..03a16afc5884f --- /dev/null +++ b/tidb-cloud/ticloud-serverless-export-list.md @@ -0,0 +1,60 @@ +--- +title: ticloud serverless export list +summary: The reference of `ticloud serverless export list`. +--- + +# ticloud serverless export list + +List data export tasks of TiDB Cloud Serverless clusters: + +```shell +ticloud serverless export list [flags] +``` + +Or use the following alias command: + +```shell +ticloud serverless export ls [flags] +``` + +## Examples + +List all export tasks in interactive mode: + +```shell +ticloud serverless export list +``` + +List export tasks for a specified cluster in non-interactive mode: + +```shell +ticloud serverless export list -c +``` + +List export tasks for a specified cluster in the JSON format in non-interactive mode: + +```shell +ticloud serverless export list -c -o json +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|--------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -o, --output string | Specifies the output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-region.md b/tidb-cloud/ticloud-serverless-region.md new file mode 100644 index 0000000000000..6e78e5d87f524 --- /dev/null +++ b/tidb-cloud/ticloud-serverless-region.md @@ -0,0 +1,48 @@ +--- +title: ticloud serverless region +summary: The reference of `ticloud serverless region`. +aliases: ['/tidbcloud/ticloud-serverless-regions'] +--- + +# ticloud serverless region + +List all available regions for TiDB Cloud Serverless: + +```shell +ticloud serverless region [flags] +``` + +## Examples + +List all available regions for TiDB Cloud Serverless: + +```shell +ticloud serverless region +``` + +List all available regions for TiDB Cloud Serverless clusters in the JSON format: + +```shell +ticloud serverless region -o json +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|---------------------|--------------------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -o, --output string | Specifies the output format (`human` by default). Valid values are `human` or `json`. To get a complete result, use the `json` format. | No | Works in both non-interactive and interactive modes. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|--------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-shell.md b/tidb-cloud/ticloud-serverless-shell.md new file mode 100644 index 0000000000000..a3188b1cc84e6 --- /dev/null +++ b/tidb-cloud/ticloud-serverless-shell.md @@ -0,0 +1,62 @@ +--- +title: ticloud serverless shell +summary: The reference of `ticloud serverless shell`. +aliases: ['/tidbcloud/ticloud-connect'] +--- + +# ticloud serverless shell + +Connect to a TiDB Cloud Serverless cluster: + +```shell +ticloud serverless shell [flags] +``` + +## Examples + +Connect to a TiDB Cloud Serverless cluster in interactive mode: + +```shell +ticloud serverless shell +``` + +Connect to a TiDB Cloud Serverless cluster with the default user in non-interactive mode: + +```shell +ticloud serverless shell -c +``` + +Connect to a TiDB Cloud Serverless cluster with the default user and password in non-interactive mode: + +```shell +ticloud serverless shell -c --password +``` + +Connect to a TiDB Cloud Serverless cluster with a specific user and password in non-interactive mode: + +```shell +ticloud serverless shell -c -u --password +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|-----------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | +| --password | Specifies the password of the user. | No | Only works in non-interactive mode. | +| -u, --user string | Specifies the user for login. | No | Only works in non-interactive mode. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-spending-limit.md b/tidb-cloud/ticloud-serverless-spending-limit.md new file mode 100644 index 0000000000000..5abca8a06666d --- /dev/null +++ b/tidb-cloud/ticloud-serverless-spending-limit.md @@ -0,0 +1,48 @@ +--- +title: ticloud serverless spending-limit +summary: The reference of `ticloud serverless spending-limit`. +--- + +# ticloud serverless spending-limit + +Set the maximum monthly [spending limit](/tidb-cloud/manage-serverless-spend-limit.md) for a TiDB Cloud Serverless cluster: + +```shell +ticloud serverless spending-limit [flags] +``` + +## Examples + +Set the spending limit for a TiDB Cloud Serverless cluster in interactive mode: + +```shell +ticloud serverless spending-limit +``` + +Set the spending limit for a TiDB Cloud Serverless cluster in non-interactive mode: + +```shell +ticloud serverless spending-limit -c --monthly +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|---------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| --monthly int32 | Specifies the maximum monthly spending limit in USD cents. | Yes | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|--------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-sql-user-create.md b/tidb-cloud/ticloud-serverless-sql-user-create.md new file mode 100644 index 0000000000000..894360c4ffdda --- /dev/null +++ b/tidb-cloud/ticloud-serverless-sql-user-create.md @@ -0,0 +1,50 @@ +--- +title: ticloud serverless sql-user create +summary: The reference of `ticloud serverless sql-user create`. +--- + +# ticloud serverless sql-user create + +Create a TiDB Cloud Serverless SQL user: + +```shell +ticloud serverless sql-user create [flags] +``` + +## Examples + +Create a TiDB Cloud Serverless SQL user in interactive mode: + +```shell +ticloud serverless sql-user create +``` + +Create a TiDB Cloud Serverless SQL user in non-interactive mode: + +```shell +ticloud serverless sql-user create --user --password --role --cluster-id +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| --password string | Specifies the password of the SQL user. | No | Only works in non-interactive mode. | +| --role strings | Specifies the roles of the SQL user. | No | Only works in non-interactive mode. | +| -u, --user string | Specifies the name of the SQL user. | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-sql-user-delete.md b/tidb-cloud/ticloud-serverless-sql-user-delete.md new file mode 100644 index 0000000000000..9f7cd35ea52d9 --- /dev/null +++ b/tidb-cloud/ticloud-serverless-sql-user-delete.md @@ -0,0 +1,48 @@ +--- +title: ticloud serverless sql-user delete +summary: The reference of `ticloud serverless sql-user delete`. +--- + +# ticloud serverless sql-user delete + +Delete a TiDB Cloud Serverless SQL user: + +```shell +ticloud serverless sql-user delete [flags] +``` + +## Examples + +Delete a TiDB Cloud Serverless SQL user in interactive mode: + +```shell +ticloud serverless sql-user delete +``` + +Delete a TiDB Cloud Serverless SQL user in non-interactive mode: + +```shell +ticloud serverless sql-user delete -c --user +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| --force | Deletes a SQL user without confirmation. | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-sql-user-list.md b/tidb-cloud/ticloud-serverless-sql-user-list.md new file mode 100644 index 0000000000000..152b0362e0701 --- /dev/null +++ b/tidb-cloud/ticloud-serverless-sql-user-list.md @@ -0,0 +1,48 @@ +--- +title: ticloud serverless sql-user list +summary: The reference of `ticloud serverless sql-user list`. +--- + +# ticloud serverless sql-user list + +List TiDB Cloud Serverless SQL users: + +```shell +ticloud serverless sql-user list [flags] +``` + +## Examples + +List TiDB Cloud Serverless SQL users in interactive mode: + +```shell +ticloud serverless sql-user list +``` + +List TiDB Cloud Serverless SQL users in non-interactive mode: + +```shell +ticloud serverless sql-user list -c +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|-------------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -o, --output string | Specifies the output format, one of ["human" "json"]. To get a complete result, use "json" format. (default "human"). | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-sql-user-update.md b/tidb-cloud/ticloud-serverless-sql-user-update.md new file mode 100644 index 0000000000000..1925b3280fc57 --- /dev/null +++ b/tidb-cloud/ticloud-serverless-sql-user-update.md @@ -0,0 +1,52 @@ +--- +title: ticloud serverless sql-user update +summary: The reference of `ticloud serverless sql-user update`. +--- + +# ticloud serverless sql-user update + +Update a TiDB Cloud Serverless SQL user: + +```shell +ticloud serverless sql-user update [flags] +``` + +## Examples + +Update a TiDB Cloud Serverless SQL user in interactive mode: + +```shell +ticloud serverless sql-user update +``` + +Update a TiDB Cloud Serverless SQL user in non-interactive mode: + +```shell +ticloud serverless sql-user update -c --user --password --role +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|-------------------------|-------------------------------------------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| --password string | Specifies the new password of the SQL user. | No | Only works in non-interactive mode. | +| --role strings | Specifies the new roles of the SQL user. Passing this flag replaces existing roles. | No | Only works in non-interactive mode. | +| --add-role strings | Specifies the roles to be added to the SQL user. | No | Only works in non-interactive mode. | +| --delete-role strings | Specifies the roles to be deleted from the SQL user. | No | Only works in non-interactive mode. | +| -u, --user string | Specifies the name of the SQL user to be updated. | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-serverless-update.md b/tidb-cloud/ticloud-serverless-update.md new file mode 100644 index 0000000000000..a8c00ce24feb1 --- /dev/null +++ b/tidb-cloud/ticloud-serverless-update.md @@ -0,0 +1,56 @@ +--- +title: ticloud serverless update +summary: The reference of `ticloud serverless update`. +--- + +# ticloud serverless update + +Update a TiDB Cloud Serverless cluster: + +```shell +ticloud serverless update [flags] +``` + +## Examples + +Update a TiDB Cloud Serverless cluster in interactive mode: + +```shell +ticloud serverless update +``` + +Update the name of a TiDB Cloud Serverless cluster in non-interactive mode: + +```shell +ticloud serverless update -c --display-name +``` + +Update labels of a TiDB Cloud Serverless cluster in non-interactive mode + +```shell +ticloud serverless update -c --labels "{\"label1\":\"value1\"}" +``` + +## Flags + +In non-interactive mode, you need to manually enter the required flags. In interactive mode, you can just follow CLI prompts to fill them in. + +| Flag | Description | Required | Note | +|---------------------------|---------------------------------------------|----------|------------------------------------------------------| +| -c, --cluster-id string | Specifies the ID of the cluster. | Yes | Only works in non-interactive mode. | +| -n --display-name string | Specifies a new name for the cluster. | No | Only works in non-interactive mode. |. +| --labels string | Specifies new labels for the cluster. | No | Only works in non-interactive mode. | +| --disable-public-endpoint | Disables the public endpoint of the cluster. | No | Only works in non-interactive mode. | +| -h, --help | Shows help information for this command. | No | Works in both non-interactive and interactive modes. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/ticloud-upgrade.md b/tidb-cloud/ticloud-upgrade.md new file mode 100644 index 0000000000000..ad033b6063338 --- /dev/null +++ b/tidb-cloud/ticloud-upgrade.md @@ -0,0 +1,39 @@ +--- +title: ticloud upgrade +summary: The reference of `ticloud upgrade`. +aliases: ['/tidbcloud/ticloud-update'] +--- + +# ticloud upgrade + +Upgrade the TiDB Cloud CLI to the latest version: + +```shell +ticloud upgrade [flags] +``` + +## Examples + +Upgrade the TiDB Cloud CLI to the latest version: + +```shell +ticloud upgrade +``` + +## Flags + +| Flag | Description | +|------------|-----------------------------------| + | -h, --help | Shows help information for this command. | + +## Inherited flags + +| Flag | Description | Required | Note | +|----------------------|------------------------------------------------------------------------------------------------------|----------|------------------------------------------------------------------------------------------------------------------| +| --no-color | Disables color in output. | No | Only works in non-interactive mode. In interactive mode, disabling color might not work with some UI components. | +| -P, --profile string | Specifies the active [user profile](/tidb-cloud/cli-reference.md#user-profile) used in this command. | No | Works in both non-interactive and interactive modes. | +| -D, --debug | Enables debug mode. | No | Works in both non-interactive and interactive modes. | + +## Feedback + +If you have any questions or suggestions on the TiDB Cloud CLI, feel free to create an [issue](https://github.com/tidbcloud/tidbcloud-cli/issues/new/choose). Also, we welcome any contributions. diff --git a/tidb-cloud/tidb-cloud-auditing.md b/tidb-cloud/tidb-cloud-auditing.md index 8527679dc3252..1cd8b00a5418a 100644 --- a/tidb-cloud/tidb-cloud-auditing.md +++ b/tidb-cloud/tidb-cloud-auditing.md @@ -21,12 +21,16 @@ The audit logging feature is disabled by default. To audit a cluster, you need t ## Prerequisites -- You are using a TiDB Dedicated cluster. Audit logging is not available for TiDB Serverless clusters. +- You are using a TiDB Cloud Dedicated cluster. Audit logging is not available for TiDB Cloud Serverless clusters. - You are in the `Organization Owner` or `Project Owner` role of your organization. Otherwise, you cannot see the database audit-related options in the TiDB Cloud console. For more information, see [User roles](/tidb-cloud/manage-user-access.md#user-roles). -## Enable audit logging for AWS or Google Cloud +## Enable audit logging -To allow TiDB Cloud to write audit logs to your cloud bucket, you need to enable audit logging first. +TiDB Cloud supports recording the audit logs of a TiDB Cloud Dedicated cluster to your cloud storage service. Before enabling database audit logging, configure your cloud storage service on the cloud provider where the cluster is located. + +> **Note:** +> +> For TiDB clusters deployed on AWS, you can choose to store audit log files in TiDB Cloud when enabling database audit logging. Currently, this feature is only available upon request. To request this feature, click **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com) and click **Request Support**. Then, fill in "Apply to store audit log files in TiDB Cloud" in the **Description** field and click **Submit**. ### Enable audit logging for AWS @@ -36,20 +40,25 @@ To enable audit logging for AWS, take the following steps: Specify an Amazon S3 bucket in your corporate-owned AWS account as a destination to which TiDB Cloud writes the audit logs. +> Note: +> +> Do not enable object lock on the AWS S3 bucket. Enabling object lock will prevent TiDB Cloud from pushing audit log files to S3. + For more information, see [Creating a bucket](https://docs.aws.amazon.com/AmazonS3/latest/userguide/create-bucket-overview.html) in the AWS User Guide. #### Step 2. Configure Amazon S3 access -> **Note:** -> -> Once the Amazon S3 access configuration is performed for one cluster in a project, you can use the same bucket as a destination for audit logs from all clusters in the same project. +1. Get the TiDB Cloud Account ID and the External ID of the TiDB cluster that you want to enable audit logging. + + 1. In the TiDB Cloud console, navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. -1. Get the TiDB Cloud account ID and the External ID of the TiDB cluster that you want to enable audit logging. + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. - 1. In the TiDB Cloud console, choose a project and a cluster deployed on AWS. - 2. Select **Settings** > **Audit Settings**. The **Audit Logging** dialog is displayed. - 3. In the **Audit Logging** dialog, click **Show AWS IAM policy settings**. The corresponding TiDB Cloud Account ID and TiDB Cloud External ID of the TiDB cluster are displayed. - 4. Record the TiDB Cloud Account ID and the External ID for later use. + 2. Click the name of your target cluster to go to its overview page, and then click **DB Audit Logging** in the left navigation pane. + 3. On the **DB Audit Logging** page, click **Enable** in the upper-right corner. + 4. In the **Enable Database Audit Logging** dialog, locate the **AWS IAM Policy Settings** section, and record **TiDB Cloud Account ID** and **TiDB Cloud External ID** for later use. 2. In the AWS Management Console, go to **IAM** > **Access Management** > **Policies**, and then check whether there is a storage bucket policy with the `s3:PutObject` write-only permission. @@ -83,23 +92,23 @@ For more information, see [Creating a bucket](https://docs.aws.amazon.com/Amazon #### Step 3. Enable audit logging -In the TiDB Cloud console, go back to the **Audit Logging** dialog box where you got the TiDB Cloud account ID and the External ID values, and then take the following steps: +In the TiDB Cloud console, go back to the **Enable Database Audit Logging** dialog box where you got the TiDB Cloud account ID and the External ID values, and then take the following steps: 1. In the **Bucket URI** field, enter the URI of your S3 bucket where the audit log files are to be written. 2. In the **Bucket Region** drop-down list, select the AWS region where the bucket locates. 3. In the **Role ARN** field, fill in the Role ARN value that you copied in [Step 2. Configure Amazon S3 access](#step-2-configure-amazon-s3-access). -4. Click **Test Connectivity** to verify whether TiDB Cloud can access and write to the bucket. +4. Click **Test Connection** to verify whether TiDB Cloud can access and write to the bucket. - If it is successful, **Pass** is displayed. Otherwise, check your access configuration. + If it is successful, **The connection is successfully** is displayed. Otherwise, check your access configuration. -5. In the upper-right corner, toggle the audit setting to **On**. +5. Click **Enable** to enable audit logging for the cluster. TiDB Cloud is ready to write audit logs for the specified cluster to your Amazon S3 bucket. > **Note:** > -> - After enabling audit logging, if you make any new changes to the bucket URI, location, or ARN, you must click **Restart** to load the changes and rerun the **Test Connectivity** check to make the changes effective. -> - To remove Amazon S3 access from TiDB Cloud, simply delete the trust policy that you added. +> - After enabling audit logging, if you make any new changes to the bucket URI, location, or ARN, you must click **Test Connection** again to verify that TiDB Cloud can connect to the bucket. Then, click **Enable** to apply the changes. +> - To remove TiDB Cloud's access to your Amazon S3, simply delete the trust policy granted to this cluster in the AWS Management Console. ### Enable audit logging for Google Cloud @@ -113,15 +122,17 @@ For more information, see [Creating storage buckets](https://cloud.google.com/st #### Step 2. Configure GCS access -> **Note:** -> -> Once the GCS access configuration is performed for one cluster in a project, you can use the same bucket as a destination for audit logs from all clusters in the same project. - 1. Get the Google Cloud Service Account ID of the TiDB cluster that you want to enable audit logging. - 1. In the TiDB Cloud console, choose a project and a cluster deployed on Google Cloud Platform. - 2. Select **Settings** > **Audit Settings**. The **Audit Logging** dialog box is displayed. - 3. Click **Show Google Cloud Service Account ID**, and then copy the Service Account ID for later use. + 1. In the TiDB Cloud console, navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project. + + > **Tip:** + > + > If you have multiple projects, you can click in the lower-left corner and switch to another project. + + 2. Click the name of your target cluster to go to its overview page, and then click **DB Audit Logging** in the left navigation pane. + 3. On the **DB Audit Logging** page, click **Enable** in the upper-right corner. + 4. In the **Enable Database Audit Logging** dialog, locate the **Google Cloud Server Account ID** section, and record **Service Account ID** for later use. 2. In the Google Cloud console, go to **IAM & Admin** > **Roles**, and then check whether a role with the following write-only permissions of the storage container exists. @@ -146,31 +157,34 @@ For more information, see [Creating storage buckets](https://cloud.google.com/st #### Step 3. Enable audit logging -In the TiDB Cloud console, go back to the **Audit Logging** dialog box where you got the TiDB Cloud account ID, and then take the following steps: +In the TiDB Cloud console, go back to the **Enable Database Audit Logging** dialog box where you got the TiDB Cloud account ID, and then take the following steps: 1. In the **Bucket URI** field, enter your full GCS bucket name. 2. In the **Bucket Region** field, select the GCS region where the bucket locates. -3. Click **Test Connectivity** to verify whether TiDB Cloud can access and write to the bucket. +3. Click **Test Connection** to verify whether TiDB Cloud can access and write to the bucket. - If it is successful, **Pass** is displayed. Otherwise, check your access configuration. + If it is successful, **The connection is successfully** is displayed. Otherwise, check your access configuration. -4. In the upper-right corner, toggle the audit setting to **On**. +4. Click **Enable** to enable audit logging for the cluster. - TiDB Cloud is ready to write audit logs for the specified cluster to your Amazon S3 bucket. + TiDB Cloud is ready to write audit logs for the specified cluster to your GCS bucket. > **Note:** > -> - After enabling audit logging, if you make any new changes to bucket URI or location, you must click **Restart** to load the changes and rerun the **Test Connectivity** check to make the changes effective. -> - To remove GCS access from TiDB Cloud, simply delete the principal that you added. +> - After enabling audit logging, if you make any new changes to the bucket URI or location, you must click **Test Connection** again to verify that TiDB Cloud can connect to the bucket. Then, click **Enable** to apply the changes. +> - To remove TiDB Cloud's access to your GCS bucket, delete the trust policy granted to this cluster in the Google Cloud console. ## Specify auditing filter rules -After enabling audit logging, you must specify auditing filter rules to control which user access events to capture and write to audit logs versus which events to ignore. If no filter rules are specified, TiDB Cloud does not log anything. +After enabling audit logging, you must specify auditing filter rules to control which user access events to capture and write to audit logs. If no filter rules are specified, TiDB Cloud does not log anything. To specify auditing filter rules for a cluster, take the following steps: -1. In the **Audit Logging** dialog box where you enable audit logging, scroll down and locate the **Filter Rules** section. -2. Add one or more filter rules, one rule per row, with each rule specifying a user expression, database expression, table expression, and access type. +1. On the **DB Audit Logging** page, click **Add Filter Rule** in the **Log Filter Rules** section to add an audit filter rule. + + You can add one audit rule at a time. Each rule specifies a user expression, database expression, table expression, and access type. You can add multiple audit rules to meet your auditing requirements. + +2.In the **Log Filter Rules** section, click **>** to expand and view the list of audit rules you have added. > **Note:** > @@ -179,6 +193,12 @@ To specify auditing filter rules for a cluster, take the following steps: ## View audit logs +By default, TiDB Cloud stores database audit log files in your storage service, so you need to read the audit log information from your storage service. + +> **Note:** +> +> If you have requested and chosen to store audit log files in TiDB Cloud, you can download them from the **Audit Log Access** section on the **Database Audit Logging** page. + TiDB Cloud audit logs are readable text files with the cluster ID, Pod ID, and log creation date incorporated into the fully qualified filenames. For example, `13796619446086334065/tidb-0/tidb-audit-2022-04-21T18-16-29.529.log`. In this example, `13796619446086334065` indicates the cluster ID and `tidb-0` indicates the Pod ID. diff --git a/tidb-cloud/tidb-cloud-billing-dm.md b/tidb-cloud/tidb-cloud-billing-dm.md index ab7dbb0a4f3ed..5ab94cd92e871 100644 --- a/tidb-cloud/tidb-cloud-billing-dm.md +++ b/tidb-cloud/tidb-cloud-billing-dm.md @@ -51,7 +51,7 @@ Note that if you are using AWS PrivateLink or VPC peering connections, and if th ![Cross-region and cross-AZ traffic charges](/media/tidb-cloud/dm-billing-cross-region-and-az-fees.png) -The cross-region and cross-AZ traffic prices are the same as those for TiDB Cloud. For more information, see [TiDB Cloud Pricing Details](https://en.pingcap.com/tidb-cloud-pricing-details/). +The cross-region and cross-AZ traffic prices are the same as those for TiDB Cloud. For more information, see [TiDB Cloud Pricing Details](https://www.pingcap.com/tidb-dedicated-pricing-details/). ## See also diff --git a/tidb-cloud/tidb-cloud-billing-recovery-group.md b/tidb-cloud/tidb-cloud-billing-recovery-group.md new file mode 100644 index 0000000000000..4197b257b90c9 --- /dev/null +++ b/tidb-cloud/tidb-cloud-billing-recovery-group.md @@ -0,0 +1,14 @@ +--- +title: Recovery Group Billing +summary: Learn about billing for recovery groups in TiDB Cloud. +--- + +# Recovery Group Billing + +TiDB Cloud bills for recovery groups based on the deployed size of your TiKV nodes in the primary cluster of the recovery group. When you [create a recovery group](/tidb-cloud/recovery-group-get-started.md) for a cluster, you can select the primary cluster for the recovery group. The larger the TiKV configuration, the higher the cost for recovery group protection. + +TiDB Cloud also bills for data processing per GiB basis. The data processing price varies depending on whether the data is replicated to a secondary cluster in another region, or within the same region. + +## Pricing + +To learn about the supported regions and the pricing for TiDB Cloud recovery groups, see [Recovery Group Cost](https://www.pingcap.com/tidb-cloud-pricing-details/#recovery-group-cost). diff --git a/tidb-cloud/tidb-cloud-billing-ticdc-rcu.md b/tidb-cloud/tidb-cloud-billing-ticdc-rcu.md index 14930b63a97ad..624d64bea3384 100644 --- a/tidb-cloud/tidb-cloud-billing-ticdc-rcu.md +++ b/tidb-cloud/tidb-cloud-billing-ticdc-rcu.md @@ -6,9 +6,11 @@ aliases: ['/tidbcloud/tidb-cloud-billing-tcu'] # Changefeed Billing +## RCU cost + TiDB Cloud measures the capacity of [changefeeds](/tidb-cloud/changefeed-overview.md) in TiCDC Replication Capacity Units (RCUs). When you [create a changefeed](/tidb-cloud/changefeed-overview.md#create-a-changefeed) for a cluster, you can select an appropriate specification. The higher the RCU, the better the replication performance. You will be charged for these TiCDC changefeed RCUs. -## Number of TiCDC RCUs +### Number of TiCDC RCUs The following table lists the specifications and corresponding replication performances for changefeeds: @@ -24,8 +26,14 @@ The following table lists the specifications and corresponding replication perfo > **Note:** > -> The preceding performance data is for reference only and might vary in different scenarios. +> The preceding performance data is for reference only and might vary in different scenarios. It is strongly recommended that you conduct a real workload test before using the changefeed feature in a production environment. For further assistance, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md). -## Price +### Price To learn about the supported regions and the price of TiDB Cloud for each TiCDC RCU, see [Changefeed Cost](https://www.pingcap.com/tidb-cloud-pricing-details/#changefeed-cost). + +## Private Data Link cost + +If you choose the **Private Link** or **Private Service Connect** network connectivity method, additional **Private Data Link** costs will be incurred. These charges fall under the [Data Transfer Cost](https://www.pingcap.com/tidb-dedicated-pricing-details/#data-transfer-cost) category. + +The price of **Private Data Link** is **$0.01/GiB**, the same as **Data Processed** of [AWS Interface Endpoint pricing](https://aws.amazon.com/privatelink/pricing/#Interface_Endpoint_pricing) and **Consumer data processing** of [Google Cloud Private Service Connect pricing](https://cloud.google.com/vpc/pricing#psc-forwarding-rules). diff --git a/tidb-cloud/tidb-cloud-billing.md b/tidb-cloud/tidb-cloud-billing.md index 081904c7cd582..a4b1d0ad712c2 100644 --- a/tidb-cloud/tidb-cloud-billing.md +++ b/tidb-cloud/tidb-cloud-billing.md @@ -7,16 +7,16 @@ summary: Learn about TiDB Cloud billing. > **Note:** > -> [TiDB Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-serverless) are free until May 31, 2023, with a 100% discount off. After that, usage beyond the [free quota](/tidb-cloud/select-cluster-tier.md#usage-quota) will be charged. +> [TiDB Cloud Serverless clusters](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) are free until May 31, 2023, with a 100% discount off. After that, usage beyond the [free quota](/tidb-cloud/select-cluster-tier.md#usage-quota) will be charged. TiDB Cloud charges according to the resources that you consume. You can visit the following pages to get more information about the pricing. -- [TiDB Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details) -- [TiDB Dedicated Pricing Details](https://en.pingcap.com/tidb-cloud-pricing-details/) +- [TiDB Cloud Serverless Pricing Details](https://www.pingcap.com/tidb-serverless-pricing-details/) +- [TiDB Cloud Dedicated Pricing Details](https://www.pingcap.com/tidb-dedicated-pricing-details/) ## Invoices -If you are in the `Organization Owner` or `Organization Billing Admin` role of your organization, you can manage the invoice information of TiDB Cloud. Otherwise, skip this section. +If you are in the `Organization Owner` or `Organization Billing Manager` role of your organization, you can manage the invoice information of TiDB Cloud. Otherwise, skip this section. After you set up the payment method, TiDB Cloud will generate an invoice once your cost reaches a quota, which is $500 by default. If you want to raise the quota or receive one invoice per month, you can [contact our sales](https://www.pingcap.com/contact-us/). @@ -46,11 +46,13 @@ To view the list of invoices, perform the following steps: > > If you are in multiple organizations, switch to your target organization by clicking its name. -2. Click **Billing**. The invoices page is displayed. +2. In the left navigation pane, click the **Billing** tab. + +3. Click the **Invoices** tab. The invoices page is displayed. ## Billing details -If you are in the `Organization Owner` or `Organization Billing Admin` role of your organization, you can view and export the billing details of TiDB Cloud. Otherwise, skip this section. +If you are in the `Organization Owner` or `Organization Billing Manager` role of your organization, you can view and export the billing details of TiDB Cloud. Otherwise, skip this section. After setting the payment method, TiDB Cloud will generate the invoice and billing details of the historical months, and generate the bill details of the current month at the beginning of each month. The billing details include your organization's TiDB cluster usage consumption, discounts, backup storage costs, data transmission costs, support service cost, credit consumption, and project splitting information. @@ -78,6 +80,48 @@ The billing details page shows the billing summary by project and by service. Yo > - The total amount in the monthly bill is rounded off to the 2nd decimal place. > - The total amount in the daily usage details is accurate to the 6th decimal place. +## Cost explorer + +If you are in the `Organization Owner` or `Organization Billing Manager` role of your organization, you can view and analyze the usage costs of TiDB Cloud. Otherwise, skip this section. + +To analyze and customize your cost reports of your organization, perform the following steps: + +1. In the lower-left corner of the [TiDB Cloud console](https://tidbcloud.com), click , and then click **Billing**. + + > **Note:** + > + > If you are in multiple organizations, switch to your target organization by clicking its name. + +2. On the **Billing** page, click the **Cost Explorer** tab. +3. On the **Cost Explorer** page, expand the **Filter** section in the upper-right corner to customize your report. You can set the time range, select a grouping option (such as by service, project, cluster, region, product type, and charge type), and apply filters by selecting specific services, projects, clusters, or regions. The cost explorer will display you with the following information: + + - **Cost Graph**: visualizes the cost trends over the selected time range. You can switch between **Monthly**, **Daily**, and **Total** views. + - **Cost Breakdown**: displays a detailed breakdown of your costs according to the selected grouping option. For further analysis, you can download the data in CSV format. + +## Billing profile + +Paid organizations can create a billing profile. Information in this profile will be used to determine the tax calculation. + +To view or update the billing profile of your organization, click in the lower-left corner and then click **Billing** > **Billing Profile**. + +There are four fields in the billing profile. + +### Company name (optional) + +If this field is specified, this name will appear on invoices instead of your organization name. + +### Billing email (optional) + +If this field is specified, invoices and other billing-related notifications will be sent to this email address. + +### Primary business address + +This is the address of the company that purchases TiDB Cloud services. It is used to calculate any applicable taxes. + +### Business tax ID (optional) + +If your business is registered for VAT/GST, fill in a valid VAT/GST ID. By providing this information, we will exempt you from charging VAT/GST if applicable. This is important for businesses operating in regions where VAT/GST registration allows for certain tax exemptions or refunds. + ## Credits TiDB Cloud offers a certain number of credits for Proof of Concept (PoC) users. One credit is equivalent to one U.S. dollar. You can use credits to pay TiDB cluster fees before the credits become expired. @@ -113,7 +157,7 @@ To view this page, perform the following steps: ## Discounts -If you are in the `Organization Owner` or `Organization Billing Admin` role of your organization, you can view the discount information of TiDB Cloud on the **Discounts** page. Otherwise, skip this section. +If you are in the `Organization Owner` or `Organization Billing Manager` role of your organization, you can view the discount information of TiDB Cloud on the **Discounts** page. Otherwise, skip this section. The discount information includes all discounts that you have received, the status, the discount percentage, and the discount start and end date. @@ -130,14 +174,12 @@ To view this page, perform the following steps: ## Payment method -If you are in the `Organization Owner` or `Organization Billing Admin` role of your organization, you can manage the payment information of TiDB Cloud. Otherwise, skip this section. +If you are in the `Organization Owner` or `Organization Billing Manager` role of your organization, you can manage the payment information of TiDB Cloud. Otherwise, skip this section. > **Note:** > > If you sign up for TiDB Cloud through [AWS Marketplace](https://aws.amazon.com/marketplace) or [Google Cloud Marketplace](https://console.cloud.google.com/marketplace), you can pay through your AWS account or Google Cloud account directly but cannot add payment methods or download invoices in the TiDB Cloud console. -### Add a credit card - The fee is deducted from a bound credit card according to your cluster usage. To add a valid credit card, you can use either of the following methods: - When you are creating a cluster: @@ -156,7 +198,9 @@ The fee is deducted from a bound credit card according to your cluster usage. To 2. Click **Billing**. 3. Under the **Payment Method** tab, click **Add a New Card**. - 4. Fill in the billing address and card information, and then click **Save**. + 4. Fill in the credit card information and credit card address, and then click **Save Card**. + + If you do not specify a primary business address in [**Billing profile**](#billing-profile), the credit card address will be used as your primary business address for tax calculation. You can update your primary business address in **Billing profile** anytime. > **Note:** > @@ -176,25 +220,9 @@ To set the default credit card, perform the following steps: 3. Click the **Payment Method** tab. 4. Select a credit card in the credit card list, and click **Set as default**. -### Edit billing profile information - -The billing profile information includes the business legal address and tax registration information. By providing your tax registration number, certain taxes might be exempted from your invoice. - -To edit the billing profile information, perform the following steps: - -1. Click in the lower-left corner of the TiDB Cloud console. - - > **Note:** - > - > If you are in multiple organizations, switch to your target organization by clicking its name. - -2. Click **Billing**. -3. Click the **Payment Method** tab. -4. Edit the billing profile information, and then click **Save**. - ## Contract -If you are in the `Organization Owner` or `Organization Billing Admin` role of your organization, you can manage your customized TiDB Cloud subscriptions in the TiDB Cloud console to meet compliance requirements. Otherwise, skip this section. +If you are in the `Organization Owner` or `Organization Billing Manager` role of your organization, you can manage your customized TiDB Cloud subscriptions in the TiDB Cloud console to meet compliance requirements. Otherwise, skip this section. If you have agreed with our sales on a contract and received an email to review and accept the contract online, you can do the following: @@ -212,7 +240,7 @@ To learn more about contracts, feel free to [contact our sales](https://www.ping ## Billing from AWS Marketplace or Google Cloud Marketplace -If you are in the `Organization Owner` or `Organization Billing Admin` role of your organization, you can link your TiDB Cloud account to an AWS billing account or Google Cloud billing account. Otherwise, skip this section. +If you are in the `Organization Owner` or `Organization Billing Manager` role of your organization, you can link your TiDB Cloud account to an AWS billing account or Google Cloud billing account. Otherwise, skip this section. If you are new to TiDB Cloud and do not have a TiDB Cloud account, you can sign up for a TiDB Cloud account through [AWS Marketplace](https://aws.amazon.com/marketplace) or [Google Cloud Marketplace](https://console.cloud.google.com/marketplace), and pay for the usage via the AWS or Google Cloud billing account. diff --git a/tidb-cloud/tidb-cloud-budget.md b/tidb-cloud/tidb-cloud-budget.md new file mode 100644 index 0000000000000..0735af9a65328 --- /dev/null +++ b/tidb-cloud/tidb-cloud-budget.md @@ -0,0 +1,113 @@ +--- +title: Manage Budgets for TiDB Cloud +summary: Learn about how to use the budget feature of TiDB Cloud to monitor your costs. +--- + +# Manage Budgets for TiDB Cloud + +In TiDB Cloud, you can use the budget feature to monitor your costs and keep your spending under control. + +When your monthly actual costs exceed the percentage thresholds of your specified budget, alert emails are sent to your organization owners and billing administrators. These notifications help you stay informed and take proactive measures to manage your spending, aligning your expenses with your budget. + +TiDB Cloud provides two types of budgets to help you track your spending: + +- **Serverless Spending Limit** budget: for each TiDB Cloud Serverless scalable cluster, TiDB Cloud automatically creates a **Serverless Spending Limit** budget. This budget helps you track the actual cost against the [spending limit](/tidb-cloud/manage-serverless-spend-limit.md) configured on that cluster. It includes three threshold rules: 75%, 90%, and 100% of the budget, which are not editable. + +- **Custom** budget: you can create custom budgets to track actual costs for an entire organization or specific projects. For each budget, you can specify a budget scope, set a target spending amount, and configure alert thresholds. After creating a custom budget, you can compare your monthly actual costs with your planned costs to ensure you stay within budget. + +## Prerequisites + +To view, create, edit, or delete budgets of your organization or projects, you must be in the `Organization Owner` or `Organization Billing Manager` role of your organization. + +## View the budget information + +To view the budget page of your organization, take the following steps: + +1. In the lower-left corner of the TiDB Cloud console, click , and then click **Billing**. + + > **Note:** + > + > If you are in multiple organizations, switch to your target organization by clicking its name. + +2. On the **Billing** page, click the **Budgets** tab. + +For each budget, you can view its name, type, status, amount used, budget amount, period, and scope. + +## Create a custom budget + +To create a custom budget to monitor the spending of your organization or specific projects, take the following steps: + +1. In the lower-left corner of the TiDB Cloud console, click , and then click **Billing**. + + > **Note:** + > + > If you are in multiple organizations, switch to your target organization by clicking its name. + +2. On the **Billing** page, click the **Budgets** tab. + +3. On the **Budgets** page, click **Create Custom Budget**. You can create up to five custom budgets. + +4. Provide the budget basic settings. + + - **Name**: enter a name for the budget. + - **Period**: select a time range for tracking costs. Currently, you can only select **Monthly**, which starts on the first day of each month and resets at the beginning of each month. TiDB Cloud tracks your actual spending during the time range against your budget amount (your planned spending). + - **Budget scope**: apply the scope to all projects (which means the entire TiDB Cloud organization) or a specific project as needed. + +5. Set the budget amount. + + - **Budget Amount**: enter a planned spending amount for the selected period. + - **Apply credits**: choose whether to apply credits to the running total cost. Credits are used to reduce the cost of your TiDB Cloud usage. When this option is enabled, the budget tracks the running total cost minus credits. + - **Apply discounts**: choose whether to apply discounts to the running total cost. Discounts are reductions in the regular price of TiDB Cloud service. When this option is enabled, the budget tracks the running total cost minus discounts. + +6. Configure alert thresholds for the budget. If your actual spending exceeds specified thresholds during the selected period, TiDB Cloud sends a budget notification email to your organization owners and billing administrators. + + - By default, TiDB Cloud provides three alert thresholds: 75%, 90%, and 100% of the budget amount. You can modify these percentages as needed. + - To add a new alert threshold, click **Add alert threshold.** + - To remove a threshold, click the delete icon next to the threshold. + +7. Click **Create**. + +## Edit a custom budget + +> **Note:** +> +> The **Serverless Spending Limit** budget cannot be edited because it is automatically created by TiDB Cloud to help you track the cost of a TiDB Cloud Serverless scalable cluster against its [spending limit](/tidb-cloud/manage-serverless-spend-limit.md). + +To edit a custom budget, take the following steps: + +1. In the lower-left corner of the TiDB Cloud console, click , and then click **Billing**. + + > **Note:** + > + > If you are in multiple organizations, switch to your target organization by clicking its name. + +2. On the **Billing** page, click the **Budgets** tab. + +3. On the **Budgets** page, locate the row of your budget, click **...** in that row, and then click **Edit**. + +4. Edit the budget name, budget scope, budget amount, and alert thresholds as needed. + + > **Note:** + > + > Editing the budget period and whether to apply credits and discounts is not supported. + +5. Click **Update**. + +## Delete a custom budget + +> **Note:** +> +> - Once a custom budget is deleted, you will no longer receive any alert emails related to it. +> - The **Serverless Spending Limit** budget cannot be deleted because it is automatically created by TiDB Cloud to help you track the cost of a TiDB Cloud Serverless scalable cluster against its [spending limit](/tidb-cloud/manage-serverless-spend-limit.md). + +To delete a custom budget, take the following steps: + +1. In the lower-left corner of the TiDB Cloud console, click , and then click **Billing**. + + > **Note:** + > + > If you are in multiple organizations, switch to your target organization by clicking its name. + +2. On the **Billing** page, click the **Budgets** tab. + +3. On the **Budgets** page, locate the row of your budget, click **...** in that row, and then click **Delete**. \ No newline at end of file diff --git a/tidb-cloud/tidb-cloud-clinic.md b/tidb-cloud/tidb-cloud-clinic.md new file mode 100644 index 0000000000000..58ddc656fb94e --- /dev/null +++ b/tidb-cloud/tidb-cloud-clinic.md @@ -0,0 +1,111 @@ +--- +title: TiDB Cloud Clinic +summary: Learn how to use TiDB Cloud Clinic for advanced monitoring and diagnostics. +--- + +# TiDB Cloud Clinic + +TiDB Cloud Clinic offers advanced monitoring and diagnostic capabilities on the TiDB Cloud, designed to help you quickly identify performance issues, optimize your database, and enhance overall performance with detailed analysis and actionable insights. + +![tidb-cloud-clinic](/media/tidb-cloud/tidb-cloud-clinic.png) + +> **Note:** +> +> Currently, TiDB Cloud Clinic is only available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters. + +## Prerequisites + +TiDB Cloud Clinic is only available for organizations that subscribe to the **Enterprise** or **Premium** support plan. + +## View the Cluster page + +To view the **Cluster** page, take the following steps: + +1. Log in to the [TiDB Cloud Clinic console](https://clinic.pingcap.com/) and select **Continue with TiDB Account** to enter the TiDB Cloud login page. + +2. From the organization list, select your target organization. The clusters in the selected project are displayed. + +3. Click the name of your target cluster. The cluster overview page is displayed, where you can view detailed information about your cluster, including: + + - Advanced Metrics + - Top Slow Queries (only supported when the TiDB version of the cluster is v8.1.1 or later, v7.5.4 or later) + - TopSQL (only supported when the TiDB version of the cluster is v8.1.1 or later, v7.5.4 or later) + - Benchmark Report + +## Monitor advanced metrics + +TiDB Cloud Clinic uses Grafana to provide a comprehensive set of metrics for TiDB clusters. These metrics include dashboards for TiDB, TiKV, TiFlash, BR, Lightning, TiCDC, TiDB resource control, and TiDB performance. + +To view the metrics dashboard, take the following steps: + +1. In the [TiDB Cloud Clinic console](https://clinic.pingcap.com/), navigate to the **Cluster** page of a cluster. + +2. Click **Metrics**. + +3. Click the name of the dashboard you want to view. The dashboard is displayed. + +The retention policy for advanced metrics is 90 days. + +## Analyze top slow queries + +By default, SQL queries that take longer than 300 milliseconds are considered slow queries. + +On the default [**Slow Queries**](/tidb-cloud/tune-performance.md#slow-query) page in the TiDB Cloud console, identifying performance-impacting queries can be difficult, especially in clusters with a large number of slow queries. The **Top Slow Queries** feature in TiDB Cloud Clinic provides aggregated analysis based on slow query logs. With this feature, you can easily pinpoint queries with performance issues, reducing overall performance tuning time by at least half. + +Top Slow Queries displays the top 10 queries aggregated by SQL digest, sorted by the following dimensions: + +- Total latency +- Maximum latency +- Average latency +- Total memory +- Maximum memory +- Average memory +- Total count + +To view slow queries in a cluster, take the following steps: + +1. In the [TiDB Cloud Clinic console](https://clinic.pingcap.com/), navigate to the **Cluster** page of a cluster. + +2. Click **Slow Query**. + +3. The top slow queries are displayed in a table. You can sort the results by different columns. + +4. (Optional) Click any slow query in the list to view its detailed execution information. + +5. (Optional) Filter slow queries by time range, database, or statement type. + +The retention policy for slow queries is 7 days. + +For more information, see [Slow Queries in TiDB Dashboard](https://docs.pingcap.com/tidb/stable/dashboard-slow-query). + +## Monitor TopSQL + +TiDB Cloud Clinic provides TopSQL information, enabling you to monitor and visually explore the CPU overhead of each SQL statement in your database in real time. This helps you optimize and resolve database performance issues. + +To view TopSQL, take the following steps: + +1. In the [TiDB Cloud Clinic console](https://clinic.pingcap.com/), navigate to the **Cluster** page of a cluster. + +2. Click **TopSQL**. + +3. Select a specific TiDB or TiKV instance to observe its load. You can use the time picker or select a time range in the chart to refine your analysis. + +4. Analyze the charts and tables displayed by TopSQL. + +For more information, see [TopSQL in TiDB Dashboard](https://docs.pingcap.com/tidb/stable/top-sql). + +## Generate benchmark reports + +The **Benchmark Report** feature helps you identify performance issues in a TiDB cluster during performance testing. After completing a stress test, you can generate a benchmark report to analyze the cluster's performance. The report highlights identified bottlenecks and provides optimization suggestions. After applying these suggestions, you can run another round of stress testing and generate a new benchmark report to compare performance improvements. + +To generate a benchmark report, take the following steps: + +1. In the [TiDB Cloud Clinic console](https://clinic.pingcap.com/), navigate to the **Cluster** page of a cluster. + +2. Click **Benchmark Report**. + +3. Select the time range to be analyzed in the benchmark report. + +4. Click **Create Report** to generate the benchmark report. + +5. Wait for report generation to complete. When the report is ready, click **View** to open it. diff --git a/tidb-cloud/tidb-cloud-connect-aws-dms.md b/tidb-cloud/tidb-cloud-connect-aws-dms.md index c1e082798b70c..6ad1a127fd72c 100644 --- a/tidb-cloud/tidb-cloud-connect-aws-dms.md +++ b/tidb-cloud/tidb-cloud-connect-aws-dms.md @@ -18,10 +18,10 @@ You are expected to have an AWS account with enough access to manage DMS-related ### A TiDB Cloud account and a TiDB cluster -You are expected to have a TiDB Cloud account and a TiDB Serverless or TiDB Dedicated cluster. If not, refer to the following documents to create one: +You are expected to have a TiDB Cloud account and a TiDB Cloud Serverless or TiDB Cloud Dedicated cluster. If not, refer to the following documents to create one: -- [Create a TiDB Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) -- [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) +- [Create a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) +- [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md) ## Configure network @@ -29,33 +29,33 @@ Before creating DMS resources, you need to configure network properly to ensure -
+
-For TiDB Serverless, your clients can connect to clusters via public endpoint or private endpoint. +For TiDB Cloud Serverless, your clients can connect to clusters via public endpoint or private endpoint. -- To [connect to a TiDB Serverless cluster via public endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md), do one of the following to make sure that the DMS replication instance can access the internet. +- To [connect to a TiDB Cloud Serverless cluster via public endpoint](/tidb-cloud/connect-via-standard-connection-serverless.md), do one of the following to make sure that the DMS replication instance can access the internet. - Deploy the replication instance in public subnets and enable **Public accessible**. For more information, see [Configuration for internet access](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Internet_Gateway.html#vpc-igw-internet-access). - Deploy the replication instance in private subnets and route traffic in the private subnets to public subnets. In this case, you need at least three subnets, two private subnets, and one public subnet. The two private subnets form a subnet group where the replication instance lives. Then you need to create a NAT gateway in the public subnet and route traffic of the two private subnets to the NAT gateway. For more information, see [Access the internet from a private subnet](https://docs.aws.amazon.com/vpc/latest/userguide/nat-gateway-scenarios.html#public-nat-internet-access). -- To connect to a TiDB Serverless cluster via private endpoint, [set up a private endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md) first and deploy the replication instance in private subnets. +- To connect to a TiDB Cloud Serverless cluster via private endpoint, [set up a private endpoint](/tidb-cloud/set-up-private-endpoint-connections-serverless.md) first and deploy the replication instance in private subnets.
-
+
-For TiDB Dedicated, your clients can connect to clusters via public endpoint, private endpoint, or VPC peering. +For TiDB Cloud Dedicated, your clients can connect to clusters via public endpoint, private endpoint, or VPC peering. -- To [connect to a TiDB Dedicated cluster via public endpoint](/tidb-cloud/connect-via-standard-connection.md), do one of the following to make sure that the DMS replication instance can access the internet. In addition, you need to add the public IP address of the replication instance or NAT gateway to the cluster's [IP access list](/tidb-cloud/configure-ip-access-list.md). +- To [connect to a TiDB Cloud Dedicated cluster via public endpoint](/tidb-cloud/connect-via-standard-connection.md), do one of the following to make sure that the DMS replication instance can access the internet. In addition, you need to add the public IP address of the replication instance or NAT gateway to the cluster's [IP access list](/tidb-cloud/configure-ip-access-list.md). - Deploy the replication instance in public subnets and enable **Public accessible**. For more information, see [Configuration for internet access](https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Internet_Gateway.html#vpc-igw-internet-access). - Deploy the replication instance in private subnets and route traffic in the private subnets to public subnets. In this case, you need at least three subnets, two private subnets, and one public subnet. The two private subnets form a subnet group where the replication instance lives. Then you need to create a NAT gateway in the public subnet and route traffic of the two private subnets to the NAT gateway. For more information, see [Access the internet from a private subnet](https://docs.aws.amazon.com/vpc/latest/userguide/nat-gateway-scenarios.html#public-nat-internet-access). -- To connect to a TiDB Dedicated cluster via private endpoint, [set up a private endpoint](/tidb-cloud/set-up-private-endpoint-connections.md) first and deploy the replication instance in private subnets. +- To connect to a TiDB Cloud Dedicated cluster via private endpoint, [set up a private endpoint](/tidb-cloud/set-up-private-endpoint-connections.md) first and deploy the replication instance in private subnets. -- To connect to a TiDB Dedicated cluster via VPC peering, [set up a VPC peering connection](/tidb-cloud/set-up-vpc-peering-connections.md) first and deploy the replication instance in private subnets. +- To connect to a TiDB Cloud Dedicated cluster via VPC peering, [set up a VPC peering connection](/tidb-cloud/set-up-vpc-peering-connections.md) first and deploy the replication instance in private subnets.
@@ -110,25 +110,25 @@ For connectivity, the steps for using TiDB Cloud clusters as a source or as a ta -
+
- - **Server name**: `HOST` of TiDB Serverless cluster. - - **Port**: `PORT` of TiDB Serverless cluster. - - **User name**: User of TiDB Serverless cluster for migration. Make sure it meets DMS requirements. - - **Password**: Password of the TiDB Serverless cluster user. + - **Server name**: `HOST` of TiDB Cloud Serverless cluster. + - **Port**: `PORT` of TiDB Cloud Serverless cluster. + - **User name**: User of TiDB Cloud Serverless cluster for migration. Make sure it meets DMS requirements. + - **Password**: Password of the TiDB Cloud Serverless cluster user. - **Secure Socket Layer (SSL) mode**: If you are connecting via public endpoint, it is highly recommended to set the mode to **verify-full** to ensure transport security. If you are connecting via private endpoint, you can set the mode to **none**. - - (Optional) **CA certificate**: Use the [ISRG Root X1 certificate](https://letsencrypt.org/certs/isrgrootx1.pem). For more information, see [TLS Connections to TiDB Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md). + - (Optional) **CA certificate**: Use the [ISRG Root X1 certificate](https://letsencrypt.org/certs/isrgrootx1.pem). For more information, see [TLS Connections to TiDB Cloud Serverless](/tidb-cloud/secure-connections-to-serverless-clusters.md).
-
+
- - **Server name**: `HOST` of TiDB Dedicated cluster. - - **Port**: `PORT` of TiDB Dedicated cluster. - - **User name**: User of TiDB Dedicated cluster for migration. Make sure it meets DMS requirements. - - **Password**: Password of TiDB Dedicated cluster user. + - **Server name**: `HOST` of TiDB Cloud Dedicated cluster. + - **Port**: `PORT` of TiDB Cloud Dedicated cluster. + - **User name**: User of TiDB Cloud Dedicated cluster for migration. Make sure it meets DMS requirements. + - **Password**: Password of TiDB Cloud Dedicated cluster user. - **Secure Socket Layer (SSL) mode**: If you are connecting via public endpoint, it is highly recommended to set the mode to **verify-full** to ensure transport security. If you are connecting via private endpoint, you can set it to **none**. - - (Optional) **CA certificate**: Get the CA certificate according to [TLS connections to TiDB Dedicated](/tidb-cloud/tidb-cloud-tls-connect-to-dedicated.md). + - (Optional) **CA certificate**: Get the CA certificate according to [TLS connections to TiDB Cloud Dedicated](/tidb-cloud/tidb-cloud-tls-connect-to-dedicated.md).
diff --git a/tidb-cloud/tidb-cloud-console-auditing.md b/tidb-cloud/tidb-cloud-console-auditing.md index 4e48ab2c2c0e8..6d81e895b6a7f 100644 --- a/tidb-cloud/tidb-cloud-console-auditing.md +++ b/tidb-cloud/tidb-cloud-console-auditing.md @@ -9,7 +9,7 @@ TiDB Cloud provides the console audit logging feature to help you track various ## Prerequisites -- You must be in the `Organization Owner` or `Organization Console Audit Admin` role of your organization in TiDB Cloud. Otherwise, you cannot see the console audit logging-related options in the TiDB Cloud console. +- You must be in the `Organization Owner` or `Organization Console Audit Manager` role of your organization in TiDB Cloud. Otherwise, you cannot see the console audit logging-related options in the TiDB Cloud console. - You can only enable and disable the console audit logging for your organization. You can only track the actions of users in your organization. - After the console audit logging is enabled, all event types of the TiDB Cloud console will be audited, and you cannot specify only auditing some of them. @@ -121,7 +121,7 @@ The console audit logs record various user activities on the TiDB Cloud console | PauseCluster | Pause a cluster | | ResumeCluster | Resume a cluster | | ScaleCluster | Scale a cluster | -| DownloadTiDBClusterCA | Download TiDB cluster CA certificate | +| DownloadTiDBClusterCA | Download CA certificate | | OpenWebSQLConsole | Connect to a TiDB cluster through Web SQL | | SetRootPassword | Set the root password of a TiDB cluster | | UpdateIPAccessList | Update the IP access list of a TiDB cluster | @@ -147,12 +147,12 @@ The console audit logs record various user activities on the TiDB Cloud console | BindSupportPlan | Bind a support plan | | CancelSupportPlan | Cancel a support plan | | UpdateOrganizationName | Update the organization name | -| SetSpendLimit | Edit the spending limit of a TiDB Serverless cluster | +| SetSpendLimit | Edit the spending limit of a TiDB Cloud Serverless scalable cluster | | UpdateMaintenanceWindow | Modify maintenance window start time | | DeferMaintenanceTask | Defer a maintenance task | -| CreateBranch | Create a TiDB Serverless branch | -| DeleteBranch | Delete a TiDB Serverless branch | -| SetBranchRootPassword | Set root password for a TiDB Serverless branch | +| CreateBranch | Create a TiDB Cloud Serverless branch | +| DeleteBranch | Delete a TiDB Cloud Serverless branch | +| SetBranchRootPassword | Set root password for a TiDB Cloud Serverless branch | | ConnectBranchGitHub | Connect the cluster with a GitHub repository to enable branching integration | | DisconnectBranchGitHub | Disconnect the cluster from a GitHub repository to disable branching integration | diff --git a/tidb-cloud/tidb-cloud-encrypt-cmek.md b/tidb-cloud/tidb-cloud-encrypt-cmek.md index 361d0b8e34fc8..e8231c3d3bf2f 100644 --- a/tidb-cloud/tidb-cloud-encrypt-cmek.md +++ b/tidb-cloud/tidb-cloud-encrypt-cmek.md @@ -5,7 +5,7 @@ summary: Learn about how to use Customer-Managed Encryption Key (CMEK) in TiDB C # Encryption at Rest Using Customer-Managed Encryption Keys -Customer-Managed Encryption Key (CMEK) allows you to secure your static data in a TiDB Dedicated cluster by utilizing a cryptographic key that is under your complete control. This key is referred to as the CMEK key. +Customer-Managed Encryption Key (CMEK) allows you to secure your static data in a TiDB Cloud Dedicated cluster by utilizing a symmetric encryption key that is under your complete control. This key is referred to as the CMEK key. Once CMEK is enabled for a project, all clusters created within that project encrypt their static data using the CMEK key. Additionally, any backup data generated by these clusters is encrypted using the same key. If CMEK is not enabled, TiDB Cloud employs an escrow key to encrypt all data in your cluster when it is at rest. @@ -17,7 +17,7 @@ Once CMEK is enabled for a project, all clusters created within that project enc - Currently, TiDB Cloud only supports using AWS KMS to provide CMEK. - To use CMEK, you need to enable CMEK when creating a project and complete CMEK-related configurations before creating a cluster. You cannot enable CMEK for existing projects. -- Currently, in CMEK-enabled projects, you can only create [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters hosted on AWS. TiDB Dedicated clusters hosted on Google Cloud and [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters are not supported. +- Currently, in CMEK-enabled projects, you can only create [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters hosted on AWS. TiDB Cloud Dedicated clusters hosted on Google Cloud and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters are not supported. - Currently, for a specific project, you can only enable CMEK for one AWS region. Once you have configured it, you cannot create clusters in other regions within the same project. ## Enable CMEK @@ -34,8 +34,8 @@ If you are in the `Organization Owner` role of your organization, you can create To create a CMEK-enabled project, take the following steps: 1. Click in the lower-left corner of the TiDB Cloud console. -2. Click **Organization Settings**. -3. On the **Organization Settings** page, click **Create New Project** to open the project creation dialog. +2. Click **Organization Settings**, and then click the **Projects** tab in the left navigation pane. The **Projects** tab is displayed. +3. Click **Create New Project** to open the project creation dialog. 4. Fill in a project name. 5. Choose to enable the CMEK capability of the project. 6. Click **Confirm** to complete the project creation. @@ -67,7 +67,7 @@ To complete the CMEK configuration of the project, take the following steps: 2. Click **Encryption Access** to enter the encryption management page of the project. 3. Click **Create Encryption Key** to enter the key creation page. 4. The key provider only supports AWS KMS. You can choose the region where the encryption key can be used. -5. Copy and save the JSON file as `ROLE-TRUST-POLICY.JSON`. This file describes the trust relationship. +5. Copy and save the JSON file as `ROLE-TRUST-POLICY.JSON`. This file describes the trust relationship. 6. Add this trust relationship to the key policy of AWS KMS. For more information, refer to [Key policies in AWS KMS](https://docs.aws.amazon.com/kms/latest/developerguide/key-policies.html). 7. In the TiDB Cloud console, scroll to the bottom of the key creation page, and then fill in the **KMS Key ARN** obtained from AWS KMS. 8. Click **Create** to create the key. @@ -141,7 +141,7 @@ To complete the CMEK configuration of the project, take the following steps: ### Step 3. Create a cluster -Under the project created in [Step 1](#step-1-create-a-cmek-enabled-project), create a TiDB Dedicated cluster hosted on AWS. For detailed steps, refer to [this document](/tidb-cloud/create-tidb-cluster.md). Ensure that the region where the cluster is located is the same as that in [Step 2](/tidb-cloud/tidb-cloud-encrypt-cmek.md#step-2-complete-the-cmek-configuration-of-the-project). +Under the project created in [Step 1](#step-1-create-a-cmek-enabled-project), create a TiDB Cloud Dedicated cluster hosted on AWS. For detailed steps, refer to [this document](/tidb-cloud/create-tidb-cluster.md). Ensure that the region where the cluster is located is the same as that in [Step 2](/tidb-cloud/tidb-cloud-encrypt-cmek.md#step-2-complete-the-cmek-configuration-of-the-project). > **Note:** > diff --git a/tidb-cloud/tidb-cloud-events.md b/tidb-cloud/tidb-cloud-events.md index a774c258e05ba..85ae6865b49d5 100644 --- a/tidb-cloud/tidb-cloud-events.md +++ b/tidb-cloud/tidb-cloud-events.md @@ -33,6 +33,7 @@ TiDB Cloud logs the following types of cluster events: | ResumeCluster | Resume a cluster | | ModifyClusterSize | Modify cluster size | | BackupCluster | Back up a cluster | +| ExportBackup | Export a backup | | RestoreFromCluster | Restore a cluster | | CreateChangefeed | Create a changefeed | | PauseChangefeed | Pause a changefeed | @@ -42,8 +43,8 @@ TiDB Cloud logs the following types of cluster events: | ScaleChangefeed | Scale the specification of a changefeed | | FailedChangefeed | Changefeed failures | | ImportData | Import data to a cluster | -| UpdateSpendingLimit | Update spending limit of a TiDB Serverless cluster | -| ResourceLimitation | Update resource limitation of a TiDB Serverless cluster | +| UpdateSpendingLimit | Update spending limit of a TiDB Cloud Serverless scalable cluster | +| ResourceLimitation | Update resource limitation of a TiDB Cloud Serverless cluster | For each event, the following information is logged: diff --git a/tidb-cloud/tidb-cloud-faq.md b/tidb-cloud/tidb-cloud-faq.md index a58acf56737c8..334943673b58b 100644 --- a/tidb-cloud/tidb-cloud-faq.md +++ b/tidb-cloud/tidb-cloud-faq.md @@ -19,7 +19,7 @@ TiDB Cloud allows developers and DBAs with little or no training to handle once- ### What is the relationship between TiDB and TiDB Cloud? -TiDB is an open-source database and is the best option for organizations who want to run TiDB Self-Hosted in their own data centers, in a self-managed cloud environment, or in a hybrid of the two. +TiDB is an open-source database and is the best option for organizations who want to run TiDB Self-Managed in their own data centers, in a self-managed cloud environment, or in a hybrid of the two. TiDB Cloud is a fully managed cloud Database as a Service of TiDB. It has an easy-to-use web-based management console to let you manage TiDB clusters for mission-critical production environments. @@ -41,18 +41,22 @@ No. ### What versions of TiDB are supported on TiDB Cloud? -- Starting from October 31, 2023, the default TiDB version for new TiDB Dedicated clusters is v7.1.2. -- Starting from March 7, 2023, the default TiDB version for new TiDB Serverless clusters is v6.6.0. +- Starting from January 2, 2025, the default TiDB version for new TiDB Cloud Dedicated clusters is [v8.1.2](https://docs.pingcap.com/tidb/v8.1/release-8.1.2). +- Starting from February 21, 2024, the TiDB version for TiDB Cloud Serverless clusters is [v7.1.3](https://docs.pingcap.com/tidb/v7.1/release-7.1.3). For more information, see [TiDB Cloud Release Notes](/tidb-cloud/tidb-cloud-release-notes.md). ### What companies are using TiDB or TiDB Cloud in production? -TiDB is trusted by over 1500 global enterprises across a variety of industries, such as financial services, gaming, and e-commerce. Our users include Square (US), Shopee (Singapore), and China UnionPay (China). See our [case studies](https://en.pingcap.com/customers/) for specific details. +TiDB is trusted by over 1500 global enterprises across a variety of industries, such as financial services, gaming, and e-commerce. Our users include Square (US), Shopee (Singapore), and China UnionPay (China). See our [case studies](https://www.pingcap.com/customers/) for specific details. ### What does the SLA look like? -TiDB Cloud provides 99.99% SLA. For details, see [Service Level Agreement for TiDB Cloud Services](https://en.pingcap.com/legal/service-level-agreement-for-tidb-cloud-services/). +TiDB Cloud provides 99.99% SLA. For details, see [Service Level Agreement for TiDB Cloud Services](https://www.pingcap.com/legal/service-level-agreement-for-tidb-cloud-services/). + +### What does BETA mean in TiDB Cloud? + +BETA is the public preview stage of a TiDB Cloud feature or offering before it becomes generally available (GA). ### How can I learn more about TiDB Cloud? @@ -60,7 +64,7 @@ The best way to learn about TiDB Cloud is to follow our step-by-step tutorial. C - [TiDB Cloud Introduction](/tidb-cloud/tidb-cloud-intro.md) - [Get Started](/tidb-cloud/tidb-cloud-quickstart.md) -- [Create a TiDB Serverless Cluster](/tidb-cloud/create-tidb-cluster-serverless.md) +- [Create a TiDB Cloud Serverless Cluster](/tidb-cloud/create-tidb-cluster-serverless.md) ### What does `XXX's Org/default project/Cluster0` refer to when deleting a cluster? @@ -92,7 +96,7 @@ Each data change is recorded as a Raft log. Through Raft log replication, data i TiDB uses the Raft consensus algorithm to ensure that data is highly available and safely replicated throughout storage in Raft Groups. Data is redundantly copied between TiKV nodes and placed in different Availability Zones to protect against machine or data center failure. With automatic failover, TiDB ensures that your service is always on. -As a Software as a Service (SaaS) provider, we take data security seriously. We have established strict information security policies and procedures required by the [Service Organization Control (SOC) 2 Type 1 compliance](https://en.pingcap.com/press-release/pingcap-successfully-completes-soc-2-type-1-examination-for-tidb-cloud/). This ensures that your data is secure, available, and confidential. +As a Software as a Service (SaaS) provider, we take data security seriously. We have established strict information security policies and procedures required by the [Service Organization Control (SOC) 2 Type 1 compliance](https://www.pingcap.com/press-release/pingcap-successfully-completes-soc-2-type-1-examination-for-tidb-cloud/). This ensures that your data is secure, available, and confidential. ## Migration FAQ @@ -104,7 +108,7 @@ TiDB is highly compatible with MySQL. You can migrate data from any MySQL-compat ### Does TiDB Cloud support incremental backups? -No. If you need to restore data to any point in time within the cluster's backup retention, you can use PITR (Point-in-time Recovery). For more information, see [Use PITR in a TiDB Dedicated cluster](/tidb-cloud/backup-and-restore.md#turn-on-auto-backup) or [Use PITR in a TiDB Serverless cluster](/tidb-cloud/backup-and-restore-serverless.md#restore). +No. If you need to restore data to any point in time within the cluster's backup retention, you can use PITR (Point-in-time Recovery). For more information, see [Use PITR in a TiDB Cloud Dedicated cluster](/tidb-cloud/backup-and-restore.md#turn-on-auto-backup) or [Use PITR in a TiDB Cloud Serverless cluster](/tidb-cloud/backup-and-restore-serverless.md#restore). ## HTAP FAQs @@ -149,16 +153,16 @@ No. TiDB Cloud is Database-as-a-Service (DBaaS) and runs only in the TiDB Cloud ### Is my TiDB cluster secure? -In TiDB Cloud, you can use either a TiDB Dedicated cluster or a TiDB Serverless cluster according to your needs. +In TiDB Cloud, you can use either a TiDB Cloud Dedicated cluster or a TiDB Cloud Serverless cluster according to your needs. -For TiDB Dedicated clusters, TiDB Cloud ensures cluster security with the following measures: +For TiDB Cloud Dedicated clusters, TiDB Cloud ensures cluster security with the following measures: - Creates independent sub-accounts and VPCs for each cluster. - Sets up firewall rules to isolate external connections. - Creates server-side TLS certificates and component-level TLS certificates for each cluster to encrypt cluster data in transit. - Provide IP access rules for each cluster to ensure that only allowed source IP addresses can access your cluster. -For TiDB Serverless clusters, TiDB Cloud ensures cluster security with the following measures: +For TiDB Cloud Serverless clusters, TiDB Cloud ensures cluster security with the following measures: - Creates independent sub-accounts for each cluster. - Sets up firewall rules to isolate external connections. @@ -167,27 +171,27 @@ For TiDB Serverless clusters, TiDB Cloud ensures cluster security with the follo ### How do I connect to my database in a TiDB cluster? -
+
-For a TiDB Dedicated cluster, the steps to connect to your cluster are simplified as follows: +For a TiDB Cloud Dedicated cluster, the steps to connect to your cluster are simplified as follows: 1. Authorize your network. 2. Set up your database users and login credentials. 3. Download and configure TLS for your cluster server. 4. Choose a SQL client, get an auto-generated connection string displayed on the TiDB Cloud UI, and then connect to your cluster through the SQL client using the string. -For more information, see [Connect to Your TiDB Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md). +For more information, see [Connect to Your TiDB Cloud Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md).
-
+
-For a TiDB Serverless cluster, the steps to connect to your cluster are simplified as follows: +For a TiDB Cloud Serverless cluster, the steps to connect to your cluster are simplified as follows: 1. Set a database user and login credential. 2. Choose a SQL client, get an auto-generated connection string displayed on the TiDB Cloud UI, and then connect to your cluster through the SQL client using the string. -For more information, see [Connect to Your TiDB Serverless Cluster](/tidb-cloud/connect-to-tidb-cluster-serverless.md). +For more information, see [Connect to Your TiDB Cloud Serverless Cluster](/tidb-cloud/connect-to-tidb-cluster-serverless.md).
@@ -197,3 +201,7 @@ For more information, see [Connect to Your TiDB Serverless Cluster](/tidb-cloud/ ### What support is available for customers? TiDB Cloud is supported by the same team behind TiDB, which has run mission-critical use cases for over 1500 global enterprises across industries including financial services, e-commerce, enterprise applications, and gaming. TiDB Cloud offers a free basic support plan for each user and you can upgrade to a paid plan for extended services. For more information, see [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). + +### How do I check if TiDB Cloud is down? + +You can check the current uptime status of TiDB Cloud on the [System Status](https://status.tidbcloud.com/) page. \ No newline at end of file diff --git a/tidb-cloud/tidb-cloud-glossary.md b/tidb-cloud/tidb-cloud-glossary.md index 703d70b33ee0c..837c066aa9437 100644 --- a/tidb-cloud/tidb-cloud-glossary.md +++ b/tidb-cloud/tidb-cloud-glossary.md @@ -25,11 +25,9 @@ ACID refers to the four key properties of a transaction: atomicity, consistency, ### Chat2Query -TiDB Cloud is powered by AI. You can use Chat2Query (beta), an AI-powered SQL editor in the [TiDB Cloud console](https://tidbcloud.com/), to maximize your data value. +Chat2Query is an AI-powered feature integrated into SQL Editor that assists users in generating, debugging, or rewriting SQL queries using natural language instructions. For more information, see [Explore your data with AI-assisted SQL Editor](/tidb-cloud/explore-data-with-chat2query.md). -In Chat2Query, you can either simply type `--` followed by your instructions to let AI generate SQL queries automatically or write SQL queries manually, and then run SQL queries against databases without a terminal. You can find the query results in tables intuitively and check the query logs easily. For more information, see [Chat2Query (beta)](/tidb-cloud/explore-data-with-chat2query.md). - -In addition, TiDB Cloud provides a Chat2Query API for TiDB Serverless clusters. After it is enabled, TiDB Cloud will automatically create a system Data App called **Chat2Query** and a Chat2Data endpoint in Data Service. You can call this endpoint to let AI generate and execute SQL statements by providing instructions. For more information, see [Get started with Chat2Query API](/tidb-cloud/use-chat2query-api.md). +In addition, TiDB Cloud provides a Chat2Query API for TiDB Cloud Serverless clusters. After it is enabled, TiDB Cloud will automatically create a system Data App called **Chat2Query** and a Chat2Data endpoint in Data Service. You can call this endpoint to let AI generate and execute SQL statements by providing instructions. For more information, see [Get started with Chat2Query API](/tidb-cloud/use-chat2query-api.md). ### Credit @@ -49,6 +47,10 @@ Data Service (beta) enables you to access TiDB Cloud data via an HTTPS request u For more information, see [Data Service Overview](/tidb-cloud/data-service-overview.md). +### Direct Customer + +A direct customer is an end customer who purchases TiDB Cloud and pays invoices directly from PingCAP. This is distinguished from an [MSP customer](#msp-customer). + ## E ### Endpoint @@ -67,6 +69,14 @@ A user that has been invited to an organization, with access to the organization Starting from v5.0, TiDB introduces Massively Parallel Processing (MPP) architecture through TiFlash nodes, which shares the execution workloads of large join queries among TiFlash nodes. When the MPP mode is enabled, TiDB, based on cost, determines whether to use the MPP framework to perform the calculation. In the MPP mode, the join keys are redistributed through the Exchange operation while being calculated, which distributes the calculation pressure to each TiFlash node and speeds up the calculation. For more information, see [Use TiFlash MPP Mode](/tiflash/use-tiflash-mpp-mode.md). +### MSP Customer + +A managed service provider (MSP) customer is an end customer who purchases TiDB Cloud and pays invoices through the MSP channel. This is distinguished from a [direct customer](#direct-customer). + +### Managed Service Provider (MSP) + +A managed service provider (MSP) is a partner who resells TiDB Cloud and provides value-added services, including but not limited to TiDB Cloud organization management, billing services, and technical support. + ## N ### node @@ -101,7 +111,7 @@ Project members are users who are invited to join one or more projects of the or ### Recycle Bin -The place where the data of deleted clusters with valid backups is stored. Once a backed-up TiDB Dedicated cluster is deleted, the existing backup files of the cluster are moved to the recycle bin. For backup files from automatic backups, the recycle bin will retain them for a specified period. You can configure the backup retention in **Backup Settings**, and the default is 7 days. For backup files from manual backups, there is no expiration date. To avoid data loss, remember to restore the data to a new cluster in time. Note that if a cluster **has no backup**, the deleted cluster will not be displayed here. +The place where the data of deleted clusters with valid backups is stored. Once a backed-up TiDB Cloud Dedicated cluster is deleted, the existing backup files of the cluster are moved to the recycle bin. For backup files from automatic backups, the recycle bin will retain them for a specified period. You can configure the backup retention in **Backup Setting**, and the default is 7 days. For backup files from manual backups, there is no expiration date. To avoid data loss, remember to restore the data to a new cluster in time. Note that if a cluster **has no backup**, the deleted cluster will not be displayed here. ### region @@ -123,13 +133,13 @@ The replication of changefeed is charged according to the computing resources, w ### Request Unit -A Request Unit (RU) is a unit of measure used to represent the amount of resources consumed by a single request to the database. The amount of RUs consumed by a request depends on various factors, such as the operation type or the amount of data being retrieved or modified. For more information, see [TiDB Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). +A Request Unit (RU) is a unit of measure used to represent the amount of resources consumed by a single request to the database. The amount of RUs consumed by a request depends on various factors, such as the operation type or the amount of data being retrieved or modified. For more information, see [TiDB Cloud Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). ## S ### Spending limit -Spending limit refers to the maximum amount of money that you are willing to spend on a particular workload in a month. It is a cost-control mechanism that allows you to set a budget for your TiDB Serverless clusters. When the spending limit of a cluster is greater than 0, the cluster is considered a paid cluster. Also, the paid cluster can have a free quota if it meets the qualifications. The paid cluster with a free quota will consume the free quota first. +Spending limit refers to the maximum amount of money that you are willing to spend on a particular workload in a month. It is a cost-control mechanism that enables you to set a budget for your TiDB Cloud Serverless clusters. For [scalable clusters](/tidb-cloud/select-cluster-tier.md#scalable-cluster-plan), the spending limit must be set to a minimum of $0.01. Also, the scalable cluster can have a free quota if it meets the qualifications. The scalable cluster with a free quota will consume the free quota first. ## T diff --git a/tidb-cloud/tidb-cloud-guide-sample-application-java.md b/tidb-cloud/tidb-cloud-guide-sample-application-java.md deleted file mode 100644 index 66b5b6b64dd7a..0000000000000 --- a/tidb-cloud/tidb-cloud-guide-sample-application-java.md +++ /dev/null @@ -1,1730 +0,0 @@ ---- -title: Build a Simple CRUD App with TiDB and Java -summary: Learn how to build a simple CRUD application with TiDB and Java. ---- - - - - -# Build a Simple CRUD App with TiDB and Java - -This document describes how to use TiDB and Java to build a simple CRUD application. - -> **Note:** -> -> It is recommended to use Java 8 or a later Java version. -> -> If you want to use Spring Boot for application development, refer to [Build the TiDB app using Spring Boot](/develop/dev-guide-sample-application-java-spring-boot.md) - -## Step 1. Launch your TiDB cluster - - - -The following introduces how to start a TiDB cluster. - -**Use a TiDB Serverless cluster** - -For detailed steps, see [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). - -**Use a local cluster** - -For detailed steps, see [Deploy a local test cluster](/quick-start-with-tidb.md#deploy-a-local-test-cluster) or [Deploy a TiDB Cluster Using TiUP](/production-deployment-using-tiup.md). - - - - - -See [Create a TiDB Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md#step-1-create-a-tidb-serverless-cluster). - - - -## Step 2. Get the code - -```shell -git clone https://github.com/pingcap-inc/tidb-example-java.git -``` - - - -
- -Compared with [Mybatis](https://mybatis.org/mybatis-3/index.html), the JDBC implementation might be not a best practice, because you need to write error handling logic manually and cannot reuse code easily, which makes your code slightly redundant. - -Mybatis is a popular open-source Java class persistence framework. The following uses [MyBatis Generator](https://mybatis.org/generator/quickstart.html) as a Maven plugin to generate the persistence layer code. - -Change to the `plain-java-mybatis` directory: - -```shell -cd plain-java-mybatis -``` - -The structure of this directory is as follows: - -``` -. -├── Makefile -├── pom.xml -└── src - └── main - ├── java - │   └── com - │   └── pingcap - │   ├── MybatisExample.java - │   ├── dao - │   │   └── PlayerDAO.java - │   └── model - │   ├── Player.java - │   ├── PlayerMapper.java - │   └── PlayerMapperEx.java - └── resources - ├── dbinit.sql - ├── log4j.properties - ├── mapper - │   ├── PlayerMapper.xml - │   └── PlayerMapperEx.xml - ├── mybatis-config.xml - └── mybatis-generator.xml -``` - -The automatically generated files are: - -- `src/main/java/com/pingcap/model/Player.java`: The `Player` entity class. -- `src/main/java/com/pingcap/model/PlayerMapper.java`: The interface of `PlayerMapper`. -- `src/main/resources/mapper/PlayerMapper.xml`: The XML mapping of `Player`. Mybatis uses this configuration to automatically generate the implementation class of the `PlayerMapper` interface. - -The strategy for generating these files is written in `mybatis-generator.xml`, which is the configuration file for [Mybatis Generator](https://mybatis.org/generator/quickstart.html). There are comments in the following configuration file to describe how to use it. - -```xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -``` - -`mybatis-generator.xml` is included in `pom.xml` as the configuration of `mybatis-generator-maven-plugin`. - -```xml - - org.mybatis.generator - mybatis-generator-maven-plugin - 1.4.1 - - src/main/resources/mybatis-generator.xml - true - true - - - - - - mysql - mysql-connector-java - 5.1.49 - - - -``` - -Once included in the Maven plugin, you can delete the old generated files and make new ones using `mvn mybatis-generate`. Or you can use `make gen` to delete the old file and generate a new one at the same time. - -> **Note:** -> -> The property `configuration.overwrite` in `mybatis-generator.xml` only ensures that the generated Java code files are overwritten. But the XML mapping files are still written as appended. Therefore, it is recommended to delete the old file before Mybaits Generator generating a new one. - -`Player.java` is a data entity class file generated using Mybatis Generator, which is a mapping of database tables in the application. Each property of the `Player` class corresponds to a field in the `player` table. - -```java -package com.pingcap.model; - -public class Player { - private String id; - - private Integer coins; - - private Integer goods; - - public Player(String id, Integer coins, Integer goods) { - this.id = id; - this.coins = coins; - this.goods = goods; - } - - public Player() { - super(); - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public Integer getCoins() { - return coins; - } - - public void setCoins(Integer coins) { - this.coins = coins; - } - - public Integer getGoods() { - return goods; - } - - public void setGoods(Integer goods) { - this.goods = goods; - } -} -``` - -`PlayerMapper.java` is a mapping interface file generated using Mybatis Generator. This file only defines the interface, and the implementation classes of interface are automatically generated using XML or annotations. - -```java -package com.pingcap.model; - -import com.pingcap.model.Player; - -public interface PlayerMapper { - int deleteByPrimaryKey(String id); - - int insert(Player row); - - int insertSelective(Player row); - - Player selectByPrimaryKey(String id); - - int updateByPrimaryKeySelective(Player row); - - int updateByPrimaryKey(Player row); -} -``` - -`PlayerMapper.xml` is a mapping XML file generated using Mybatis Generator. Mybatis uses this to automatically generate the implementation class of the `PlayerMapper` interface. - -```xml - - - - - - - - - - - - id, coins, goods - - - - delete from player - where id = #{id,jdbcType=VARCHAR} - - - insert into player (id, coins, goods - ) - values (#{id,jdbcType=VARCHAR}, #{coins,jdbcType=INTEGER}, #{goods,jdbcType=INTEGER} - ) - - - insert into player - - - id, - - - coins, - - - goods, - - - - - #{id,jdbcType=VARCHAR}, - - - #{coins,jdbcType=INTEGER}, - - - #{goods,jdbcType=INTEGER}, - - - - - update player - - - coins = #{coins,jdbcType=INTEGER}, - - - goods = #{goods,jdbcType=INTEGER}, - - - where id = #{id,jdbcType=VARCHAR} - - - update player - set coins = #{coins,jdbcType=INTEGER}, - goods = #{goods,jdbcType=INTEGER} - where id = #{id,jdbcType=VARCHAR} - - -``` - -Since Mybatis Generator needs to generate the source code from the table definition, the table needs to be created first. To create the table, you can use `dbinit.sql`. - -```sql -USE test; -DROP TABLE IF EXISTS player; - -CREATE TABLE player ( - `id` VARCHAR(36), - `coins` INTEGER, - `goods` INTEGER, - PRIMARY KEY (`id`) -); -``` - -Split the interface `PlayerMapperEx` additionally to extend from `PlayerMapper` and write a matching `PlayerMapperEx.xml` file. Avoid changing `PlayerMapper.java` and `PlayerMapper.xml` directly. This is to avoid overwrite by Mybatis Generator. - -Define the added interface in `PlayerMapperEx.java`: - -```java -package com.pingcap.model; - -import java.util.List; - -public interface PlayerMapperEx extends PlayerMapper { - Player selectByPrimaryKeyWithLock(String id); - - List selectByLimit(Integer limit); - - Integer count(); -} -``` - -Define the mapping rules in `PlayerMapperEx.xml`: - -```xml - - - - - - - - - - - - id, coins, goods - - - - - - - - - -``` - -`PlayerDAO.java` is a class used to manage data, in which `DAO` means [Data Access Object](https://en.wikipedia.org/wiki/Data_access_object). The class defines a set of data manipulation methods for writing data. In it, Mybatis encapsulates a large number of operations such as object mapping and CRUD of basic objects, which greatly simplifies the code. - -```java -package com.pingcap.dao; - -import com.pingcap.model.Player; -import com.pingcap.model.PlayerMapperEx; -import org.apache.ibatis.session.SqlSession; -import org.apache.ibatis.session.SqlSessionFactory; - -import java.util.List; -import java.util.function.Function; - -public class PlayerDAO { - public static class NotEnoughException extends RuntimeException { - public NotEnoughException(String message) { - super(message); - } - } - - // Run SQL code in a way that automatically handles the - // transaction retry logic, so we don't have to duplicate it in - // various places. - public Object runTransaction(SqlSessionFactory sessionFactory, Function fn) { - Object resultObject = null; - SqlSession session = null; - - try { - // open a session with autoCommit is false - session = sessionFactory.openSession(false); - - // get player mapper - PlayerMapperEx playerMapperEx = session.getMapper(PlayerMapperEx.class); - - resultObject = fn.apply(playerMapperEx); - session.commit(); - System.out.println("APP: COMMIT;"); - } catch (Exception e) { - if (e instanceof NotEnoughException) { - System.out.printf("APP: ROLLBACK BY LOGIC; \n%s\n", e.getMessage()); - } else { - System.out.printf("APP: ROLLBACK BY ERROR; \n%s\n", e.getMessage()); - } - - if (session != null) { - session.rollback(); - } - } finally { - if (session != null) { - session.close(); - } - } - - return resultObject; - } - - public Function createPlayers(List players) { - return playerMapperEx -> { - Integer addedPlayerAmount = 0; - for (Player player: players) { - playerMapperEx.insert(player); - addedPlayerAmount ++; - } - System.out.printf("APP: createPlayers() --> %d\n", addedPlayerAmount); - return addedPlayerAmount; - }; - } - - public Function buyGoods(String sellId, String buyId, Integer amount, Integer price) { - return playerMapperEx -> { - Player sellPlayer = playerMapperEx.selectByPrimaryKeyWithLock(sellId); - Player buyPlayer = playerMapperEx.selectByPrimaryKeyWithLock(buyId); - - if (buyPlayer == null || sellPlayer == null) { - throw new NotEnoughException("sell or buy player not exist"); - } - - if (buyPlayer.getCoins() < price || sellPlayer.getGoods() < amount) { - throw new NotEnoughException("coins or goods not enough, rollback"); - } - - int affectRows = 0; - buyPlayer.setGoods(buyPlayer.getGoods() + amount); - buyPlayer.setCoins(buyPlayer.getCoins() - price); - affectRows += playerMapperEx.updateByPrimaryKey(buyPlayer); - - sellPlayer.setGoods(sellPlayer.getGoods() - amount); - sellPlayer.setCoins(sellPlayer.getCoins() + price); - affectRows += playerMapperEx.updateByPrimaryKey(sellPlayer); - - System.out.printf("APP: buyGoods --> sell: %s, buy: %s, amount: %d, price: %d\n", sellId, buyId, amount, price); - return affectRows; - }; - } - - public Function getPlayerByID(String id) { - return playerMapperEx -> playerMapperEx.selectByPrimaryKey(id); - } - - public Function printPlayers(Integer limit) { - return playerMapperEx -> { - List players = playerMapperEx.selectByLimit(limit); - - for (Player player: players) { - System.out.println("\n[printPlayers]:\n" + player); - } - return 0; - }; - } - - public Function countPlayers() { - return PlayerMapperEx::count; - } -} -``` - -`MybatisExample` is the main class of the `plain-java-mybatis` sample application. It defines the entry functions: - -```java -package com.pingcap; - -import com.pingcap.dao.PlayerDAO; -import com.pingcap.model.Player; -import org.apache.ibatis.io.Resources; -import org.apache.ibatis.session.SqlSessionFactory; -import org.apache.ibatis.session.SqlSessionFactoryBuilder; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import java.util.Collections; - -public class MybatisExample { - public static void main( String[] args ) throws IOException { - // 1. Create a SqlSessionFactory based on our mybatis-config.xml configuration - // file, which defines how to connect to the database. - InputStream inputStream = Resources.getResourceAsStream("mybatis-config.xml"); - SqlSessionFactory sessionFactory = new SqlSessionFactoryBuilder().build(inputStream); - - // 2. And then, create DAO to manager your data - PlayerDAO playerDAO = new PlayerDAO(); - - // 3. Run some simple examples. - - // Create a player who has 1 coin and 1 goods. - playerDAO.runTransaction(sessionFactory, playerDAO.createPlayers( - Collections.singletonList(new Player("test", 1, 1)))); - - // Get a player. - Player testPlayer = (Player)playerDAO.runTransaction(sessionFactory, playerDAO.getPlayerByID("test")); - System.out.printf("PlayerDAO.getPlayer:\n => id: %s\n => coins: %s\n => goods: %s\n", - testPlayer.getId(), testPlayer.getCoins(), testPlayer.getGoods()); - - // Count players amount. - Integer count = (Integer)playerDAO.runTransaction(sessionFactory, playerDAO.countPlayers()); - System.out.printf("PlayerDAO.countPlayers:\n => %d total players\n", count); - - // Print 3 players. - playerDAO.runTransaction(sessionFactory, playerDAO.printPlayers(3)); - - // 4. Getting further. - - // Player 1: id is "1", has only 100 coins. - // Player 2: id is "2", has 114514 coins, and 20 goods. - Player player1 = new Player("1", 100, 0); - Player player2 = new Player("2", 114514, 20); - - // Create two players "by hand", using the INSERT statement on the backend. - int addedCount = (Integer)playerDAO.runTransaction(sessionFactory, - playerDAO.createPlayers(Arrays.asList(player1, player2))); - System.out.printf("PlayerDAO.createPlayers:\n => %d total inserted players\n", addedCount); - - // Player 1 wants to buy 10 goods from player 2. - // It will cost 500 coins, but player 1 cannot afford it. - System.out.println("\nPlayerDAO.buyGoods:\n => this trade will fail"); - Integer updatedCount = (Integer)playerDAO.runTransaction(sessionFactory, - playerDAO.buyGoods(player2.getId(), player1.getId(), 10, 500)); - System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); - - // So player 1 has to reduce the incoming quantity to two. - System.out.println("\nPlayerDAO.buyGoods:\n => this trade will success"); - updatedCount = (Integer)playerDAO.runTransaction(sessionFactory, - playerDAO.buyGoods(player2.getId(), player1.getId(), 2, 100)); - System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); - } -} -``` - - - -
- -Compared with Hibernate, the JDBC implementation might be not a best practice, because you need to write error handling logic manually and cannot reuse code easily, which makes your code slightly redundant. - -Hibernate is a popular open-source Java ORM, and it supports TiDB dialect starting from `v6.0.0.Beta2`, which fits TiDB features well. The following instructions take `v6.0.0.Beta2` as an example. - -Change to the `plain-java-hibernate` directory: - -```shell -cd plain-java-hibernate -``` - -The structure of this directory is as follows: - -``` -. -├── Makefile -├── plain-java-hibernate.iml -├── pom.xml -└── src - └── main - ├── java - │ └── com - │ └── pingcap - │ └── HibernateExample.java - └── resources - └── hibernate.cfg.xml -``` - -`hibernate.cfg.xml` is the Hibernate configuration file: - -```xml - - - - - - - com.mysql.cj.jdbc.Driver - org.hibernate.dialect.TiDBDialect - jdbc:mysql://localhost:4000/test - root - - false - - - create-drop - - - true - true - - -``` - -`HibernateExample.java` is the main body of the `plain-java-hibernate`. Compared with JDBC, when using Hibernate, you only need to write the path of the configuration file, because Hibernate avoids differences in database creation between different databases. - -`PlayerDAO` is a class used to manage data, in which `DAO` means [Data Access Object](https://en.wikipedia.org/wiki/Data_access_object). The class defines a set of data manipulation methods for writing data. Compared with JDBC, Hibernate encapsulates a large number of operations such as object mapping and CRUD of basic objects, which greatly simplifies the code. - -`PlayerBean` is a data entity class that is a mapping for tables. Each property of a `PlayerBean` corresponds to a field in the `player` table. Compared with JDBC, `PlayerBean` in Hibernate adds annotations to indicate mapping relationships for more information. - -```java -package com.pingcap; - -import jakarta.persistence.Column; -import jakarta.persistence.Entity; -import jakarta.persistence.Id; -import jakarta.persistence.Table; -import org.hibernate.JDBCException; -import org.hibernate.Session; -import org.hibernate.SessionFactory; -import org.hibernate.Transaction; -import org.hibernate.cfg.Configuration; -import org.hibernate.query.NativeQuery; -import org.hibernate.query.Query; - -import java.util.Arrays; -import java.util.Collections; -import java.util.List; -import java.util.function.Function; - -@Entity -@Table(name = "player_hibernate") -class PlayerBean { - @Id - private String id; - @Column(name = "coins") - private Integer coins; - @Column(name = "goods") - private Integer goods; - - public PlayerBean() { - } - - public PlayerBean(String id, Integer coins, Integer goods) { - this.id = id; - this.coins = coins; - this.goods = goods; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public Integer getCoins() { - return coins; - } - - public void setCoins(Integer coins) { - this.coins = coins; - } - - public Integer getGoods() { - return goods; - } - - public void setGoods(Integer goods) { - this.goods = goods; - } - - @Override - public String toString() { - return String.format(" %-8s => %10s\n %-8s => %10s\n %-8s => %10s\n", - "id", this.id, "coins", this.coins, "goods", this.goods); - } -} - -/** - * Main class for the basic Hibernate example. - **/ -public class HibernateExample -{ - public static class PlayerDAO { - public static class NotEnoughException extends RuntimeException { - public NotEnoughException(String message) { - super(message); - } - } - - // Run SQL code in a way that automatically handles the - // transaction retry logic so we don't have to duplicate it in - // various places. - public Object runTransaction(Session session, Function fn) { - Object resultObject = null; - - Transaction txn = session.beginTransaction(); - try { - resultObject = fn.apply(session); - txn.commit(); - System.out.println("APP: COMMIT;"); - } catch (JDBCException e) { - System.out.println("APP: ROLLBACK BY JDBC ERROR;"); - txn.rollback(); - } catch (NotEnoughException e) { - System.out.printf("APP: ROLLBACK BY LOGIC; %s", e.getMessage()); - txn.rollback(); - } - return resultObject; - } - - public Function createPlayers(List players) throws JDBCException { - return session -> { - Integer addedPlayerAmount = 0; - for (PlayerBean player: players) { - session.persist(player); - addedPlayerAmount ++; - } - System.out.printf("APP: createPlayers() --> %d\n", addedPlayerAmount); - return addedPlayerAmount; - }; - } - - public Function buyGoods(String sellId, String buyId, Integer amount, Integer price) throws JDBCException { - return session -> { - PlayerBean sellPlayer = session.get(PlayerBean.class, sellId); - PlayerBean buyPlayer = session.get(PlayerBean.class, buyId); - - if (buyPlayer == null || sellPlayer == null) { - throw new NotEnoughException("sell or buy player not exist"); - } - - if (buyPlayer.getCoins() < price || sellPlayer.getGoods() < amount) { - throw new NotEnoughException("coins or goods not enough, rollback"); - } - - buyPlayer.setGoods(buyPlayer.getGoods() + amount); - buyPlayer.setCoins(buyPlayer.getCoins() - price); - session.persist(buyPlayer); - - sellPlayer.setGoods(sellPlayer.getGoods() - amount); - sellPlayer.setCoins(sellPlayer.getCoins() + price); - session.persist(sellPlayer); - - System.out.printf("APP: buyGoods --> sell: %s, buy: %s, amount: %d, price: %d\n", sellId, buyId, amount, price); - return 0; - }; - } - - public Function getPlayerByID(String id) throws JDBCException { - return session -> session.get(PlayerBean.class, id); - } - - public Function printPlayers(Integer limit) throws JDBCException { - return session -> { - NativeQuery limitQuery = session.createNativeQuery("SELECT * FROM player_hibernate LIMIT :limit", PlayerBean.class); - limitQuery.setParameter("limit", limit); - List players = limitQuery.getResultList(); - - for (PlayerBean player: players) { - System.out.println("\n[printPlayers]:\n" + player); - } - return 0; - }; - } - - public Function countPlayers() throws JDBCException { - return session -> { - Query countQuery = session.createQuery("SELECT count(player_hibernate) FROM PlayerBean player_hibernate", Long.class); - return countQuery.getSingleResult(); - }; - } - } - - public static void main(String[] args) { - // 1. Create a SessionFactory based on our hibernate.cfg.xml configuration - // file, which defines how to connect to the database. - SessionFactory sessionFactory - = new Configuration() - .configure("hibernate.cfg.xml") - .addAnnotatedClass(PlayerBean.class) - .buildSessionFactory(); - - try (Session session = sessionFactory.openSession()) { - // 2. And then, create DAO to manager your data. - PlayerDAO playerDAO = new PlayerDAO(); - - // 3. Run some simple example. - - // Create a player who has 1 coin and 1 goods. - playerDAO.runTransaction(session, playerDAO.createPlayers(Collections.singletonList( - new PlayerBean("test", 1, 1)))); - - // Get a player. - PlayerBean testPlayer = (PlayerBean)playerDAO.runTransaction(session, playerDAO.getPlayerByID("test")); - System.out.printf("PlayerDAO.getPlayer:\n => id: %s\n => coins: %s\n => goods: %s\n", - testPlayer.getId(), testPlayer.getCoins(), testPlayer.getGoods()); - - // Count players amount. - Long count = (Long)playerDAO.runTransaction(session, playerDAO.countPlayers()); - System.out.printf("PlayerDAO.countPlayers:\n => %d total players\n", count); - - // Print 3 players. - playerDAO.runTransaction(session, playerDAO.printPlayers(3)); - - // 4. Getting further. - - // Player 1: id is "1", has only 100 coins. - // Player 2: id is "2", has 114514 coins, and 20 goods. - PlayerBean player1 = new PlayerBean("1", 100, 0); - PlayerBean player2 = new PlayerBean("2", 114514, 20); - - // Create two players "by hand", using the INSERT statement on the backend. - int addedCount = (Integer)playerDAO.runTransaction(session, - playerDAO.createPlayers(Arrays.asList(player1, player2))); - System.out.printf("PlayerDAO.createPlayers:\n => %d total inserted players\n", addedCount); - - // Player 1 wants to buy 10 goods from player 2. - // It will cost 500 coins, but player 1 can't afford it. - System.out.println("\nPlayerDAO.buyGoods:\n => this trade will fail"); - Integer updatedCount = (Integer)playerDAO.runTransaction(session, - playerDAO.buyGoods(player2.getId(), player1.getId(), 10, 500)); - System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); - - // So player 1 have to reduce his incoming quantity to two. - System.out.println("\nPlayerDAO.buyGoods:\n => this trade will success"); - updatedCount = (Integer)playerDAO.runTransaction(session, - playerDAO.buyGoods(player2.getId(), player1.getId(), 2, 100)); - System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); - } finally { - sessionFactory.close(); - } - } -} -``` - -
- -
- -Change to the `plain-java-jdbc` directory: - -```shell -cd plain-java-jdbc -``` - -The structure of this directory is as follows: - -``` -. -├── Makefile -├── plain-java-jdbc.iml -├── pom.xml -└── src - └── main - ├── java - │ └── com - │ └── pingcap - │ └── JDBCExample.java - └── resources - └── dbinit.sql -``` - -You can find initialization statements for the table creation in `dbinit.sql`: - -```sql -USE test; -DROP TABLE IF EXISTS player; - -CREATE TABLE player ( - `id` VARCHAR(36), - `coins` INTEGER, - `goods` INTEGER, - PRIMARY KEY (`id`) -); -``` - -`JDBCExample.java` is the main body of the `plain-java-jdbc`. TiDB is highly compatible with the MySQL protocol, so you need to initialize a MySQL source instance `MysqlDataSource` to connect to TiDB. Then, you can initialize `PlayerDAO` for object management and use it to read, edit, add, and delete data. - -`PlayerDAO` is a class used to manage data, in which `DAO` means [Data Access Object](https://en.wikipedia.org/wiki/Data_access_object). The class defines a set of data manipulation methods to provide the ability to write data. - -`PlayerBean` is a data entity class that is a mapping for tables. Each property of a `PlayerBean` corresponds to a field in the `player` table. - -```java -package com.pingcap; - -import com.mysql.cj.jdbc.MysqlDataSource; - -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.*; - -/** - * Main class for the basic JDBC example. - **/ -public class JDBCExample -{ - public static class PlayerBean { - private String id; - private Integer coins; - private Integer goods; - - public PlayerBean() { - } - - public PlayerBean(String id, Integer coins, Integer goods) { - this.id = id; - this.coins = coins; - this.goods = goods; - } - - public String getId() { - return id; - } - - public void setId(String id) { - this.id = id; - } - - public Integer getCoins() { - return coins; - } - - public void setCoins(Integer coins) { - this.coins = coins; - } - - public Integer getGoods() { - return goods; - } - - public void setGoods(Integer goods) { - this.goods = goods; - } - - @Override - public String toString() { - return String.format(" %-8s => %10s\n %-8s => %10s\n %-8s => %10s\n", - "id", this.id, "coins", this.coins, "goods", this.goods); - } - } - - /** - * Data access object used by 'ExampleDataSource'. - * Example for CURD and bulk insert. - */ - public static class PlayerDAO { - private final MysqlDataSource ds; - private final Random rand = new Random(); - - PlayerDAO(MysqlDataSource ds) { - this.ds = ds; - } - - /** - * Create players by passing in a List of PlayerBean. - * - * @param players Will create players list - * @return The number of create accounts - */ - public int createPlayers(List players){ - int rows = 0; - - Connection connection = null; - PreparedStatement preparedStatement = null; - try { - connection = ds.getConnection(); - preparedStatement = connection.prepareStatement("INSERT INTO player (id, coins, goods) VALUES (?, ?, ?)"); - } catch (SQLException e) { - System.out.printf("[createPlayers] ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - e.printStackTrace(); - - return -1; - } - - try { - for (PlayerBean player : players) { - preparedStatement.setString(1, player.getId()); - preparedStatement.setInt(2, player.getCoins()); - preparedStatement.setInt(3, player.getGoods()); - - preparedStatement.execute(); - rows += preparedStatement.getUpdateCount(); - } - } catch (SQLException e) { - System.out.printf("[createPlayers] ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - e.printStackTrace(); - } finally { - try { - connection.close(); - } catch (SQLException e) { - e.printStackTrace(); - } - } - - System.out.printf("\n[createPlayers]:\n '%s'\n", preparedStatement); - return rows; - } - - /** - * Buy goods and transfer funds between one player and another in one transaction. - * @param sellId Sell player id. - * @param buyId Buy player id. - * @param amount Goods amount, if sell player has not enough goods, the trade will break. - * @param price Price should pay, if buy player has not enough coins, the trade will break. - * - * @return The number of effected players. - */ - public int buyGoods(String sellId, String buyId, Integer amount, Integer price) { - int effectPlayers = 0; - - Connection connection = null; - try { - connection = ds.getConnection(); - } catch (SQLException e) { - System.out.printf("[buyGoods] ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - e.printStackTrace(); - return effectPlayers; - } - - try { - connection.setAutoCommit(false); - - PreparedStatement playerQuery = connection.prepareStatement("SELECT * FROM player WHERE id=? OR id=? FOR UPDATE"); - playerQuery.setString(1, sellId); - playerQuery.setString(2, buyId); - playerQuery.execute(); - - PlayerBean sellPlayer = null; - PlayerBean buyPlayer = null; - - ResultSet playerQueryResultSet = playerQuery.getResultSet(); - while (playerQueryResultSet.next()) { - PlayerBean player = new PlayerBean( - playerQueryResultSet.getString("id"), - playerQueryResultSet.getInt("coins"), - playerQueryResultSet.getInt("goods") - ); - - System.out.println("\n[buyGoods]:\n 'check goods and coins enough'"); - System.out.println(player); - - if (sellId.equals(player.getId())) { - sellPlayer = player; - } else { - buyPlayer = player; - } - } - - if (sellPlayer == null || buyPlayer == null) { - throw new SQLException("player not exist."); - } - - if (sellPlayer.getGoods().compareTo(amount) < 0) { - throw new SQLException(String.format("sell player %s goods not enough.", sellId)); - } - - if (buyPlayer.getCoins().compareTo(price) < 0) { - throw new SQLException(String.format("buy player %s coins not enough.", buyId)); - } - - PreparedStatement transfer = connection.prepareStatement("UPDATE player set goods = goods + ?, coins = coins + ? WHERE id=?"); - transfer.setInt(1, -amount); - transfer.setInt(2, price); - transfer.setString(3, sellId); - transfer.execute(); - effectPlayers += transfer.getUpdateCount(); - - transfer.setInt(1, amount); - transfer.setInt(2, -price); - transfer.setString(3, buyId); - transfer.execute(); - effectPlayers += transfer.getUpdateCount(); - - connection.commit(); - - System.out.println("\n[buyGoods]:\n 'trade success'"); - } catch (SQLException e) { - System.out.printf("[buyGoods] ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - - try { - System.out.println("[buyGoods] Rollback"); - - connection.rollback(); - } catch (SQLException ex) { - // do nothing - } - } finally { - try { - connection.close(); - } catch (SQLException e) { - // do nothing - } - } - - return effectPlayers; - } - - /** - * Get the player info by id. - * - * @param id Player id. - * @return The player of this id. - */ - public PlayerBean getPlayer(String id) { - PlayerBean player = null; - - try (Connection connection = ds.getConnection()) { - PreparedStatement preparedStatement = connection.prepareStatement("SELECT * FROM player WHERE id = ?"); - preparedStatement.setString(1, id); - preparedStatement.execute(); - - ResultSet res = preparedStatement.executeQuery(); - if(!res.next()) { - System.out.printf("No players in the table with id %s", id); - } else { - player = new PlayerBean(res.getString("id"), res.getInt("coins"), res.getInt("goods")); - } - } catch (SQLException e) { - System.out.printf("PlayerDAO.getPlayer ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - } - - return player; - } - - /** - * Insert randomized account data (id, coins, goods) using the JDBC fast path for - * bulk inserts. The fastest way to get data into TiDB is using the - * TiDB Lightning(https://docs.pingcap.com/tidb/stable/tidb-lightning-overview). - * However, if you must bulk insert from the application using INSERT SQL, the best - * option is the method shown here. It will require the following: - * - * Add `rewriteBatchedStatements=true` to your JDBC connection settings. - * Setting rewriteBatchedStatements to true now causes CallableStatements - * with batched arguments to be re-written in the form "CALL (...); CALL (...); ..." - * to send the batch in as few client/server round trips as possible. - * https://dev.mysql.com/doc/relnotes/connector-j/5.1/en/news-5-1-3.html - * - * You can see the `rewriteBatchedStatements` param effect logic at - * implement function: `com.mysql.cj.jdbc.StatementImpl.executeBatchUsingMultiQueries` - * - * @param total Add players amount. - * @param batchSize Bulk insert size for per batch. - * - * @return The number of new accounts inserted. - */ - public int bulkInsertRandomPlayers(Integer total, Integer batchSize) { - int totalNewPlayers = 0; - - try (Connection connection = ds.getConnection()) { - // We're managing the commit lifecycle ourselves, so we can - // control the size of our batch inserts. - connection.setAutoCommit(false); - - // In this example we are adding 500 rows to the database, - // but it could be any number. What's important is that - // the batch size is 128. - try (PreparedStatement pstmt = connection.prepareStatement("INSERT INTO player (id, coins, goods) VALUES (?, ?, ?)")) { - for (int i=0; i<=(total/batchSize);i++) { - for (int j=0; j %s row(s) updated in this batch\n", count.length); - } - connection.commit(); - } catch (SQLException e) { - System.out.printf("PlayerDAO.bulkInsertRandomPlayers ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - } - } catch (SQLException e) { - System.out.printf("PlayerDAO.bulkInsertRandomPlayers ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - } - return totalNewPlayers; - } - - - /** - * Print a subset of players from the data store by limit. - * - * @param limit Print max size. - */ - public void printPlayers(Integer limit) { - try (Connection connection = ds.getConnection()) { - PreparedStatement preparedStatement = connection.prepareStatement("SELECT * FROM player LIMIT ?"); - preparedStatement.setInt(1, limit); - preparedStatement.execute(); - - ResultSet res = preparedStatement.executeQuery(); - while (!res.next()) { - PlayerBean player = new PlayerBean(res.getString("id"), - res.getInt("coins"), res.getInt("goods")); - System.out.println("\n[printPlayers]:\n" + player); - } - } catch (SQLException e) { - System.out.printf("PlayerDAO.printPlayers ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - } - } - - - /** - * Count players from the data store. - * - * @return All players count - */ - public int countPlayers() { - int count = 0; - - try (Connection connection = ds.getConnection()) { - PreparedStatement preparedStatement = connection.prepareStatement("SELECT count(*) FROM player"); - preparedStatement.execute(); - - ResultSet res = preparedStatement.executeQuery(); - if(res.next()) { - count = res.getInt(1); - } - } catch (SQLException e) { - System.out.printf("PlayerDAO.countPlayers ERROR: { state => %s, cause => %s, message => %s }\n", - e.getSQLState(), e.getCause(), e.getMessage()); - } - - return count; - } - } - - public static void main(String[] args) { - // 1. Configure the example database connection. - - // 1.1 Create a mysql data source instance. - MysqlDataSource mysqlDataSource = new MysqlDataSource(); - - // 1.2 Set server name, port, database name, username and password. - mysqlDataSource.setServerName("localhost"); - mysqlDataSource.setPortNumber(4000); - mysqlDataSource.setDatabaseName("test"); - mysqlDataSource.setUser("root"); - mysqlDataSource.setPassword(""); - - // Or you can use jdbc string instead. - // mysqlDataSource.setURL("jdbc:mysql://{host}:{port}/test?user={user}&password={password}"); - - // 2. And then, create DAO to manager your data. - PlayerDAO dao = new PlayerDAO(mysqlDataSource); - - // 3. Run some simple example. - - // Create a player, has a coin and a goods. - dao.createPlayers(Collections.singletonList(new PlayerBean("test", 1, 1))); - - // Get a player. - PlayerBean testPlayer = dao.getPlayer("test"); - System.out.printf("PlayerDAO.getPlayer:\n => id: %s\n => coins: %s\n => goods: %s\n", - testPlayer.getId(), testPlayer.getCoins(), testPlayer.getGoods()); - - // Create players with bulk inserts, insert 1919 players totally, and per batch for 114 players. - int addedCount = dao.bulkInsertRandomPlayers(1919, 114); - System.out.printf("PlayerDAO.bulkInsertRandomPlayers:\n => %d total inserted players\n", addedCount); - - // Count players amount. - int count = dao.countPlayers(); - System.out.printf("PlayerDAO.countPlayers:\n => %d total players\n", count); - - // Print 3 players. - dao.printPlayers(3); - - // 4. Getting further. - - // Player 1: id is "1", has only 100 coins. - // Player 2: id is "2", has 114514 coins, and 20 goods. - PlayerBean player1 = new PlayerBean("1", 100, 0); - PlayerBean player2 = new PlayerBean("2", 114514, 20); - - // Create two players "by hand", using the INSERT statement on the backend. - addedCount = dao.createPlayers(Arrays.asList(player1, player2)); - System.out.printf("PlayerDAO.createPlayers:\n => %d total inserted players\n", addedCount); - - // Player 1 wants to buy 10 goods from player 2. - // It will cost 500 coins, but player 1 can't afford it. - System.out.println("\nPlayerDAO.buyGoods:\n => this trade will fail"); - int updatedCount = dao.buyGoods(player2.getId(), player1.getId(), 10, 500); - System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); - - // So player 1 have to reduce his incoming quantity to two. - System.out.println("\nPlayerDAO.buyGoods:\n => this trade will success"); - updatedCount = dao.buyGoods(player2.getId(), player1.getId(), 2, 100); - System.out.printf("PlayerDAO.buyGoods:\n => %d total update players\n", updatedCount); - } -} -``` - -
- - - -## Step 3. Run the code - -The following content introduces how to run the code step by step. - -### Step 3.1 Table initialization - - - -
- -When using Mybatis, you need to initialize the database tables manually. If you are using a local cluster, and MySQL client has been installed locally, you can run it directly in the `plain-java-mybatis` directory: - -```shell -make prepare -``` - -Or you can execute the following command: - -```shell -mysql --host 127.0.0.1 --port 4000 -u root < src/main/resources/dbinit.sql -``` - -If you are using a non-local cluster or MySQL client has not been installed, connect to your cluster and run the statement in the `src/main/resources/dbinit.sql` file. - -
- -
- -No need to initialize tables manually. - -
- -
- - - -When using JDBC, you need to initialize the database tables manually. If you are using a local cluster, and MySQL client has been installed locally, you can run it directly in the `plain-java-jdbc` directory: - -```shell -make mysql -``` - -Or you can execute the following command: - -```shell -mysql --host 127.0.0.1 --port 4000 -u root - - - -When using JDBC, you need to connect to your cluster and run the statement in the `src/main/resources/dbinit.sql` file to initialize the database tables manually. - - - -
- - - -### Step 3.2 Modify parameters for TiDB Cloud - - - -
- -If you are using a TiDB Serverless cluster, modify the `dataSource.url`, `dataSource.username`, `dataSource.password` in `mybatis-config.xml`. - -```xml - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -``` - -Suppose that the password you set is `123456`, and the connection parameters you get from the cluster details page are the following: - -- Endpoint: `xxx.tidbcloud.com` -- Port: `4000` -- User: `2aEp24QWEDLqRFs.root` - -In this case, you can modify the parameters in `dataSource` node as follows: - -```xml - - - - - ... - - - - - - - - ... - - -``` - -
- -
- -If you are using a TiDB Serverless cluster, modify the `hibernate.connection.url`, `hibernate.connection.username`, `hibernate.connection.password` in `hibernate.cfg.xml`. - -```xml - - - - - - - com.mysql.cj.jdbc.Driver - org.hibernate.dialect.TiDBDialect - jdbc:mysql://localhost:4000/test - root - - false - - - create-drop - - - true - true - - -``` - -Suppose that the password you set is `123456`, and the connection parameters you get from the cluster details page are the following: - -- Endpoint: `xxx.tidbcloud.com` -- Port: `4000` -- User: `2aEp24QWEDLqRFs.root` - -In this case, you can modify the parameters as follows: - -```xml - - - - - - - com.mysql.cj.jdbc.Driver - org.hibernate.dialect.TiDBDialect - jdbc:mysql://xxx.tidbcloud.com:4000/test?sslMode=VERIFY_IDENTITY&enabledTLSProtocols=TLSv1.2,TLSv1.3 - 2aEp24QWEDLqRFs.root - 123456 - false - - - create-drop - - - true - true - - -``` - -
- -
- -If you are using a TiDB Serverless cluster, modify the parameters of the host, port, user, and password in `JDBCExample.java`: - -```java -mysqlDataSource.setServerName("localhost"); -mysqlDataSource.setPortNumber(4000); -mysqlDataSource.setDatabaseName("test"); -mysqlDataSource.setUser("root"); -mysqlDataSource.setPassword(""); -``` - -Suppose that the password you set is `123456`, and the connection parameters you get from the cluster details page are the following: - -- Endpoint: `xxx.tidbcloud.com` -- Port: `4000` -- User: `2aEp24QWEDLqRFs.root` - -In this case, you can modify the parameters as follows: - -```java -mysqlDataSource.setServerName("xxx.tidbcloud.com"); -mysqlDataSource.setPortNumber(4000); -mysqlDataSource.setDatabaseName("test"); -mysqlDataSource.setUser("2aEp24QWEDLqRFs.root"); -mysqlDataSource.setPassword("123456"); -mysqlDataSource.setSslMode(PropertyDefinitions.SslMode.VERIFY_IDENTITY.name()); -mysqlDataSource.setEnabledTLSProtocols("TLSv1.2,TLSv1.3"); -``` - -
- - - -### Step 3.3 Run - - - -
- -To run the code, you can run `make prepare`, `make gen`, `make build` and `make run` respectively: - -```shell -make prepare -# this command executes : -# - `mysql --host 127.0.0.1 --port 4000 -u root < src/main/resources/dbinit.sql` -# - `mysql --host 127.0.0.1 --port 4000 -u root -e "TRUNCATE test.player"` - -make gen -# this command executes : -# - `rm -f src/main/java/com/pingcap/model/Player.java` -# - `rm -f src/main/java/com/pingcap/model/PlayerMapper.java` -# - `rm -f src/main/resources/mapper/PlayerMapper.xml` -# - `mvn mybatis-generator:generate` - -make build # this command executes `mvn clean package` -make run # this command executes `java -jar target/plain-java-mybatis-0.0.1-jar-with-dependencies.jar` -``` - -Or you can use the native commands: - -```shell -mysql --host 127.0.0.1 --port 4000 -u root < src/main/resources/dbinit.sql -mysql --host 127.0.0.1 --port 4000 -u root -e "TRUNCATE test.player" -rm -f src/main/java/com/pingcap/model/Player.java -rm -f src/main/java/com/pingcap/model/PlayerMapper.java -rm -f src/main/resources/mapper/PlayerMapper.xml -mvn mybatis-generator:generate -mvn clean package -java -jar target/plain-java-mybatis-0.0.1-jar-with-dependencies.jar -``` - -Or run the `make` command directly, which is a combination of `make prepare`, `make gen`, `make build` and `make run`. - -
- -
- -To run the code, you can run `make build` and `make run` respectively: - -```shell -make build # this command executes `mvn clean package` -make run # this command executes `java -jar target/plain-java-jdbc-0.0.1-jar-with-dependencies.jar` -``` - -Or you can use the native commands: - -```shell -mvn clean package -java -jar target/plain-java-jdbc-0.0.1-jar-with-dependencies.jar -``` - -Or run the `make` command directly, which is a combination of `make build` and `make run`. - -
- -
- -To run the code, you can run `make build` and `make run` respectively: - -```shell -make build # this command executes `mvn clean package` -make run # this command executes `java -jar target/plain-java-jdbc-0.0.1-jar-with-dependencies.jar` -``` - -Or you can use the native commands: - -```shell -mvn clean package -java -jar target/plain-java-jdbc-0.0.1-jar-with-dependencies.jar -``` - -Or run the `make` command directly, which is a combination of `make build` and `make run`. - -
- - - -## Step 4. Expected output - - - -
- -[Mybatis Expected Output](https://github.com/pingcap-inc/tidb-example-java/blob/main/Expected-Output.md#plain-java-mybatis) - -
- -
- -[Hibernate Expected Output](https://github.com/pingcap-inc/tidb-example-java/blob/main/Expected-Output.md#plain-java-hibernate) - -
- -
- -[JDBC Expected Output](https://github.com/pingcap-inc/tidb-example-java/blob/main/Expected-Output.md#plain-java-jdbc) - -
- - diff --git a/tidb-cloud/tidb-cloud-htap-quickstart.md b/tidb-cloud/tidb-cloud-htap-quickstart.md index 0427f7a97a0b6..e7628aba99b57 100644 --- a/tidb-cloud/tidb-cloud-htap-quickstart.md +++ b/tidb-cloud/tidb-cloud-htap-quickstart.md @@ -12,7 +12,7 @@ This tutorial guides you through an easy way to experience the Hybrid Transactio ## Before you begin -Before experiencing the HTAP feature, follow [TiDB Cloud Quick Start](/tidb-cloud/tidb-cloud-quickstart.md) to create a cluster with TiFlash nodes, connect to the TiDB cluster, and import the Capital Bikeshare sample data to the cluster. +Before experiencing the HTAP feature, follow [TiDB Cloud Quick Start](/tidb-cloud/tidb-cloud-quickstart.md) to create a TiDB Cloud Serverless cluster and import the **Steam Game Stats** sample dataset to the cluster. ## Steps @@ -20,28 +20,28 @@ Before experiencing the HTAP feature, follow [TiDB Cloud Quick Start](/tidb-clou After a cluster with TiFlash nodes is created, TiKV does not replicate data to TiFlash by default. You need to execute DDL statements in a MySQL client of TiDB to specify the tables to be replicated. After that, TiDB will create the specified table replicas in TiFlash accordingly. -For example, to replicate the `trips` table (in the Capital Bikeshare sample data) to TiFlash, execute the following statements: +For example, to replicate the `games` table (in the **Steam Game Stats** sample dataset) to TiFlash, execute the following statements: ```sql -USE bikeshare; +USE game; ``` ```sql -ALTER TABLE trips SET TIFLASH REPLICA 1; +ALTER TABLE games SET TIFLASH REPLICA 2; ``` To check the replication progress, execute the following statement: ```sql -SELECT * FROM information_schema.tiflash_replica WHERE TABLE_SCHEMA = 'bikeshare' and TABLE_NAME = 'trips'; +SELECT TABLE_SCHEMA, TABLE_NAME, TABLE_ID, REPLICA_COUNT, LOCATION_LABELS, AVAILABLE, PROGRESS FROM information_schema.tiflash_replica WHERE TABLE_SCHEMA = 'game' and TABLE_NAME = 'games'; ``` ```sql -+--------------+------------+----------+---------------+-----------------+-----------+----------+------------+ -| TABLE_SCHEMA | TABLE_NAME | TABLE_ID | REPLICA_COUNT | LOCATION_LABELS | AVAILABLE | PROGRESS | TABLE_MODE | -+--------------+------------+----------+---------------+-----------------+-----------+----------+------------+ -| bikeshare | trips | 88 | 1 | | 1 | 1 | NORMAL | -+--------------+------------+----------+---------------+-----------------+-----------+----------+------------+ ++--------------+------------+----------+---------------+-----------------+-----------+----------+ +| TABLE_SCHEMA | TABLE_NAME | TABLE_ID | REPLICA_COUNT | LOCATION_LABELS | AVAILABLE | PROGRESS | ++--------------+------------+----------+---------------+-----------------+-----------+----------+ +| game | games | 88 | 2 | | 1 | 1 | ++--------------+------------+----------+---------------+-----------------+-----------+----------+ 1 row in set (0.20 sec) ``` @@ -54,12 +54,20 @@ In the result of the preceding statement: When the process of replication is completed, you can start to run some queries. -For example, you can check the number of trips by different start and end stations: +For example, you can check the number of games released every year, as well as the average price and average playtime: ```sql -SELECT start_station_name, end_station_name, COUNT(ride_id) as count from `trips` -GROUP BY start_station_name, end_station_name -ORDER BY count ASC; +SELECT + YEAR(`release_date`) AS `release_year`, + COUNT(*) AS `games_released`, + AVG(`price`) AS `average_price`, + AVG(`average_playtime_forever`) AS `average_playtime` +FROM + `games` +GROUP BY + `release_year` +ORDER BY + `release_year` DESC; ``` ### Step 3. Compare the query performance between row-based storage and columnar storage @@ -69,12 +77,20 @@ In this step, you can compare the execution statistics between TiKV (row-based s - To get the execution statistics of this query using TiKV, execute the following statement: ```sql - EXPLAIN ANALYZE SELECT /*+ READ_FROM_STORAGE(TIKV[trips]) */ start_station_name, end_station_name, COUNT(ride_id) as count from `trips` - GROUP BY start_station_name, end_station_name - ORDER BY count ASC; + EXPLAIN ANALYZE SELECT /*+ READ_FROM_STORAGE(TIKV[games]) */ + YEAR(`release_date`) AS `release_year`, + COUNT(*) AS `games_released`, + AVG(`price`) AS `average_price`, + AVG(`average_playtime_forever`) AS `average_playtime` + FROM + `games` + GROUP BY + `release_year` + ORDER BY + `release_year` DESC; ``` - For tables with TiFlash replicas, the TiDB optimizer automatically determines whether to use either TiKV or TiFlash replicas based on the cost estimation. In the preceding `EXPLAIN ANALYZE` statement, `HINT /*+ READ_FROM_STORAGE(TIKV[trips]) */` is used to force the optimizer to choose TiKV so you can check the execution statistics of TiKV. + For tables with TiFlash replicas, the TiDB optimizer automatically determines whether to use either TiKV or TiFlash replicas based on the cost estimation. In the preceding `EXPLAIN ANALYZE` statement, the `/*+ READ_FROM_STORAGE(TIKV[games]) */` hint is used to force the optimizer to choose TiKV so you can check the execution statistics of TiKV. > **Note:** > @@ -83,41 +99,50 @@ In this step, you can compare the execution statistics between TiKV (row-based s In the output, you can get the execution time from the `execution info` column. ```sql - id | estRows | actRows | task | access object | execution info | operator info | memory | disk - ---------------------------+-----------+---------+-----------+---------------+-------------------------------------------+-----------------------------------------------+---------+--------- - Sort_5 | 633.00 | 73633 | root | | time:1.62s, loops:73 | Column#15 | 6.88 MB | 0 Bytes - └─Projection_7 | 633.00 | 73633 | root | | time:1.57s, loops:76, Concurrency:OFF... | bikeshare.trips.start_station_name... | 6.20 MB | N/A | 6.20 MB | N/A - └─HashAgg_15 | 633.00 | 73633 | root | | time:1.57s, loops:76, partial_worker:... | group by:bikeshare.trips.end_station_name... | 58.0 MB | N/A - └─TableReader_16 | 633.00 | 111679 | root | | time:1.34s, loops:3, cop_task: {num: ... | data:HashAgg_8 | 7.55 MB | N/A - └─HashAgg_8 | 633.00 | 111679 | cop[tikv] | | tikv_task:{proc max:830ms, min:470ms,... | group by:bikeshare.trips.end_station_name... | N/A | N/A - └─TableFullScan_14 | 816090.00 | 816090 | cop[tikv] | table:trips | tikv_task:{proc max:490ms, min:310ms,... | keep order:false | N/A | N/A + id | estRows | actRows | task | access object | execution info | operator info | memory | disk + ---------------------------+----------+---------+-----------+---------------+--------------------------------------------+-----------------------------------------------+---------+--------- + Sort_5 | 4019.00 | 28 | root | | time:672.7ms, loops:2, RU:1159.679690 | Column#36:desc | 18.0 KB | 0 Bytes + └─Projection_7 | 4019.00 | 28 | root | | time:672.7ms, loops:6, Concurrency:5 | year(game.games.release_date)->Column#36, ... | 35.5 KB | N/A + └─HashAgg_15 | 4019.00 | 28 | root | | time:672.6ms, loops:6, partial_worker:... | group by:Column#38, funcs:count(Column#39)... | 56.7 KB | N/A + └─TableReader_16 | 4019.00 | 28 | root | | time:672.4ms, loops:2, cop_task: {num:... | data:HashAgg_9 | 3.60 KB | N/A + └─HashAgg_9 | 4019.00 | 28 | cop[tikv] | | tikv_task:{proc max:300ms, min:0s, avg... | group by:year(game.games.release_date), ... | N/A | N/A + └─TableFullScan_14 | 68223.00 | 68223 | cop[tikv] | table:games | tikv_task:{proc max:290ms, min:0s, avg... | keep order:false | N/A | N/A (6 rows) ``` - To get the execution statistics of this query using TiFlash, execute the following statement: ```sql - EXPLAIN ANALYZE SELECT start_station_name, end_station_name, COUNT(ride_id) as count from `trips` - GROUP BY start_station_name, end_station_name - ORDER BY count ASC; + EXPLAIN ANALYZE SELECT + YEAR(`release_date`) AS `release_year`, + COUNT(*) AS `games_released`, + AVG(`price`) AS `average_price`, + AVG(`average_playtime_forever`) AS `average_playtime` + FROM + `games` + GROUP BY + `release_year` + ORDER BY + `release_year` DESC; ``` In the output, you can get the execution time from the `execution info` column. ```sql - id | estRows | actRows | task | access object | execution info | operator info | memory | disk - -----------------------------------+-----------+---------+--------------+---------------+-------------------------------------------+------------------------------------+---------+--------- - Sort_5 | 633.00 | 73633 | root | | time:420.2ms, loops:73 | Column#15 | 5.61 MB | 0 Bytes - └─Projection_7 | 633.00 | 73633 | root | | time:368.7ms, loops:73, Concurrency:OFF | bikeshare.trips.start_station_... | 4.94 MB | N/A - └─TableReader_34 | 633.00 | 73633 | root | | time:368.6ms, loops:73, cop_task: {num... | data:ExchangeSender_33 | N/A | N/A - └─ExchangeSender_33 | 633.00 | 73633 | mpp[tiflash] | | tiflash_task:{time:360.7ms, loops:1,... | ExchangeType: PassThrough | N/A | N/A - └─Projection_29 | 633.00 | 73633 | mpp[tiflash] | | tiflash_task:{time:330.7ms, loops:1,... | Column#15, bikeshare.trips.star... | N/A | N/A - └─HashAgg_30 | 633.00 | 73633 | mpp[tiflash] | | tiflash_task:{time:330.7ms, loops:1,... | group by:bikeshare.trips.end_st... | N/A | N/A - └─ExchangeReceiver_32 | 633.00 | 73633 | mpp[tiflash] | | tiflash_task:{time:280.7ms, loops:12,... | | N/A | N/A - └─ExchangeSender_31 | 633.00 | 73633 | mpp[tiflash] | | tiflash_task:{time:272.3ms, loops:256,... | ExchangeType: HashPartition, Ha... | N/A | N/A - └─HashAgg_12 | 633.00 | 73633 | mpp[tiflash] | | tiflash_task:{time:252.3ms, loops:256,... | group by:bikeshare.trips.end_st... | N/A | N/A - └─TableFullScan_28 | 816090.00 | 816090 | mpp[tiflash] | table:trips | tiflash_task:{time:92.3ms, loops:16,... | keep order:false | N/A | N/A - (10 rows) + id | estRows | actRows | task | access object | execution info | operator info | memory | disk + -------------------------------------+----------+---------+--------------+---------------+-------------------------------------------------------+--------------------------------------------+---------+--------- + Sort_5 | 4019.00 | 28 | root | | time:222.2ms, loops:2, RU:25.609675 | Column#36:desc | 3.77 KB | 0 Bytes + └─TableReader_39 | 4019.00 | 28 | root | | time:222.1ms, loops:2, cop_task: {num: 2, max: 0s,... | MppVersion: 1, data:ExchangeSender_38 | 4.64 KB | N/A + └─ExchangeSender_38 | 4019.00 | 28 | mpp[tiflash] | | tiflash_task:{time:214.8ms, loops:1, threads:1} | ExchangeType: PassThrough | N/A | N/A + └─Projection_8 | 4019.00 | 28 | mpp[tiflash] | | tiflash_task:{time:214.8ms, loops:1, threads:1} | year(game.games.release_date)->Column#3... | N/A | N/A + └─Projection_34 | 4019.00 | 28 | mpp[tiflash] | | tiflash_task:{time:214.8ms, loops:1, threads:1} | Column#33, div(Column#34, cast(case(eq(... | N/A | N/A + └─HashAgg_35 | 4019.00 | 28 | mpp[tiflash] | | tiflash_task:{time:214.8ms, loops:1, threads:1} | group by:Column#63, funcs:sum(Column#64... | N/A | N/A + └─ExchangeReceiver_37 | 4019.00 | 28 | mpp[tiflash] | | tiflash_task:{time:214.8ms, loops:1, threads:8} | | N/A | N/A + └─ExchangeSender_36 | 4019.00 | 28 | mpp[tiflash] | | tiflash_task:{time:210.6ms, loops:1, threads:1} | ExchangeType: HashPartition, Compressio... | N/A | N/A + └─HashAgg_33 | 4019.00 | 28 | mpp[tiflash] | | tiflash_task:{time:210.6ms, loops:1, threads:1} | group by:Column#75, funcs:count(1)->Col... | N/A | N/A + └─Projection_40 | 68223.00 | 68223 | mpp[tiflash] | | tiflash_task:{time:210.6ms, loops:2, threads:8} | game.games.price, game.games.price, gam... | N/A | N/A + └─TableFullScan_23 | 68223.00 | 68223 | mpp[tiflash] | table:games | tiflash_task:{time:210.6ms, loops:2, threads:8}, ... | keep order:false | N/A | N/A + (11 rows) ``` > **Note:** diff --git a/tidb-cloud/tidb-cloud-import-local-files.md b/tidb-cloud/tidb-cloud-import-local-files.md index 89c0e514a1a7c..5c7362d89d88d 100644 --- a/tidb-cloud/tidb-cloud-import-local-files.md +++ b/tidb-cloud/tidb-cloud-import-local-files.md @@ -1,25 +1,19 @@ --- -title: Import Local Files to TiDB Cloud -summary: Learn how to import local files to TiDB Cloud. +title: Import Local Files to TiDB Cloud Serverless +summary: Learn how to import local files to TiDB Cloud Serverless. --- -# Import Local Files to TiDB Cloud +# Import Local Files to TiDB Cloud Serverless -You can import local files to TiDB Cloud directly. It only takes a few clicks to complete the task configuration, and then your local CSV data will be quickly imported to your TiDB cluster. Using this method, you do not need to provide the cloud storage bucket path and Role ARN. The whole importing process is quick and smooth. +You can import local files to TiDB Cloud Serverless directly. It only takes a few clicks to complete the task configuration, and then your local CSV data will be quickly imported to your TiDB cluster. Using this method, you do not need to provide the cloud storage and credentials. The whole importing process is quick and smooth. -Currently, this method supports importing one CSV file for one task into either an existing table or a new table. +Currently, this method supports importing one CSV file for one task into either an existing empty table or a new table. ## Limitations -- Currently, TiDB Cloud only supports importing a local file in CSV format within 50 MiB for one task. -- Importing local files is supported only for TiDB Serverless clusters, not for TiDB Dedicated clusters. +- Currently, TiDB Cloud only supports importing a local file in CSV format within 250 MiB for one task. +- Importing local files is supported only for TiDB Cloud Serverless clusters, not for TiDB Cloud Dedicated clusters. - You cannot run more than one import task at the same time. -- When you import a CSV file into an existing table in TiDB Cloud and the target table has more columns than the source file, the extra columns are handled differently depending on the situation: - - If the extra columns are not the primary keys or the unique keys, no error will be reported. Instead, these extra columns will be populated with their [default values](/data-type-default-values.md). - - If the extra columns are the primary keys or the unique keys and do not have the `auto_increment` or `auto_random` attribute, an error will be reported. In that case, it is recommended that you choose one of the following strategies: - - Provide a source file that includes these the primary keys or the unique keys columns. - - Set the attributes of the primary key or the unique key columns to `auto_increment` or `auto_random`. -- If a column name is a reserved [keyword](/keywords.md) in TiDB, TiDB Cloud automatically adds backticks `` ` `` to enclose the column name. For example, if the column name is `order`, TiDB Cloud automatically adds backticks `` ` `` to change it to `` `order` `` and imports the data into the target table. ## Import local files @@ -33,9 +27,9 @@ Currently, this method supports importing one CSV file for one task into either 2. Click the name of your target cluster to go to its overview page, and then click **Import** in the left navigation pane. -2. On the **Import** page, you can directly drag and drop your local file to the upload area, or click the upload area to select and upload the target local file. Note that you can upload only one CSV file of less than 50 MiB for one task. If your local file is larger than 50 MiB, see [How to import a local file larger than 50 MiB?](#how-to-import-a-local-file-larger-than-50-mib). +2. On the **Import** page, you can directly drag and drop your local file to the upload area, or click **Upload a local file** to select and upload the target local file. Note that you can upload only one CSV file of less than 250 MiB for one task. If your local file is larger than 250 MiB, see [How to import a local file larger than 250 MiB?](#how-to-import-a-local-file-larger-than-250-mib). -3. In the **Target** area, select the target database and the target table, or enter a name directly to create a new database or a new table. The name must start with letters (a-z and A-Z) or numbers (0-9), and can contain letters (a-z and A-Z), numbers (0-9), and the underscore (_) character. Click **Preview**. +3. In the **Destination** section, select the target database and the target table, or enter a name directly to create a new database or a new table. The name must only contain characters in Unicode BMP (Basic Multilingual Plane), excluding the null character `\u0000` and whitespace characters, and can be up to 64 characters in length. Click **Define Table**, the **Table Definition** section is displayed. 4. Check the table. @@ -43,7 +37,7 @@ Currently, this method supports importing one CSV file for one task into either - If you import data into an existing table in TiDB Cloud, the column list is extracted from the table definition, and the previewed data is mapped to the corresponding columns by column names. - - If you want to create a new table, the column list is extracted from the CSV file, and the column type is inferred by TiDB Cloud. For example, if the previewed data is all integers, the inferred column type will be **int** (integer). + - If you want to create a new table, the column list is extracted from the CSV file, and the column type is inferred by TiDB Cloud. For example, if the previewed data is all integers, the inferred column type will be integer. 5. Configure the column names and data types. @@ -55,30 +49,37 @@ Currently, this method supports importing one CSV file for one task into either - If you need TiDB Cloud to create the target table, input the name for each column. The column name must meet the following requirements: - * The name must be composed of only letters (a-z and A-Z), numbers (0-9), characters (such as Chinese and Japanese), and the underscore (`_`) character. - * Other special characters are not supported. + * The name must be composed of characters in Unicode BMP, excluding the null character `\u0000` and whitespace characters. * The length of the name must be less than 65 characters. You can also change the data type if needed. + > **Note:** + > + > When you import a CSV file into an existing table in TiDB Cloud and the target table has more columns than the source file, the extra columns are handled differently depending on the situation: + > - If the extra columns are not the primary keys or the unique keys, no error will be reported. Instead, these extra columns will be populated with their [default values](/data-type-default-values.md). + > - If the extra columns are the primary keys or the unique keys and do not have the `auto_increment` or `auto_random` attribute, an error will be reported. In that case, it is recommended that you choose one of the following strategies: + > - Provide a source file that includes these primary key or unique key columns. + > - Modify the target table's primary key and unique key columns to match the existing columns in the source file. + > - Set the attributes of the primary key or the unique key columns to `auto_increment` or `auto_random`. + 6. For a new target table, you can set the primary key. You can select a column as the primary key, or select multiple columns to create a composite primary key. The composite primary key will be formed in the order in which you select the column names. > **Note:** > - > - The primary key of the table is a clustered index and cannot be deleted after creation. - > - Ensure that the data corresponding to the primary key field is unique and not empty. Otherwise, the import task will result in data inconsistency. + > The primary key of the table is a clustered index and cannot be dropped after creation. 7. Edit the CSV configuration if needed. You can also click **Edit CSV configuration** to configure Backslash Escape, Separator, and Delimiter for more fine-grained control. For more information about the CSV configuration, see [CSV Configurations for Importing Data](/tidb-cloud/csv-config-for-import-data.md). -8. On the **Preview** page, you can have a preview of the data. Click **Start Import**. +8. Click **Start Import**. You can view the import progress on the **Import Task Detail** page. If there are warnings or failed tasks, you can check to view the details and solve them. -9. After the import task is completed, you can click **Explore your data by Chat2Query** to query your imported data. For more information about how to use Chat2Query, see [Explore Your Data with AI-Powered Chat2Query](/tidb-cloud/explore-data-with-chat2query.md). +9. After the import task is completed, you can click **Explore your data by SQL Editor** to query your imported data. For more information about how to use SQL Editor, see [Explore your data with AI-assisted SQL Editor](/tidb-cloud/explore-data-with-chat2query.md). -10. On the **Import** page, you can click **View** in the **Action** column to check the import task detail. +10. On the **Import** page, you can click **...** > **View** in the **Action** column to check the import task detail. ## FAQ @@ -98,15 +99,17 @@ CREATE TABLE `import_test` ( LOAD DATA LOCAL INFILE 'load.txt' INTO TABLE import_test FIELDS TERMINATED BY ',' (name, address); ``` -If you use the `mysql` command-line client and encounter `ERROR 2068 (HY000): LOAD DATA LOCAL INFILE file request rejected due to restrictions on access.`, you can add `--local-infile=true` in the connection string. +If you use `mysql` and encounter `ERROR 2068 (HY000): LOAD DATA LOCAL INFILE file request rejected due to restrictions on access.`, you can add `--local-infile=true` in the connection string. ### Why can't I query a column with a reserved keyword after importing data into TiDB Cloud? -If a column name is a reserved [keyword](/keywords.md) in TiDB, TiDB Cloud automatically adds backticks `` ` `` to enclose the column name and then imports the data into the target table. When you query the column, you need to add backticks `` ` `` to enclose the column name. For example, if the column name is `order`, you need to query the column with `` `order` ``. +If a column name is a reserved [keyword](/keywords.md) in TiDB, when you query the column, you need to add backticks `` ` `` to enclose the column name. For example, if the column name is `order`, you need to query the column with `` `order` ``. + +### How to import a local file larger than 250 MiB? -### How to import a local file larger than 50 MiB? +If the file is larger than 250 MiB, you can use [TiDB Cloud CLI](/tidb-cloud/get-started-with-cli.md) to import the file. For more information, see [`ticloud serverless import start`](/tidb-cloud/ticloud-import-start.md). -If the file is larger than 50 MiB, you can use the `split [-l ${line_count}]` utility to split it into multiple smaller files (for Linux or macOS only). For example, run `split -l 100000 tidb-01.csv small_files` to split a file named `tidb-01.csv` by line length `100000`, and the split files are named `small_files${suffix}`. Then, you can import these smaller files to TiDB Cloud one by one. +Alternatively, you can use the `split [-l ${line_count}]` utility to split it into multiple smaller files (for Linux or macOS only). For example, run `split -l 100000 tidb-01.csv small_files` to split a file named `tidb-01.csv` by line length `100000`, and the split files are named `small_files${suffix}`. Then, you can import these smaller files to TiDB Cloud one by one. Refer to the following script: diff --git a/tidb-cloud/tidb-cloud-intro.md b/tidb-cloud/tidb-cloud-intro.md index 03a38a2e03bd9..292a5c7d0a5ca 100644 --- a/tidb-cloud/tidb-cloud-intro.md +++ b/tidb-cloud/tidb-cloud-intro.md @@ -1,10 +1,10 @@ --- -title: TiDB Cloud Introduction +title: What is TiDB Cloud summary: Learn about TiDB Cloud and its architecture. category: intro --- -# TiDB Cloud Introduction +# What is TiDB Cloud [TiDB Cloud](https://www.pingcap.com/tidb-cloud/) is a fully-managed Database-as-a-Service (DBaaS) that brings [TiDB](https://docs.pingcap.com/tidb/stable/overview), an open-source Hybrid Transactional and Analytical Processing (HTAP) database, to your cloud. TiDB Cloud offers an easy way to deploy and manage databases to let you focus on your applications, not the complexities of the databases. You can create TiDB Cloud clusters to quickly build mission-critical applications on Google Cloud and Amazon Web Services (AWS). @@ -64,15 +64,15 @@ With TiDB Cloud, you can get the following key features: TiDB Cloud provides the following two deployment options: -- [TiDB Serverless](https://www.pingcap.com/tidb-serverless) +- [TiDB Cloud Serverless](https://www.pingcap.com/tidb-cloud-serverless) - TiDB Serverless is a fully managed, multi-tenant TiDB offering. It delivers an instant, autoscaling MySQL-compatible database and offers a generous free tier and consumption based billing once free limits are exceeded. + TiDB Cloud Serverless is a fully managed, multi-tenant TiDB offering. It delivers an instant, autoscaling MySQL-compatible database and offers a generous free tier and consumption based billing once free limits are exceeded. -- [TiDB Dedicated](https://www.pingcap.com/tidb-dedicated) +- [TiDB Cloud Dedicated](https://www.pingcap.com/tidb-cloud-dedicated) - TiDB Dedicated is for production use with the benefits of cross-zone high availability, horizontal scaling, and [HTAP](https://en.wikipedia.org/wiki/Hybrid_transactional/analytical_processing). + TiDB Cloud Dedicated is for production use with the benefits of cross-zone high availability, horizontal scaling, and [HTAP](https://en.wikipedia.org/wiki/Hybrid_transactional/analytical_processing). -For feature comparison between TiDB Serverless and TiDB Dedicated, see [TiDB: An advanced, open source, distributed SQL database](https://www.pingcap.com/get-started-tidb). +For feature comparison between TiDB Cloud Serverless and TiDB Cloud Dedicated, see [TiDB: An advanced, open source, distributed SQL database](https://www.pingcap.com/get-started-tidb). ## Architecture @@ -80,7 +80,7 @@ For feature comparison between TiDB Serverless and TiDB Dedicated, see [TiDB: An - TiDB VPC (Virtual Private Cloud) - For each TiDB Cloud cluster, all TiDB nodes and auxiliary nodes, including TiDB Operator nodes and logging nodes, are deployed in an independent VPC. + For each TiDB Cloud cluster, all TiDB nodes and auxiliary nodes, including TiDB Operator nodes and logging nodes, are deployed in the same VPC. - TiDB Cloud Central Services diff --git a/tidb-cloud/tidb-cloud-migration-overview.md b/tidb-cloud/tidb-cloud-migration-overview.md index 9ff553e3ebe36..e6dfcd50e4bb3 100644 --- a/tidb-cloud/tidb-cloud-migration-overview.md +++ b/tidb-cloud/tidb-cloud-migration-overview.md @@ -27,9 +27,9 @@ When you migrate data from a MySQL-compatible database, you can perform full dat If your application uses MySQL shards for data storage, you can migrate these shards into TiDB Cloud as one table. For more information, see [Migrate and Merge MySQL Shards of Large Datasets to TiDB Cloud](/tidb-cloud/migrate-sql-shards.md). -- Migrate from TiDB Self-Hosted +- Migrate from TiDB Self-Managed - You can migrate data from your TiDB Self-Hosted clusters to TiDB Cloud (AWS) through Dumpling and TiCDC. For more information, see [Migrate from TiDB Self-Hosted to TiDB Cloud](/tidb-cloud/migrate-from-op-tidb.md). + You can migrate data from your TiDB Self-Managed clusters to TiDB Cloud (AWS) through Dumpling and TiCDC. For more information, see [Migrate from TiDB Self-Managed to TiDB Cloud](/tidb-cloud/migrate-from-op-tidb.md). ## Import data from files to TiDB Cloud diff --git a/tidb-cloud/tidb-cloud-org-sso-authentication.md b/tidb-cloud/tidb-cloud-org-sso-authentication.md index 760dc50020f41..534af36f5fa84 100644 --- a/tidb-cloud/tidb-cloud-org-sso-authentication.md +++ b/tidb-cloud/tidb-cloud-org-sso-authentication.md @@ -9,13 +9,13 @@ Single Sign-On (SSO) is an authentication scheme that enables members in your Ti TiDB Cloud supports the following two types of SSO authentication: -- [Basic SSO](/tidb-cloud/tidb-cloud-sso-authentication.md): members can log in to the [TiDB Cloud console](https://tidbcloud.com/) using their GitHub, Google, or Microsoft authentication methods. The basic SSO is enabled by default for all organizations in TiDB Cloud. +- [Standard SSO](/tidb-cloud/tidb-cloud-sso-authentication.md): members can log in to the [TiDB Cloud console](https://tidbcloud.com/) using their GitHub, Google, or Microsoft authentication methods. The standard SSO is enabled by default for all organizations in TiDB Cloud. - Cloud Organization SSO: members can log in to a custom login page of TiDB Cloud using the authentication methods specified by your organization. The Cloud Organization SSO is disabled by default. -Compared with basic SSO, Cloud Organization SSO provides more flexibility and customization so you can better meet your organization's security and compliance requirements. For example, you can specify which authentication methods are displayed on the login page, limit which email address domains are allowed for login, and let your members log in to TiDB Cloud with your identity provider (IdP) that uses the [OpenID Connect (OIDC)](https://openid.net/connect/) or [Security Assertion Markup Language (SAML)](https://en.wikipedia.org/wiki/Security_Assertion_Markup_Language) identity protocol. +Compared with standard SSO, Cloud Organization SSO provides more flexibility and customization so you can better meet your organization's security and compliance requirements. For example, you can specify which authentication methods are displayed on the login page, limit which email address domains are allowed for login, and let your members log in to TiDB Cloud with your identity provider (IdP) that uses the [OpenID Connect (OIDC)](https://openid.net/connect/) or [Security Assertion Markup Language (SAML)](https://en.wikipedia.org/wiki/Security_Assertion_Markup_Language) identity protocol. -In this document, you will learn how to migrate the authentication scheme of your organization from basic SSO to Cloud Organization SSO. +In this document, you will learn how to migrate the authentication scheme of your organization from standard SSO to Cloud Organization SSO. > **Note:** > @@ -77,7 +77,7 @@ To enable Cloud Organization SSO, take the following steps: 1. Log in to [TiDB Cloud console](https://tidbcloud.com) as a user with the `Organization Owner` role. 2. In the lower-left corner of the TiDB Cloud console, click , and then click **Organization Settings**. -3. On the **Organization Settings** page, click the **Authentication** tab, and then click **Enable**. +3. In the left navigation pane, click the **Authentication** tab, and then click **Enable**. 4. In the dialog, fill in the custom URL for your organization, which must be unique in TiDB Cloud. > **Note:** @@ -168,7 +168,7 @@ In TiDB Cloud, the SAML authentication method is disabled by default. After enab - Sign on URL - Signing Certificate -2. On the **Organization Settings** page, click the **Authentication** tab, locate the row of SAML in the **Authentication Methods** area, and then click to show the SAML method details. +2. On the **Organization Settings** page, click the **Authentication** tab in the left navigation pane, locate the row of SAML in the **Authentication Methods** area, and then click to show the SAML method details. 3. In the method details, you can configure the following: - **Name** @@ -232,8 +232,9 @@ In TiDB Cloud, the SAML authentication method is disabled by default. After enab 3. In TiDB Cloud, view groups pushed from your identity provider. 1. In the lower-left corner of the [TiDB Cloud console](https://tidbcloud.com), click , and then click **Organization Settings**. - 2. On the **Organization Settings** page, click the **Groups** tab. The groups synchronized from your identity provider are displayed. - 3. To view users in a group, click **View**. + 2. In the left navigation pane, click the **Authentication** tab. + 3. Click the **Groups** tab. The groups synchronized from your identity provider are displayed. + 4. To view users in a group, click **View**. 4. In TiDB Cloud, grant roles to the groups pushed from your identity provider. diff --git a/tidb-cloud/tidb-cloud-partners.md b/tidb-cloud/tidb-cloud-partners.md new file mode 100644 index 0000000000000..fc0c442b520fc --- /dev/null +++ b/tidb-cloud/tidb-cloud-partners.md @@ -0,0 +1,72 @@ +--- +title: TiDB Cloud Partner Web Console +summary: Learn how to use the TiDB Cloud Partner web console as a reseller and Managed Service Provider (MSP). +aliases: ['/tidbcloud/managed-service-provider'] +--- + +# TiDB Cloud Partner Web Console + +TiDB Cloud Partner Web Console is designed for partners focused on SaaS solutions, with the goal of building and nurturing a strong partnership between PingCAP and our partners to better serve our customers. + +There are two types of TiDB Cloud partners: + +- Reseller: resells TiDB Cloud through AWS Marketplace Channel Partner Private Offer (CPPO) +- Managed Service Provider (MSP): resells TiDB Cloud and provides value-added services + +## Reseller through AWS Channel Partner Private Offer (CPPO) + +The reseller through [AWS CPPO](https://aws.amazon.com/marketplace/features/cpprivateoffers) allows customers to purchase TiDB Cloud through the AWS Marketplace directly from a reseller. This enables customers to benefit from the partner's business knowledge, localized support, and expertise, while still enjoying the fast and seamless purchasing experience they expect from AWS Marketplace. + +### Become a reseller of PingCAP + +If you are interested in the reseller program and would like to join as a partner, [contact sales](https://www.pingcap.com/partners/become-a-partner/) to enroll. + +### Manage daily tasks for a reseller + +As a reseller, you have two ways to manage your daily management tasks: + +- [TiDB Cloud Partner console](https://partner-console.tidbcloud.com) +- Partner Management API. You can find the open API documentation on the **Support** page of the TiDB Cloud Partner console. + +## Managed Service Provider (MSP) + +An MSP is a partner who resells TiDB Cloud and provides value-added services, including but not limited to TiDB Cloud organization management, billing services, and technical support. + +Benefits of becoming a managed service provider include: + +- Discounts and incentive programs +- Enablement training +- Increased visibility through certification +- Joint marketing opportunities + +### Become an MSP of PingCAP + +If you are interested in the MSP program and would like to join as a partner, [contact sales](https://www.pingcap.com/partners/become-a-partner/) to enroll. Please provide the following information: + +- Company name +- Company contact email +- Company official website URL +- Company logo (One SVG file for light mode and one SVG file for dark mode; a horizontal logo with 256 x 48 pixels is preferred) + +The preceding information is used to generate an exclusive sign-up URL and page with your company logo for your customers. + +We will carefully evaluate your request and get back to you soon. + +### Manage daily tasks for an MSP + +As a TiDB Cloud MSP partner, there are two methods for you to manage your daily management tasks: + +- [TiDB Cloud Partner console](https://partner-console.tidbcloud.com) +- [MSP Management API (deprecated)](https://docs.pingcap.com/tidbcloud/api/v1beta1/msp) + +After your complete the registration as a TiDB Cloud partner, you will receive an email notification to activate the account in the TiDB Cloud Partner console, and receive an API key for the MSP Management API. + +You can use the MSP management API to manage the following daily tasks: + +- Query the MSP monthly bill for a specific month +- Query credits applied to an MSP +- Query discounts applied to an MSP +- Query the monthly bill for a specific MSP customer +- Create a new sign-up URL for an MSP customer +- List all MSP customers +- Retrieve MSP customer information by the customer organization ID diff --git a/tidb-cloud/tidb-cloud-password-authentication.md b/tidb-cloud/tidb-cloud-password-authentication.md index 56e79b1f1a9e2..b484543b597b8 100644 --- a/tidb-cloud/tidb-cloud-password-authentication.md +++ b/tidb-cloud/tidb-cloud-password-authentication.md @@ -78,15 +78,14 @@ If you forget your password, you can reset it by email as follows: 3. Click the **Change Password** tab, click **Change Password**, and then check your email for TiDB Cloud to reset the password. -## Enable or disable MFA (optional) +## Manage multi-factor authentication (optional) > **Note:** > -> This section applies only when you [sign up](https://tidbcloud.com/free-trial) for TiDB Cloud with emails and passwords. If you sign up for TiDB Cloud with Google, GitHub, or Microsoft SSO, you can enable MFA on your chosen identity management platform. +> - This section applies only when you [sign up](https://tidbcloud.com/free-trial) for TiDB Cloud with emails and passwords. If you sign up for TiDB Cloud with Google, GitHub, or Microsoft SSO, you can enable MFA on your chosen identity management platform. +> - If you have enabled TiDB Cloud MFA in an SSO login scenario, migrate your MFA management to your SSO identity management platform before **September 30, 2025**, to ensure account security. -After logging in to TiDB Cloud, you can enable MFA in accordance with laws and regulations. - -Two-factor authentication adds additional security by requiring an Authenticator app to generate a one-time password for login. You can use any Authenticator app from the iOS or Android App Store to generate this password, such as Google Authenticator and Authy. +Multi-factor authentication (MFA) adds additional security by requiring an Authenticator app to generate a one-time verification code for login. When you log in, TiDB Cloud verifies both your password and the MFA verification code. You can use any Authenticator app from the iOS or Android App Store to generate this password, such as Google Authenticator and Authy. ### Enable MFA @@ -94,16 +93,35 @@ Two-factor authentication adds additional security by requiring an Authenticator 2. Click **Account Settings**. -3. Click the **Two Factor Authentication** tab. +3. Click the **Multi-Factor Authentication** tab. 4. Click **Enable**. +5. Enter your account password to confirm your identity. + +6. In the **Set Up Authenticator App** section, use your authenticator app to scan the QR code and associate your MFA device. + +7. Enter the authentication code generated by your app to complete the MFA device association. + +8. Save the one-time recovery code, which helps you authenticate if your MFA device is unavailable. + +> **Note:** +> +> - You need to store the recovery code securely to maintain account security. +> - If you use the recovery code for MFA verification during login, the system automatically generates a new recovery code after successful verification. + ### Disable MFA 1. Click in the lower-left corner of the TiDB Cloud console. 2. Click **Account Settings**. -3. Click the **Two Factor Authentication** tab. +3. Click the **Multi-Factor Authentication** tab. 4. Click **Disable**. + +5. In the **Disable Multi-Factor Authentication** section, enter your account password and MFA authentication code to confirm the operation. + +> **Note:** +> +> Disabling MFA reduces the security of your account. Proceed with caution. diff --git a/tidb-cloud/tidb-cloud-performance-reference.md b/tidb-cloud/tidb-cloud-performance-reference.md index d94905929c80c..d16663ebde40c 100644 --- a/tidb-cloud/tidb-cloud-performance-reference.md +++ b/tidb-cloud/tidb-cloud-performance-reference.md @@ -9,9 +9,25 @@ This document provides [Sysbench](https://github.com/akopytov/sysbench) performa > **Note:** > -> The tests are performed on TiDB v6.1.1, and the test results are based on the condition that the P95 latency is below 105 ms. - -In this document, the transaction models `Read Only`, `Read Write`, and `Write Only` represent read workloads, mixed workloads, and write workloads. +> The tests are performed on TiDB v6.1.1, and the test results are based on the condition that the P95 transaction latency is below 105 ms. + +Here is an example of a Sysbench configuration file: + +```txt +mysql-host={TIDB_HOST} +mysql-port=4000 +mysql-user=root +mysql-password=password +mysql-db=sbtest +time=1200 +threads={100} +report-interval=10 +db-driver=mysql +mysql-ignore-errors=1062,2013,8028,9002,9007 +auto-inc=false +``` + +In this document, the transaction models `Read Only`, `Read Write`, and `Write Only` represent read workloads, mixed workloads, and write workloads. ## 4 vCPU performance @@ -24,7 +40,7 @@ Test results: **TiDB (4 vCPU, 16 GiB) \* 1; TiKV (4 vCPU, 16 GiB) \* 3** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|----------|----------|----------------------|------------------| | Read Only | 35 | 8,064.89 | 504.06 | 69.43 | 104.84 | | Read Write | 25 | 6,747.60 | 337.38 | 74.10 | 102.97 | @@ -32,7 +48,7 @@ Test results: **TiDB (4 vCPU, 16 GiB) \* 2; TiKV (4 vCPU, 16 GiB) \* 3** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|-----------|----------|----------------------|------------------| | Read Only | 65 | 16,805.76 | 1,050.36 | 61.88 | 95.81 | | Read Write | 45 | 12,940.36 | 647.02 | 69.55 | 99.33 | @@ -53,7 +69,7 @@ Test results: **TiDB (8 vCPU, 16 GiB) \* 2; TiKV (8 vCPU, 32 GiB) \* 3** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|-----------|----------|----------------------|------------------| | Read Only | 150 | 37,863.64 | 2,366.48 | 63.38 | 99.33 | | Read Write | 100 | 30,218.42 | 1,510.92 | 66.18 | 94.10 | @@ -61,7 +77,7 @@ Test results: **TiDB (8 vCPU, 16 GiB) \* 4; TiKV (8 vCPU, 32 GiB) \* 3** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|-----------|----------|----------------------|------------------| | Read Only | 300 | 74,190.40 | 4,636.90 | 64.69 | 104.84 | | Read Write | 200 | 53,351.84 | 2,667.59 | 74.97 | 97.55 | @@ -69,7 +85,7 @@ Test results: **TiDB (8 vCPU, 16 GiB) \* 4; TiKV (8 vCPU, 32 GiB) \* 6** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|-----------|-----------|----------------------|------------------| | Read Only | 300 | 75,713.04 | 4,732.06 | 63.39 | 102.97 | | Read Write | 200 | 62,640.62 | 3,132.03 | 63.85 | 95.81 | @@ -77,7 +93,7 @@ Test results: **TiDB (8 vCPU, 16 GiB) \* 6; TiKV (8 vCPU, 32 GiB) \* 9** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|------------|-----------|----------------------|------------------| | Read Only | 450 | 113,407.94 | 7,088.00 | 63.48 | 104.84 | | Read Write | 300 | 92,387.31 | 4,619.37 | 64.93 | 99.33 | @@ -85,7 +101,7 @@ Test results: **TiDB (8 vCPU, 16 GiB) \* 9; TiKV (8 vCPU, 32 GiB) \* 6** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|------------|-----------|----------------------|------------------| | Read Only | 650 | 168,486.65 | 10,530.42 | 61.72 | 101.13 | | Read Write | 400 | 106,853.63 | 5,342.68 | 74.86 | 101.13 | @@ -93,7 +109,7 @@ Test results: **TiDB (8 vCPU, 16 GiB) \* 12; TiKV (8 vCPU, 32 GiB) \* 9** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|------------|-----------|----------------------|------------------| | Read Only | 800 | 211,882.77 | 13,242.67 | 60.40 | 101.13 | | Read Write | 550 | 139,393.46 | 6,969.67 | 78.90 | 104.84 | @@ -110,7 +126,7 @@ Test results: **TiDB (16 vCPU, 32 GiB) \* 1; TiKV (16 vCPU, 64 GiB) \* 3** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|----------|---------|----------------------|------------------| | Read Only | 125 | 37448.41 | 2340.53 | 53.40 | 89.16 | | Read Write | 100 | 28903.99 | 1445.20 | 69.19 | 104.84 | @@ -118,8 +134,33 @@ Test results: **TiDB (16 vCPU, 32 GiB) \* 2; TiKV (16 vCPU, 64 GiB) \* 3** -| Transaction model | Threads | QPS | TPS | Average latency (ms) | P95 latency (ms) | +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | |-------------------|---------|----------|----------|----------------------|------------------| | Read Only | 300 | 77238.30 | 4827.39 | 62.14 | 102.97 | | Read Write | 200 | 58241.15 | 2912.06 | 68.67 | 97.55 | | Write Only | 700 | 68829.89 | 11471.65 | 61.01 | 101.13 | + +## 32 vCPU performance + +Test scales: + +- TiDB (32 vCPU, 64 GiB) \* 1; TiKV (32 vCPU, 128 GiB) \* 3 +- TiDB (32 vCPU, 64 GiB) \* 2; TiKV (32 vCPU, 128 GiB) \* 3 + +Test results: + +**TiDB (32 vCPU, 64 GiB) \* 1; TiKV (32 vCPU, 128 GiB) \* 3** + +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | +|-------------------|---------|----------|---------|----------------------|------------------| +| Read Only | 300 | 83941.16 | 5246 | 57.20 | 87.6 | +| Read Write | 250 | 71290.31 | 3565 | 70.10 | 105.0 | +| Write Only | 700 | 72199.56 | 12033 | 58.20 | 101.0 | + +**TiDB (32 vCPU, 64 GiB) \* 2; TiKV (32 vCPU, 128 GiB) \* 3** + +| Transaction model | Threads | QPS | TPS | Average transaction latency (ms) | P95 transaction latency (ms) | +|-------------------|---------|-----------|----------|----------------------|------------------| +| Read Only | 650 | 163101.68 | 10194 | 63.8 | 99.3 | +| Read Write | 450 | 123152.74 | 6158 | 73.1 | 101 | +| Write Only | 1200 | 112333.16 | 18722 | 64.1 | 101 | diff --git a/tidb-cloud/tidb-cloud-poc.md b/tidb-cloud/tidb-cloud-poc.md index 0afbfef8f87a1..3d2265be830a1 100644 --- a/tidb-cloud/tidb-cloud-poc.md +++ b/tidb-cloud/tidb-cloud-poc.md @@ -13,7 +13,7 @@ This document describes the typical PoC procedures and aims to help you quickly If you are interested in doing a PoC, feel free to contact PingCAP before you get started. The support team can help you create a test plan and walk you through the PoC procedures smoothly. -Alternatively, you can [create a TiDB Serverless](/tidb-cloud/tidb-cloud-quickstart.md#step-1-create-a-tidb-cluster) to get familiar with TiDB Cloud for a quick evaluation. Note that the TiDB Serverless has some [special terms and conditions](/tidb-cloud/select-cluster-tier.md#tidb-serverless-special-terms-and-conditions). +Alternatively, you can [create a TiDB Cloud Serverless](/tidb-cloud/tidb-cloud-quickstart.md#step-1-create-a-tidb-cluster) to get familiar with TiDB Cloud for a quick evaluation. Note that the TiDB Cloud Serverless has some [special terms and conditions](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless-special-terms-and-conditions). ## Overview of the PoC procedures @@ -23,7 +23,7 @@ A typical TiDB Cloud PoC consists of the following steps: 1. Define success criteria and create a test plan 2. Identify characteristics of your workload -3. Sign up and create a TiDB Dedicated cluster for the PoC +3. Sign up and create a TiDB Cloud Dedicated cluster for the PoC 4. Adapt your schemas and SQL 5. Import data 6. Run your workload and evaluate results @@ -56,9 +56,9 @@ TiDB Cloud is suitable for various use cases that require high availability and You might also be interested in using [TiFlash](https://docs.pingcap.com/tidb/stable/tiflash-overview), a columnar storage engine that helps speed up analytical processing. During the PoC, you can use the TiFlash feature at any time. -## Step 3. Sign up and create a TiDB Dedicated cluster for the PoC +## Step 3. Sign up and create a TiDB Cloud Dedicated cluster for the PoC -To create a [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) cluster for the PoC, take the following steps: +To create a [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) cluster for the PoC, take the following steps: 1. Fill in the PoC application form by doing one of the following: @@ -67,16 +67,16 @@ To create a [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) Once you submit the form, the TiDB Cloud Support team will review your application, contact you, and transfer credits to your account once the application is approved. You can also contact a PingCAP support engineer to assist with your PoC procedures to ensure the PoC runs as smoothly as possible. -2. Refer to [Create a TiDB Dedicated Cluster](/tidb-cloud/create-tidb-cluster.md) to create a TiDB Dedicated cluster for the PoC. +2. Refer to [Create a TiDB Cloud Dedicated Cluster](/tidb-cloud/create-tidb-cluster.md) to create a TiDB Cloud Dedicated cluster for the PoC. Capacity planning is recommended for cluster sizing before you create a cluster. You can start with estimated numbers of TiDB, TiKV, or TiFlash nodes, and scale out the cluster later to meet performance requirements. You can find more details in the following documents or consult our support team. - For more information about estimation practice, see [Size Your TiDB](/tidb-cloud/size-your-cluster.md). -- For configurations of the TiDB Dedicated cluster, see [Create a TiDB Dedicated Cluster](/tidb-cloud/create-tidb-cluster.md). Configure the cluster size for TiDB, TiKV, and TiFlash (optional) respectively. +- For configurations of the TiDB Cloud Dedicated cluster, see [Create a TiDB Cloud Dedicated Cluster](/tidb-cloud/create-tidb-cluster.md). Configure the cluster size for TiDB, TiKV, and TiFlash (optional) respectively. - For how to plan and optimize your PoC credits consumption effectively, see [FAQ](#faq) in this document. - For more information about scaling, see [Scale Your TiDB Cluster](/tidb-cloud/scale-tidb-cluster.md). -Once a dedicated PoC cluster is created, you are ready to load data and perform a series of tests. For how to connect to a TiDB cluster, see [Connect to Your TiDB Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md). +Once a dedicated PoC cluster is created, you are ready to load data and perform a series of tests. For how to connect to a TiDB cluster, see [Connect to Your TiDB Cloud Dedicated Cluster](/tidb-cloud/connect-to-tidb-cluster.md). For a newly created cluster, note the following configurations: @@ -87,7 +87,7 @@ For a newly created cluster, note the following configurations: Next, you can load your database schemas to the TiDB cluster, including tables and indexes. -Because the amount of PoC credits is limited, to maximize the value of credits, it is recommended that you create a [TiDB Serverless cluster](/tidb-cloud/select-cluster-tier.md#tidb-serverless) for compatibility tests and preliminary analysis on TiDB Cloud. +Because the amount of PoC credits is limited, to maximize the value of credits, it is recommended that you create a [TiDB Cloud Serverless cluster](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) for compatibility tests and preliminary analysis on TiDB Cloud. TiDB Cloud is highly compatible with MySQL 8.0. You can directly import your data into TiDB if it is MySQL-compatible or can be adapted to be compatible with MySQL. @@ -175,13 +175,13 @@ Now the workload testing is finished, you can explore more features, for example - Backup - To avoid vendor lock-in, you can use daily full backup to migrate data to a new cluster and use [Dumpling](https://docs.pingcap.com/tidb/stable/dumpling-overview) to export data. For more information, see [Back Up and Restore TiDB Dedicated Data](/tidb-cloud/backup-and-restore.md#turn-on-auto-backup) and [Back Up and Restore TiDB Dedicated Data](/tidb-cloud/backup-and-restore-serverless.md#backup). + To avoid vendor lock-in, you can use daily full backup to migrate data to a new cluster and use [Dumpling](https://docs.pingcap.com/tidb/stable/dumpling-overview) to export data. For more information, see [Back Up and Restore TiDB Cloud Dedicated Data](/tidb-cloud/backup-and-restore.md#turn-on-auto-backup) and [Back Up and Restore TiDB Cloud Serverless Data](/tidb-cloud/backup-and-restore-serverless.md). ## Step 8. Clean up the environment and finish the PoC You have completed the full cycle of a PoC after you test TiDB Cloud using real workloads and get the testing results. These results help you determine if TiDB Cloud meets your expectations. Meanwhile, you have accumulated best practices for using TiDB Cloud. -If you want to try TiDB Cloud on a larger scale, for a new round of deployments and tests, such as deploying with other node storage sizes offered by TiDB Cloud, get full access to TiDB Cloud by creating a [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) cluster. +If you want to try TiDB Cloud on a larger scale, for a new round of deployments and tests, such as deploying with other node storage sizes offered by TiDB Cloud, get full access to TiDB Cloud by creating a [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) cluster. If your credits are running out and you want to continue with the PoC, contact the [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md) for consultation. diff --git a/tidb-cloud/tidb-cloud-quickstart.md b/tidb-cloud/tidb-cloud-quickstart.md index 7ffedf227e8eb..2b698fe2fdcc3 100644 --- a/tidb-cloud/tidb-cloud-quickstart.md +++ b/tidb-cloud/tidb-cloud-quickstart.md @@ -14,7 +14,7 @@ Additionally, you can try out TiDB features on [TiDB Playground](https://play.ti ## Step 1: Create a TiDB cluster -[TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) is the best way to get started with TiDB Cloud. To create a TiDB Serverless cluster, follow these steps: +[TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) is the best way to get started with TiDB Cloud. To create a TiDB Cloud Serverless cluster, follow these steps: 1. If you do not have a TiDB Cloud account, click [here](https://tidbcloud.com/free-trial) to sign up. @@ -24,25 +24,27 @@ Additionally, you can try out TiDB features on [TiDB Playground](https://play.ti The [**Clusters**](https://tidbcloud.com/console/clusters) page is displayed by default. -3. For new sign-up users, TiDB Cloud automatically creates a default TiDB Serverless cluster named `Cluster0` for you. +3. For new sign-up users, TiDB Cloud automatically creates a default TiDB Cloud Serverless cluster named `Cluster0` for you. - - To instantly try out TiDB Cloud features with this default cluster, proceed to [Step 2: Try AI-powered Chat2Query (beta)](#step-2-try-ai-powered-chat2query-beta). - - To create a new TiDB Serverless cluster on your own, follow these steps: + - To instantly try out TiDB Cloud features with this default cluster, proceed to [Step 2: Try AI-assisted SQL Editor](#step-2-try-ai-assisted-sql-editor). + - To create a new TiDB Cloud Serverless cluster on your own, follow these steps: 1. Click **Create Cluster**. - 2. On the **Create Cluster** page, **Serverless** is selected by default. Select the target region for your cluster, update the default cluster name if necessary, and then click **Create**. Your TiDB Serverless cluster will be created in approximately 30 seconds. + 2. On the **Create Cluster** page, **Serverless** is selected by default. Select the target region for your cluster, update the default cluster name if necessary, select your [cluster plan](/tidb-cloud/select-cluster-tier.md#cluster-plans), and then click **Create**. Your TiDB Cloud Serverless cluster will be created in approximately 30 seconds. -## Step 2: Try AI-powered Chat2Query (beta) +## Step 2: Try AI-assisted SQL Editor -TiDB Cloud is powered by AI. You can use Chat2Query (beta), an AI-powered SQL editor in the TiDB Cloud console, to maximize the value of your data. +You can use the built-in AI-assisted SQL Editor in the TiDB Cloud console to maximize your data value. This enables you to run SQL queries against databases without a local SQL client. You can intuitively view the query results in tables or charts and easily check the query logs. -In Chat2Query, you can either simply type `--` followed by your instructions to let AI automatically generate SQL queries, or write SQL queries manually and run them against databases without using a terminal. +1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click on a cluster name to go to its overview page, and then click **SQL Editor** in the left navigation pane. -1. On the [**Clusters**](https://tidbcloud.com/console/clusters) page, click on a cluster name to go to its overview page, and then click **Chat2Query** in the left navigation pane. +2. To try the AI capacity of TiDB Cloud, follow the on-screen instructions to allow PingCAP and AWS Bedrock to use your code snippets for research and service improvement, and then click **Save and Get Started**. -2. To try the AI capacity of TiDB Cloud, follow the on-screen instructions to allow PingCAP and OpenAI to use your code snippets for research and service improvement, and then click **Save and Get Started**. +3. In SQL Editor, press ⌘ + I on macOS (or Control + I on Windows or Linux) to instruct [Chat2Query (beta)](/tidb-cloud/tidb-cloud-glossary.md#chat2query) to generate SQL queries automatically. -3. In the editor, you can either simply type `--` followed by your instructions to let AI automatically generate SQL queries, or write SQL queries manually. + For example, to create a new table `test.t` with two columns (column `id` and column `name`), you can type `use test;` to specify the database, press ⌘ + I, type `create a new table t with id and name` as the instruction, and then press **Enter** to let AI generate a SQL statement accordingly. + + For the generated statement, you can accept it by clicking **Accept** and then further edit it if needed, or reject it by clicking **Discard**. > **Note:** > @@ -76,7 +78,32 @@ In Chat2Query, you can either simply type `--` followed by your instructions to -After running the queries, you can immediately see the query logs and results at the bottom of the page. +After running the queries, you can immediately see the query logs and results at the bottom of the page. + +To let AI generate more SQL statements, you can type more instructions as shown in the following example: + +```sql +use test; + +-- create a new table t with id and name +CREATE TABLE + `t` (`id` INT, `name` VARCHAR(255)); + +-- add 3 rows +INSERT INTO + `t` (`id`, `name`) +VALUES + (1, 'row1'), + (2, 'row2'), + (3, 'row3'); + +-- query all +SELECT + `id`, + `name` +FROM + `t`; +``` ## Step 3: Try interactive tutorials @@ -84,12 +111,12 @@ TiDB Cloud offers interactive tutorials with carefully crafted sample datasets t 1. Click on the **?** icon in the lower-right corner of the console and select **Interactive Tutorials**. 2. In the tutorials list, select a tutorial card to start, such as **Steam Game Stats**. -3. Choose a TiDB Serverless cluster that you want to use for the tutorial, and click **Import Dataset**. The import process might take approximately one minute. +3. Choose a TiDB Cloud Serverless cluster that you want to use for the tutorial, and click **Import Dataset**. The import process might take approximately one minute. 4. Once the sample data is imported, follow the on-screen instructions to complete the tutorial. ## What's next -- To learn how to connect to your cluster using different methods, see [Connect to a TiDB Serverless cluster](/tidb-cloud/connect-to-tidb-cluster-serverless.md). -- For more information about how to use Chat2Query to explore your data, see [Chat2Query](/tidb-cloud/explore-data-with-chat2query.md). +- To learn how to connect to your cluster using different methods, see [Connect to a TiDB Cloud Serverless cluster](/tidb-cloud/connect-to-tidb-cluster-serverless.md). +- For more information about how to use SQL Editor and Chat2Query to explore your data, see [Explore your data with AI-assisted SQL Editor](/tidb-cloud/explore-data-with-chat2query.md). - For TiDB SQL usage, see [Explore SQL with TiDB](/basic-sql-operations.md). -- For production use with the benefits of cross-zone high availability, horizontal scaling, and [HTAP](https://en.wikipedia.org/wiki/Hybrid_transactional/analytical_processing), see [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). +- For production use with the benefits of cross-zone high availability, horizontal scaling, and [HTAP](https://en.wikipedia.org/wiki/Hybrid_transactional/analytical_processing), see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). diff --git a/tidb-cloud/tidb-cloud-release-notes.md b/tidb-cloud/tidb-cloud-release-notes.md index 2737ac68c1c9f..d905a8c35c329 100644 --- a/tidb-cloud/tidb-cloud-release-notes.md +++ b/tidb-cloud/tidb-cloud-release-notes.md @@ -1,1004 +1,241 @@ --- -title: TiDB Cloud Release Notes in 2023 -summary: Learn about the release notes of TiDB Cloud in 2023. +title: TiDB Cloud Release Notes in 2025 +summary: Learn about the release notes of TiDB Cloud in 2025. aliases: ['/tidbcloud/supported-tidb-versions','/tidbcloud/release-notes'] --- -# TiDB Cloud Release Notes in 2023 +# TiDB Cloud Release Notes in 2025 -This page lists the release notes of [TiDB Cloud](https://www.pingcap.com/tidb-cloud/) in 2023. +This page lists the release notes of [TiDB Cloud](https://www.pingcap.com/tidb-cloud/) in 2025. -## December 5, 2023 +## May 13, 2025 **General changes** -- [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) enables you to resume a failed changefeed, which saves your effort to recreate a new one. +- Full-text search (beta) now available in [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) for AI applications. - For more information, see [Changefeed states](/tidb-cloud/changefeed-overview.md#changefeed-states). + TiDB Cloud Serverless now supports full-text search (beta), enabling AI and Retrieval-Augmented Generation (RAG) applications to retrieve content by exact keywords. This complements vector search, which retrieves content by semantic similarity. Combining both methods significantly improves retrieval accuracy and answer quality in RAG workflows. Key features include: -**Console changes** - -- Enhance the connection experience for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless). - - Refine the **Connect** dialog interface to offer TiDB Serverless users a smoother and more efficient connection experience. In addition, TiDB Serverless introduces more client types and allows you to select the desired branch for connection. - - For more information, see [Connect to TiDB Serverless](/tidb-cloud/connect-via-standard-connection-serverless.md). - -## November 28, 2023 - -**General changes** - -- [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) supports restoring SQL bindings from backups. - - TiDB Dedicated now restores user accounts and SQL bindings by default when restoring from a backup. This enhancement is available for clusters of v6.2.0 or later versions, streamlining the data restoration process. The restoration of SQL bindings ensures the smooth reintegration of query-related configurations and optimizations, providing you with a more comprehensive and efficient recovery experience. - - For more information, see [Back up and restore TiDB Dedicated data](/tidb-cloud/backup-and-restore.md). - -**Console changes** - -- [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) supports monitoring SQL statement RU costs. - - TiDB Serverless now provides detailed insights into each SQL statement's [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit). You can view both the **Total RU** and **Mean RU** costs per SQL statement. This feature helps you identify and analyze RU costs, offering opportunities for potential cost savings in your operations. - - To check your SQL statement RU details, navigate to the **Diagnosis** page of [your TiDB Serverless cluster](https://tidbcloud.com/console/clusters) and then click the **SQL Statement** tab. - -## November 21, 2023 - -**General changes** - -- [Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md) supports high-speed physical mode for TiDB clusters deployed on Google Cloud. - - Now you can use physical mode for TiDB clusters deployed on AWS and Google Cloud. The migration speed of physical mode can reach up to 110 MiB/s, which is 2.4 times faster than logical mode. The improved performance is suitable for quickly migrating large datasets to TiDB Cloud. - - For more information, see [Migrate existing data and incremental data](/tidb-cloud/migrate-from-mysql-using-data-migration.md#migrate-existing-data-and-incremental-data). - -## November 14, 2023 - -**General changes** - -- When you restore data from TiDB Dedicated clusters, the default behavior is now modified from restoring without user accounts to restoring with all user accounts, including the `cloud_admin@'%'` account. - - For more information, see [BackUp and Restore TiDB Dedicated Data](/tidb-cloud/backup-and-restore.md). - -- Introduce event filters for changefeeds. - - This enhancement empowers you to easily manage event filters for changefeeds directly through the [TiDB Cloud console](https://tidbcloud.com/), streamlining the process of excluding specific events from changefeeds and providing better control over data replication downstream. - - For more information, see [Changefeed](/tidb-cloud/changefeed-overview.md#edit-a-changefeed). - -## November 7, 2023 - -**General changes** - -- Add the following resource usage alerts. The new alerts are disabled by default. You can enable them as needed. - - - Max memory utilization across TiDB nodes exceeded 70% for 10 minutes - - Max memory utilization across TiKV nodes exceeded 70% for 10 minutes - - Max CPU utilization across TiDB nodes exceeded 80% for 10 minutes - - Max CPU utilization across TiKV nodes exceeded 80% for 10 minutes - - For more information, see [TiDB Cloud Built-in Alerting](/tidb-cloud/monitor-built-in-alerting.md#resource-usage-alerts). - -## October 31, 2023 - -**General changes** - -- Support directly upgrading to the Enterprise support plan in the TiDB Cloud console without contacting sales. - - For more information, see [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). - -## October 25, 2023 - -**General changes** - -- [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) supports dual region backup (beta) on Google Cloud. - - TiDB Dedicated clusters hosted on Google Cloud work seamlessly with Google Cloud Storage. Similar to the [Dual-regions](https://cloud.google.com/storage/docs/locations#location-dr) feature of Google Cloud Storage, the pair of regions that you use for the dual-region in TiDB Dedicated must be within the same multi-region. For example, Tokyo and Osaka are in the same multi-region `ASIA` so they can be used together for dual-region storage. - - For more information, see [Back Up and Restore TiDB Dedicated Data](/tidb-cloud/backup-and-restore.md#turn-on-dual-region-backup-beta). - -- The feature of [streaming data change logs to Apache Kafka](/tidb-cloud/changefeed-sink-to-apache-kafka.md) is now in General Availability (GA). - - After a successful 10-month beta trial, the feature of streaming data change logs from TiDB Cloud to Apache Kafka becomes generally available. Streaming data from TiDB to a message queue is a common need in data integration scenarios. You can use Kafka sink to integrate with other data processing systems (such as Snowflake) or support business consumption. - - For more information, see [Changefeed overview](/tidb-cloud/changefeed-overview.md). - -## October 11, 2023 - -**General changes** - -- Support [dual region backup (beta)](/tidb-cloud/backup-and-restore.md#turn-on-dual-region-backup-beta) for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters deployed on AWS. - - You can now replicate backups across geographic regions within your cloud provider. This feature provides an additional layer of data protection and disaster recovery capabilities. - - For more information, see [Back up and restore TiDB Dedicated data](/tidb-cloud/backup-and-restore.md). - -- Data Migration now supports both physical mode and logical mode for migrating existing data. - - In physical mode, the migration speed can reach up to 110 MiB/s. Compared with 45 MiB/s in logical mode, the migration performance has improved significantly. - - For more information, see [Migrate existing data and incremental data](/tidb-cloud/migrate-from-mysql-using-data-migration.md#migrate-existing-data-and-incremental-data). - -## October 10, 2023 - -**General changes** - -- Support using TiDB Serverless branches in [Vercel Preview Deployments](https://vercel.com/docs/deployments/preview-deployments), with TiDB Cloud Vercel integration. - - For more information, see [Connect with TiDB Serverless branching](/tidb-cloud/integrate-tidbcloud-with-vercel.md#connect-with-tidb-serverless-branching). - -## September 28, 2023 - -**API changes** - -- Introduce a TiDB Cloud Billing API endpoint to retrieve the bill for the given month of a specific organization. - - This Billing API endpoint is released in TiDB Cloud API v1beta1, which is the latest API version of TiDB Cloud. For more information, refer to the [API documentation (v1beta1)](https://docs.pingcap.com/tidbcloud/api/v1beta1#tag/Billing). - -## September 19, 2023 - -**General changes** - -- Remove 2 vCPU TiDB and TiKV nodes from [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. - - The 2 vCPU option is no longer available on the **Create Cluster** page or the **Modify Cluster** page. - -- Release [TiDB Cloud serverless driver (beta)](/tidb-cloud/serverless-driver.md) for JavaScript. - - TiDB Cloud serverless driver for JavaScript allows you to connect to your [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster over HTTPS. It is particularly useful in edge environments where TCP connections are limited, such as [Vercel Edge Function](https://vercel.com/docs/functions/edge-functions) and [Cloudflare Workers](https://workers.cloudflare.com/). - - For more information, see [TiDB Cloud serverless driver (beta)](/tidb-cloud/serverless-driver.md). - -**Console changes** - -- For [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters, you can get an estimation of cost in the **Usage This Month** panel or while setting up the spending limit. - -## September 5, 2023 - -**General changes** - -- [Data Service (beta)](https://tidbcloud.com/console/data-service) supports customizing the rate limit for each API key to meet specific rate-limiting requirements in different situations. - - You can adjust the rate limit of an API key when you [create](/tidb-cloud/data-service-api-key.md#create-an-api-key) or [edit](/tidb-cloud/data-service-api-key.md#edit-an-api-key) the key. - - For more information, see [Rate limiting](/tidb-cloud/data-service-api-key.md#rate-limiting). - -- Support a new AWS region for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters: São Paulo (sa-east-1). - -- Support adding up to 100 IP addresses to the IP access list for each [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) cluster. - - For more information, see [Configure an IP access list](/tidb-cloud/configure-ip-access-list.md). - -**Console changes** - -- Introduce the **Events** page for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters, which provides the records of main changes to your cluster. - - On this page, you can view the event history for the last 7 days and track important details such as the trigger time and the user who initiated an action. - - For more information, see [TiDB Cloud cluster events](/tidb-cloud/tidb-cloud-events.md). - -**API changes** - -- Release several TiDB Cloud API endpoints for managing the [AWS PrivateLink](https://aws.amazon.com/privatelink/?privatelink-blogs.sort-by=item.additionalFields.createdDate&privatelink-blogs.sort-order=desc) or [Google Cloud Private Service Connect](https://cloud.google.com/vpc/docs/private-service-connect) for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters: - - - Create a private endpoint service for a cluster - - Retrieve the private endpoint service information of a cluster - - Create a private endpoint for a cluster - - List all private endpoints of a cluster - - List all private endpoints in a project - - Delete a private endpoint of a cluster - - For more information, refer to the [API documentation](https://docs.pingcap.com/tidbcloud/api/v1beta#tag/Cluster). - -## August 23, 2023 - -**General changes** - -- Support Google Cloud [Private Service Connect](https://cloud.google.com/vpc/docs/private-service-connect) for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. - - You can now create a private endpoint and establish a secure connection to a TiDB Dedicated cluster hosted on Google Cloud. - - Key benefits: - - - Intuitive operations: helps you create a private endpoint with only several steps. - - Enhanced security: establishes a secure connection to protect your data. - - Improved performance: provides low-latency and high-bandwidth connectivity. - - For more information, see [Connect via Private Endpoint with Google Cloud](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md). - -- Support using a changefeed to stream data from a [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) cluster to [Google Cloud Storage (GCS)](https://cloud.google.com/storage). - - You can now stream data from TiDB Cloud to GCS by using your own account's bucket and providing precisely tailored permissions. After replicating data to GCS, you can analyze the changes in your data as you wish. - - For more information, see [Sink to Cloud Storage](/tidb-cloud/changefeed-sink-to-cloud-storage.md). - -## August 15, 2023 - -**General changes** - -- [Data Service (beta)](https://tidbcloud.com/console/data-service) supports pagination for `GET` requests to improve the development experience. - - For `GET` requests, you can paginate results by enabling **Pagination** in **Advance Properties** and specifying `page` and `page_size` as query parameters when calling the endpoint. For example, to get the second page with 10 items per page, you can use the following command: - - ```bash - curl --digest --user ':' \ - --request GET 'https://.data.tidbcloud.com/api/v1beta/app//endpoint/?page=2&page_size=10' - ``` - - Note that this feature is available only for `GET` requests where the last query is a `SELECT` statement. - - For more information, see [Call an endpoint](/tidb-cloud/data-service-manage-endpoint.md#call-an-endpoint). - -- [Data Service (beta)](https://tidbcloud.com/console/data-service) supports caching endpoint response of `GET` requests for a specified time-to-live (TTL) period. - - This feature decreases database load and optimizes endpoint latency. - - For an endpoint using the `GET` request method, you can enable **Cache Response** and configure the TTL period for the cache in **Advance Properties**. - - For more information, see [Advanced properties](/tidb-cloud/data-service-manage-endpoint.md#advanced-properties). - -- Disable the load balancing improvement for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters that are hosted on AWS and created after August 15, 2023, including: - - - Disable automatically migrating existing connections to new TiDB nodes when you scale out TiDB nodes hosted on AWS. - - Disable automatically migrating existing connections to available TiDB nodes when you scale in TiDB nodes hosted on AWS. - - This change avoids resource contention of hybrid deployments and does not affect existing clusters with this improvement enabled. If you want to enable the load balancing improvement for your new clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). - -## August 8, 2023 - -**General changes** - -- [Data Service (beta)](https://tidbcloud.com/console/data-service) now supports Basic Authentication. - - You can provide your public key as the username and private key as the password in requests using the ['Basic' HTTP Authentication](https://datatracker.ietf.org/doc/html/rfc7617). Compared with Digest Authentication, the Basic Authentication is simpler, enabling more straightforward usage when calling Data Service endpoints. - - For more information, see [Call an endpoint](/tidb-cloud/data-service-manage-endpoint.md#call-an-endpoint). - -## August 1, 2023 - -**General changes** - -- Support the OpenAPI Specification for Data Apps in TiDB Cloud [Data Service](https://tidbcloud.com/console/data-service). - - TiDB Cloud Data Service provides autogenerated OpenAPI documentation for each Data App. In the documentation, you can view the endpoints, parameters, and responses, and try out the endpoints. - - You can also download an OpenAPI Specification (OAS) for a Data App and its deployed endpoints in YAML or JSON format. The OAS provides standardized API documentation, simplified integration, and easy code generation, which enables faster development and improved collaboration. - - For more information, see [Use the OpenAPI Specification](/tidb-cloud/data-service-manage-data-app.md#use-the-openapi-specification) and [Use the OpenAPI Specification with Next.js](/tidb-cloud/data-service-oas-with-nextjs.md). - -- Support running Data App in [Postman](https://www.postman.com/). - - The Postman integration empowers you to import a Data App's endpoints as a collection into your preferred workspace. Then you can benefit from enhanced collaboration and seamless API testing with support for both Postman web and desktop apps. - - For more information, see [Run Data App in Postman](/tidb-cloud/data-service-postman-integration.md). - -- Introduce a new **Pausing** status for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters, allowing cost-effective pauses with no charges during this period. - - When you click **Pause** for a TiDB Dedicated cluster, the cluster will enter the **Pausing** status first. Once the pause operation is completed, the cluster status will transition to **Paused**. - - A cluster can only be resumed after its status transitions to **Paused**, which resolves the abnormal resumption issue caused by rapid clicks of **Pause** and **Resume**. - - For more information, see [Pause or resume a TiDB Dedicated cluster](/tidb-cloud/pause-or-resume-tidb-cluster.md). - -## July 26, 2023 - -**General changes** - -- Introduce a powerful feature in TiDB Cloud [Data Service](https://tidbcloud.com/console/data-service): Automatic endpoint generation. - - Developers can now effortlessly create HTTP endpoints with minimal clicks and configurations. Eliminate repetitive boilerplate code, simplify and accelerate endpoint creation, and reduce potential errors. - - For more information on how to use this feature, see [Generate an endpoint automatically](/tidb-cloud/data-service-manage-endpoint.md#generate-an-endpoint-automatically). - -- Support `PUT` and `DELETE` request methods for endpoints in TiDB Cloud [Data Service](https://tidbcloud.com/console/data-service). - - - Use the `PUT` method to update or modify data, similar to an `UPDATE` statement. - - Use the `DELETE` method to delete data, similar to a `DELETE` statement. - - For more information, see [Configure properties](/tidb-cloud/data-service-manage-endpoint.md#configure-properties). - -- Support **Batch Operation** for `POST`, `PUT`, and `DELETE` request methods in TiDB Cloud [Data Service](https://tidbcloud.com/console/data-service). - - When **Batch Operation** is enabled for an endpoint, you gain the ability to perform operations on multiple rows in a single request. For instance, you can insert multiple rows of data using a single `POST` request. - - For more information, see [Advanced properties](/tidb-cloud/data-service-manage-endpoint.md#advanced-properties). - -## July 25, 2023 - -**General changes** - -- Upgrade the default TiDB version of new [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters from [v6.5.3](https://docs.pingcap.com/tidb/v6.5/release-6.5.3) to [v7.1.1](https://docs.pingcap.com/tidb/v7.1/release-7.1.1). - -**Console changes** - -- Simplify access to PingCAP Support for TiDB Cloud users by optimizing support entries. Improvements include: - - - Add an entrance for **Support** in the in the lower-left corner. - - Revamp the menus of the **?** icon in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com/) to make them more intuitive. - - For more information, see [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). - -## July 18, 2023 - -**General changes** - -- Refine role-based access control at both the organization level and project level, which lets you grant roles with minimum permissions to users for better security, compliance, and productivity. - - - The organization roles include `Organization Owner`, `Organization Billing Admin`, `Organization Console Audit Admin`, and `Organization Member`. - - The project roles include `Project Owner`, `Project Data Access Read-Write`, and `Project Data Access Read-Only`. - - To manage clusters in a project (such as cluster creation, modification, and deletion), you need to be in the `Organization Owner` or `Project Owner` role. - - For more information about permissions of different roles, see [User roles](/tidb-cloud/manage-user-access.md#user-roles). - -- Support the Customer-Managed Encryption Key (CMEK) feature (beta) for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters hosted on AWS. - - You can create CMEK based on AWS KMS to encrypt data stored in EBS and S3 directly from the TiDB Cloud console. This ensures that customer data is encrypted with a key managed by the customer, which enhances security. - - Note that this feature still has restrictions and is only available upon request. To apply for this feature, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). - -- Optimize the Import feature in TiDB Cloud, aimed at enhancing the data import experience. The following improvements have been made: - - - Unified Import entry for TiDB Serverless: consolidate the entries for importing data, allowing you to seamlessly switch between importing local files and importing files from Amazon S3. - - Streamlined configuration: importing data from Amazon S3 now only requires a single step, saving time and effort. - - Enhanced CSV configuration: the CSV configuration settings are now located under the file type option, making it easier for you to quickly configure the necessary parameters. - - Enhanced target table selection: support choosing the desired target tables for data import by clicking checkboxes. This improvement eliminates the need for complex expressions and simplifies the target table selection. - - Refined display information: resolve issues related to inaccurate information displayed during the import process. In addition, the Preview feature has been removed to prevent incomplete data display and avoid misleading information. - - Improved source files mapping: support defining mapping relationships between source files and target tables. It addresses the challenge of modifying source file names to meet specific naming requirements. - -## July 11, 2023 - -**General changes** - -- [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) now is Generally Available. - -- Introduce TiDB Bot (beta), an OpenAI-powered chatbot that offers multi-language support, 24/7 real-time response, and integrated documentation access. - - TiDB Bot provides you with the following benefits: - - - Continuous support: always available to assist and answer your questions for an enhanced support experience. - - Improved efficiency: automated responses reduce latency, improving overall operations. - - Seamless documentation access: direct access to TiDB Cloud documentation for easy information retrieval and quick issue resolution. - - To use TiDB Bot, click **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com), and select **Ask TiDB Bot** to start a chat. - -- Support [the branching feature (beta)](/tidb-cloud/branch-overview.md) for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. - - TiDB Cloud lets you create branches for TiDB Serverless clusters. A branch for a cluster is a separate instance that contains a diverged copy of data from the original cluster. It provides an isolated environment, allowing you to connect to it and experiment freely without worrying about affecting the original cluster. - - You can create branches for TiDB Serverless clusters created after July 5, 2023 by using either [TiDB Cloud console](/tidb-cloud/branch-manage.md) or [TiDB Cloud CLI](/tidb-cloud/ticloud-branch-create.md). - - If you use GitHub for application development, you can integrate TiDB Serverless branching into your GitHub CI/CD pipeline, which lets you automatically test your pull requests with branches without affecting the production database. For more information, see [Integrate TiDB Serverless Branching (Beta) with GitHub](/tidb-cloud/branch-github-integration.md). - -- Support weekly backup for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. For more information, see [Back up and restore TiDB Dedicated data](/tidb-cloud/backup-and-restore.md#turn-on-auto-backup). - -## July 4, 2023 - -**General changes** - -- Support point-in-time recovery (PITR) (beta) for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. - - You can now restore your TiDB Serverless cluster to any point in time within the last 90 days. This feature enhances the data recovery capability of TiDB Serverless clusters. For example, you can use PITR when data write errors occur and you want to restore the data to an earlier state. - - For more information, see [Back up and restore TiDB Serverless data](/tidb-cloud/backup-and-restore-serverless.md#restore). - -**Console changes** - -- Enhance the **Usage This Month** panel on the cluster overview page for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters to provide a clearer view of your current resource usage. - -- Enhance the overall navigation experience by making the following changes: - - - Consolidate **Organization** and **Account** in the upper-right corner into the left navigation bar. - - Consolidate **Admin** in the left navigation bar into **Project** in the left navigation bar, and remove the ☰ hover menu in the upper-left corner. Now you can click to switch between projects and modify project settings. - - Consolidate all the help and support information for TiDB Cloud into the menu of the **?** icon in the lower-right corner, such as documentation, interactive tutorials, self-paced training, and support entries. - -- TiDB Cloud console now supports Dark Mode, which provides a more comfortable, eye-friendly experience. You can switch between light mode and dark mode from the bottom of the left navigation bar. - -## June 27, 2023 - -**General changes** - -- Remove the pre-built sample dataset for newly created [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. - -## June 20, 2023 - -**General changes** - -- Upgrade the default TiDB version of new [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters from [v6.5.2](https://docs.pingcap.com/tidb/v6.5/release-6.5.2) to [v6.5.3](https://docs.pingcap.com/tidb/v6.5/release-6.5.3). + - Direct text search: query string columns directly without the need for embeddings. + - Multilingual support: automatically detects and analyzes text in multiple languages, even within the same table, without requiring language specification. + - Relevance-based ranking: results are ranked using the industry-standard BM25 algorithm for optimal relevance. + - Native SQL compatibility: seamlessly use SQL features such as filtering, grouping, and joining with full-text search. -## June 13, 2023 + To get started, see [Full Text Search with SQL](/tidb-cloud/vector-search-full-text-search-sql.md) or [Full Text Search with Python](/tidb-cloud/vector-search-full-text-search-python.md). -**General changes** - -- Support using changefeeds to stream data to Amazon S3. +- Increase the maximum TiFlash node storage for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) cluster: - This enables seamless integration between TiDB Cloud and Amazon S3. It allows real-time data capture and replication from [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters to Amazon S3, ensuring that downstream applications and analytics have access to up-to-date data. + - For 8 vCPU TiFlash, from 2048 GiB to 4096 GiB + - For 32 vCPU TiFlash, from 4096 GiB to 8192 GiB - For more information, see [Sink to cloud storage](/tidb-cloud/changefeed-sink-to-cloud-storage.md). + This enhancement increases the analytics data storage capacity of your TiDB Cloud Dedicated cluster, improves workload scaling efficiency, and accommodates growing data requirements. -- Increase the maximum node storage of 16 vCPU TiKV for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters from 4 TiB to 6 TiB. + For more information, see [TiFlash node storage](/tidb-cloud/size-your-cluster.md#tiflash-node-storage). - This enhancement increases the data storage capacity of your TiDB Dedicated cluster, improves workload scaling efficiency, and accommodates growing data requirements. +- Enhance the maintenance window configuration experience by providing intuitive options to configure and reschedule maintenance tasks. - For more information, see [Size your cluster](/tidb-cloud/size-your-cluster.md). + For more information, see [Configure maintenance window](/tidb-cloud/configure-maintenance-window.md). -- Extend the [monitoring metrics retention period](/tidb-cloud/built-in-monitoring.md#metrics-retention-policy) for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters from 3 days to 7 days. - - By extending the metrics retention period, now you have access to more historical data. This helps you identify trends and patterns of the cluster for better decision-making and faster troubleshooting. +- Extend the discount period for TiKV [Standard](/tidb-cloud/size-your-cluster.md#standard-storage) and [Performance](/tidb-cloud/size-your-cluster.md#performance-and-plus-storage) storage types. The promotion now ends on June 5, 2025. After this date, pricing will return to the standard rate. **Console changes** -- Release a new native web infrastructure for the [**Key Visualizer**](/tidb-cloud/tune-performance.md#key-visualizer) page of [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. - - With the new infrastructure, you can easily navigate through the **Key Visualizer** page and access the necessary information in a more intuitive and efficient manner. The new infrastructure also resolves many problems on UX, making the SQL diagnosis process more user-friendly. - -## June 6, 2023 - -**General changes** - -- Introduce [Index Insight (beta)](/tidb-cloud/index-insight.md) for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters, which optimizes query performance by providing index recommendations for slow queries. - - With Index Insight, you can improve the overall application performance and efficiency of your database operations in the following ways: - - - Enhanced query performance: Index Insight identifies slow queries and suggests appropriate indexes for them, thereby speeding up query execution, reducing response time, and improving user experience. - - Cost efficiency: By using Index Insight to optimize query performance, the need for extra computing resources is reduced, enabling you to use existing infrastructure more effectively. This can potentially lead to operational cost savings. - - Simplified optimization process: Index Insight simplifies the identification and implementation of index improvements, eliminating the need for manual analysis and guesswork. As a result, you can save time and effort with accurate index recommendations. - - Improved application efficiency: By using Index Insight to optimize database performance, applications running on TiDB Cloud can handle larger workloads and serve more users concurrently, which makes scaling operations of applications more efficient. - - To use Index Insight, navigate to the **Diagnosis** page of your TiDB Dedicated cluster and click the **Index Insight BETA** tab. - - For more information, see [Use Index Insight (beta)](/tidb-cloud/index-insight.md). - -- Introduce [TiDB Playground](https://play.tidbcloud.com/?utm_source=docs&utm_medium=tidb_cloud_release_notes), an interactive platform for experiencing the full capabilities of TiDB, without registration or installation. +- Refine the **Backup Setting** page layout to improve the backup configuration experience in [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters. - TiDB Playground is an interactive platform designed to provide a one-stop-shop experience for exploring the capabilities of TiDB, such as scalability, MySQL compatibility, and real-time analytics. + For more information, see [Back Up and Restore TiDB Cloud Dedicated Data](/tidb-cloud/backup-and-restore.md). - With TiDB Playground, you can try out TiDB features in a controlled environment free from complex configurations in real-time, making it ideal to understand the features in TiDB. - - To get started with TiDB Playground, go to the [**TiDB Playground**](https://play.tidbcloud.com/?utm_source=docs&utm_medium=tidb_cloud_release_notes) page, select a feature you want to explore, and begin your exploration. - -## June 5, 2023 +## April 22, 2025 **General changes** -- Support connecting your [Data App](/tidb-cloud/tidb-cloud-glossary.md#data-app) to GitHub. - - By [connecting your Data App to GitHub](/tidb-cloud/data-service-manage-github-connection.md), you can manage all configurations of the Data App as [code files](/tidb-cloud/data-service-app-config-files.md) on Github, which integrates TiDB Cloud Data Service seamlessly with your system architecture and DevOps process. - - With this feature, you can easily accomplish the following tasks, which improves the CI/CD experience of developing Data Apps: +- Data export to Alibaba Cloud OSS is now supported. - - Automatically deploy Data App changes with GitHub. - - Configure CI/CD pipelines of your Data App changes on GitHub with version control. - - Disconnect from a connected GitHub repository. - - Review endpoint changes before the deployment. - - View deployment history and take necessary actions in the event of a failure. - - Re-deploy a commit to roll back to an earlier deployment. + [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters now support exporting data to [Alibaba Cloud Object Storage Service (OSS)](https://www.alibabacloud.com/en/product/object-storage-service) using an [AccessKey pair](https://www.alibabacloud.com/help/en/ram/user-guide/create-an-accesskey-pair). - For more information, see [Deploy Data App automatically with GitHub](/tidb-cloud/data-service-manage-github-connection.md). + For more information, see [Export Data from TiDB Cloud Serverless](/tidb-cloud/serverless-export.md#alibaba-cloud-oss). -## June 2, 2023 +## April 15, 2025 **General changes** -- In our pursuit to simplify and clarify, we have updated the names of our products: +- Support importing data from [Alibaba Cloud Object Storage Service (OSS)](https://www.alibabacloud.com/en/product/object-storage-service) into [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters. - - "TiDB Cloud Serverless Tier" is now called "TiDB Serverless". - - "TiDB Cloud Dedicated Tier" is now called "TiDB Dedicated". - - "TiDB On-Premises" is now called "TiDB Self-Hosted". + This feature simplifies data migration to TiDB Cloud Serverless. You can use an AccessKey pair to authenticate. - Enjoy the same great performance under these refreshed names. Your experience is our priority. - -## May 30, 2023 - -**General changes** + For more information, see the following documentation: -- Enhance support for incremental data migration for the Data Migration feature in TiDB Cloud. + - [Import CSV Files from Amazon S3, GCS, Azure Blob Storage, or Alibaba Cloud OSS into TiDB Cloud Serverless](/tidb-cloud/import-csv-files-serverless.md) + - [Import Apache Parquet Files from Amazon S3, GCS, Azure Blob Storage, or Alibaba Cloud OSS into TiDB Cloud Serverless](/tidb-cloud/import-parquet-files-serverless.md) - You can now specify a binlog position or a global transaction identifier (GTID) to replicate only incremental data generated after the specified position to TiDB Cloud. This enhancement empowers you with greater flexibility to select and replicate the data you need, aligning with your specific requirements. - - For details, refer to [Migrate Only Incremental Data from MySQL-Compatible Databases to TiDB Cloud Using Data Migration](/tidb-cloud/migrate-incremental-data-from-mysql-using-data-migration.md). - -- Add a new event type (`ImportData`) to the [**Events**](/tidb-cloud/tidb-cloud-events.md) page. - -- Remove **Playground** from the TiDB Cloud console. - - Stay tuned for the new standalone Playground with an optimized experience. - -## May 23, 2023 +## April 1, 2025 **General changes** -- When uploading a CSV file to TiDB, you can use not only English letters and numbers, but also characters such as Chinese and Japanese to define column names. However, for special characters, only underscore (`_`) is supported. - - For details, refer to [Import Local Files to TiDB Cloud](/tidb-cloud/tidb-cloud-import-local-files.md). +- The [TiDB Node Groups](/tidb-cloud/tidb-node-group-overview.md) feature is now generally available (GA) for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters hosted on AWS and Google Cloud. -## May 16, 2023 + This feature enables **fine-grained computing resource isolation** within a single cluster, helping you optimize performance and resource allocation for multi-tenant or multi-workload scenarios. -**Console changes** - -- Introduce the left navigation entries organized by functional categories for both Dedicated and Serverless tiers. + **Key benefits:** - The new navigation makes it easier and more intuitive for you to discover the feature entries. To view the new navigation, access the overview page of your cluster. + - **Resource isolation**: -- Release a new native web infrastructure for the following two tabs on the **Diagnosis** page of Dedicated Tier clusters. + - Group TiDB nodes into logically isolated units, ensuring workloads in one group do not affect other groups. + - Prevent resource contention between applications or business units. - - [Slow Query](/tidb-cloud/tune-performance.md#slow-query) - - [SQL Statement](/tidb-cloud/tune-performance.md#statement-analysis) + - **Simplified management**: - With the new infrastructure, you can easily navigate through the two tabs and access the necessary information in a more intuitive and efficient manner. The new infrastructure also improves user experience, making the SQL diagnosis process more user-friendly. + - Manage all node groups within a single cluster, reducing operational overhead. + - Scale groups independently based on demand. -## May 9, 2023 - -**General changes** + For more information about the benefits, see [the technical blog](https://www.pingcap.com/blog/tidb-cloud-node-groups-scaling-workloads-predictable-performance/). To get started, see [Manage TiDB Node Groups](/tidb-cloud/tidb-node-group-management.md). -- Support changing node sizes for GCP-hosted clusters created after April 26, 2023. +- Introduce the [Standard storage](/tidb-cloud/size-your-cluster.md#standard-storage) type for TiKV nodes in [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters hosted on AWS. - With this feature, you can upgrade to higher-performance nodes for increased demand or downgrade to lower-performance nodes for cost savings. With this added flexibility, you can adjust your cluster's capacity to align with your workloads and optimize costs. + The Standard storage type is ideal for most workloads, providing a balance between performance and cost efficiency. - For detailed steps, see [Change node size](/tidb-cloud/scale-tidb-cluster.md#change-vcpu-and-ram). + **Key benefits:** -- Support importing compressed files. You can import CSV and SQL files in the following formats: `.gzip`, `.gz`, `.zstd`, `.zst`, and `.snappy`. This feature provides a more efficient and cost-effective way to import data and reduces your data transfer costs. + - **Improved performance**: Reserves sufficient disk resources for Raft logs, reducing I/O contention between Raft and data storage, thereby improving both the read and write performance of TiKV. + - **Enhanced stability**: Isolates critical Raft operations from data workloads, ensuring more predictable performance. + - **Cost efficiency**: Delivers higher performance at a competitive price compared with the previous storage type. - For more information, see [Import CSV Files from Amazon S3 or GCS into TiDB Cloud](/tidb-cloud/import-csv-files.md) and [Import Sample Data](/tidb-cloud/import-sample-data.md). + **Availability:** -- Support AWS PrivateLink-powered endpoint connection as a new network access management option for TiDB Cloud [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. + The Standard storage type is automatically applied to new [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters created on or after April 1, 2025, hosted on AWS, with supported versions (versions >= 7.5.5, 8.1.2, or 8.5.0). Existing clusters still use the previous [Basic storage](/tidb-cloud/size-your-cluster.md#basic-storage) type, and no migration is needed. - The private endpoint connection does not expose your data to the public internet. In addition, the endpoint connection supports CIDR overlap and is easier for network management. + The price of the Standard storage differs from that of the Basic storage. For more information, see [Pricing](https://www.pingcap.com/tidb-dedicated-pricing-details/). - For more information, see [Set Up Private Endpoint Connections](/tidb-cloud/set-up-private-endpoint-connections.md). +## March 25, 2025 **Console changes** -- Add new event types to the [**Event**](/tidb-cloud/tidb-cloud-events.md) page to record backup, restore, and changefeed actions for [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. - - To get a full list of the events that can be recorded, see [Logged events](/tidb-cloud/tidb-cloud-events.md#logged-events). - -- Introduce the **SQL Statement** tab on the [**SQL Diagnosis**](/tidb-cloud/tune-performance.md) page for [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. - - The **SQL Statement** tab provides the following: - - - A comprehensive overview of all SQL statements executed by your TiDB database, allowing you to easily identify and diagnose slow queries. - - Detailed information on each SQL statement, such as the query time, execution plan, and the database server response, helping you optimize your database performance. - - A user-friendly interface that makes it easy to sort, filter, and search through large amounts of data, enabling you to focus on the most critical queries. - - For more information, see [Statement Analysis](/tidb-cloud/tune-performance.md#statement-analysis). - -## May 6, 2023 - -**General changes** - -- Support directly accessing the [Data Service endpoint](/tidb-cloud/tidb-cloud-glossary.md#endpoint) in the region where a TiDB [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster is located. +- Support firewall rules for public endpoints in [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters. - For newly created Serverless Tier clusters, the endpoint URL now includes the cluster region information. By requesting the regional domain `.data.tidbcloud.com`, you can directly access the endpoint in the region where the TiDB cluster is located. + You can now configure firewall rules for TiDB Cloud Serverless clusters to control access via public endpoints. Specify allowed IP addresses or ranges directly in the [TiDB Cloud console](https://tidbcloud.com/) to enhance security. - Alternatively, you can also request the global domain `data.tidbcloud.com` without specifying a region. In this way, TiDB Cloud will internally redirect the request to the target region, but this might result in additional latency. If you choose this way, make sure to add the `--location-trusted` option to your curl command when calling an endpoint. + For more information, see [Configure TiDB Cloud Serverless Firewall Rules for Public Endpoints](/tidb-cloud/configure-serverless-firewall-rules-for-public-endpoints.md). - For more information, see [Call an endpoint](/tidb-cloud/data-service-manage-endpoint.md#call-an-endpoint). - -## April 25, 2023 +## March 18, 2025 **General changes** -- For the first five [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters in your organization, TiDB Cloud provides a free usage quota for each of them as follows: - - - Row storage: 5 GiB - - [Request Units (RUs)](/tidb-cloud/tidb-cloud-glossary.md#request-unit): 50 million RUs per month - - Until May 31, 2023, Serverless Tier clusters are still free, with a 100% discount off. After that, usage beyond the free quota will be charged. - - You can easily [monitor your cluster usage or increase your usage quota](/tidb-cloud/manage-serverless-spend-limit.md#manage-spending-limit-for-tidb-serverless-clusters) in the **Usage This Month** area of your cluster **Overview** page. Once the free quota of a cluster is reached, the read and write operations on this cluster will be throttled until you increase the quota or the usage is reset upon the start of a new month. - - For more information about the RU consumption of different resources (including read, write, SQL CPU, and network egress), the pricing details, and the throttled information, see [TiDB Cloud Serverless Tier Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details). - -- Support backup and restore for TiDB Cloud [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. - - For more information, see [Back up and Restore TiDB Cluster Data](/tidb-cloud/backup-and-restore-serverless.md). - -- Upgrade the default TiDB version of new [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters from [v6.5.1](https://docs.pingcap.com/tidb/v6.5/release-6.5.1) to [v6.5.2](https://docs.pingcap.com/tidb/v6.5/release-6.5.2). - -- Provide a maintenance window feature to enable you to easily schedule and manage planned maintenance activities for [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. - - A maintenance window is a designated timeframe during which planned maintenance activities, such as operating system updates, security patches, and infrastructure upgrades, are performed automatically to ensure the reliability, security, and performance of the TiDB Cloud service. - - During a maintenance window, temporary connection disruptions or QPS fluctuations might occur, but the clusters remain available, and SQL operations, the existing data import, backup, restore, migration, and replication tasks can still run normally. See [a list of allowed and disallowed operations](/tidb-cloud/configure-maintenance-window.md#allowed-and-disallowed-operations-during-a-maintenance-window) during maintenance. - - We will strive to minimize the frequency of maintenance. If a maintenance window is planned, the default start time is 03:00 Wednesday (based on the time zone of your TiDB Cloud organization) of the target week. To avoid potential disruptions, it is important to be aware of the maintenance schedules and plan your operations accordingly. - - - To keep you informed, TiDB Cloud will send you three email notifications for every maintenance window: one before, one starting, and one after the maintenance tasks. - - To minimize the maintenance impact, you can modify the maintenance start time to your preferred time or defer maintenance activities on the **Maintenance** page. +- Support creating TiDB node groups for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters deployed on Google Cloud to enhance resource management flexibility. - For more information, see [Configure maintenance window](/tidb-cloud/configure-maintenance-window.md). + For more information, see [Overview of TiDB Node Group](/tidb-cloud/tidb-node-group-overview.md). -- Improve load balancing of TiDB and reduce connection drops when you scale TiDB nodes of [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters that are hosted on AWS and created after April 25, 2023. +- Support storing database audit log files in TiDB Cloud for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters deployed on AWS. - - Support automatically migrating existing connections to new TiDB nodes when you scale out TiDB nodes. - - Support automatically migrating existing connections to available TiDB nodes when you scale in TiDB nodes. + You can download these audit log files directly from TiDB Cloud. Note that this feature is only available upon request. - Currently, this feature is provided for all Dedicated Tier clusters hosted on AWS. + For more information, see [Database Audit Logging](/tidb-cloud/tidb-cloud-auditing.md). -**Console changes** - -- Release a new native web infrastructure for the [Monitoring](/tidb-cloud/built-in-monitoring.md#view-the-metrics-page) page of [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. - - With the new infrastructure, you can easily navigate through the [Monitoring](/tidb-cloud/built-in-monitoring.md#view-the-metrics-page) page and access the necessary information in a more intuitive and efficient manner. The new infrastructure also resolves many problems on UX, making the monitoring process more user-friendly. - -## April 18, 2023 - -**General changes** - -- Support scaling up or down [Data Migration job specifications](/tidb-cloud/tidb-cloud-billing-dm.md#specifications-for-data-migration) for [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. +- Enhance TiDB Cloud account security by improving the management of multi-factor authentication (MFA). This feature applies to password-based logins for TiDB Cloud. - With this feature, you can improve migration performance by scaling up specifications or reduce costs by scaling down specifications. + For more information, see [Password Authentication](/tidb-cloud/tidb-cloud-password-authentication.md). - For more information, see [Migrate MySQL-Compatible Databases to TiDB Cloud Using Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md#scale-a-migration-job-specification). +## February 18, 2025 **Console changes** -- Revamp the UI to make [cluster creation](https://tidbcloud.com/console/clusters/create-cluster) experience more user-friendly, enabling you to create and configure clusters with just a few clicks. - - The new design focuses on simplicity, reducing visual clutter, and providing clear instructions. After clicking **Create** on the cluster creation page, you will be directed to the cluster overview page without having to wait for the cluster creation to be completed. - - For more information, see [Create a cluster](/tidb-cloud/create-tidb-cluster.md). - -- Introduce the **Discounts** tab on the **Billing** page to show the discount information for organization owners and billing administrators. - - For more information, see [Discounts](/tidb-cloud/tidb-cloud-billing.md#discounts). - -## April 11, 2023 - -**General changes** - -- Improve the load balance of TiDB and reduce connection drops when you scale TiDB nodes of [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters hosted on AWS. - - - Support automatically migrating existing connections to new TiDB nodes when you scale out TiDB nodes. - - Support automatically migrating existing connections to available TiDB nodes when you scale in TiDB nodes. - - Currently, this feature is only provided for Dedicated Tier clusters that are hosted on the AWS `Oregon (us-west-2)` region. +- Introduce Connected Care, the new support services for TiDB Cloud. -- Support the [New Relic](https://newrelic.com/) integration for [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. + The Connected Care services are designed to strengthen your connection with TiDB Cloud through modern communication tools, proactive support, and advanced AI capabilities, delivering a seamless and customer-centric experience. - With the New Relic integration, you can configure TiDB Cloud to send metric data of your TiDB clusters to [New Relic](https://newrelic.com/). Then, you can monitor and analyze both your application performance and your TiDB database performance on [New Relic](https://newrelic.com/). This feature can help you quickly identify and troubleshoot potential issues and reduce the resolution time. + The Connected Care services introduce the following features: - For integration steps and available metrics, see [Integrate TiDB Cloud with New Relic](/tidb-cloud/monitor-new-relic-integration.md). + - **Clinic service**: Advanced monitoring and diagnostics to optimize performance. + - **AI chat in IM**: Get immediate AI assistance through an instant message (IM) tool. + - **IM subscription for alerts and ticket updates**: Stay informed with alerts and ticket progress via IM. + - **IM interaction for support tickets**: Create and interact with support tickets through an IM tool. -- Add the following [changefeed](/tidb-cloud/changefeed-overview.md) metrics to the Prometheus integration for Dedicated Tier clusters. + For more information, see [Connected Care Overview](/tidb-cloud/connected-care-overview.md). - - `tidbcloud_changefeed_latency` - - `tidbcloud_changefeed_replica_rows` - - If you have [integrated TiDB Cloud with Prometheus](/tidb-cloud/monitor-prometheus-and-grafana-integration.md), you can monitor the performance and health of changefeeds in real time using these metrics. Additionally, you can easily create alerts to monitor the metrics using Prometheus. - -**Console changes** +- Support importing data from GCS and Azure Blob Storage into [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters. -- Update the [Monitoring](/tidb-cloud/built-in-monitoring.md#view-the-metrics-page) page for [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters to use [node-level resource metrics](/tidb-cloud/built-in-monitoring.md#server). - - With node-level resource metrics, you can see a more accurate representation of resource consumption to better understand the actual usage of purchased services. - - To access these metrics, navigate to the [Monitoring](/tidb-cloud/built-in-monitoring.md#view-the-metrics-page) page of your cluster, and then check the **Server** category under the **Metrics** tab. - -- Optimize the [Billing](/tidb-cloud/tidb-cloud-billing.md#billing-details) page by reorganizing the billing items in **Summary by Project** and **Summary by Service**, which makes the billing information clearer. - -## April 4, 2023 - -**General changes** + TiDB Cloud Serverless now supports importing data from Google Cloud Storage (GCS) and Azure Blob Storage. You can use a Google Cloud service account key or an Azure shared access signature (SAS) token to authenticate. This feature simplifies data migration to TiDB Cloud Serverless. -- Remove the following two alerts from [TiDB Cloud built-in alerts](/tidb-cloud/monitor-built-in-alerting.md#tidb-cloud-built-in-alert-conditions) to prevent false positives. This is because temporary offline or out-of-memory (OOM) issues on one of the nodes do not significantly affect the overall health of a cluster. + For more information, see [Import CSV Files from Amazon S3, GCS, or Azure Blob Storage into TiDB Cloud Serverless](/tidb-cloud/import-csv-files-serverless.md) and [Import Apache Parquet Files from Amazon S3, GCS, or Azure Blob Storage into TiDB Cloud Serverless](/tidb-cloud/import-parquet-files-serverless.md). - - At least one TiDB node in the cluster has run out of memory. - - One or more cluster nodes are offline. +## January 21, 2025 **Console changes** -- Introduce the [Alerts](/tidb-cloud/monitor-built-in-alerting.md) page for [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters, which lists both active and closed alerts for each Dedicated Tier cluster. +- Support importing a single local CSV file of up to 250 MiB per task to [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters, increased from the previous limit of 50 MiB. - The **Alerts** page provides the following: + For more information, see [Import Local Files to TiDB Cloud](/tidb-cloud/tidb-cloud-import-local-files.md). - - An intuitive and user-friendly user interface. You can view alerts for your clusters on this page even if you have not subscribed to the alert notification emails. - - Advanced filtering options to help you quickly find and sort alerts based on their severity, status, and other attributes. It also allows you to view the historical data for the last 7 days, which eases the alert history tracking. - - The **Edit Rule** feature. You can customize alert rule settings to meet your cluster's specific needs. - - For more information, see [TiDB Cloud built-in alerts](/tidb-cloud/monitor-built-in-alerting.md). - -- Consolidate the help-related information and actions of TiDB Cloud into a single place. - - Now, you can get all the [TiDB Cloud help information](/tidb-cloud/tidb-cloud-support.md) and contact support by clicking **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com/). - -- Introduce the [Getting Started](https://tidbcloud.com/console/getting-started) page to help you learn about TiDB Cloud. - - The **Getting Started** page provides you with interactive tutorials, essential guides, and useful links. By following interactive tutorials, you can easily explore TiDB Cloud features and HTAP capabilities with pre-built industry-specific datasets (Steam Game Dataset and S&P 500 Dataset). - - To access the **Getting Started** page, click **Getting Started** in the left navigation bar of the [TiDB Cloud console](https://tidbcloud.com/). On this page, you can click **Query Sample Dataset** to open the interactive tutorials or click other links to explore TiDB Cloud. Alternatively, you can click **?** in the lower-right corner and click **Interactive Tutorials**. - -## March 29, 2023 +## January 14, 2025 **General changes** -- [Data Service (beta)](/tidb-cloud/data-service-overview.md) supports more fine-grained access control for Data Apps. +- Support a new AWS region for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters: `Jakarta (ap-southeast-3)`. - On the Data App details page, now you can link clusters to your Data App and specify the role for each API key. The role controls whether the API key can read or write data to the linked clusters and can be set to `ReadOnly` or `ReadAndWrite`. This feature provides cluster-level and permission-level access control for Data Apps, giving you more flexibility to control the access scope according to your business needs. +- Introduce the [Notifications](https://tidbcloud.com/console/notifications) feature, which enables you to stay informed instantly with TiDB Cloud updates and alerts through the [TiDB Cloud console](https://tidbcloud.com/). - For more information, see [Manage linked clusters](/tidb-cloud/data-service-manage-data-app.md#manage-linked-data-sources) and [Manage API keys](/tidb-cloud/data-service-api-key.md). + For more information, see [Notifications](/tidb-cloud/notifications.md). -## March 28, 2023 +## January 2, 2025 **General changes** -- Add 2 RCUs, 4 RCUs, and 8 RCUs specifications for [changefeeds](/tidb-cloud/changefeed-overview.md), and support choosing your desired specification when you [create a changefeed](/tidb-cloud/changefeed-overview.md#create-a-changefeed). +- Support creating TiDB node groups for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters to enhance resource management flexibility. - Using these new specifications, the data replication costs can be reduced by up to 87.5% compared to scenarios where 16 RCUs were previously required. + For more information, see [Overview of TiDB Node Group](/tidb-cloud/tidb-node-group-overview.md). -- Support scaling up or down specifications for [changefeeds](/tidb-cloud/changefeed-overview.md) created after March 28, 2023. +- Support connecting [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters to generic Kafka in AWS and Google Cloud through Private Connect (beta). - You can improve replication performance by choosing a higher specification or reduce replication costs by choosing a lower specification. + Private Connect leverages Private Link or Private Service Connect technologies from cloud providers to enable changefeeds in the TiDB Cloud VPC to connect to Kafka in customers' VPCs using private IP addresses, as if those Kafkas were hosted directly within the TiDB Cloud VPC. This feature helps prevent VPC CIDR conflicts and meets security compliance requirements. - For more information, see [Scale a changefeed](/tidb-cloud/changefeed-overview.md#scale-a-changefeed). + - For Apache Kafka in AWS, follow the instructions in [Set Up Self-Hosted Kafka Private Link Service in AWS](/tidb-cloud/setup-self-hosted-kafka-private-link-service.md) to configure the network connection. -- Support replicating incremental data in real-time from a [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) cluster in AWS to a [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster in the same project and same region. - - For more information, see [Sink to TiDB Cloud](/tidb-cloud/changefeed-sink-to-tidb-cloud.md). - -- Support two new GCP regions for the [Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md) feature of [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters: `Singapore (asia-southeast1)` and `Oregon (us-west1)`. - - With these new regions, you have more options for migrating your data to TiDB Cloud. If your upstream data is stored in or near these regions, you can now take advantage of faster and more reliable data migration from GCP to TiDB Cloud. - - For more information, see [Migrate MySQL-compatible databases to TiDB Cloud using Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md). - -**Console changes** + - For Apache Kafka in Google Cloud, follow the instructions in [Set Up Self-Hosted Kafka Private Service Connect in Google Cloud](/tidb-cloud/setup-self-hosted-kafka-private-service-connect.md) to configure the network connection. + + Note that using this feature incurs additional [Private Data Link costs](/tidb-cloud/tidb-cloud-billing-ticdc-rcu.md#private-data-link-cost). -- Release a new native web infrastructure for the [Slow Query](/tidb-cloud/tune-performance.md#slow-query) page of [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. + For more information, see [Changefeed Sink to Apache Kafka](/tidb-cloud/changefeed-sink-to-apache-kafka.md#network). - With this new infrastructure, you can easily navigate through the [Slow Query](/tidb-cloud/tune-performance.md#slow-query) page and access the necessary information in a more intuitive and efficient manner. The new infrastructure also resolves many problems on UX, making the SQL diagnosis process more user-friendly. +- Introduce additional configurable options for Kafka changefeeds: -## March 21, 2023 + - Support using the Debezium protocol. Debezium is a tool for capturing database changes. It converts each captured database change into a message called an event, and sends these events to Kafka. For more information, see [TiCDC Debezium Protocol](https://docs.pingcap.com/tidb/v8.1/ticdc-debezium). -**General changes** + - Support defining a single partition dispatcher for all tables, or different partition dispatchers for different tables. -- Introduce [Data Service (beta)](https://tidbcloud.com/console/data-service) for [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters, which enables you to access data via an HTTPS request using a custom API endpoint. + - Introduce two new dispatcher types for the partition distribution of Kafka messages: timestamp and column value. - With Data Service, you can seamlessly integrate TiDB Cloud with any application or service that is compatible with HTTPS. The following are some common scenarios: + For more information, see [Sink to Apache Kafka](/tidb-cloud/changefeed-sink-to-apache-kafka.md). - - Access the database of your TiDB cluster directly from a mobile or web application. - - Use serverless edge functions to call endpoints and avoid scalability issues caused by database connection pooling. - - Integrate TiDB Cloud with data visualization projects by using Data Service as a data source. - - Connect to your database from an environment that MySQL interface does not support. +- Enhance roles in TiDB Cloud: - In addition, TiDB Cloud provides the [Chat2Query API](/tidb-cloud/use-chat2query-api.md), a RESTful interface that allows you to generate and execute SQL statements using AI. + - Introduce the `Project Viewer` and `Organization Billing Viewer` roles to enhance granular access control on TiDB Cloud. - To access Data Service, navigate to the [**Data Service**](https://tidbcloud.com/console/data-service) page in the left navigation pane. For more information, see the following documentation: + - Rename the following roles: - - [Data Service Overview](/tidb-cloud/data-service-overview.md) - - [Get Started with Data Service](/tidb-cloud/data-service-get-started.md) - - [Get Started with Chat2Query API](/tidb-cloud/use-chat2query-api.md) + - `Organization Member` to `Organization Viewer` + - `Organization Billing Admin` to `Organization Billing Manager` + - `Organization Console Audit Admin` to `Organization Console Audit Manager` -- Support decreasing the size of TiDB, TiKV, and TiFlash nodes to scale in a [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) cluster that is hosted on AWS and created after December 31, 2022. + For more information, see [Identity Access Management](/tidb-cloud/manage-user-access.md#organization-roles). - You can decrease the node size [via the TiDB Cloud console](/tidb-cloud/scale-tidb-cluster.md#change-vcpu-and-ram) or [via the TiDB Cloud API (beta)](https://docs.pingcap.com/tidbcloud/api/v1beta#tag/Cluster/operation/UpdateCluster). +- Regional high availability (beta) for [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters. -- Support a new GCP region for the [Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md) feature of [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters: `Tokyo (asia-northeast1)`. + This feature is designed for workloads that require maximum infrastructure redundancy and business continuity. Key functions include: - The feature can help you migrate data from MySQL-compatible databases in Google Cloud Platform (GCP) to your TiDB cluster easily and efficiently. + - Nodes are distributed across multiple availability zones to ensure high availability in the event of a zone failure. + - Critical OLTP (Online Transactional Processing) components, such as PD and TiKV, are replicated across availability zones for redundancy. + - Automatic failover minimizes service disruption during a primary zone failure. + + This feature is currently available only in the AWS Tokyo (ap-northeast-1) region and can be enabled only during cluster creation. + + For more information, see [High Availability in TiDB Cloud Serverless](/tidb-cloud/serverless-high-availability.md). - For more information, see [Migrate MySQL-compatible databases to TiDB Cloud using Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md). +- Upgrade the default TiDB version of new [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters from [v8.1.1](https://docs.pingcap.com/tidb/v8.1/release-8.1.1) to [v8.1.2](https://docs.pingcap.com/tidb/v8.1/release-8.1.2). **Console changes** -- Introduce the **Events** page for [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters, which provides the records of main changes to your cluster. - - On this page, you can view the event history for the last 7 days and track important details such as the trigger time and the user who initiated an action. For example, you can view events such as when a cluster was paused or who modified the cluster size. - - For more information, see [TiDB Cloud cluster events](/tidb-cloud/tidb-cloud-events.md). - -- Add the **Database Status** tab to the **Monitoring** page for [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters, which displays the following database-level metrics: - - - QPS Per DB - - Average Query Duration Per DB - - Failed Queries Per DB - - With these metrics, you can monitor the performance of individual databases, make data-driven decisions, and take actions to improve the performance of your applications. - - For more information, see [Monitoring metrics for Serverless Tier clusters](/tidb-cloud/built-in-monitoring.md). - -## March 14, 2023 - -**General changes** - -- Upgrade the default TiDB version of new [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters from [v6.5.0](https://docs.pingcap.com/tidb/v6.5/release-6.5.0) to [v6.5.1](https://docs.pingcap.com/tidb/v6.5/release-6.5.1). - -- Support modifying column names of the target table to be created by TiDB Cloud when uploading a local CSV file with a header row. - - When importing a local CSV file with a header row to a [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster, if you need TiDB Cloud to create the target table and the column names in the header row do not follow the TiDB Cloud column naming conventions, you will see a warning icon next to the corresponding column name. To resolve the warning, you can move the cursor over the icon and follow the message to edit the existing column names or enter new column names. +- Strengthen the data export service: - For information about column naming conventions, see [Import local files](/tidb-cloud/tidb-cloud-import-local-files.md#import-local-files). - -## March 7, 2023 - -**General changes** - -- Upgrade the default TiDB version of all [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters from [v6.4.0](https://docs.pingcap.com/tidb/v6.4/release-6.4.0) to [v6.6.0](https://docs.pingcap.com/tidb/v6.6/release-6.6.0). - -## February 28, 2023 - -**General changes** - -- Add the [SQL Diagnosis](/tidb-cloud/tune-performance.md) feature for [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. - - With SQL Diagnosis, you can gain deep insights into SQL-related runtime status, which makes the SQL performance tuning more efficient. Currently, the SQL Diagnosis feature for Serverless Tier only provides slow query data. - - To use SQL Diagnosis, click **SQL Diagnosis** on the left navigation bar of your Serverless Tier cluster page. - -**Console changes** - -- Optimize the left navigation. - - You can navigate pages more efficiently, for example: - - - You can hover the mouse in the upper-left corner to quickly switch between clusters or projects. - - You can switch between the **Clusters** page and the **Admin** page. - -**API changes** - -- Release several TiDB Cloud API endpoints for data import： - - - List all import tasks - - Get an import task - - Create an import task - - Update an import task - - Upload a local file for an import task - - Preview data before starting an import task - - Get the role information for import tasks - - For more information, refer to the [API documentation](https://docs.pingcap.com/tidbcloud/api/v1beta#tag/Import). - -## February 22, 2023 - -**General changes** - -- Support using the [console audit logging](/tidb-cloud/tidb-cloud-console-auditing.md) feature to track various activities performed by members within your organization in the [TiDB Cloud console](https://tidbcloud.com/). - - The console audit logging feature is only visible to users with the `Owner` or `Audit Admin` role and is disabled by default. To enable it, click **Organization** > **Console Audit Logging** in the upper-right corner of the [TiDB Cloud console](https://tidbcloud.com/). - - By analyzing console audit logs, you can identify suspicious operations performed within your organization, thereby improving the security of your organization's resources and data. - - For more information, see [Console audit logging](/tidb-cloud/tidb-cloud-console-auditing.md). - -**CLI changes** - -- Add a new command [`ticloud cluster connect-info`](/tidb-cloud/ticloud-cluster-connect-info.md) for [TiDB Cloud CLI](/tidb-cloud/cli-reference.md). - - `ticloud cluster connect-info` is a command that allows you to get the connection string of a cluster. To use this command, [update `ticloud`](/tidb-cloud/ticloud-update.md) to v0.3.2 or a later version. - -## February 21, 2023 - -**General changes** - -- Support using the AWS access keys of an IAM user to access your Amazon S3 bucket when importing data to TiDB Cloud. - - This method is simpler than using Role ARN. For more information, refer to [Configure Amazon S3 access](/tidb-cloud/config-s3-and-gcs-access.md#configure-amazon-s3-access). - -- Extend the [monitoring metrics retention period](/tidb-cloud/built-in-monitoring.md#metrics-retention-policy) from 2 days to a longer period: - - - For Dedicated Tier clusters, you can view metrics data for the past 7 days. - - For Serverless Tier clusters, you can view metrics data for the past 3 days. - - By extending the metrics retention period, now you have access to more historical data. This helps you identify trends and patterns of the cluster for better decision-making and faster troubleshooting. - -**Console changes** - -- Release a new native web infrastructure on the Monitoring page of [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. - - With the new infrastructure, you can easily navigate through the Monitoring page and access the necessary information in a more intuitive and efficient manner. The new infrastructure also resolves many problems on UX, making the monitoring process a lot more user-friendly. - -## February 17, 2023 - -**CLI changes** - -- Add a new command [`ticloud connect`](/tidb-cloud/ticloud-connect.md) for [TiDB Cloud CLI](/tidb-cloud/cli-reference.md). - - `ticloud connect` is a command that allows you to connect to your TiDB Cloud cluster from your local machine without installing any SQL clients. After connecting to your TiDB Cloud cluster, you can execute SQL statements in the TiDB Cloud CLI. - -## February 14, 2023 - -**General changes** - -- Support decreasing the number of TiKV and TiFlash nodes to scale in a TiDB [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) cluster. - - You can decrease the node number [via the TiDB Cloud console](/tidb-cloud/scale-tidb-cluster.md#change-node-number) or [via the TiDB Cloud API (beta)](https://docs.pingcap.com/tidbcloud/api/v1beta#tag/Cluster/operation/UpdateCluster). - -**Console changes** - -- Introduce the **Monitoring** page for [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. - - The **Monitoring** page provides a range of metrics and data, such as the number of SQL statements executed per second, the average duration of queries, and the number of failed queries, which helps you better understand the overall performance of SQL statements in your Serverless Tier cluster. - - For more information, see [TiDB Cloud built-in monitoring](/tidb-cloud/built-in-monitoring.md). - -## February 2, 2023 - -**CLI changes** - -- Introduce the TiDB Cloud CLI client [`ticloud`](/tidb-cloud/cli-reference.md). - - Using `ticloud`, you can easily manage your TiDB Cloud resources from a terminal or other automatic workflows with a few lines of commands. Especially for GitHub Actions, we have provided [`setup-tidbcloud-cli`](https://github.com/marketplace/actions/set-up-tidbcloud-cli) for you to easily set up `ticloud`. - - For more information, see [TiDB Cloud CLI Quick Start](/tidb-cloud/get-started-with-cli.md) and [TiDB Cloud CLI Reference](/tidb-cloud/cli-reference.md). - -## January 18, 2023 - -**General changes** - -* Support [signing up](https://tidbcloud.com/free-trial) TiDB Cloud with a Microsoft account. - -## January 17, 2023 - -**General changes** - -- Upgrade the default TiDB version of new [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters from [v6.1.3](https://docs.pingcap.com/tidb/stable/release-6.1.3) to [v6.5.0](https://docs.pingcap.com/tidb/stable/release-6.5.0). - -- For new sign-up users, TiDB Cloud will automatically create a free [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) cluster so that you can quickly start a data exploration journey with TiDB Cloud. - -- Support a new AWS region for [Dedicated Tier](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters: `Seoul (ap-northeast-2)`. - - The following features are enabled for this region: - - - [Migrate MySQL-compatible databases to TiDB Cloud using Data Migration](/tidb-cloud/migrate-from-mysql-using-data-migration.md) - - [Stream data from TiDB Cloud to other data services using changefeed](/tidb-cloud/changefeed-overview.md) - - [Back up and restore TiDB cluster data](/tidb-cloud/backup-and-restore.md) - -## January 10, 2023 - -**General changes** - -- Optimize the feature of importing data from local CSV files to TiDB to improve the user experience for [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. - - - To upload a CSV file, now you can simply drag and drop it to the upload area on the **Import** page. - - When creating an import task, if your target database or table does not exist, you can enter a name to let TiDB Cloud create it for you automatically. For the target table to be created, you can specify a primary key or select multiple fields to form a composite primary key. - - After the import is completed, you can explore your data with [AI-powered Chat2Query](/tidb-cloud/explore-data-with-chat2query.md) by clicking **Explore your data by Chat2Query** or clicking the target table name in the task list. - - For more information, see [Import local files to TiDB Cloud](/tidb-cloud/tidb-cloud-import-local-files.md). - -**Console changes** - -- Add the **Get Support** option for each cluster to simplify the process of requesting support for a specific cluster. - - You can request support for a cluster in either of the following ways: - - - On the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, click **...** in the row of your cluster and select **Get Support**. - - On your cluster overview page, click **...** in the upper-right corner and select **Get Support**. - -## January 5, 2023 - -**Console changes** - -- Rename SQL Editor (beta) to Chat2Query (beta) for [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters and support generating SQL queries using AI. - - In Chat2Query, you can either let AI generate SQL queries automatically or write SQL queries manually, and run SQL queries against databases without a terminal. - - To access Chat2Query, go to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, click your cluster name, and then click **Chat2Query** in the left navigation pane. - -## January 4, 2023 - -**General changes** - -- Support scaling up TiDB, TiKV, and TiFlash nodes by increasing the **Node Size(vCPU + RAM)** for TiDB Dedicated clusters hosted on AWS and created after December 31, 2022. - - You can increase the node size [using the TiDB Cloud console](/tidb-cloud/scale-tidb-cluster.md#change-vcpu-and-ram) or [using the TiDB Cloud API (beta)](https://docs.pingcap.com/tidbcloud/api/v1beta#tag/Cluster/operation/UpdateCluster). - -- Extend the metrics retention period on the [**Monitoring**](/tidb-cloud/built-in-monitoring.md) page to two days. - - Now you have access to metrics data of the last two days, giving you more flexibility and visibility into your cluster performance and trends. - - This improvement comes at no additional cost and can be accessed on the **Diagnosis** tab of the [**Monitoring**](/tidb-cloud/built-in-monitoring.md) page for your cluster. This will help you identify and troubleshoot performance issues and monitor the overall health of your cluster more effectively. - -- Support customizing Grafana dashboard JSON for Prometheus integration. - - If you have [integrated TiDB Cloud with Prometheus](/tidb-cloud/monitor-prometheus-and-grafana-integration.md), you can now import a pre-built Grafana dashboard to monitor TiDB Cloud clusters and customize the dashboard to your needs. This feature enables easy and fast monitoring of your TiDB Cloud clusters and helps you identify any performance issues quickly. - - For more information, see [Use Grafana GUI dashboards to visualize the metrics](/tidb-cloud/monitor-prometheus-and-grafana-integration.md#step-3-use-grafana-gui-dashboards-to-visualize-the-metrics). - -- Upgrade the default TiDB version of all [Serverless Tier](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters from [v6.3.0](https://docs.pingcap.com/tidb/v6.3/release-6.3.0) to [v6.4.0](https://docs.pingcap.com/tidb/v6.4/release-6.4.0). The cold start issue after upgrading the default TiDB version of Serverless Tier clusters to v6.4.0 has been resolved. - -**Console changes** + - Support exporting data from [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) to Google Cloud Storage and Azure Blob Storage through the [TiDB Cloud console](https://tidbcloud.com/). -- Simplify the display of the [**Clusters**](https://tidbcloud.com/console/clusters) page and the cluster overview page. + - Support exporting data in Parquet files through the [TiDB Cloud console](https://tidbcloud.com/). - - You can click the cluster name on the [**Clusters**](https://tidbcloud.com/console/clusters) page to enter the cluster overview page and start operating the cluster. - - Remove the **Connection** and **Import** panes from the cluster overview page. You can click **Connect** in the upper-right corner to get the connection information and click **Import** in the left navigation pane to import data. + For more information, see [Export Data from TiDB Cloud Serverless](/tidb-cloud/serverless-export.md) and [Configure External Storage Access for TiDB Cloud Serverless](/tidb-cloud/serverless-external-storage.md). diff --git a/tidb-cloud/tidb-cloud-roadmap.md b/tidb-cloud/tidb-cloud-roadmap.md index 95cc3bc30f4b3..202280b0c1204 100644 --- a/tidb-cloud/tidb-cloud-roadmap.md +++ b/tidb-cloud/tidb-cloud-roadmap.md @@ -5,6 +5,10 @@ summary: Learn about TiDB Cloud's roadmap for the next few months. See the new f # TiDB Cloud Roadmap +> **Warning:** +> +> This roadmap might contain outdated information. We are working on updating it to reflect the latest product plans and development priorities. + The TiDB Cloud roadmap brings you what's coming in the near future, so you can see the new features or improvements in advance, follow the progress, and learn about the key milestones on the way. In the course of development, this roadmap is subject to change based on user needs, feedback, and our assessment. ✅: The feature or improvement is already available in TiDB Cloud. diff --git a/tidb-cloud/tidb-cloud-sql-tuning-overview.md b/tidb-cloud/tidb-cloud-sql-tuning-overview.md index 1f7d130561cb8..0058552892dfc 100644 --- a/tidb-cloud/tidb-cloud-sql-tuning-overview.md +++ b/tidb-cloud/tidb-cloud-sql-tuning-overview.md @@ -27,13 +27,21 @@ The TiDB Cloud console provides a **[SQL Statement](/tidb-cloud/tune-performance Note that on this sub-tab, SQL queries with the same structure (even if the query parameters do not match) are grouped into the same SQL statement. For example, `SELECT * FROM employee WHERE id IN (1, 2, 3)` and `select * from EMPLOYEE where ID in (4, 5)` are both part of the same SQL statement `select * from employee where id in (...)`. -You can view some key information in **Statement**. +You can view some key information in **SQL Statement**. + +- **SQL Template**: including SQL digest, SQL template ID, the time range currently viewed, the number of execution plans, and the database where the execution takes place. + + ![Details0](/media/dashboard/dashboard-statement-detail0.png) -- SQL statement overview: including SQL digest, SQL template ID, the time range currently viewed, the number of execution plans, and the database where the execution takes place. - Execution plan list: if a SQL statement has more than one execution plan, the list is displayed. You can select different execution plans and the details of the selected execution plan are displayed at the bottom of the list. If there is only one execution plan, the list will not be displayed. -- Execution plan details: shows the details of the selected execution plan. It collects the execution plans of such SQL type and the corresponding execution time from several perspectives to help you get more information. See [Execution plan in details](https://docs.pingcap.com/tidb/stable/dashboard-statement-details#statement-execution-details-of-tidb-dashboard) (area 3 in the image below). -![Details](/media/dashboard/dashboard-statement-detail.png) + ![Details1](/media/dashboard/dashboard-statement-detail1.png) + +- Execution plan details: shows the details of the selected execution plan. It collects the execution plans of each SQL type and the corresponding execution time from several perspectives to help you get more information. See [Execution plans](https://docs.pingcap.com/tidb/stable/dashboard-statement-details#execution-plans). + + ![Details2](/media/dashboard/dashboard-statement-detail2.png) + +- Related slow query In addition to the information in the **Statement** dashboard, there are also some SQL best practices for TiDB Cloud as described in the following sections. diff --git a/tidb-cloud/tidb-cloud-sso-authentication.md b/tidb-cloud/tidb-cloud-sso-authentication.md index 9c5735aa7f266..85701d2481bfb 100644 --- a/tidb-cloud/tidb-cloud-sso-authentication.md +++ b/tidb-cloud/tidb-cloud-sso-authentication.md @@ -1,9 +1,9 @@ --- -title: Basic SSO Authentication +title: Standard SSO Authentication summary: Learn how to log in to the TiDB Cloud console via your Google, GitHub, or Microsoft account. --- -# Basic SSO Authentication +# Standard SSO Authentication This document describes how to log in to the [TiDB Cloud console](https://tidbcloud.com/) via basic Single Sign-on (SSO) authentication, which is quick and convenient. diff --git a/tidb-cloud/tidb-cloud-support.md b/tidb-cloud/tidb-cloud-support.md index 4283cf27e1d54..cf6f45c2f939f 100644 --- a/tidb-cloud/tidb-cloud-support.md +++ b/tidb-cloud/tidb-cloud-support.md @@ -5,76 +5,100 @@ summary: Learn how to contact the support team of TiDB Cloud. # TiDB Cloud Support -The **?** icon in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com/) consolidates all the help and support information for TiDB Cloud, such as documentation, interactive tutorials, self-paced training, and support entries. +TiDB Cloud offers tiered support plan offerings tailored to meet customers' needs. For more information, see [Connected Care Details](/tidb-cloud/connected-care-detail.md). -## Request PoC, demo, or free trial credits +> **Note:** +> +> To request a [Proof of Concept (PoC)](/tidb-cloud/tidb-cloud-poc.md), a demo, or free trial credits, click **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com/), and click **Contact Sales**. -To request a Proof of Concept (PoC), a demo, or free trial credits, click **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com/), and click **Contact Sales**. +## Access PingCAP Help Center -> **Tip:** -> -> For more information about PoC, see [Perform a Proof of Concept (PoC) with TiDB Cloud](/tidb-cloud/tidb-cloud-poc.md). +The [PingCAP Help Center](https://tidb.support.pingcap.com/servicedesk/customer/portals) is the central hub for TiDB Cloud users to access support services and manage support tickets. -## Get support +You can access the PingCAP Help Center via directly, or through the [TiDB Cloud console](https://tidbcloud.com/) in the following ways: -You can get support for your clusters, projects, and organizations. After receiving your request, the support team will contact you via email. +- Click **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com/), and then click **Request Support**. +- Click in the lower-left corner of [TiDB Cloud console](https://tidbcloud.com/), click **Support**, and then click **Create a Support Ticket**. +- On the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, click **...** in the row of your cluster, and then select **Get Support**. +- On your cluster overview page, click **...** in the upper-right corner, and then select **Get Support**. -> **Tip:** -> -> If you want to apply for a beta feature that is available upon request, fill in your application information in the **Description** field of the request. For example, "apply for the xxx feature". +## Create an account or billing support ticket + +To create a support ticket about account or billing issues, take the following steps: + +1. Log in to the [PingCAP Help Center](https://tidb.support.pingcap.com/servicedesk/customer/portals), and then click [TiDB Cloud Account/Billing Support](https://tidb.support.pingcap.com/servicedesk/customer/portal/16). +2. Click **Submit a request**. +3. Fill in the following fields: -### Get support for a cluster + - **Summary**: provide a brief summary of your question. + - **TiDB Cloud Org**: select the relevant TiDB Cloud organization, if applicable. + - **TiDB Cloud Cluster**: select the relevant TiDB Cloud cluster, if applicable. + - **Description**: provide the details about the issue. + - **Severity**: estimate the business impact of the issue and choose the proper severity for it. (S1 is not applicable to billing or account issues.) -You can get support for a cluster in either of the following ways: +4. Click **Submit**. -- On the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, click **...** in the row of your cluster and select **Get Support**. +## Create a technical support ticket -- On your cluster overview page, click **...** in the upper-right corner and select **Get Support**. +To create a support ticket about technical issues, take the following steps: -### Get support for a project or organization +1. Log in to the [PingCAP Help Center](https://tidb.support.pingcap.com/servicedesk/customer/portals), and then click [TiDB Cloud Technical Support](https://tidb.support.pingcap.com/servicedesk/customer/portal/6). -To get support for a project or organization, click **?** in the lower-right corner of the [TiDB Cloud console](https://tidbcloud.com/), and then click **Request Support**. + > **Note:** + > + > The [TiDB Cloud Technical Support](https://tidb.support.pingcap.com/servicedesk/customer/portal/6) entry is only available for customers subscribed to **Developer**, **Enterprise**, or **Premium** [support plans](/tidb-cloud/connected-care-detail.md). -Alternatively, you can click in the lower-left corner, and then click **Support** to go to the support page. +2. Click **Submit a request**. -## Check your submitted request +3. Fill in the following fields: -After you have submitted a request, to check the status of your request, perform the following steps: + - **Summary**: provide a brief summary of your question. + - **TiDB Cloud Org**: select the relevant TiDB Cloud organization for the issue. + - **TiDB Cloud Cluster**: select the relevant TiDB Cloud cluster, if applicable. + - **Environment**: select the corresponding environment in which you use the TiDB Cloud cluster. + - **Description**: describe the issue you encountered as detailed as possible. For example, share the exact timestamp when you encountered the issue, attach the detailed error messages and call stack of the issue, and add your troubleshooting or analysis of the issue. + - **Severity**: estimate the business impact of the issue and choose the proper severity for it. -1. In the [TiDB Cloud console](https://tidbcloud.com/), click **?** in the lower-right corner, and then click **View Support**. The support page is displayed. + | Severity | Description | + | --- | --- | + | S1 | Complete loss of production environmental functionality | + | S2 | High impact on operations in production environments | + | S3 | Non-critical database usage issues in production or non-production environments | + | S4 | General question on how a particular feature or function performs or should be configured. An issue that has minimal impact on business and can be tolerated for a reasonable period. | - Alternatively, you can click in the lower-left corner, and then click **Support** to go to the support page. + - **Components**: select the relevant TiDB Cloud component to report the issue for, such as TiDB, TiKV, PD, or TiFlash. + - **Affects versions**: specify the TiDB Cloud cluster version related to the issue. -2. In the **Request Support** area, click **View Support Portal**. The **My request** page is displayed. +4. Click **Submit**. - On the **My request** page, you can view the status of your request. +## View support tickets -Alternatively, you can also access the [Customer Support](https://support.pingcap.com/hc/en-us) page, log in to the support portal, and then click **View my requests**. +To view all the historical support tickets, log in to the [PingCAP Help Center](https://tidb.support.pingcap.com/servicedesk/customer/portals), click your avatar in the upper-right corner, and then click **Requests**. ## Check or upgrade your support plan -TiDB Cloud offers a free basic support plan for each user and you can upgrade to a paid plan for extended services. +TiDB Cloud offers a free basic support plan by default. For extended services, you can upgrade to a paid plan. To check or upgrade your support plan, perform the following steps: -1. In the [TiDB Cloud console](https://tidbcloud.com/), click **?** in the lower-right corner, and then click **View Support**. +1. In the [TiDB Cloud console](https://tidbcloud.com/), click in the lower-left corner, and then click **Support** to go to the support page. - The support page is displayed. On this page, you can see your **Current Plan**. By default, the **Basic** free plan is selected. + On this page, you can find your **Current Plan**. By default, the **Basic** free plan is selected. 2. Choose your desired support plan. -
+
- To upgrade to **Standard** or **Enterprise**: + To upgrade to **Developer** or **Enterprise**: - 1. Click **Select Plan** in the **Standard** or **Enterprise** pane. A **Finish and Start Using Support** page is displayed. - 2. Check the billing information in the lower-left corner of the page. - 3. Add your payment information in the **Billing Profile** and **Credit Card** areas. + 1. Click **Upgrade** in the **Developer** or **Enterprise** pane. An **Upgrade to Developer Plan** or **Upgrade to Enterprise Plan** page is displayed. + 2. Check the corresponding support service information on the page. For a complete version of each support plan, see [Connected Care Details](/tidb-cloud/connected-care-detail.md). + 3. Click **Add Credit Card and Upgrade**, and then fill in the **Credit Card** details. For more information about billing, see [TiDB Cloud Payment Method](/tidb-cloud/tidb-cloud-billing.md#payment-method). - 4. Click **Confirm and Start Using Support** in the lower-right corner of the page. + 4. Click **Save Card** in the lower-right corner of the page. After the payment is finished, you have upgraded your plan to **Standard** or **Enterprise**. @@ -91,7 +115,7 @@ To check or upgrade your support plan, perform the following steps: ## Downgrade your support plan -To downgrade your support plan to **Basic**, perform the following steps: +To downgrade your support plan, perform the following steps: -1. In the [TiDB Cloud console](https://tidbcloud.com/), click **?** in the lower-right corner, and then click **View Support**. -2. At the bottom of the **Support** page, click **Downgrade to basic plan**. +1. In the [TiDB Cloud console](https://tidbcloud.com/), click in the lower-left corner, and then click **Support** to go to the support page. +2. Choose the support plan you want to switch to, and then click **Downgrade**. \ No newline at end of file diff --git a/tidb-cloud/tidb-cloud-tls-connect-to-dedicated.md b/tidb-cloud/tidb-cloud-tls-connect-to-dedicated.md index 93eebf02a3cb4..1af58081d15fd 100644 --- a/tidb-cloud/tidb-cloud-tls-connect-to-dedicated.md +++ b/tidb-cloud/tidb-cloud-tls-connect-to-dedicated.md @@ -1,53 +1,50 @@ --- -title: TLS Connections to TiDB Dedicated -summary: Introduce TLS connections in TiDB Dedicated. +title: TLS Connections to TiDB Cloud Dedicated +summary: Introduce TLS connections in TiDB Cloud Dedicated. aliases: ['/tidbcloud/tidb-cloud-tls-connect-to-dedicated-tier'] --- -# TLS Connections to TiDB Dedicated +# TLS Connections to TiDB Cloud Dedicated -On TiDB Cloud, establishing TLS connections is one of the basic security practices for connecting to TiDB Dedicated clusters. You can configure multiple TLS connections from your client, application, and development tools to your TiDB Dedicated cluster to protect data transmission security. For security reasons, TiDB Dedicated only supports TLS 1.2 and TLS 1.3, and does not support TLS 1.0 and TLS 1.1 versions. +On TiDB Cloud, establishing TLS connections is one of the basic security practices for connecting to TiDB Cloud Dedicated clusters. You can configure multiple TLS connections from your client, application, and development tools to your TiDB Cloud Dedicated cluster to protect data transmission security. For security reasons, TiDB Cloud Dedicated only supports TLS 1.2 and TLS 1.3, and does not support TLS 1.0 and TLS 1.1 versions. -To ensure data security, TiDB cluster CA for your TiDB Dedicated cluster is hosted on [AWS Certificate Manager (ACM)](https://aws.amazon.com/certificate-manager/), and TiDB cluster private keys are stored in AWS-managed hardware security modules (HSMs) that meet [FIPS 140-2 Level 3](https://csrc.nist.gov/projects/cryptographic-module-validation-program/Certificate/3139) security standards. +To ensure data security, TiDB cluster CA for your TiDB Cloud Dedicated cluster is hosted on [AWS Certificate Manager (ACM)](https://aws.amazon.com/certificate-manager/), and TiDB cluster private keys are stored in AWS-managed hardware security modules (HSMs) that meet [FIPS 140-2 Level 3](https://csrc.nist.gov/projects/cryptographic-module-validation-program/Certificate/3139) security standards. ## Prerequisites -- Log in to TiDB Cloud via [Password Authentication](/tidb-cloud/tidb-cloud-password-authentication.md) or [SSO Authentication](/tidb-cloud/tidb-cloud-sso-authentication.md), and then [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). +- Log in to TiDB Cloud via [Password Authentication](/tidb-cloud/tidb-cloud-password-authentication.md) or [SSO Authentication](/tidb-cloud/tidb-cloud-sso-authentication.md), and then [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). - Set a password to access your cluster in secure settings. - To do so, you can navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, click **...** in the row of your TiDB Dedicated cluster, and then select **Security Settings**. In security settings, you can click **Generate** to automatically generate a root password with a length of 16 characters, including numbers, uppercase and lowercase characters, and special characters. + To do so, you can navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, click **...** in the row of your TiDB Cloud Dedicated cluster, and then select **Password Settings**. In password settings, you can click **Auto-generate Password** to automatically generate a root password with a length of 16 characters, including numbers, uppercase and lowercase characters, and special characters. -## Secure connection to a TiDB Dedicated cluster +## Secure connection to a TiDB Cloud Dedicated cluster -In the [TiDB Cloud console](https://tidbcloud.com/), you can get examples of different connection methods and connect to your TiDB Dedicated cluster as follows: +In the [TiDB Cloud console](https://tidbcloud.com/), you can get examples of different connection methods and connect to your TiDB Cloud Dedicated cluster as follows: -1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your TiDB Dedicated cluster to go to its overview page. +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your TiDB Cloud Dedicated cluster to go to its overview page. 2. Click **Connect** in the upper-right corner. A dialog is displayed. -3. On the **Standard Connection** tab of this dialog, follow the three steps to set up the TLS connection. - - Step 1：Create traffic filter - - Step 2：Download CA cert - - Step 3：Connect with an SQL client +3. In the connection dialog, select **Public** from the **Connection Type** drop-down list. -4. Under **Step 1: Create traffic filter** in the dialog, configure the IP addresses that are allowed to access your cluster. For more information, see [Configure an IP access list in standard connection](/tidb-cloud/configure-ip-access-list.md#configure-an-ip-access-list-in-standard-connection). + If you have not configured the IP access list, click **Configure IP Access List** to configure it before your first connection. For more information, see [Configure an IP access list](/tidb-cloud/configure-ip-access-list.md). -5. Under **Step 2: Download CA cert**, click **Download CA cert** to download it locally for client TLS configuration. The CA cert ensures that the TLS connection is secure and reliable. +4. Click **CA cert** to download CA cert for TLS connection to TiDB clusters. The CA cert supports TLS 1.2 version by default. > **Note:** > - > - You can store the downloaded CA cert in the default storage path of your operating system, or specify another storage path. You need to replace the CA cert path in the code example with your own CA cert path in the subsequent steps. - > - TiDB Dedicated does not force clients to use TLS connections, and user-defined configuration of the [`require_secure_transport`](/system-variables.md#require_secure_transport-new-in-v610) variable is currently not supported on TiDB Dedicated. + > - You can store the downloaded CA cert in the default storage path of your operating system, or specify another storage path. You need to replace the CA cert path in the code example with your own CA cert path in the subsequent steps. + > - TiDB Cloud Dedicated does not force clients to use TLS connections, and user-defined configuration of the [`require_secure_transport`](/system-variables.md#require_secure_transport-new-in-v610) variable is currently not supported on TiDB Cloud Dedicated. -6. Under **Step 3: Connect with an SQL client** in the dialog, click the tab of your preferred connection method, and then refer to the connection string and sample code on the tab to connect to your cluster. +5. Choose your preferred connection method, and then refer to the connection string and sample code on the tab to connect to your cluster. The following examples show the connection strings in MySQL, MyCLI, JDBC, Python, Go, and Node.js:
-MySQL CLI client attempts to establish a TLS connection by default. When you connect to TiDB Dedicated clusters, you need to set `ssl-mode` and `ssl-ca`. +MySQL CLI client attempts to establish a TLS connection by default. When you connect to TiDB Cloud Dedicated clusters, you need to set `ssl-mode` and `ssl-ca`. ```shell mysql --connect-timeout 15 --ssl-mode=VERIFY_IDENTITY --ssl-ca=ca.pem --tls-version="TLSv1.2" -u root -h tidb.eqlfbdgthh8.clusters.staging.tidb-cloud.com -P 4000 -D test -p @@ -55,7 +52,7 @@ mysql --connect-timeout 15 --ssl-mode=VERIFY_IDENTITY --ssl-ca=ca.pem --tls-vers Parameter description： -- With `--ssl-mode=VERIFY_IDENTITY`, MySQL CLI client forces to enable TLS and validate TiDB Dedicated clusters. +- With `--ssl-mode=VERIFY_IDENTITY`, MySQL CLI client forces to enable TLS and validate TiDB Cloud Dedicated clusters. - Use `--ssl-ca=` to specify your local path of the downloaded TiDB cluster `ca.pem`. - Use `--tls-version=TLSv1.2` to restrict the versions of the TLS protocol. If you want to use TLS 1.3, you can set the version to `TLSv1.3`. @@ -63,7 +60,7 @@ Parameter description：
-[MyCLI](https://www.mycli.net/) automatically enables TLS when using TLS related parameters. When you connect to TiDB Dedicated clusters, you need to set `ssl-ca` and `ssl-verify-server-cert`. +[MyCLI](https://www.mycli.net/) automatically enables TLS when using TLS related parameters. When you connect to TiDB Cloud Dedicated clusters, you need to set `ssl-ca` and `ssl-verify-server-cert`. ```shell mycli --ssl-ca=ca.pem --ssl-verify-server-cert -u root -h tidb.eqlfbdgthh8.clusters.staging.tidb-cloud.com -P 4000 -D test @@ -72,7 +69,7 @@ mycli --ssl-ca=ca.pem --ssl-verify-server-cert -u root -h tidb.eqlfbdgthh8.clust Parameter descriptions： - Use `--ssl-ca=` to specify your local path of the downloaded TiDB cluster `ca.pem`. -- With `--ssl-verify-server-cert` to validate TiDB Dedicated clusters. +- With `--ssl-verify-server-cert` to validate TiDB Cloud Dedicated clusters.
@@ -117,7 +114,7 @@ class Main { Parameter description： -- Set `sslMode=VERIFY_IDENTITY` to enable TLS and validate TiDB Dedicated clusters. +- Set `sslMode=VERIFY_IDENTITY` to enable TLS and validate TiDB Cloud Dedicated clusters. - Set `enabledTLSProtocols=TLSv1.2` to restrict the versions of the TLS protocol. If you want to use TLS 1.3, you can set the version to `TLSv1.3`. - Set `trustCertificateKeyStoreUrl` to your custom truststore path. - Set `trustCertificateKeyStorePassword` to your truststore password. @@ -148,7 +145,7 @@ with connection: Parameter descriptions： -- Set `ssl_mode="VERIFY_IDENTITY"` to enable TLS and validate TiDB Dedicated clusters. +- Set `ssl_mode="VERIFY_IDENTITY"` to enable TLS and validate TiDB Cloud Dedicated clusters. - Use `ssl={"ca": ""}` to specify your local path of the downloaded TiDB cluster `ca.pem`.
@@ -220,9 +217,9 @@ func main() { Parameter descriptions： -- Register `tls.Config` in the TLS connection configuration to enable TLS and validate TiDB Dedicated clusters. +- Register `tls.Config` in the TLS connection configuration to enable TLS and validate TiDB Cloud Dedicated clusters. - Set `MinVersion: tls.VersionTLS12` to restrict the versions of TLS protocol. -- Set `ServerName: ""` to verify TiDB Dedicated's hostname. +- Set `ServerName: ""` to verify TiDB Cloud Dedicated's hostname. - If you do not want to register a new TLS configuration, you can just set `tls=true` in the connection string.
@@ -285,18 +282,18 @@ Parameter descriptions：
-## Manage root certificates for TiDB Dedicated +## Manage root certificates for TiDB Cloud Dedicated -TiDB Dedicated uses certificates from [AWS Certificate Manager (ACM)](https://aws.amazon.com/certificate-manager/) as a Certificate Authority (CA) for TLS connections between clients and TiDB Dedicated clusters. Usually, the root certificates of ACM are stored securely in AWS-managed hardware security modules (HSMs) that meet [FIPS 140-2 Level 3](https://csrc.nist.gov/projects/cryptographic-module-validation-program/Certificate/3139) security standards. +TiDB Cloud Dedicated uses certificates from [AWS Certificate Manager (ACM)](https://aws.amazon.com/certificate-manager/) as a Certificate Authority (CA) for TLS connections between clients and TiDB Cloud Dedicated clusters. Usually, the root certificates of ACM are stored securely in AWS-managed hardware security modules (HSMs) that meet [FIPS 140-2 Level 3](https://csrc.nist.gov/projects/cryptographic-module-validation-program/Certificate/3139) security standards. ## FAQs -### Which TLS versions are supported to connect to my TiDB Dedicated cluster? +### Which TLS versions are supported to connect to my TiDB Cloud Dedicated cluster? -For security reasons, TiDB Dedicated only supports TLS 1.2 and TLS 1.3, and does not support TLS 1.0 and TLS 1.1 versions. See IETF [Deprecating TLS 1.0 and TLS 1.1](https://datatracker.ietf.org/doc/rfc8996/) for details. +For security reasons, TiDB Cloud Dedicated only supports TLS 1.2 and TLS 1.3, and does not support TLS 1.0 and TLS 1.1 versions. See IETF [Deprecating TLS 1.0 and TLS 1.1](https://datatracker.ietf.org/doc/rfc8996/) for details. -### Is two-way TLS authentication between my client and TiDB Dedicated supported? +### Is two-way TLS authentication between my client and TiDB Cloud Dedicated supported? No. -TiDB Dedicated only supports one-way TLS authentication, and does not support two-way TLS authentication currently. If you need two-way TLS authentication, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). +TiDB Cloud Dedicated only supports one-way TLS authentication, and does not support two-way TLS authentication currently. If you need two-way TLS authentication, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). diff --git a/tidb-cloud/tidb-node-group-management.md b/tidb-cloud/tidb-node-group-management.md new file mode 100644 index 0000000000000..b175c5fe95786 --- /dev/null +++ b/tidb-cloud/tidb-node-group-management.md @@ -0,0 +1,155 @@ +--- +title: Manage TiDB Node Groups +summary: Learn about how to manage TiDB node groups and their endpoints to isolate your business workload. +--- + +# Manage TiDB Node Groups + +This document describes how to manage TiDB node groups and their endpoints to isolate your business workload using the [TiDB Cloud console](https://tidbcloud.com/). + +> **Note**: +> +> The TiDB Node Group feature is **NOT** available for TiDB Cloud Serverless clusters. + +## Terms + +- TiDB Node Group: A TiDB node group manages the grouping of TiDB nodes and maintains the mapping between endpoints and TiDB nodes. + + - Each TiDB node group has a unique endpoint. + - When you delete a TiDB node group, the related network setting (such as private link and IP access list) will be deleted too. + +- Default Group: When a cluster is created, a default TiDB node group is created. Therefore, each cluster has a default group. The default group cannot be deleted. + +## Prerequisites + +- You have a [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) cluster deployed on AWS or Google Cloud. +- You are in the **Organization Owner** or **Project Owner** role of your organization. For more information, see [User roles](/tidb-cloud/manage-user-access.md#user-roles). + +> **Note**: +> +> The TiDB node groups cannot be created during cluster creation. You need to add the groups after the cluster is created and in the **Available** state. + +## Create a TiDB node group + +To create a TiDB node group, perform the following steps: + +1. In the [TiDB Cloud console](https://tidbcloud.com/), navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page of your project, and then click the name of your target cluster to go to its overview page. +2. In the left navigation pane, click **Nodes**. +3. Click **Modify** in the upper-right corner. The **Modify Cluster** page is displayed. +4. On the **Modify Cluster** page, click **+** to add a new TiDB node group as follows. You can also use the default group directly. + + - TiDB + - **vCPU + RAM**: choose the [TiDB size](/tidb-cloud/size-your-cluster.md#size-tidb) you need. Only TiDB nodes with 8 vCPU and 16 GiB memory or higher specifications are supported. + - **Node Groups**: click **+** to create a new TiDB node group. You can also use the default group and enter the number of TiDB nodes in the **DefaultGroup** field. + + - TiKV + - **vCPU + RAM**: choose the [TiKV size](/tidb-cloud/size-your-cluster.md#size-tikv) you need. + - **Storage x Nodes**: choose the storage size and the number of TiKV nodes. + + - TiFlash (optional) + - **vCPU + RAM**: choose the [TiFlash size](/tidb-cloud/size-your-cluster.md#size-tiflash) you need. + - **Storage x Nodes**: choose the storage size and the number of TiFlash nodes. + + ![Create TiDB Node Group](/media/tidb-cloud/tidb-node-group-create.png) + +5. New TiDB nodes are added along with the new TiDB node group, which affects the cluster's billing. Review the cluster size in the right pane, then click **Confirm**. + +By default, you can create up to five TiDB node groups for a TiDB Cloud Dedicated cluster. If you need more groups, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). + +If you create a TiDB node group but still use the endpoint of the default group to connect to the cluster, the TiDB nodes in the TiDB node group will not take any workload, which is a waste of the resource. You need to create new connection to the TiDB nodes in the new TiDB node group. See [Connect to a TiDB node group](#connect-to-a-tidb-node-group). + +## Connect to a TiDB node group + +### Connect via public connection + +Public connection for the new TiDB node group is disabled by default. You need to enable it first. + +To enable public connection, do the following: + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. +2. In the upper-right corner, click **Connect**. A connection dialog is displayed. +3. Select your TiDB node group from the **TiDB Node Group** list and **Public** from the **Connection Type** list. + + If you have not configured the IP access list, click **Configure IP Access List** or follow the steps in [Configure an IP Access List](https://docs.pingcap.com/tidbcloud/configure-ip-access-list) to configure it before your first connection. + +4. On the **Networking** page, select your TiDB node group from the **TiDB Node Group** list in the upper-right corner. +5. Click **Enable** in the **Public Endpoint** section, then click **Add IP Address** in the **IP Access List** section. +6. In the upper-right corner of the **Networking** page, click **Connect** to get the connection string. + +![Connect to the new TiDB node group via Public Endpoint](/media/tidb-cloud/tidb-node-group-connect-public-endpoint.png) + +For more information, see [Connect to TiDB Cloud Dedicated via Public Connection](/tidb-cloud/connect-via-standard-connection.md). + +### Connect via private endpoint + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. +2. In the upper-right corner, click **Connect**. A connection dialog is displayed. +3. Select your TiDB node group from the **TiDB Node Group** list and **Private Endpoint** from the **Connection Type** list. +4. In the left navigation pane, click **Networking**, and then select your TiDB node group from the **TiDB Node Group** list in the upper-right corner. +5. Click **Create Private Endpoint Connection** to create a new connection for this node group. For clusters deployed on AWS, refer to [Connect to a TiDB Cloud Dedicated Cluster via Private Endpoint with AWS](/tidb-cloud/set-up-private-endpoint-connections.md). For clusters deployed on Google Cloud, refer to [Connect to a TiDB Cloud Dedicated Cluster via Google Cloud Private Service Connect](/tidb-cloud/set-up-private-endpoint-connections-on-google-cloud.md). + + > **Note**: + > + > If you use Private Link to connect different node groups, you need to create separated private endpoint connection for each node group. + +6. After creating the private endpoint connection, click **Connect** in the upper-right corner of the page to get the connection string. + +### Connect via VPC peering + +Because all TiDB node groups share the same VPC as the cluster, you only need to create one VPC peering connection to enable access for all groups. + +1. Follow the instructions in [Connect to TiDB Cloud Dedicated via VPC Peering](/tidb-cloud/set-up-vpc-peering-connections.md) to create a VPC peering for this cluster. +2. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. +3. In the left navigation pane, click **Networking**. +4. In the upper-right corner of the **Networking** page, click **Connect** to get the connection string. + +## View TiDB node groups + +To view the details of TiDB node groups, perform the following steps: + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. +2. In the left navigation pane, click **Nodes** to view the list of TiDB node groups. + + To switch to the table view, click . + +## Modify a TiDB node group + +You can modify the group name and node configurations in the group. + +### Change the group name + +To change the group name, perform the following steps: + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. +2. In the left navigation pane, click **Nodes**. +3. Click and enter a new name for the TiDB node group. + +### Update the node configuration + +To update TiDB, TiKV, or TiFlash node configurations in the group, perform the following steps: + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. +2. In the left navigation pane, click **Nodes**. +3. On the **Node Map** page, click **Modify** in the upper-right corner. The **Modify Cluster** page is displayed. +4. On the **Modify Cluster** page, you can: + + - Change the number of TiDB nodes. + - Add new node groups. + - Update the size and **Storage x Nodes** configuration for TiKV and TiFlash nodes. + +![Change TiDB node group node count](/media/tidb-cloud/tidb-node-group-change-node-count.png) + +## Delete a TiDB node group + +> **Note**: +> +> When you delete a TiDB node group, its nodes and network configurations are also removed, including private endpoint connections and the IP list for public access. + +To delete a TiDB node group, perform the following steps: + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. +2. In the left navigation pane, click **Nodes**. +3. On the **Node Map** page, click **Modify** in the upper-right corner. The **Modify Cluster** page is displayed. +4. On the **Modify Cluster** page, click to delete the TiDB node group. + +![Delete the TiDB node group](/media/tidb-cloud/tidb-node-group-delete.png) diff --git a/tidb-cloud/tidb-node-group-overview.md b/tidb-cloud/tidb-node-group-overview.md new file mode 100644 index 0000000000000..6dacc886eab67 --- /dev/null +++ b/tidb-cloud/tidb-node-group-overview.md @@ -0,0 +1,59 @@ +--- +title: Overview of TiDB Node Group +summary: Learn about the implementation and usage scenarios of the TiDB Node Group feature. +--- + +# Overview of TiDB Node Group + +You can create TiDB node groups for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters. A TiDB node group physically groups the computing nodes (TiDB layer) of a cluster, with each group containing a specific number of TiDB nodes. This configuration provides physical isolation of computing resources between groups, enabling efficient resource allocation in multi-business scenarios. + +With TiDB node groups, you can divide computing nodes into multiple TiDB node groups based on business requirements and configure unique connection endpoints for each TiDB node group. Your applications connect to the cluster through their respective endpoints, and requests route to the corresponding node group for processing. This ensures that resource overuse in one group does not affect other groups. + +> **Note**: +> +> The TiDB Node Group feature is **NOT** available for TiDB Cloud Serverless clusters. + +## Implementation + +TiDB node groups manage the grouping of TiDB nodes and maintain the mapping between endpoints and their corresponding TiDB nodes. + +Each TiDB node group is associated with a dedicated load balancer. When a user sends a SQL request to the endpoint of a TiDB node group, the request first passes through that group's load balancer, which then routes it exclusively to TiDB nodes within the group. + +The following diagram illustrates the implementation of the TiDB Node Group feature. + +![The implementation of the TiDB Node Group feature](/media/tidb-cloud/implementation-of-tidb-node-group.png) + +All nodes in a TiDB node group respond to requests from the corresponding endpoint. You can perform the following tasks: + +- Create a TiDB node group and assign TiDB nodes to it. +- Set up connection endpoints for each group. Supported connection types include [public connection](/tidb-cloud/tidb-node-group-management.md#connect-via-public-connection), [private endpoint](/tidb-cloud/tidb-node-group-management.md#connect-via-private-endpoint), and [VPC peering](/tidb-cloud/tidb-node-group-management.md#connect-via-vpc-peering). +- Route applications to specific groups using distinct endpoints to achieve resource isolation. + +## Scenarios + +The TiDB Node Group feature significantly enhances resource allocation for TiDB Cloud Dedicated clusters. TiDB nodes are dedicated to computation and do not store data. By organizing nodes into multiple physical groups, the feature ensures that resource overuse in one group does not impact other groups. + +With this feature, you can: + +- Consolidate multiple applications from different systems into a single TiDB Cloud Dedicated cluster. As an application's workload grows, it will not affect the normal operation of other applications. The TiDB Node Group feature ensures that the response time of transactional applications is not impacted by data analysis or batch applications. + +- Perform import or DDL tasks on the TiDB Cloud Dedicated cluster without affecting the performance of existing production workloads. You can create a separate TiDB node group for importing or DDL tasks. Even though these tasks consume significant CPU or memory resources, they only use the resources in their own TiDB node group, ensuring the workloads in other TiDB node groups are not impacted. + +- Combine all test environments into a single TiDB cluster or group resource-intensive batch tasks into a dedicated TiDB node group. This approach improves hardware utilization, reduces operating costs, and ensures that critical applications always have access to necessary resources. + +In addition, TiDB node groups are easy to scale in or out. For key applications with high performance requirements, you can allocate TiDB nodes to the group as needed. For less demanding applications, you can start with a small number of TiDB nodes and scale out as needed. Efficient use of the TiDB Node Group feature reduces the number of clusters, simplifies operations and maintenance, and lowers management costs. + +## Limitations and quotas + +Currently, the TiDB Node Group feature is free of charge. The following are limitations and quotas: + +- You can only create TiDB node groups for TiDB Cloud Dedicated clusters on AWS or Google Cloud. Support for other cloud providers is planned for the near future. +- TiDB clusters with 4 vCPUs and 16 GiB of memory do not support the TiDB Node Group feature. +- By default, you can create up to five TiDB node groups for a TiDB Cloud Dedicated cluster. If you need more groups, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). +- Each TiDB node group must contain at least one TiDB node. While there is no limit to the number of nodes in a group, the total number of TiDB nodes in a TiDB Cloud Dedicated cluster must not exceed 150. + +## SLA impact + +According to TiDB Cloud [Service Level Agreement (SLA)](https://www.pingcap.com/legal/service-level-agreement-for-tidb-cloud-services/), the Monthly Uptime Percentage of TiDB Cloud Dedicated clusters with multiple TiDB nodes deployment can reach up to 99.99%. However, after introducing TiDB Node Group, if you create multiple TiDB Node Groups with only 1 TiDB node in each group, you will lose the high availability for the groups and your cluster's monthly uptime percentage will downgrade to a single TiDB node deployment model (namely, up to 99.9%). + +For high availability, it is recommended that you configure at least two TiDB nodes for each TiDB node group. diff --git a/tidb-cloud/transaction-concepts.md b/tidb-cloud/transaction-concepts.md new file mode 100644 index 0000000000000..d5ec3638f363c --- /dev/null +++ b/tidb-cloud/transaction-concepts.md @@ -0,0 +1,40 @@ +--- +title: Transactions +summary: Learn about transaction concepts for TiDB Cloud. +--- + +# Transactions + +TiDB provides complete distributed transactions, and the model has some optimizations on the basis of [Google Percolator](https://research.google.com/pubs/pub36726.html). + +## Optimistic transaction mode + +TiDB's optimistic transaction model does not detect conflicts until the commit phase. If there are conflicts, the transaction needs a retry. But this model is inefficient if the conflict is severe, because operations before the retry are invalid and need to repeat. + +Assume that the database is used as a counter. High access concurrency might lead to severe conflicts, resulting in multiple retries or even timeouts. Therefore, in the scenario of severe conflicts, it is recommended to use the pessimistic transaction mode or to solve problems at the system architecture level, such as placing a counter in Redis. Nonetheless, the optimistic transaction model is efficient if the access conflict is not very severe. + +For more information, see [TiDB Optimistic Transaction Model](/optimistic-transaction.md). + +## Pessimistic transaction mode + +In TiDB, the pessimistic transaction mode has almost the same behavior as in MySQL. The transaction applies a lock during the execution phase, which avoids retries in conflict situations and ensures a higher success rate. By applying the pessimistic locking, you can also lock data in advance using `SELECT FOR UPDATE`. + +However, if the application scenario has fewer conflicts, the optimistic transaction model has better performance. + +For more information, see [TiDB Pessimistic Transaction Mode](/pessimistic-transaction.md). + +## Transaction isolation levels + +Transaction isolation is one of the foundations of database transaction processing. Isolation is one of the four key properties of a transaction (commonly referred to as [ACID](/tidb-cloud/tidb-cloud-glossary.md#acid)). + +TiDB implements Snapshot Isolation (SI) consistency, which it advertises as `REPEATABLE-READ` for compatibility with MySQL. This differs from the [ANSI Repeatable Read isolation level](/transaction-isolation-levels.md#difference-between-tidb-and-ansi-repeatable-read) and the [MySQL Repeatable Read level](/transaction-isolation-levels.md#difference-between-tidb-and-mysql-repeatable-read). + +For more information, see [TiDB Transaction Isolation Levels](/transaction-isolation-levels.md). + +## Non-transactional DML statements + +A non-transactional DML statement is a DML statement split into multiple SQL statements (which is, multiple batches) to be executed in sequence. It enhances the performance and ease of use in batch data processing at the expense of transactional atomicity and isolation. + +Usually, memory-consuming transactions need to be split into multiple SQL statements to bypass the transaction size limit. Non-transactional DML statements integrate this process into the TiDB kernel to achieve the same effect. It is helpful to understand the effect of non-transactional DML statements by splitting SQL statements. The `DRY RUN` syntax can be used to preview the split statements. + +For more information, see [Non-Transactional DML Statements](/non-transactional-dml.md). \ No newline at end of file diff --git a/tidb-cloud/troubleshoot-import-access-denied-error.md b/tidb-cloud/troubleshoot-import-access-denied-error.md index d418157f2f5cd..4a861ce6ee0ac 100644 --- a/tidb-cloud/troubleshoot-import-access-denied-error.md +++ b/tidb-cloud/troubleshoot-import-access-denied-error.md @@ -229,7 +229,7 @@ To solve the `AccessDenied` error in this situation, click the key ARN or manual > **Note:** > -> If the objects in your bucket have been copied from an existing encrypted bucket, you also need to include the key of the source bucket in the AWS KMS key ARN. This is because the objects in the your bucket use the same encryption method as the source object encryption. For more information, see the AWS document [Using default encryption with replication](https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucket-encryption.html). +> If the objects in your bucket have been copied from an existing encrypted bucket, you also need to include the key of the source bucket in the AWS KMS key ARN. This is because the objects in your bucket use the same encryption method as the source object encryption. For more information, see the AWS document [Using default encryption with replication](https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucket-encryption.html). ### Check the AWS article for instruction diff --git a/tidb-cloud/tune-performance.md b/tidb-cloud/tune-performance.md index d7da01d3e9854..53083b9334821 100644 --- a/tidb-cloud/tune-performance.md +++ b/tidb-cloud/tune-performance.md @@ -17,7 +17,7 @@ TiDB Cloud provides [Slow Query](#slow-query), [Statement Analysis](#statement-a > **Note:** > -> Currently, **Key Visualizer** and **Index Insight (beta)** are unavailable for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. +> Currently, **Key Visualizer** and **Index Insight (beta)** are unavailable for [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters. ## Slow Query @@ -57,7 +57,7 @@ For more information, see [Statement Execution Details in TiDB Dashboard](https: > **Note:** > -> Key Visualizer is only available for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. +> Key Visualizer is only available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters. To view the key analytics, perform the following steps: @@ -75,6 +75,6 @@ The Index Insight feature in TiDB Cloud provides powerful capabilities to optimi > **Note:** > -> Index Insight is currently in beta and only available for [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters. +> Index Insight is currently in beta and only available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters. For more information, see [Index Insight](/tidb-cloud/index-insight.md). diff --git a/tidb-cloud/use-chat2query-api.md b/tidb-cloud/use-chat2query-api.md index a8b2a8dd6fa52..b02985f86366b 100644 --- a/tidb-cloud/use-chat2query-api.md +++ b/tidb-cloud/use-chat2query-api.md @@ -11,9 +11,13 @@ Chat2Query API can only be accessed through HTTPS, ensuring that all data transm > **Note:** > -> Chat2Query API is available for [TiDB Serverless](/tidb-cloud/select-cluster-tier.md#tidb-serverless) clusters. To use the Chat2Query API on [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) clusters, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md). +> Chat2Query API is available for [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters. To use the Chat2Query API on [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md). -## Step 1. Create a Chat2Query Data App +## Before you begin + +Before calling Chat2Query endpoints, you need to create a Chat2Query Data App and create an API key for the Data App. + +### Create a Chat2Query Data App To create a Data App for your project, perform the following steps: @@ -21,7 +25,7 @@ To create a Data App for your project, perform the following steps: > **Tip:** > - > If you are on the **Chat2Query** page of your cluster, you can also open the data app creation dialog by clicking **...** in the upper-right corner, choosing **Access Chat2Query via API**, and clicking **New Chat2Query Data App**. + > If you are on the **SQL Editor** page of your cluster, you can also open the data app creation dialog by clicking **...** in the upper-right corner, choosing **Access Chat2Query via API**, and clicking **New Chat2Query Data App**. 2. In the dialog, define a name for your Data App, choose the desired clusters as the data sources, and select **Chat2Query Data App** as the **Data App** type. Optionally, you can also write a description for the App. @@ -29,7 +33,7 @@ To create a Data App for your project, perform the following steps: The newly created Chat2Query Data App is displayed in the left pane. Under this Data App, you can find a list of Chat2Query endpoints. -## Step 2. Create an API key +### Create an API key Before calling an endpoint, you need to create an API key for the Chat2Query Data App, which is used by the endpoint to access data in your TiDB Cloud clusters. @@ -49,26 +53,29 @@ To create an API key, perform the following steps: - `Chat2Query SQL ReadOnly`: only allows the API key to generate SQL statements based on provided instructions and execute `SELECT` SQL statements. - `Chat2Query SQL ReadWrite`: allows the API key to generate SQL statements based on provided instructions and execute any SQL statements. -4. Click **Next**. The public key and private key are displayed. +4. By default, an API key never expires. If you prefer to set an expiration time for the key, click **Expires in**, select a time unit (`Minutes`, `Days`, or `Months`), and then fill in a desired number for the time unit. + +5. Click **Next**. The public key and private key are displayed. Make sure that you have copied and saved the private key in a secure location. After leaving this page, you will not be able to get the full private key again. -5. Click **Done**. +6. Click **Done**. -## Step 3. Call Chat2Query endpoints +## Call Chat2Query endpoints > **Note:** > -> Each Chat2Query Data App has a rate limit of 100 requests per day. If you exceed the rate limit, the API returns a `429` error. For more quota, you can [submit a request](https://support.pingcap.com/hc/en-us/requests/new?ticket_form_id=7800003722519) to our support team. +> Each Chat2Query Data App has a rate limit of 100 requests per day. If you exceed the rate limit, the API returns a `429` error. For more quota, you can [submit a request](https://tidb.support.pingcap.com/) to our support team. In each Chat2Query Data App, you can find the following endpoints: -- Chat2Query v1 endpoint: `/v1/chat2data` +- Chat2Query v3 endpoints: the endpoints whose names starting with `/v3`, such as `/v3/dataSummaries` and `/v3/chat2data`(recommended) - Chat2Query v2 endpoints: the endpoints whose names starting with `/v2`, such as `/v2/dataSummaries` and `/v2/chat2data` +- Chat2Query v1 endpoint: `/v1/chat2data`(deprecated) > **Tip:** > -> Compared with `/v1/chat2data`, `/v2/chat2data` requires you to analyze your database first by calling `/v2/dataSummaries`, so the results returned by `/v2/chat2data` are generally more accurate. +> Compared with `/v1/chat2data`, `/v3/chat2data` and `/v2/chat2data` requires you to analyze your database first by calling `/v3/dataSummaries` or `/v2/dataSummaries`. Consequently, the results returned by `/v3/chat2data` and `/v2/chat2data` are generally more accurate. ### Get the code example of an endpoint @@ -84,34 +91,60 @@ TiDB Cloud provides code examples to help you quickly call Chat2Query endpoints. > **Note:** > - > For `/v2/chat2data` and `/v2/jobs/{job_id}`, you only need to select the authentication method. + > For some of the endpoints such as `/v2/jobs/{job_id}`, you only need to select the authentication method. 4. To call the endpoint, you can paste the example in your application, replace the parameters in the example with your own (such as replacing the `${PUBLIC_KEY}` and `${PRIVATE_KEY}` placeholders with your API key), and then run it. -### Call Chat2Query v2 endpoints - -TiDB Cloud Data Service provides the following Chat2Query v2 endpoints: - -| Method | Endpoint| Description | -| ---- | ---- |---- | -| POST | `/v2/dataSummaries` | This endpoint generates a data summary for your database schema, table schema, and column schema by using artificial intelligence for analysis. | -| POST | `/v2/chat2data` | This endpoint enables you to generate and execute SQL statements using artificial intelligence by providing the data summary ID and instructions. | -| GET | `/v2/jobs/{job_id}` | This endpoint enables you to query the status of the data summary generation job. | - -In the subsequent sections, you will learn how to call these endpoints. - -#### 1. Generate a data summary by calling `/v2/dataSummaries` - -Before calling `/v2/chat2data`, let AI analyze the database and generate a data summary first by calling `/v2/dataSummaries`, so `/v2/chat2data` can get a better performance in SQL generation later. - -The following is a code example of calling `/v2/chat2data` to analyze the `sp500insight` database and generate a data summary for the database: +### Call Chat2Query v3 endpoints or v2 endpoints + +TiDB Cloud Data Service provides the following Chat2Query v3 endpoints and v2 endpoints: + +| Method | Endpoint | Description | +| ------ | -------- | ----------- | +| POST | `/v3/dataSummaries` | This endpoint generates a data summary for your database schema, table schema, and column schema by using artificial intelligence for analysis. | +| GET | `/v3/dataSummaries` | This endpoint retrieves all data summaries of your database. | +| GET | `/v3/dataSummaries/{data_summary_id}` | This endpoint retrieves a specific data summary. | +| PUT | `/v3/dataSummaries/{data_summary_id}` | This endpoint updates a specific data summary. | +| PUT | `/v3/dataSummaries/{data_summary_id}/tables/{table_name}` | This endpoint updates the description of a specific table in a specific data summary. | +| PUT | `/v3/dataSummaries/{data_summary_id}/tables/{table_name}/columns` | This endpoint updates the description of columns for a specific table in a specific data summary. | +| POST | `/v3/knowledgeBases` | This endpoint creates a new knowledge base. For more information about the usage of knowledge base related endpoints, see [Use knowledge bases](/tidb-cloud/use-chat2query-knowledge.md). | +| GET | `/v3/knowledgeBases` | This endpoint retrieves all knowledge bases. | +| GET | `/v3/knowledgeBases/{knowledge_base_id}` | This endpoint retrieves a specific knowledge base. | +| PUT | `/v3/knowledgeBases/{knowledge_base_id}` | This endpoint updates a specific knowledge base. | +| POST | `/v3/knowledgeBases/{knowledge_base_id}/data` | This endpoint adds data to a specific knowledge base. | +| GET | `/v3/knowledgeBases/{knowledge_base_id}/data` | This endpoint retrieves data from a specific knowledge base. | +| PUT | `/v3/knowledgeBases/{knowledge_base_id}/data/{knowledge_data_id}` | This endpoint updates specific data in a knowledge base. | +| DEL | `/v3/knowledgeBases/{knowledge_base_id}/data/{knowledge_data_id}` | This endpoint deletes specific data from a knowledge base. | +| POST | `/v3/sessions` | This endpoint creates a new session. For more information about the usage of session-related endpoints, see [Start multi-round Chat2Query](/tidb-cloud/use-chat2query-sessions.md). | +| GET | `/v3/sessions` | This endpoint retrieves a list of all sessions. | +| GET | `/v3/sessions/{session_id}` | This endpoint retrieves the details of a specific session. | +| PUT | `/v3/sessions/{session_id}` | This endpoint updates a specific session. | +| PUT | `/v3/sessions/{session_id}/reset` | This endpoint resets a specific session. | +| POST | `/v3/sessions/{session_id}/chat2data` | This endpoint generates and executes SQL statements within a specific session using artificial intelligence. For more information, see [Start multi-round Chat2Query by using sessions](/tidb-cloud/use-chat2query-sessions.md). | +| POST | `/v3/chat2data` | This endpoint enables you to generate and execute SQL statements using artificial intelligence by providing the data summary ID and instructions. | +| POST | `/v3/refineSql` | This endpoint refines existing SQL queries using artificial intelligence. | +| POST | `/v3/suggestQuestions` | This endpoint suggests questions based on the provided data summary. | +| POST | `/v2/dataSummaries` | This endpoint generates a data summary for your database schema, table schema, and column schema using artificial intelligence. | +| GET | `/v2/dataSummaries` | This endpoint retrieves all data summaries. | +| POST | `/v2/chat2data` | This endpoint enables you to generate and execute SQL statements using artificial intelligence by providing the data summary ID and instructions. | +| GET | `/v2/jobs/{job_id}` | This endpoint enables you to query the status of a specific data summary generation job. | + +The steps to call `/v3/chat2data` and `/v2/chat2data` are the same. The following sections take `/v3/chat2data` as an example to show how to call it. + +#### 1. Generate a data summary by calling `/v3/dataSummaries` + +Before calling `/v3/chat2data`, let AI analyze the database and generate a data summary first by calling `/v3/dataSummaries`, so `/v3/chat2data` can get a better performance in SQL generation later. + +The following is a code example of calling `/v3/dataSummaries` to analyze the `sp500insight` database and generate a data summary for the database: ```bash -curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://.data.dev.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v2/dataSummaries'\ +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://.data.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v3/dataSummaries'\ --header 'content-type: application/json'\ --data-raw '{ - "cluster_id": "10939961583884005252", - "database": "sp500insight" + "cluster_id": "10140100115280519574", + "database": "sp500insight", + "description": "Data summary for SP500 Insight", + "reuse": false }' ``` @@ -119,23 +152,25 @@ In the preceding example, the request body is a JSON object with the following p - `cluster_id`: _string_. A unique identifier of the TiDB cluster. - `database`: _string_. The name of the database. +- `description`: _string_. A description of the data summary. +- `reuse`: _boolean_. Specifies whether to reuse an existing data summary. If you set it to `true`, the API will reuse an existing data summary. If you set it to `false`, the API will generate a new data summary. An example response is as follows: -```json +```js { "code": 200, "msg": "", "result": { - "data_summary_id": 481235, - "job_id": "79c2b3d36c074943ab06a29e45dd5887" + "data_summary_id": 304823, + "job_id": "fb99ef785da640ab87bf69afed60903d" } } ``` #### 2. Check the analysis status by calling `/v2/jobs/{job_id}` -The `/v2/dataSummaries` API is asynchronous. For a database with a large dataset, it might take a few minutes to complete the database analysis and return the full data summary. +The `/v3/dataSummaries` API is asynchronous. For a database with a large dataset, it might take a few minutes to complete the database analysis and return the full data summary. To check the analysis status of your database, you can call the `/v2/jobs/{job_id}` endpoint as follows: @@ -146,103 +181,102 @@ curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request GET 'https://.data.dev.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v2/chat2data'\ +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://.data.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v3/chat2data'\ --header 'content-type: application/json'\ --data-raw '{ - "cluster_id": "10939961583884005252", - "database": "sp500insight", - "raw_question": "" + "cluster_id": "10140100115280519574", + "database": "sp500insight", + "question": "", + "sql_generate_mode": "direct" }' ``` -In the preceding code, the request body is a JSON object with the following properties: +The request body is a JSON object with the following properties: - `cluster_id`: _string_. A unique identifier of the TiDB cluster. - `database`: _string_. The name of the database. -- `raw_question`: _string_. A natural language describing the query you want. +- `data_summary_id`: _integer_. The ID of the data summary used to generate SQL. This property only takes effect if `cluster_id` and `database` are not provided. If you specify both `cluster_id` and `database`, the API uses the default data summary of the database. +- `question`: _string_. A question in natural language describing the query you want. +- `sql_generate_mode`: _string_. The mode to generate SQL statements. The value can be `direct` or `auto_breakdown`. If you set it to `direct`, the API will generate SQL statements directly based on the `question` you provided. If you set it to `auto_breakdown`, the API will break down the `question` into multiple tasks and generate SQL statements for each task. An example response is as follows: -```json +```js { "code": 200, "msg": "", "result": { - "job_id": "3966d5bd95324a6283445e3a02ccd97c" + "cluster_id": "10140100115280519574", + "database": "sp500insight", + "job_id": "20f7577088154d7889964f1a5b12cb26", + "session_id": 304832 } } ``` If you receive a response with the status code `400` as follows, it means that you need to wait a moment for the data summary to be ready. -```json +```js { "code": 400, "msg": "Data summary is not ready, please wait for a while and retry", @@ -250,7 +284,7 @@ If you receive a response with the status code `400` as follows, it means that y } ``` -The `/v2/chat2data` API is asynchronous. You can check the job status by calling the `/v2/jobs/{job_id}` endpoint: +The `/v3/chat2data` API is asynchronous. You can check the job status by calling the `/v2/jobs/{job_id}` endpoint: ```bash curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request GET 'https://.data.dev.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v2/jobs/{job_id}'\ @@ -259,44 +293,55 @@ curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request GET 'https:// **Note:** +> +> The Chat2Data v1 endpoint is deprecated. It is recommended that you call Chat2Data v3 endpoints instead. TiDB Cloud Data Service provides the following Chat2Query v1 endpoint: @@ -316,9 +361,8 @@ When calling `/v1/chat2data`, you need to replace the following parameters: > **Note:** > -> Each Chat2Query Data App has a rate limit of 100 requests per day. If you exceed the rate limit, the API returns a `429` error. For more quota, you can [submit a request](https://support.pingcap.com/hc/en-us/requests/new?ticket_form_id=7800003722519) to our support team. -> An API Key with the role `Chat2Query Data Summary Management Role` cannot call the Chat2Data v1 endpoint. - +> - Each Chat2Query Data App has a rate limit of 100 requests per day. If you exceed the rate limit, the API returns a `429` error. For more quota, you can [submit a request](https://tidb.support.pingcap.com/) to our support team. +> - An API Key with the role `Chat2Query Data Summary Management Role` cannot call the Chat2Data v1 endpoint. The following code example is used to count how many users are in the `sp500insight.users` table: ```bash @@ -337,7 +381,7 @@ In the preceding example, the request body is a JSON object with the following p - `cluster_id`: _string_. A unique identifier of the TiDB cluster. - `database`: _string_. The name of the database. - `tables`: _array_. (optional) A list of table names to be queried. -- `instruction`: _string_. A natural language instruction describing the query you want. +- `instruction`: _string_. An instruction in natural language describing the query you want. The response is as follows: @@ -398,4 +442,6 @@ If your API call is not successful, you will receive a status code other than `2 ## Learn more - [Manage an API key](/tidb-cloud/data-service-api-key.md) +- [Start Multi-round Chat2Query](/tidb-cloud/use-chat2query-sessions.md) +- [Use Knowledge Bases](/tidb-cloud/use-chat2query-knowledge.md) - [Response and Status Codes of Data Service](/tidb-cloud/data-service-response-and-status-code.md) diff --git a/tidb-cloud/use-chat2query-knowledge.md b/tidb-cloud/use-chat2query-knowledge.md new file mode 100644 index 0000000000000..99c0d614b5c29 --- /dev/null +++ b/tidb-cloud/use-chat2query-knowledge.md @@ -0,0 +1,218 @@ +--- +title: Use Knowledge Bases +summary: Learn how to improve your Chat2Query results by using Chat2Query knowledge base APIs. +--- + +# Use Knowledge Bases + +A knowledge base is a collection of structured data that can be used to enhance the SQL generation capabilities of Chat2Query. + +Starting from v3, the Chat2Query API enables you to add or modify knowledge bases by calling knowledge base related endpoints of your Chat2Query Data App. + +> **Note:** +> +> Knowledge base related endpoints are available for [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless) clusters by default. To use knowledge base related endpoints on [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) clusters, contact [TiDB Cloud support](/tidb-cloud/tidb-cloud-support.md). + +## Before you begin + +Before creating a knowledge base for your database, make sure that you have the following: + +- A [Chat2Query Data App](/tidb-cloud/use-chat2query-api.md#create-a-chat2query-data-app) +- An [API key for the Chat2Query Data App](/tidb-cloud/use-chat2query-api.md#create-an-api-key) + +## Step 1. Create a knowledge base for the linked database + +> **Note:** +> +> The knowledge used by Chat2Query is **structured according to the database dimension**. You can connect multiple Chat2Query Data Apps to the same database, but each Chat2Query Data App can only use knowledge from a specific database it is linked to. + +In your Chat2Query Data App, you can create a knowledge base for a specific database by calling the `/v3/knowledgeBases` endpoint. After creation, you will get a `knowledge_base_id` for future knowledge management. + +The following is a general code example for calling this endpoint. + +> **Tip:** +> +> To get a specific code example for your endpoint, click the endpoint name in the left pane of your Data App, and then click **Show Code Example**. For more information, see [Get the example code of an endpoint](/tidb-cloud/use-chat2query-api.md#get-the-code-example-of-an-endpoint). + +```bash +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://.data.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v3/knowledgeBases'\ + --header 'content-type: application/json'\ + --data-raw '{ + "cluster_id": "", + "database": "", + "description": "" +}' +``` + +An example response is as follows: + +```json +{ + "code":200, + "msg":"", + "result": + { + "default":true, + "description":"", + "knowledge_base_id":2 + } +} +``` + +After getting the response, record the `knowledge_base_id` value in your response for later use. + +## Step 2. Choose a knowledge type + +The knowledge base of each database can contain multiple types of knowledge. Before adding knowledge to your knowledge base, you need to choose a knowledge type that best suits your use case. + +Currently, Chat2Query knowledge bases support the following knowledge types. Each type is specifically designed for different scenarios and has a unique knowledge structure. + +- [Few-shot example](#few-shot-example) +- [Term-sheet explanation](#term-sheet-explanation) +- [Instruction](#instruction) + +### Few-shot example + +Few-shot example refers to the Q&A learning samples provided to Chat2Query, which include sample questions and their corresponding answers. These examples help Chat2Query handle new tasks more effectively. + +> **Note:** +> +> Make sure the accuracy of newly added examples, because the quality of examples affects how well Chat2Query learns. Poor examples, such as mismatched questions and answers, can degrade the performance of Chat2Query on new tasks. + +#### Knowledge structure + +Each example consists of a sample question and its corresponding answer. + +For example: + +```json +{ + "question": "How many records are in the 'test' table?", + "answer": "SELECT COUNT(*) FROM `test`;" +} +``` + +#### Use cases + +Few-Shot examples can significantly improve the performance of Chat2Query in various scenarios, including but not limited to the following: + +1. **When dealing with rare or complex questions**: if Chat2Query encounters infrequent or complex questions, adding few-shot examples can enhance its understanding and improve the accuracy of the results. + +2. **When struggling with a certain type of question**: if Chat2Query frequently makes mistakes or has difficulty with specific questions, adding few-shot examples can help improve its performance on these questions. + +### Term-sheet explanation + +Term-sheet explanation refers to a comprehensive explanation of a specific term or a group of similar terms, helping Chat2Query understand the meaning and usage of these terms. + +> **Note:** +> +> Make sure the accuracy of newly added term explanations, because the quality of explanations affects how well Chat2Query learns. Incorrect interpretations do not improve Chat2Query results but also potentially lead to adverse effects. + +#### Knowledge structure + +Each explanation includes either a single term or a list of similar terms and their detailed descriptions. + +For example: + +```json +{ + "term": ["OSS"], + "description": "OSS Insight is a powerful tool that provides online data analysis for users based on nearly 6 billion rows of GitHub event data." +} +``` + +#### Use cases + +Term-sheet explanation is primarily used to improve Chat2Query's comprehension of user queries, especially in these situations: + +- **Dealing with industry-specific terminology or acronyms**: when your query contains industry-specific terminology or acronyms that might not be universally recognized, using a term-sheet explanation can help Chat2Query understand the meaning and usage of these terms. +- **Dealing with ambiguities in user queries**: when your query contains ambiguous concepts that is confusing, using a term-sheet explanation can help Chat2Query clarify these ambiguities. +- **Dealing with terms with various meanings**: when your query contains terms that carry different meanings in various contexts, using a term-sheet explanation can assist Chat2Query in discerning the correct interpretation. + +### Instruction + +Instruction is a piece of textual command. It is used to guide or control the behavior of Chat2Query, specifically instructing it on how to generate SQL according to specific requirements or conditions. + +> **Note:** +> +> - The instruction has a length limit of 512 characters. +> - Make sure to provide as clear and specific instructions as possible to ensure that Chat2Query can understand and execute the instructions effectively. + +#### Knowledge structure + +Instruction only includes a piece of textual command. + +For example: + +```json +{ + "instruction": "If the task requires calculating the sequential growth rate, use the LAG function with the OVER clause in SQL" +} +``` + +#### Use cases + +Instruction can be used in many scenarios to guide Chat2Query to output according to your requirements, including but not limited to the following: + +- **Limiting query scope**: if you want the SQL to consider only certain tables or columns, use an instruction to specify this. +- **Guiding SQL structure**: if you have specific requirements for the SQL structure, use an instruction to guide Chat2Query. + +## Step 3. Add knowledge to the newly created knowledge base + +To add new knowledge, you can call the `/v3/knowledgeBases/{knowledge_base_id}/data` endpoint. + +### Add a few-shot example type of knowledge + +For example, if you want Chat2Query to generate SQL statements of the count of rows in a table in a specific structure, you can add a few-shot example type of knowledge by calling `/v3/knowledgeBases/{knowledge_base_id}/data` as follows: + +```bash +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://.data.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v3/knowledgeBases//data'\ + --header 'content-type: application/json'\ + --data-raw '{ + "type": "few-shot", + "meta_data": {}, + "raw_data": { + "question": "How many records are in the 'test' table?", + "answer": "SELECT COUNT(*) FROM `test`;" + } +}' +``` + +In the preceding example code, `"type": "few-shot"` represents the few-shot example knowledge type. + +### Add a term-sheet explanation type of knowledge + +For example, if you want Chat2Query to comprehend the meaning of the term `OSS` using your provided explanation, you can add a term-sheet explanation type of knowledge by calling `/v3/knowledgeBases/{knowledge_base_id}/data` as follows: + +```bash +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://.data.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v3/knowledgeBases//data'\ + --header 'content-type: application/json'\ + --data-raw '{ + "type": "term-sheet", + "meta_data": {}, + "raw_data": { + "term": ["OSS"], + "description": "OSS Insight is a powerful tool that provides online data analysis for users based on nearly 6 billion rows of GitHub event data." + } +}' +``` + +In the preceding example code, `"type": "term-sheet"` represents the term-sheet explanation knowledge type. + +### Add an instruction type of knowledge + +For example, if you want Chat2Query to consistently use the `LAG` function with the `OVER` clause in SQL queries when dealing with questions about sequential growth rate calculation, you can add an instruction type of knowledge by calling `/v3/knowledgeBases/{knowledge_base_id}/data` as follows: + +```bash +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://.data.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v3/knowledgeBases//data'\ + --header 'content-type: application/json'\ + --data-raw '{ + "type": "instruction", + "meta_data": {}, + "raw_data": { + "instruction": "If the task requires calculating the sequential growth rate, use the LAG function with the OVER clause in SQL" + } +}' +``` + +In the preceding example code, `"type": "instruction"` represents the instruction knowledge type. \ No newline at end of file diff --git a/tidb-cloud/use-chat2query-sessions.md b/tidb-cloud/use-chat2query-sessions.md new file mode 100644 index 0000000000000..de8dbd8deb8b5 --- /dev/null +++ b/tidb-cloud/use-chat2query-sessions.md @@ -0,0 +1,101 @@ +--- +title: Start Multi-round Chat2Query +summary: Learn how to start multi-round chat by using Chat2Query session-related APIs. +--- + +# Start Multi-round Chat2Query + +Starting from v3, the Chat2Query API enables you to start multi-round chats by calling session related endpoints. You can use the `session_id` returned by the `/v3/chat2data` endpoint to continue your conversation in the next round. + +## Before you begin + +Before starting multi-round Chat2Query, make sure that you have the following: + +- A [Chat2Query Data App](/tidb-cloud/use-chat2query-api.md#create-a-chat2query-data-app). +- An [API key for the Chat2Query Data App](/tidb-cloud/use-chat2query-api.md#create-an-api-key). +- A [data summary for your target database](/tidb-cloud/use-chat2query-api.md#1-generate-a-data-summary-by-calling-v3datasummaries). + +## Step 1. Start a session + +To start a session, you can call the `/v3/sessions` endpoint of your Chat2Query Data App. + +The following is a general code example for calling this endpoint. + +> **Tip:** +> +> To get a specific code example for your endpoint, click the endpoint name in the left pane of your Data App, and then click **Show Code Example**. For more information, see [Get the example code of an endpoint](/tidb-cloud/use-chat2query-api.md#get-the-code-example-of-an-endpoint). + +```bash +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://.data.tidbcloud.com/api/v1beta/app/chat2query-/endpoint/v3/sessions'\ + --header 'content-type: application/json'\ + --data-raw '{ + "cluster_id": "10140100115280519574", + "database": "sp500insight", + "name": "" +}' +``` + +In the preceding code, the request body is a JSON object with the following properties: + +- `cluster_id`: _string_. A unique identifier of the TiDB cluster. +- `database`: _string_. The name of the database. +- `name`: _string_. The name of the session. + +An example response is as follows: + +```json +{ + "code": 200, + "msg": "", + "result": { + "messages": [], + "meta": { + "created_at": 1718948875, // A UNIX timestamp indicating when the session is created + "creator": "", // The creator of the session + "name": "", // The name of the session + "org_id": "1", // The organization ID + "updated_at": 1718948875 // A UNIX timestamp indicating when the session is updated + }, + "session_id": 305685 // The session ID + } +} +``` + +## Step 2. Call Chat2Data endpoints with the session + +After starting a session, you can call `/v3/sessions/{session_id}/chat2data` to continue your conversation in the next round. + +The following is a general code example: + +```bash +curl --digest --user ${PUBLIC_KEY}:${PRIVATE_KEY} --request POST 'https://eu-central-1.data.tidbcloud.com/api/v1beta/app/chat2query-YqAvnlRj/endpoint/v3/sessions/{session_id}/chat2data'\ + --header 'content-type: application/json'\ + --data-raw '{ + "question": "", + "feedback_answer_id": "", + "feedback_task_id": "", + "sql_generate_mode": "direct" +}' +``` + +In the preceding code, the request body is a JSON object with the following properties: + +- `question`: _string_. A question in natural language describing the query you want. +- `feedback_answer_id`: _string_. The feedback answer ID. This field is optional and is only used for feedback. +- `feedback_task_id`: _string_. The feedback task ID. This field is optional and is only used for feedback. +- `sql_generate_mode`: _string_. The mode to generate SQL statements. The value can be `direct` or `auto_breakdown`. If you set it to `direct`, the API will generate SQL statements directly based on the `question` you provided. If you set it to `auto_breakdown`, the API will break down the `question` into multiple tasks and generate SQL statements for each task. + +An example response is as follows: + +```json +{ + "code": 200, + "msg": "", + "result": { + "job_id": "d96b6fd23c5f445787eb5fd067c14c0b", + "session_id": 305685 + } +} +``` + +The response is similar to the response of the `/v3/chat2data` endpoint. You can check the job status by calling the `/v2/jobs/{job_id}` endpoint. For more information, see [Check the analysis status by calling `/v2/jobs/{job_id}`](/tidb-cloud/use-chat2query-api.md#2-check-the-analysis-status-by-calling-v2jobsjob_id). \ No newline at end of file diff --git a/tidb-cloud/use-htap-cluster.md b/tidb-cloud/use-htap-cluster.md index c39f00c8de520..6b818678b79a6 100644 --- a/tidb-cloud/use-htap-cluster.md +++ b/tidb-cloud/use-htap-cluster.md @@ -11,7 +11,7 @@ With TiDB Cloud, you can create an HTAP cluster easily by specifying one or more > **Note:** > -> TiFlash is always enabled for TiDB Serverless clusters. You cannot disable it. +> TiFlash is always enabled for TiDB Cloud Serverless clusters. You cannot disable it. TiKV data is not replicated to TiFlash by default. You can select which table to replicate to TiFlash using the following SQL statement: diff --git a/tidb-cloud/v6.5-performance-benchmarking-with-sysbench.md b/tidb-cloud/v6.5-performance-benchmarking-with-sysbench.md new file mode 100644 index 0000000000000..d8c4f4179fa6c --- /dev/null +++ b/tidb-cloud/v6.5-performance-benchmarking-with-sysbench.md @@ -0,0 +1,155 @@ +--- +title: TiDB Cloud Sysbench Performance Test Report for TiDB v6.5.6 +summary: Introduce the Sysbench performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v6.5.6. +--- + +# TiDB Cloud Sysbench Performance Test Report for TiDB v6.5.6 + +This document provides the Sysbench performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v6.5.6. This report can also be used as a reference for the performance of TiDB Self-Managed v6.5.6 clusters. + +## Test overview + +This test aims at showing the Sysbench performance of TiDB v6.5.6 in the Online Transactional Processing (OLTP) scenario. + +## Test environment + +### TiDB cluster + +The test is conducted on a TiDB cluster with the following settings: + +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) +- Cluster version: v6.5.6 +- Cloud provider: AWS (us-west-2) +- Cluster configuration: + + | Node type | Node size | Node quantity | Node storage | + | :-------- | :-------------- | :------------ | :----------- | + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + +### Benchmark executor + +The benchmark executor sends SQL queries to the TiDB cluster. In this test, its hardware configuration is as follows: + +- Machine type: Amazon EC2 (us-west-2) +- Instance type: c6a.2xlarge +- Sysbench version: sysbench 1.0.20 (using bundled LuaJIT 2.1.0-beta2) + +## Test steps + +This section introduces how to perform the Sysbench performance test step by step. + +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. + + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + +2. On the benchmark executor, connect to the newly created cluster and create a database named `sbtest`. + + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + + To create the `sbtest` database, execute the following SQL statement: + + ```sql + CREATE DATABASE sbtest; + ``` + +3. Load Sysbench data to the `sbtest` database. + + 1. The test in this document is implemented based on [sysbench](https://github.com/akopytov/sysbench). To install sysbench, see [Building and installing from source](https://github.com/akopytov/sysbench#building-and-installing-from-source). + + 2. Run the following `sysbench prepare` command to import 32 tables and 10,000,000 rows to the `sbtest` database. Replace `${HOST}`, `${PORT}`, `${THREAD}`, and `${PASSWORD}` with your actual values. + + ```shell + sysbench oltp_common \ + --threads=${THREAD} \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --mysql-host=${HOST} \ + --mysql-port=${PORT} \ + --mysql-user=root \ + --mysql-password=${PASSWORD} \ + prepare --tables=32 --table-size=10000000 + ``` + +4. Run the following `sysbench run` command to conduct Sysbench performance tests on different workloads. This document conducts tests on five workloads: `oltp_point_select`, `oltp_read_write`, `oltp_update_non_index`, `oltp_update_index`, and `oltp_insert`. For each workload, this document conducts three tests with the `${THREAD}` value of `100`, `200`, and `400`. For each concurrency, the test takes 20 minutes. + + ```shell + sysbench ${WORKLOAD} run \ + --mysql-host=${HOST} \ + --mysql-port=${PORT} \ + --mysql-user=root \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --threads=${THREAD} \ + --time=1200 \ + --report-interval=10 \ + --tables=32 \ + --table-size=10000000 \ + --mysql-ignore-errors=1062,2013,8028,9007 \ + --auto-inc=false \ + --mysql-password=${PASSWORD} + ``` + +## Test results + +This section introduces the Sysbench performance of v6.5.6 in the [test environment](#test-environment). + +### Point select performance + +The performance on the `oltp_point_select` workload is as follows: + +| Threads | TPS | 95% latency (ms) | +| :------ | :----- | :--------------- | +| 50 | 34125 | 2.03 | +| 100 | 64987 | 2.07 | +| 200 | 121656 | 2.14 | + +![Sysbench point select performance](/media/tidb-cloud/v6.5.6-oltp_select_point.png) + +### Read write performance + +The performance on the `oltp_read_write` workload is as follows: + +| Threads | TPS | 95% latency (ms) | +| :------ | :--- | :--------------- | +| 50 | 1232 | 46.6 | +| 100 | 2266 | 51.9 | +| 200 | 3578 | 81.5 | + +![Sysbench read write performance](/media/tidb-cloud/v6.5.6-oltp_read_write.png) + +### Update non-index performance + +The performance on the `oltp_update_non_index` workload is as follows: + +| Threads | TPS | 95% latency (ms) | +| :------ | :---- | :--------------- | +| 100 | 11016 | 11.0 | +| 200 | 20640 | 12.1 | +| 400 | 36830 | 13.5 | + +![Sysbench update non-index performance](/media/tidb-cloud/v6.5.6-oltp_update_non_index.png) + +### Update index performance + +The performance on the `oltp_update_index` workload is as follows: + +| Threads | TPS | 95% latency (ms) | +| :------ | :---- | :--------------- | +| 100 | 9270 | 14.0 | +| 200 | 14466 | 18.0 | +| 400 | 22194 | 24.8 | + +![Sysbench update index performance](/media/tidb-cloud/v6.5.6-oltp_update_index.png) + +### Insert performance + +The performance on the `oltp_insert` workload is as follows: + +| Threads | TPS | 95% latency (ms) | +| :------ | :---- | :--------------- | +| 100 | 16008 | 8.13 | +| 200 | 27143 | 10.1 | +| 400 | 40884 | 15.0 | + +![Sysbench insert performance](/media/tidb-cloud/v6.5.6-oltp_insert.png) diff --git a/tidb-cloud/v7.1.0-performance-benchmarking-with-tpcc.md b/tidb-cloud/v6.5-performance-benchmarking-with-tpcc.md similarity index 67% rename from tidb-cloud/v7.1.0-performance-benchmarking-with-tpcc.md rename to tidb-cloud/v6.5-performance-benchmarking-with-tpcc.md index eb4f988afea16..3538c4001f194 100644 --- a/tidb-cloud/v7.1.0-performance-benchmarking-with-tpcc.md +++ b/tidb-cloud/v6.5-performance-benchmarking-with-tpcc.md @@ -1,15 +1,15 @@ --- -title: TiDB Cloud TPC-C Performance Test Report for TiDB v7.1.0 -summary: Introduce the TPC-C performance test results for a TiDB Dedicated cluster with the TiDB version of v7.1.0. +title: TiDB Cloud TPC-C Performance Test Report for TiDB v6.5.6 +summary: Introduce the TPC-C performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v6.5.6. --- -# TiDB Cloud TPC-C Performance Test Report for TiDB v7.1.0 +# TiDB Cloud TPC-C Performance Test Report for TiDB v6.5.6 -This document provides the TPC-C performance test steps and results for a TiDB Dedicated cluster with the TiDB version of v7.1.0. This report can also be used as a reference for the performance of TiDB Self-Hosted v7.1.0 clusters. +This document provides the TPC-C performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v6.5.6. This report can also be used as a reference for the performance of TiDB Self-Managed v6.5.6 clusters. ## Test overview -This test aims at showing the TPC-C performance of TiDB v7.1.0 in the Online Transactional Processing (OLTP) scenario. +This test aims at showing the TPC-C performance of TiDB v6.5.6 in the Online Transactional Processing (OLTP) scenario. ## Test environment @@ -17,15 +17,15 @@ This test aims at showing the TPC-C performance of TiDB v7.1.0 in the Online Tra The test is conducted on a TiDB cluster with the following settings: -- Cluster type: [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) -- Cluster version: v7.1.0 +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) +- Cluster version: v6.5.6 - Cloud provider: AWS (us-west-2) - Cluster configuration: - | Node type | Node size | Node quantity | Node storage | - |:----------|:----------|:----------|:----------| - | TiDB | 16 vCPU, 32 GiB | 2 | N/A | - | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + | Node type | Node size | Node quantity | Node storage | + | :-------- | :-------------- | :------------ | :----------- | + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | ### Benchmark executor @@ -38,13 +38,13 @@ The benchmark executor sends SQL queries to the TiDB cluster. In this test, its This section introduces how to perform the TPC-C performance test step by step. -1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. - For more information, see [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). 2. On the benchmark executor, connect to the newly created cluster and create a database named `tpcc`. - To connect to the cluster, see [Connect to TiDB Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). To create the `tpcc` database, execute the following SQL statement: @@ -88,7 +88,7 @@ This section introduces how to perform the TPC-C performance test step by step. SET tidb_index_serial_scan_concurrency=16; ``` -5. Run the following `go-tpc tpcc` command to conduct stress tests on the TiDB Dedicated cluster. For each concurrency, the test takes two hours. +5. Run the following `go-tpc tpcc` command to conduct stress tests on the TiDB Cloud Dedicated cluster. For each concurrency, the test takes two hours. ```shell go-tpc tpcc --host ${HOST} -P 4000 --warehouses 1000 run -D tpcc -T ${THREAD} --time 2h0m0s -p ${PASSWORD} --ignore-error @@ -100,12 +100,12 @@ This section introduces how to perform the TPC-C performance test step by step. ## Test results -The TPC-C performance of v7.1.0 in the [test environment](#test-environment) is as follows: +The TPC-C performance of v6.5.6 in the [test environment](#test-environment) is as follows: -| Threads | v7.1.0 tpmC | -|:--------|:----------| -| 50 | 36,159 | -| 100 | 66,742 | -| 200 | 94,406 | +| Threads | v6.5.6 tpmC | +| :------ | :---------- | +| 50 | 44183 | +| 100 | 74424 | +| 200 | 101545 | -![TPC-C](/media/tidb-cloud/v7.1.0-tpmC.png) +![TPC-C](/media/tidb-cloud/v6.5.6-tpmC.png) diff --git a/tidb-cloud/v7.1.0-performance-benchmarking-with-sysbench.md b/tidb-cloud/v7.1-performance-benchmarking-with-sysbench.md similarity index 64% rename from tidb-cloud/v7.1.0-performance-benchmarking-with-sysbench.md rename to tidb-cloud/v7.1-performance-benchmarking-with-sysbench.md index 5f11db1dd6080..66f28cc222005 100644 --- a/tidb-cloud/v7.1.0-performance-benchmarking-with-sysbench.md +++ b/tidb-cloud/v7.1-performance-benchmarking-with-sysbench.md @@ -1,15 +1,16 @@ --- -title: TiDB Cloud Sysbench Performance Test Report for TiDB v7.1.0 -summary: Introduce the Sysbench performance test results for a TiDB Dedicated cluster with the TiDB version of v7.1.0. +title: TiDB Cloud Sysbench Performance Test Report for TiDB v7.1.3 +summary: Introduce the Sysbench performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v7.1.3. +aliases: ['/tidbcloud/v7.1.0-performance-benchmarking-with-sysbench'] --- -# TiDB Cloud Sysbench Performance Test Report for TiDB v7.1.0 +# TiDB Cloud Sysbench Performance Test Report for TiDB v7.1.3 -This document provides the Sysbench performance test steps and results for a TiDB Dedicated cluster with the TiDB version of v7.1.0. This report can also be used as a reference for the performance of TiDB Self-Hosted v7.1.0 clusters. +This document provides the Sysbench performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v7.1.3. This report can also be used as a reference for the performance of TiDB Self-Managed v7.1.3 clusters. ## Test overview -This test aims at showing the Sysbench performance of TiDB v7.1.0 in the Online Transactional Processing (OLTP) scenario. +This test aims at showing the Sysbench performance of TiDB v7.1.3 in the Online Transactional Processing (OLTP) scenario. ## Test environment @@ -17,15 +18,15 @@ This test aims at showing the Sysbench performance of TiDB v7.1.0 in the Online The test is conducted on a TiDB cluster with the following settings: -- Cluster type: [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) -- Cluster version: v7.1.0 +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) +- Cluster version: v7.1.3 - Cloud provider: AWS (us-west-2) - Cluster configuration: - | Node type | Node size | Node quantity | Node storage | - |:----------|:----------|:----------|:----------| - | TiDB | 16 vCPU, 32 GiB | 2 | N/A | - | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + | Node type | Node size | Node quantity | Node storage | + | :-------- | :-------------- | :------------ | :----------- | + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | ### Parameter configuration @@ -59,13 +60,13 @@ The benchmark executor sends SQL queries to the TiDB cluster. In this test, its This section introduces how to perform the Sysbench performance test step by step. -1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. - For more information, see [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). 2. On the benchmark executor, connect to the newly created cluster and create a database named `sbtest`. - To connect to the cluster, see [Connect to TiDB Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). To create the `sbtest` database, execute the following SQL statement: @@ -112,64 +113,64 @@ This section introduces how to perform the Sysbench performance test step by ste ## Test results -This section introduces the Sysbench performance of v7.1.0 in the [test environment](#test-environment). +This section introduces the Sysbench performance of v7.1.3 in the [test environment](#test-environment). ### Point select performance The performance on the `oltp_point_select` workload is as follows: -| Threads | TPS | 95% latency (ms)| -|:--------|:----------|:----------| -| 100 | 56,039 | 2.34 | -| 200 | 95,908 | 2.78 | -| 400 | 111,810 | 5.57 | +| Threads | TPS | 95% latency (ms) | +| :------ | :----- | :--------------- | +| 50 | 35309 | 1.93 | +| 100 | 64853 | 2.00 | +| 200 | 118462 | 2.22 | -![Sysbench point select performance](/media/tidb-cloud/v7.1.0-oltp_select_point.png) +![Sysbench point select performance](/media/tidb-cloud/v7.1.3-oltp_select_point.png) ### Read write performance The performance on the `oltp_read_write` workload is as follows: -| Threads | TPS | 95% latency (ms)| -|:--------|:----------|:----------| -| 100 | 1,789 | 66.8 | -| 200 | 2,842 | 97.6 | -| 400 | 3,090 | 191 | +| Threads | TPS | 95% latency (ms) | +| :------ | :--- | :--------------- | +| 50 | 1218 | 48.3 | +| 100 | 2235 | 53.9 | +| 200 | 3380 | 87.6 | -![Sysbench read write performance](/media/tidb-cloud/v.7.1.0-oltp_read_write.png) +![Sysbench read write performance](/media/tidb-cloud/v7.1.3-oltp_read_write.png) ### Update non-index performance The performance on the `oltp_update_non_index` workload is as follows: -| Threads | TPS | 95% latency (ms)| -|:--------|:----------|:----------| -| 100 | 7,944 | 16.7 | -| 200 | 13,844 | 19.0 | -| 400 | 29,063 | 20.4 | +| Threads | TPS | 95% latency (ms) | +| :------ | :---- | :--------------- | +| 100 | 10928 | 11.7 | +| 200 | 19985 | 12.8 | +| 400 | 35621 | 14.7 | -![Sysbench update non-index performance](/media/tidb-cloud/v7.1.0-oltp_update_non_index.png) +![Sysbench update non-index performance](/media/tidb-cloud/v7.1.3-oltp_update_non_index.png) ### Update index performance The performance on the `oltp_update_index` workload is as follows: -| Threads | TPS | 95% latency (ms)| -|:--------|:----------|:----------| -| 100 | 6,389 | 20 | -| 200 | 12,583 | 22.3 | -| 400 | 22,393 | 25.7 | +| Threads | TPS | 95% latency (ms) | +| :------ | :---- | :--------------- | +| 100 | 8854 | 14.7 | +| 200 | 14414 | 18.6 | +| 400 | 21997 | 25.3 | -![Sysbench update index performance](/media/tidb-cloud/v7.1.0-oltp_update_index.png) +![Sysbench update index performance](/media/tidb-cloud/v7.1.3-oltp_update_index.png) ### Insert performance The performance on the `oltp_insert` workload is as follows: -| Threads | TPS | 95% latency (ms)| -|:--------|:----------|:----------| -| 100 | 7,671 | 17.3 | -| 200 | 13,584 | 19.7 | -| 400 | 31,252 | 20 | +| Threads | TPS | 95% latency (ms) | +| :------ | :---- | :--------------- | +| 100 | 15575 | 8.13 | +| 200 | 25078 | 11.0 | +| 400 | 38436 | 15.6 | -![Sysbench insert performance](/media/tidb-cloud/v7.1.0-oltp_insert.png) +![Sysbench insert performance](/media/tidb-cloud/v7.1.3-oltp_insert.png) diff --git a/tidb-cloud/v7.1-performance-benchmarking-with-tpcc.md b/tidb-cloud/v7.1-performance-benchmarking-with-tpcc.md new file mode 100644 index 0000000000000..4cd0f9f38710e --- /dev/null +++ b/tidb-cloud/v7.1-performance-benchmarking-with-tpcc.md @@ -0,0 +1,112 @@ +--- +title: TiDB Cloud TPC-C Performance Test Report for TiDB v7.1.3 +summary: Introduce the TPC-C performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v7.1.3. +aliases: ['/tidbcloud/v7.1.0-performance-benchmarking-with-tpcc'] +--- + +# TiDB Cloud TPC-C Performance Test Report for TiDB v7.1.3 + +This document provides the TPC-C performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v7.1.3. This report can also be used as a reference for the performance of TiDB Self-Managed v7.1.3 clusters. + +## Test overview + +This test aims at showing the TPC-C performance of TiDB v7.1.3 in the Online Transactional Processing (OLTP) scenario. + +## Test environment + +### TiDB cluster + +The test is conducted on a TiDB cluster with the following settings: + +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) +- Cluster version: v7.1.3 +- Cloud provider: AWS (us-west-2) +- Cluster configuration: + + | Node type | Node size | Node quantity | Node storage | + | :-------- | :-------------- | :------------ | :----------- | + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + +### Benchmark executor + +The benchmark executor sends SQL queries to the TiDB cluster. In this test, its hardware configuration is as follows: + +- Machine type: Amazon EC2 (us-west-2) +- Instance type: c6a.2xlarge + +## Test steps + +This section introduces how to perform the TPC-C performance test step by step. + +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. + + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + +2. On the benchmark executor, connect to the newly created cluster and create a database named `tpcc`. + + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + + To create the `tpcc` database, execute the following SQL statement: + + ```sql + CREATE DATABASE tpcc; + ``` + +3. Load TPC-C data to the `tpcc` database. + + 1. The test in this document is implemented based on [go-tpc](https://github.com/pingcap/go-tpc). You can download the test program using the following command: + + ```shell + curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/pingcap/go-tpc/master/install.sh | sh + ``` + + 2. Run the following `go-tpc tpcc` command to import 1,000 warehouses to the `tpcc` database. Replace `${HOST}`, `${THREAD}`, and `${PASSWORD}` with your actual values. This document conducts three tests with the `${THREAD}` value of `50`, `100`, and `200`. + + ```shell + go-tpc tpcc --host ${HOST} --warehouses 1000 prepare -P 4000 -D tpcc -T ${THREAD} --time 2h0m0s -p ${PASSWORD} --ignore-error + ``` + +4. To ensure that the TiDB optimizer can generate the optimal execution plan, execute the following SQL statements to collect statistics before conducting the TPC-C test: + + ```sql + ANALYZE TABLE customer; + ANALYZE TABLE district; + ANALYZE TABLE history; + ANALYZE TABLE item; + ANALYZE TABLE new_order; + ANALYZE TABLE order_line; + ANALYZE TABLE orders; + ANALYZE TABLE stock; + ANALYZE TABLE warehouse; + ``` + + To accelerate the collection of statistics, execute the following SQL statements before collecting: + + ```sql + SET tidb_build_stats_concurrency=16; + SET tidb_distsql_scan_concurrency=16; + SET tidb_index_serial_scan_concurrency=16; + ``` + +5. Run the following `go-tpc tpcc` command to conduct stress tests on the TiDB Cloud Dedicated cluster. For each concurrency, the test takes two hours. + + ```shell + go-tpc tpcc --host ${HOST} -P 4000 --warehouses 1000 run -D tpcc -T ${THREAD} --time 2h0m0s -p ${PASSWORD} --ignore-error + ``` + +6. Extract the tpmC data of `NEW_ORDER` from the result. + + TPC-C uses tpmC (transactions per minute) to measure the maximum qualified throughput (MQTh, Max Qualified Throughput). The transactions are the NewOrder transactions and the final unit of measure is the number of new orders processed per minute. + +## Test results + +The TPC-C performance of v7.1.3 in the [test environment](#test-environment) is as follows: + +| Threads | v7.1.3 tpmC | +| :------ | :---------- | +| 50 | 42839 | +| 100 | 72895 | +| 200 | 97924 | + +![TPC-C](/media/tidb-cloud/v7.1.3-tpmC.png) diff --git a/tidb-cloud/v7.5.0-performance-benchmarking-with-sysbench.md b/tidb-cloud/v7.5-performance-benchmarking-with-sysbench.md similarity index 67% rename from tidb-cloud/v7.5.0-performance-benchmarking-with-sysbench.md rename to tidb-cloud/v7.5-performance-benchmarking-with-sysbench.md index 3e797f1d6072e..3ceb83626726d 100644 --- a/tidb-cloud/v7.5.0-performance-benchmarking-with-sysbench.md +++ b/tidb-cloud/v7.5-performance-benchmarking-with-sysbench.md @@ -1,11 +1,12 @@ --- title: TiDB Cloud Sysbench Performance Test Report for TiDB v7.5.0 -summary: Introduce the Sysbench performance test results for a TiDB Dedicated cluster with the TiDB version of v7.5.0. +summary: Introduce the Sysbench performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v7.5.0. +aliases: ['/tidbcloud/v7.5.0-performance-benchmarking-with-sysbench'] --- # TiDB Cloud Sysbench Performance Test Report for TiDB v7.5.0 -This document provides the Sysbench performance test steps and results for a TiDB Dedicated cluster with the TiDB version of v7.5.0. This report can also be used as a reference for the performance of TiDB Self-Hosted v7.5.0 clusters. +This document provides the Sysbench performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v7.5.0. This report can also be used as a reference for the performance of TiDB Self-Managed v7.5.0 clusters. ## Test overview @@ -17,15 +18,15 @@ This test aims at showing the Sysbench performance of TiDB v7.5.0 in the Online The test is conducted on a TiDB cluster with the following settings: -- Cluster type: [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) - Cluster version: v7.5.0 - Cloud provider: AWS (us-west-2) - Cluster configuration: - | Node type | Node size | Node quantity | Node storage | - |:----------|:----------|:----------|:----------| - | TiDB | 16 vCPU, 32 GiB | 2 | N/A | - | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + | Node type | Node size | Node quantity | Node storage | + | :-------- | :-------------- | :------------ | :----------- | + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | ### Parameter configuration @@ -59,13 +60,13 @@ The benchmark executor sends SQL queries to the TiDB cluster. In this test, its This section introduces how to perform the Sysbench performance test step by step. -1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. - For more information, see [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). 2. On the benchmark executor, connect to the newly created cluster and create a database named `sbtest`. - To connect to the cluster, see [Connect to TiDB Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). To create the `sbtest` database, execute the following SQL statement: @@ -118,58 +119,58 @@ This section introduces the Sysbench performance of v7.5.0 in the [test environm The performance on the `oltp_point_select` workload is as follows: -| Threads | TPS | 95% latency (ms)| -|:--------|:----------|:----------| -| 100 | 64,810 | 2.03 | -| 200 | 118,651 | 2.22 | -| 400 | 153,580 | 3.96 | +| Threads | TPS | 95% latency (ms) | +| :------ | :------ | :--------------- | +| 50 | 33,344 | 1.96 | +| 100 | 64,810 | 2.03 | +| 200 | 118,651 | 2.22 | -![Sysbench point select performance](/media/tidb-cloud/v7.5.0_oltp_point_select.png) +![Sysbench point select performance](/media/tidb-cloud/v7.5.0-oltp_point_select.png) ### Read write performance The performance on the `oltp_read_write` workload is as follows: -| Threads | TPS | 95% latency (ms)| -|:--------|:----------|:----------| -| 100 | 2,134 | 54.8 | -| 200 | 3,020 | 99.3 | -| 400 | 3,251 | 193 | +| Threads | TPS | 95% latency (ms) | +| :------ | :---- | :--------------- | +| 50 | 1,181 | 49.2 | +| 100 | 2,162 | 54.8 | +| 200 | 3,169 | 92.4 | -![Sysbench read write performance](/media/tidb-cloud/v7.5.0_oltp_read_write.png) +![Sysbench read write performance](/media/tidb-cloud/v7.5.0-oltp_read_write.png) ### Update non-index performance The performance on the `oltp_update_non_index` workload is as follows: -| Threads | TPS | 95% latency (ms)| -|:--------|:----------|:----------| -| 100 | 10,567 | 11.7 | -| 200 | 20,223 | 13.0 | -| 400 | 34,011 | 14.7 | +| Threads | TPS | 95% latency (ms) | +| :------ | :----- | :--------------- | +| 100 | 10,567 | 11.7 | +| 200 | 20,223 | 13.0 | +| 400 | 34,011 | 14.7 | -![Sysbench update non-index performance](/media/tidb-cloud/v7.5.0_oltp_update_non_index.png) +![Sysbench update non-index performance](/media/tidb-cloud/v7.5.0-oltp_update_non_index.png) ### Update index performance The performance on the `oltp_update_index` workload is as follows: -| Threads | TPS | 95% latency (ms)| -|:--------|:----------|:----------| -| 100 | 8,896 | 14.7 | -| 200 | 1,3718 | 19.0 | -| 400 | 2,0377 | 26.9 | +| Threads | TPS | 95% latency (ms) | +| :------ | :----- | :--------------- | +| 100 | 8,896 | 14.7 | +| 200 | 13,718 | 19.0 | +| 400 | 20,377 | 26.9 | -![Sysbench update index performance](/media/tidb-cloud/v7.5.0_oltp_update_index.png) +![Sysbench update index performance](/media/tidb-cloud/v7.5.0-oltp_update_index.png) ### Insert performance The performance on the `oltp_insert` workload is as follows: -| Threads | TPS | 95% latency (ms)| -|:--------|:----------|:----------| -| 100 | 15,132 | 8.58 | -| 200 | 24,756 | 10.8 | -| 400 | 37,247 | 16.4 | +| Threads | TPS | 95% latency (ms) | +| :------ | :----- | :--------------- | +| 100 | 15,132 | 8.58 | +| 200 | 24,756 | 10.8 | +| 400 | 37,247 | 16.4 | -![Sysbench insert performance](/media/tidb-cloud/v7.5.0_oltp_insert.png) +![Sysbench insert performance](/media/tidb-cloud/v7.5.0-oltp_insert.png) diff --git a/tidb-cloud/v7.5.0-performance-benchmarking-with-tpcc.md b/tidb-cloud/v7.5-performance-benchmarking-with-tpcc.md similarity index 73% rename from tidb-cloud/v7.5.0-performance-benchmarking-with-tpcc.md rename to tidb-cloud/v7.5-performance-benchmarking-with-tpcc.md index 545fb8eb8a62a..63ec70b77b0f1 100644 --- a/tidb-cloud/v7.5.0-performance-benchmarking-with-tpcc.md +++ b/tidb-cloud/v7.5-performance-benchmarking-with-tpcc.md @@ -1,11 +1,12 @@ --- title: TiDB Cloud TPC-C Performance Test Report for TiDB v7.5.0 -summary: Introduce the TPC-C performance test results for a TiDB Dedicated cluster with the TiDB version of v7.5.0. +summary: Introduce the TPC-C performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v7.5.0. +aliases: ['/tidbcloud/v7.5.0-performance-benchmarking-with-tpcc'] --- # TiDB Cloud TPC-C Performance Test Report for TiDB v7.5.0 -This document provides the TPC-C performance test steps and results for a TiDB Dedicated cluster with the TiDB version of v7.5.0. This report can also be used as a reference for the performance of TiDB Self-Hosted v7.5.0 clusters. +This document provides the TPC-C performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v7.5.0. This report can also be used as a reference for the performance of TiDB Self-Managed v7.5.0 clusters. ## Test overview @@ -17,15 +18,15 @@ This test aims at showing the TPC-C performance of TiDB v7.5.0 in the Online Tra The test is conducted on a TiDB cluster with the following settings: -- Cluster type: [TiDB Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-dedicated) +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) - Cluster version: v7.5.0 - Cloud provider: AWS (us-west-2) - Cluster configuration: - | Node type | Node size | Node quantity | Node storage | - |:----------|:----------|:----------|:----------| - | TiDB | 16 vCPU, 32 GiB | 2 | N/A | - | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + | Node type | Node size | Node quantity | Node storage | + | :-------- | :-------------- | :------------ | :----------- | + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | ### Benchmark executor @@ -38,13 +39,13 @@ The benchmark executor sends SQL queries to the TiDB cluster. In this test, its This section introduces how to perform the TPC-C performance test step by step. -1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. - For more information, see [Create a TiDB Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). 2. On the benchmark executor, connect to the newly created cluster and create a database named `tpcc`. - To connect to the cluster, see [Connect to TiDB Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). To create the `tpcc` database, execute the following SQL statement: @@ -88,7 +89,7 @@ This section introduces how to perform the TPC-C performance test step by step. SET tidb_index_serial_scan_concurrency=16; ``` -5. Run the following `go-tpc tpcc` command to conduct stress tests on the TiDB Dedicated cluster. For each concurrency, the test takes two hours. +5. Run the following `go-tpc tpcc` command to conduct stress tests on the TiDB Cloud Dedicated cluster. For each concurrency, the test takes two hours. ```shell go-tpc tpcc --host ${HOST} -P 4000 --warehouses 1000 run -D tpcc -T ${THREAD} --time 2h0m0s -p ${PASSWORD} --ignore-error @@ -102,10 +103,10 @@ This section introduces how to perform the TPC-C performance test step by step. The TPC-C performance of v7.5.0 in the [test environment](#test-environment) is as follows: -| Threads | v7.5.0 tpmC | -|:--------|:----------| -| 50 | 37,443 | -| 100 | 67,899 | -| 200 | 93,038 | +| Threads | v7.5.0 tpmC | +| :------ | :---------- | +| 50 | 41,426 | +| 100 | 71,499 | +| 200 | 97,389 | ![TPC-C](/media/tidb-cloud/v7.5.0_tpcc.png) diff --git a/tidb-cloud/v8.1-performance-benchmarking-with-sysbench.md b/tidb-cloud/v8.1-performance-benchmarking-with-sysbench.md new file mode 100644 index 0000000000000..b3bd825524fa2 --- /dev/null +++ b/tidb-cloud/v8.1-performance-benchmarking-with-sysbench.md @@ -0,0 +1,181 @@ +--- +title: TiDB Cloud Sysbench Performance Test Report for TiDB v8.1.0 +summary: Introduce the Sysbench performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v8.1.0. +--- + +# TiDB Cloud Sysbench Performance Test Report for TiDB v8.1.0 + +This document provides the Sysbench performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v8.1.0. This report can also be used as a reference for the performance of TiDB Self-Managed v8.1.0 clusters. + +## Test overview + +This test aims at showing the Sysbench performance of TiDB v8.1.0 in the Online Transactional Processing (OLTP) scenario. + +## Test environment + +### TiDB cluster + +The test is conducted on a TiDB cluster with the following settings: + +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) +- Cluster version: v8.1.0 +- Cloud provider: AWS (us-west-2) +- Cluster configuration: + + | Node type | Node size | Node quantity | Node storage | + |:----------|:----------|:----------|:----------| + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + +### Parameter configuration + +The system variable [`tidb_session_plan_cache_size`](https://docs.pingcap.com/tidb/stable/system-variables#tidb_session_plan_cache_size-new-in-v710) controls the maximum number of plans that can be cached. The default value is `100`. For each workload, this document conducts tests with `tidb_session_plan_cache_size` set to `1000`: + +```sql +SET GLOBAL tidb_session_plan_cache_size = 1000; +``` + +> **Note:** +> +> For TiDB Cloud, to modify the TiKV parameters of your cluster, you can contact [PingCAP Support](/tidb-cloud/tidb-cloud-support.md) for help. + +The TiKV parameter [`prefill-for-recycle`](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#prefill-for-recycle-new-in-v700) can make log recycling effective immediately after initialization. This document conducts tests based on different workloads with the following `prefill-for-recycle` configuration: + +- For the `oltp_point_select` workload, use the default value of the [`prefill-for-recycle`](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#prefill-for-recycle-new-in-v700) parameter: + + ```yaml + raft-engine.prefill-for-recycle = false + ``` + +- For `oltp_insert`, `oltp_read_write`, `oltp_update_index`, and `oltp_update_non_index` workloads, enable the [`prefill-for-recycle`](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#prefill-for-recycle-new-in-v700) parameter: + + ```yaml + raft-engine.prefill-for-recycle = true + ``` + +### Benchmark executor + +The benchmark executor sends SQL queries to the TiDB cluster. In this test, its hardware configuration is as follows: + +- Machine type: Amazon EC2 (us-west-2) +- Instance type: c6a.2xlarge +- Sysbench version: sysbench 1.0.20 (using bundled LuaJIT 2.1.0-beta2) + +## Test steps + +This section introduces how to perform the Sysbench performance test step by step. + +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. + + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + +2. On the benchmark executor, connect to the newly created cluster and create a database named `sbtest`. + + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + + To create the `sbtest` database, execute the following SQL statement: + + ```sql + CREATE DATABASE sbtest; + ``` + +3. Load Sysbench data to the `sbtest` database. + + 1. The test in this document is implemented based on [sysbench](https://github.com/akopytov/sysbench). To install sysbench, see [Building and installing from source](https://github.com/akopytov/sysbench#building-and-installing-from-source). + + 2. Run the following `sysbench prepare` command to import 32 tables and 10,000,000 rows to the `sbtest` database. Replace `${HOST}`, `${PORT}`, `${THREAD}`, and `${PASSWORD}` with your actual values. + + ```shell + sysbench oltp_common \ + --threads=${THREAD} \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --mysql-host=${HOST} \ + --mysql-port=${PORT} \ + --mysql-user=root \ + --mysql-password=${PASSWORD} \ + prepare --tables=32 --table-size=10000000 + ``` + +4. Run the following `sysbench run` command to conduct Sysbench performance tests on different workloads. This document conducts tests on five workloads: `oltp_point_select`, `oltp_read_write`, `oltp_update_non_index`, `oltp_update_index`, and `oltp_insert`. For each workload, this document conducts three tests with different values for the `${THREAD}` variable. For `oltp_point_select` and `oltp_read_write`, the values are `50`, `100`, and `200`. For other workloads, the values are `100`, `200`, and `400`. For each concurrency, the test takes 20 minutes. + + ```shell + sysbench ${WORKLOAD} run \ + --mysql-host=${HOST} \ + --mysql-port=${PORT} \ + --mysql-user=root \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --threads=${THREAD} \ + --time=1200 \ + --report-interval=10 \ + --tables=32 \ + --table-size=10000000 \ + --mysql-ignore-errors=1062,2013,8028,9007 \ + --auto-inc=false \ + --mysql-password=${PASSWORD} + ``` + +## Test results + +This section introduces the Sysbench performance of v8.1.0 in the [test environment](#test-environment). + +### Point select performance + +The performance on the `oltp_point_select` workload is as follows: + +| Threads | TPS | 95% latency (ms)| +|:--------|:----------|:----------| +| 50 | 32,741 | 1.96 | +| 100 | 62,545 | 2.03 | +| 200 | 111,470 | 2.48 | + +![Sysbench point select performance](/media/tidb-cloud/v8.1.0_oltp_point_select.png) + +### Read write performance + +The performance on the `oltp_read_write` workload is as follows: + +| Threads | TPS | 95% latency (ms)| +|:--------|:----------|:----------| +| 50 | 1,232 | 46.6 | +| 100 | 2,341 | 51 | +| 200 | 3,240 | 109 | + +![Sysbench read write performance](/media/tidb-cloud/v8.1.0_oltp_read_write.png) + +### Update non-index performance + +The performance on the `oltp_update_non_index` workload is as follows: + +| Threads | TPS | 95% latency (ms)| +|:--------|:----------|:----------| +| 100 | 14,000 | 9.39 | +| 200 | 25,215 | 10.5 | +| 400 | 42,550 | 12.8 | + +![Sysbench update non-index performance](/media/tidb-cloud/v8.1.0_oltp_update_non_index.png) + +### Update index performance + +The performance on the `oltp_update_index` workload is as follows: + +| Threads | TPS | 95% latency (ms)| +|:--------|:----------|:----------| +| 100 | 11,188 | 11.7 | +| 200 | 17,805 | 14.7 | +| 400 | 24,575 | 23.5 | + +![Sysbench update index performance](/media/tidb-cloud/v8.1.0_oltp_update_index.png) + +### Insert performance + +The performance on the `oltp_insert` workload is as follows: + +| Threads | TPS | 95% latency (ms)| +|:--------|:----------|:----------| +| 100 | 18,339 | 7.3| +| 200 | 29,387 | 9.73 | +| 400 | 42,712 | 14.2 | + +![Sysbench insert performance](/media/tidb-cloud/v8.1.0_oltp_insert.png) diff --git a/tidb-cloud/v8.1-performance-benchmarking-with-tpcc.md b/tidb-cloud/v8.1-performance-benchmarking-with-tpcc.md new file mode 100644 index 0000000000000..8aad399883823 --- /dev/null +++ b/tidb-cloud/v8.1-performance-benchmarking-with-tpcc.md @@ -0,0 +1,123 @@ +--- +title: TiDB Cloud TPC-C Performance Test Report for TiDB v8.1.0 +summary: Introduce the TPC-C performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v8.1.0. +--- + +# TiDB Cloud TPC-C Performance Test Report for TiDB v8.1.0 + +This document provides the TPC-C performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v8.1.0. This report can also be used as a reference for the performance of TiDB Self-Managed v8.1.0 clusters. + +## Test overview + +This test aims at showing the TPC-C performance of TiDB v8.1.0 in the Online Transactional Processing (OLTP) scenario. + +## Test environment + +### TiDB cluster + +The test is conducted on a TiDB cluster with the following settings: + +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) +- Cluster version: v8.1.0 +- Cloud provider: AWS (us-west-2) +- Cluster configuration: + + | Node type | Node size | Node quantity | Node storage | + |:----------|:----------|:----------|:----------| + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + +### Parameter configuration + +> **Note:** +> +> For TiDB Cloud, to modify the TiKV parameters of your cluster, you can contact [PingCAP Support](/tidb-cloud/tidb-cloud-support.md) for help. + +The TiKV parameter [`prefill-for-recycle`](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#prefill-for-recycle-new-in-v700) can make log recycling effective immediately after initialization. This document conducts tests with `prefill-for-recycle` enabled: + +```yaml +raft-engine.prefill-for-recycle = true +``` + +### Benchmark executor + +The benchmark executor sends SQL queries to the TiDB cluster. In this test, its hardware configuration is as follows: + +- Machine type: Amazon EC2 (us-west-2) +- Instance type: c6a.2xlarge + +## Test steps + +This section introduces how to perform the TPC-C performance test step by step. + +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. + + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + +2. On the benchmark executor, connect to the newly created cluster and create a database named `tpcc`. + + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + + To create the `tpcc` database, execute the following SQL statement: + + ```sql + CREATE DATABASE tpcc; + ``` + +3. Load TPC-C data to the `tpcc` database. + + 1. The test in this document is implemented based on [go-tpc](https://github.com/pingcap/go-tpc). You can download the test program using the following command: + + ```shell + curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/pingcap/go-tpc/master/install.sh | sh + ``` + + 2. Run the following `go-tpc tpcc` command to import 1,000 warehouses to the `tpcc` database. Replace `${HOST}`, `${THREAD}`, and `${PASSWORD}` with your actual values. This document conducts three tests with the `${THREAD}` value of `50`, `100`, and `200`. + + ```shell + go-tpc tpcc --host ${HOST} --warehouses 1000 prepare -P 4000 -D tpcc -T ${THREAD} --time 2h0m0s -p ${PASSWORD} --ignore-error + ``` + +4. To ensure that the TiDB optimizer can generate the optimal execution plan, execute the following SQL statements to collect statistics before conducting the TPC-C test: + + ```sql + ANALYZE TABLE customer; + ANALYZE TABLE district; + ANALYZE TABLE history; + ANALYZE TABLE item; + ANALYZE TABLE new_order; + ANALYZE TABLE order_line; + ANALYZE TABLE orders; + ANALYZE TABLE stock; + ANALYZE TABLE warehouse; + ``` + + To accelerate the collection of statistics, execute the following SQL statements before collecting: + + ```sql + SET tidb_build_stats_concurrency=16; + SET tidb_distsql_scan_concurrency=16; + SET tidb_index_serial_scan_concurrency=16; + ``` + +5. Run the following `go-tpc tpcc` command to conduct stress tests on the TiDB Cloud Dedicated cluster. For each concurrency, the test takes two hours. + + ```shell + go-tpc tpcc --host ${HOST} -P 4000 --warehouses 1000 run -D tpcc -T ${THREAD} --time 2h0m0s -p ${PASSWORD} --ignore-error + ``` + +6. Extract the tpmC data of `NEW_ORDER` from the result. + + TPC-C uses tpmC (transactions per minute) to measure the maximum qualified throughput (MQTh, Max Qualified Throughput). The transactions are the NewOrder transactions and the final unit of measure is the number of new orders processed per minute. + +## Test results + +The TPC-C performance of v8.1.0 in the [test environment](#test-environment) is as follows: + +| Threads | v8.1.0 tpmC | +|:--------|:----------| +| 50 | 43,660 | +| 100 | 75,495 | +| 200 | 102,013 | + +![TPC-C](/media/tidb-cloud/v8.1.0_tpcc.png) diff --git a/tidb-cloud/v8.5-performance-benchmarking-with-sysbench.md b/tidb-cloud/v8.5-performance-benchmarking-with-sysbench.md new file mode 100644 index 0000000000000..add0417dac31d --- /dev/null +++ b/tidb-cloud/v8.5-performance-benchmarking-with-sysbench.md @@ -0,0 +1,181 @@ +--- +title: TiDB Cloud Sysbench Performance Test Report for TiDB v8.5.0 +summary: Introduce the Sysbench performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v8.5.0. +--- + +# TiDB Cloud Sysbench Performance Test Report for TiDB v8.5.0 + +This document provides the Sysbench performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v8.5.0. This report can also be used as a reference for the performance of TiDB Self-Managed v8.5.0 clusters. + +## Test overview + +This test aims at showing the Sysbench performance of TiDB v8.5.0 in the Online Transactional Processing (OLTP) scenario. + +## Test environment + +### TiDB cluster + +The test is conducted on a TiDB cluster with the following settings: + +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) +- Cluster version: v8.5.0 +- Cloud provider: AWS (us-west-2) +- Cluster configuration: + + | Node type | Node size | Node quantity | Node storage | + |:----------|:----------|:----------|:----------| + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + +### Parameter configuration + +The system variable [`tidb_session_plan_cache_size`](https://docs.pingcap.com/tidb/stable/system-variables#tidb_session_plan_cache_size-new-in-v710) controls the maximum number of plans that can be cached. The default value is `100`. For each workload, this document conducts tests with `tidb_session_plan_cache_size` set to `1000`: + +```sql +SET GLOBAL tidb_session_plan_cache_size = 1000; +``` + +> **Note:** +> +> For TiDB Cloud, to modify the TiKV parameters of your cluster, you can contact [PingCAP Support](/tidb-cloud/tidb-cloud-support.md) for help. + +The TiKV parameter [`prefill-for-recycle`](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#prefill-for-recycle-new-in-v700) can make log recycling effective immediately after initialization. This document conducts tests based on different workloads with the following `prefill-for-recycle` configuration: + +- For the `oltp_point_select` workload, use the default value of the [`prefill-for-recycle`](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#prefill-for-recycle-new-in-v700) parameter: + + ```yaml + raft-engine.prefill-for-recycle = false + ``` + +- For `oltp_insert`, `oltp_read_write`, `oltp_update_index`, and `oltp_update_non_index` workloads, enable the [`prefill-for-recycle`](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#prefill-for-recycle-new-in-v700) parameter: + + ```yaml + raft-engine.prefill-for-recycle = true + ``` + +### Benchmark executor + +The benchmark executor sends SQL queries to the TiDB cluster. In this test, its hardware configuration is as follows: + +- Machine type: Amazon EC2 (us-west-2) +- Instance type: c6a.2xlarge +- Sysbench version: sysbench 1.0.20 (using bundled LuaJIT 2.1.0-beta2) + +## Test steps + +This section introduces how to perform the Sysbench performance test step by step. + +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. + + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + +2. On the benchmark executor, connect to the newly created cluster and create a database named `sbtest`. + + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + + To create the `sbtest` database, execute the following SQL statement: + + ```sql + CREATE DATABASE sbtest; + ``` + +3. Load Sysbench data to the `sbtest` database. + + 1. The test in this document is implemented based on [sysbench](https://github.com/akopytov/sysbench). To install sysbench, see [Building and installing from source](https://github.com/akopytov/sysbench#building-and-installing-from-source). + + 2. Run the following `sysbench prepare` command to import 32 tables and 10,000,000 rows to the `sbtest` database. Replace `${HOST}`, `${PORT}`, `${THREAD}`, and `${PASSWORD}` with your actual values. + + ```shell + sysbench oltp_common \ + --threads=${THREAD} \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --mysql-host=${HOST} \ + --mysql-port=${PORT} \ + --mysql-user=root \ + --mysql-password=${PASSWORD} \ + prepare --tables=32 --table-size=10000000 + ``` + +4. Run the following `sysbench run` command to conduct Sysbench performance tests on different workloads. This document conducts tests on five workloads: `oltp_point_select`, `oltp_read_write`, `oltp_update_non_index`, `oltp_update_index`, and `oltp_insert`. For each workload, this document conducts three tests with different values for the `${THREAD}` variable. For `oltp_point_select` and `oltp_read_write`, the values are `50`, `100`, and `200`. For other workloads, the values are `100`, `200`, and `400`. For each concurrency, the test takes 20 minutes. + + ```shell + sysbench ${WORKLOAD} run \ + --mysql-host=${HOST} \ + --mysql-port=${PORT} \ + --mysql-user=root \ + --db-driver=mysql \ + --mysql-db=sbtest \ + --threads=${THREAD} \ + --time=1200 \ + --report-interval=10 \ + --tables=32 \ + --table-size=10000000 \ + --mysql-ignore-errors=1062,2013,8028,9007 \ + --auto-inc=false \ + --mysql-password=${PASSWORD} + ``` + +## Test results + +This section introduces the Sysbench performance of v8.5.0 in the [test environment](#test-environment). + +### Point select performance + +The performance on the `oltp_point_select` workload is as follows: + +| Threads | TPS | 95% latency (ms)| +|:--------|:----------|:----------| +| 50 | 32,733 | 1.94 | +| 100 | 62,123 | 2.04 | +| 200 | 107,795 | 2.56 | + +![Sysbench point select performance](/media/tidb-cloud/v8.5.0_oltp_point_select.png) + +### Read write performance + +The performance on the `oltp_read_write` workload is as follows: + +| Threads | TPS | 95% latency (ms)| +|:--------|:----------|:----------| +| 50 | 1,258 | 45.8 | +| 100 | 2,243 | 52 | +| 200 | 3,187 | 94.7 | + +![Sysbench read write performance](/media/tidb-cloud/v8.5.0_oltp_read_write.png) + +### Update non-index performance + +The performance on the `oltp_update_non_index` workload is as follows: + +| Threads | TPS | 95% latency (ms)| +|:--------|:----------|:----------| +| 100 | 12,969 | 10.10 | +| 200 | 23,265 | 11.2 | +| 400 | 39,547 | 13.2 | + +![Sysbench update non-index performance](/media/tidb-cloud/v8.5.0_oltp_update_non_index.png) + +### Update index performance + +The performance on the `oltp_update_index` workload is as follows: + +| Threads | TPS | 95% latency (ms)| +|:--------|:----------|:----------| +| 100 | 11,678 | 11.0 | +| 200 | 17,685 | 14.8 | +| 400 | 25,369 | 20.4 | + +![Sysbench update index performance](/media/tidb-cloud/v8.5.0_oltp_update_index.png) + +### Insert performance + +The performance on the `oltp_insert` workload is as follows: + +| Threads | TPS | 95% latency (ms)| +|:--------|:----------|:----------| +| 100 | 18,409 | 7.28| +| 200 | 30,943 | 8.28 | +| 400 | 48,621 | 11.2 | + +![Sysbench insert performance](/media/tidb-cloud/v8.5.0_oltp_insert.png) diff --git a/tidb-cloud/v8.5-performance-benchmarking-with-tpcc.md b/tidb-cloud/v8.5-performance-benchmarking-with-tpcc.md new file mode 100644 index 0000000000000..53abd9096c0cb --- /dev/null +++ b/tidb-cloud/v8.5-performance-benchmarking-with-tpcc.md @@ -0,0 +1,123 @@ +--- +title: TiDB Cloud TPC-C Performance Test Report for TiDB v8.5.0 +summary: Introduce the TPC-C performance test results for a TiDB Cloud Dedicated cluster with the TiDB version of v8.5.0. +--- + +# TiDB Cloud TPC-C Performance Test Report for TiDB v8.5.0 + +This document provides the TPC-C performance test steps and results for a TiDB Cloud Dedicated cluster with the TiDB version of v8.5.0. This report can also be used as a reference for the performance of TiDB Self-Managed v8.5.0 clusters. + +## Test overview + +This test aims at showing the TPC-C performance of TiDB v8.5.0 in the Online Transactional Processing (OLTP) scenario. + +## Test environment + +### TiDB cluster + +The test is conducted on a TiDB cluster with the following settings: + +- Cluster type: [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated) +- Cluster version: v8.5.0 +- Cloud provider: AWS (us-west-2) +- Cluster configuration: + + | Node type | Node size | Node quantity | Node storage | + |:----------|:----------|:----------|:----------| + | TiDB | 16 vCPU, 32 GiB | 2 | N/A | + | TiKV | 16 vCPU, 64 GiB | 3 | 1000 GiB | + +### Parameter configuration + +> **Note:** +> +> For TiDB Cloud, to modify the TiKV parameters of your cluster, you can contact [PingCAP Support](/tidb-cloud/tidb-cloud-support.md) for help. + +The TiKV parameter [`prefill-for-recycle`](https://docs.pingcap.com/tidb/stable/tikv-configuration-file#prefill-for-recycle-new-in-v700) can make log recycling effective immediately after initialization. This document conducts tests with `prefill-for-recycle` enabled: + +```yaml +raft-engine.prefill-for-recycle = true +``` + +### Benchmark executor + +The benchmark executor sends SQL queries to the TiDB cluster. In this test, its hardware configuration is as follows: + +- Machine type: Amazon EC2 (us-west-2) +- Instance type: c6a.2xlarge + +## Test steps + +This section introduces how to perform the TPC-C performance test step by step. + +1. In the [TiDB Cloud console](https://tidbcloud.com/), create a TiDB Cloud Dedicated cluster that meets the [test environment](#tidb-cluster) requirements. + + For more information, see [Create a TiDB Cloud Dedicated cluster](/tidb-cloud/create-tidb-cluster.md). + +2. On the benchmark executor, connect to the newly created cluster and create a database named `tpcc`. + + To connect to the cluster, see [Connect to TiDB Cloud Dedicated via Private Endpoint](/tidb-cloud/set-up-private-endpoint-connections.md). + + To create the `tpcc` database, execute the following SQL statement: + + ```sql + CREATE DATABASE tpcc; + ``` + +3. Load TPC-C data to the `tpcc` database. + + 1. The test in this document is implemented based on [go-tpc](https://github.com/pingcap/go-tpc). You can download the test program using the following command: + + ```shell + curl --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/pingcap/go-tpc/master/install.sh | sh + ``` + + 2. Run the following `go-tpc tpcc` command to import 1,000 warehouses to the `tpcc` database. Replace `${HOST}`, `${THREAD}`, and `${PASSWORD}` with your actual values. This document conducts three tests with the `${THREAD}` value of `50`, `100`, and `200`. + + ```shell + go-tpc tpcc --host ${HOST} --warehouses 1000 prepare -P 4000 -D tpcc -T ${THREAD} --time 2h0m0s -p ${PASSWORD} --ignore-error + ``` + +4. To ensure that the TiDB optimizer can generate the optimal execution plan, execute the following SQL statements to collect statistics before conducting the TPC-C test: + + ```sql + ANALYZE TABLE customer; + ANALYZE TABLE district; + ANALYZE TABLE history; + ANALYZE TABLE item; + ANALYZE TABLE new_order; + ANALYZE TABLE order_line; + ANALYZE TABLE orders; + ANALYZE TABLE stock; + ANALYZE TABLE warehouse; + ``` + + To accelerate the collection of statistics, execute the following SQL statements before collecting: + + ```sql + SET tidb_build_stats_concurrency=16; + SET tidb_distsql_scan_concurrency=16; + SET tidb_index_serial_scan_concurrency=16; + ``` + +5. Run the following `go-tpc tpcc` command to conduct stress tests on the TiDB Cloud Dedicated cluster. For each concurrency, the test takes two hours. + + ```shell + go-tpc tpcc --host ${HOST} -P 4000 --warehouses 1000 run -D tpcc -T ${THREAD} --time 2h0m0s -p ${PASSWORD} --ignore-error + ``` + +6. Extract the tpmC data of `NEW_ORDER` from the result. + + TPC-C uses tpmC (transactions per minute) to measure the maximum qualified throughput (MQTh, Max Qualified Throughput). The transactions are the NewOrder transactions and the final unit of measure is the number of new orders processed per minute. + +## Test results + +The TPC-C performance of v8.5.0 in the [test environment](#test-environment) is as follows: + +| Threads | v8.5.0 tpmC | +|:--------|:----------| +| 50 | 43,146 | +| 100 | 73,875 | +| 200 | 103,395 | + +![TPC-C](/media/tidb-cloud/v8.5.0_tpcc.png) diff --git a/tidb-cloud/v8.5-performance-highlights.md b/tidb-cloud/v8.5-performance-highlights.md new file mode 100644 index 0000000000000..9deb7dab0a52f --- /dev/null +++ b/tidb-cloud/v8.5-performance-highlights.md @@ -0,0 +1,248 @@ +--- +title: TiDB Cloud Performance Highlights for TiDB v8.5.0 +summary: Introduce the performance improvements for TiDB Cloud Dedicated clusters with the TiDB version of v8.5.0. +--- + +# TiDB Cloud Performance Highlights for TiDB v8.5.0 + +[TiDB v8.5.0](https://docs.pingcap.com/tidb/v8.5/release-8.5.0) is an important Long-Term Support (LTS) release, which delivers notable improvements in performance, scalability, and operational efficiency. + +This document outlines the performance improvements in v8.5.0 across the following areas: + +- General performance +- Hotspot reads with excessive MVCC versions +- IO latency jitter +- Batch processing +- TiKV scaling performance + +## General performance + +With the default Region size increased from 96 MiB to 256 MiB and some other improvements in v8.5.0, significant performance improvements are observed: + +- `oltp_insert` performance improves by 27%. +- `Analyze` performance shows a major boost of approximately 45%. + +## Hotspot reads with excessive MVCC versions + +### Challenge + +In some user scenarios, the following challenges might occur: + +- Frequent version updates: in some workloads, the data is updated and read very frequently. +- Long retention of historical versions: to meet business requirements, such as supporting flashback to specific time points, users might configure overly long GC (Garbage Collection) times (such as 24 hours). This results in an excessive accumulation of Multi-Version Concurrency Control (MVCC) versions, which significantly reduces query efficiency. + +The accumulation of MVCC versions creates a large gap between the requested data and the processed data, leading to degraded read performance. + +### Solution + +To address this issue, TiKV introduces [In-Memory Engine (IME)](https://docs.pingcap.com/tidb/stable/tikv-in-memory-engine). By caching the latest versions in memory, TiKV reduces the impact of historical versions on read performance, significantly improving query efficiency. + +### Test environment + +- Cluster topology: TiDB (16 vCPU, 32 GiB) \* 1 + TiKV (16 vCPU, 32 GiB) \* 6 +- Cluster configurations: + + ``` + tikv_configs: + [in-memory-engine] + enable = true + ``` + +- Dataset: 24 GiB storage size with about 340 Regions, containing frequently updated data +- Workload: rows requiring scans contain numerous MVCC versions introduced by frequent updates + +### Test results + +The query latency decreases by 50%, and the throughput increases by 100%. + +| Configuration | Duration (s) | Threads | TPS | QPS | Average latency (ms) | P95 latency (ms) | +| --- | --- | --- | --- | --- | --- | --- | +| Disable IME | 3600 | 10 | 123.8 | 1498.3 | 80.76 | 207.82 | +| Enable IME | 3600 | 10 | 238.2 | 2881.5 | 41.99 | 78.60 | + +## IO latency jitter + +### Challenge + +In cloud environments, transient or sustained IO latency fluctuations on cloud disks are a common challenge. These fluctuations can increase request latency, causing timeouts, errors, and disruptions to normal business operations, ultimately degrading service quality. + +### Solution + +TiDB v8.5.0 introduces multiple enhancements to mitigate the impact of cloud disk IO jitter on performance: + +- **Leader write optimization**: allows the leader to apply committed but not yet persisted Raft logs early, reducing IO jitter's effect on write latency for the leader peer. +- **Enhanced slow-node detection**: improves the slow-node detection algorithm, enabling slow score detection by default. This triggers the evict-leader scheduler to restore performance when a slow node is identified. The [slow node detection mechanism](https://docs.pingcap.com/tidb/v8.5/pd-scheduling-best-practices#troubleshoot-tikv-node) uses [evict-slow-store-scheduler](https://docs.pingcap.com/tidb/v8.5/pd-control#scheduler-show--add--remove--pause--resume--config--describe) to detect and manage slow nodes, reducing the effects of cloud disk jitter. + +- **Unified health controller**: adds a unified health controller in TiKV and a feedback mechanism to the KV client. The KV client optimizes error handling and replica selection based on TiKV node health and performance. +- **Improved replica selector**: introduces Replica Selector V2 in the KV client, with refined state transitions that eliminate unnecessary retries and backoff operations. +- **Additional fixes and improvements**: includes enhancements to critical components such as the region cache and KV client health checker, while avoiding unnecessary IO operations in TiKV's store loop. + +### Test environment + +- Cluster topology: TiDB (32 vCPU, 64 GiB) \* 3 + TiKV (32 vCPU, 64 GiB) \* 6 +- Workload: a read/write ratio of 2:1, with simulated cloud disk IO delays or hangs on one TiKV node + +### Test results + +Based on the preceding test setup, failovers are now improved in multiple IO delay scenarios, and P99/999 latency during impacts is reduced by up to 98%. + +In the following table of test results, the **Current** column shows the results with improvements to reduce IO latency jitter, while the **Original** column shows the results without these improvements: + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Workload descriptionFailover timeMaximum latency during impacts (P999)Maximum latency during impacts (P99)
CurrentOriginalCurrentOriginalCurrentOriginal
IO delay of 2 ms lasts for 10 minsAlmost no effectFailover not available14 ms232 ms7.9 ms22.9 ms
IO delay of 5 ms lasts for 10 mins2 minsFailover not available37.9 ms462 ms10 ms246 ms
IO delay of 10 ms lasts for 10 mins3 minsFailover not available69 ms3 s25 ms1.45 s
IO delay of 50 ms lasts for 10 mins3 minsFailover not available1.36 s13.2 s238 ms6.7 s
IO delay of 100 ms lasts for 10 mins3 minsFailover not available7.53 s32 s1.7 s26 s
+ +### Further improvements + +The severity of disk jitter might also be highly related to users' workload profiles. In latency-sensitive scenarios, designing applications in conjunction with TiDB features can further minimize the impact of IO jitter on applications. For example, in read-heavy and latency-sensitive environments, adjusting the [`tikv_client_read_timeout`](/system-variables.md#tikv_client_read_timeout-new-in-v740) system variable according to latency requirements and using stale reads or follower reads can enable faster failover retries to other replica peers for KV requests sent from TiDB. This reduces the impact of IO jitter on a single TiKV node and helps improve query latency. Note that the effectiveness of this feature depends on the workload profile, which should be evaluated before implementation. + +Additionally, users [deploying TiDB on public cloud](https://docs.pingcap.com/tidb/dev/best-practices-on-public-cloud) can reduce the probability of jitter by choosing cloud disks with higher performance. + +## Batch processing + +### Challenge + +Large-scale transactions, such as bulk data updates, system migrations, and ETL workflows, involve processing millions of rows and are vital for supporting critical operations. While TiDB excels as a distributed SQL database, handling such transactions at scale presents two significant challenges: + +- Memory limits: in versions earlier than TiDB v8.1.0, all transaction mutations are held in memory throughout the transaction lifecycle, which strains resources and reduces performance. For operations involving millions of rows, this could lead to excessive memory usage and, in some cases, Out of Memory (OOM) errors when resources are insufficient. + +- Performance slowdowns: managing large in-memory buffers relies on red-black trees, which introduces computational overhead. As buffers grow, their operations slow down due to the *O(N log N)* complexity inherent in these data structures. + +### Solution + +These challenges highlight a clear opportunity to improve scalability, reduce complexity, and enhance reliability. With the rise of modern data workloads, TiDB introduces the [Pipelined DML](https://docs.pingcap.com/tidb/stable/system-variables#tidb_dml_type-new-in-v800) feature, designed to optimize the handling of large transactions, and improve resource utilization and overall performance. + +### Test environment + +- Cluster topology: TiDB (16 vCPU, 32 GiB) \* 1 + TiKV (16 vCPU, 32 GiB) \* 3 +- Dataset: YCSB non-clustered table with 10 million rows (about 10 GiB of data). Primary keys are selectively removed in certain tests to isolate and evaluate the impact of hotspot patterns. +- Workload: DML operations including `INSERT`, `UPDATE`, and `DELETE`. + +### Test results + +The execution speed increases by 2x, the maximum TiDB memory usage decreases by 50%, and the TiKV write flow becomes much smoother. + +- Latency (in seconds) + + | Workload (10 GiB) | Standard DML | Pipelined DML | Improvement | + | --- | --- | --- | --- | + | YCSB-insert-10M | 368 | 159 | 131.45% | + | YCSB-update-10M | 255 | 131 | 94.66% | + | YCSB-delete-10M | 136 | 42 | 223.81% | + +- TiDB memory usage peak (GiB) + + | Workload (10 GiB) | Standard DML | Pipelined DML | Reduction | + | --- | --- | --- | --- | + | YCSB-insert-10M | 25.8 | 12 | 53.49% | + | YCSB-update-10M | 23.1 | 12.9 | 44.16% | + | YCSB-delete-10M | 10.1 | 8.08 | 20.00% | + +## TiKV scaling performance + +### Challenge + +Horizontal scaling is a core capability of TiKV, enabling the system to scale in or out as needed. As business demands grow and the number of tenants increases, TiDB clusters experience rapid growth in databases, tables, and data volume. Scaling out TiKV nodes quickly becomes essential to maintaining service quality. + +In some scenarios, TiDB hosts a large number of databases and tables. When these tables are small or empty, TiKV accumulates a significant number of tiny Regions, especially when the number of tables grows to a large scale (such as 1 million or more). These small Regions introduce a substantial maintenance burden, increase resource overhead, and reduce efficiency. + +### Solution + +To address this issue, TiDB v8.5.0 improves the performance of merging small Regions, reducing internal overhead and improving resource utilization. Additionally, TiDB v8.5.0 includes several other enhancements to further improve TiKV scaling performance. + +### Test environment + +#### Merging small Regions + +- Cluster topology: TiDB (16 vCPU, 32 GiB) \* 1 + TiKV (16 vCPU, 32 GiB) \* 3 +- Dataset: nearly 1 million small tables, with the size of each table < 2 MiB +- Workload: automatic merging of small Regions + +#### Scaling out TiKV nodes + +- Cluster topology: TiDB (16 vCPU, 32 GiB) \* 1 + TiKV (16 vCPU, 32 GiB) \* 4 +- Dataset: TPC-C dataset with 20,000 warehouses +- Workload: scaling out TiKV nodes from 4 to 7 + +### Test results + +The small Region merge speed improves by about 10 times. + +| Metric | Without improvement | With improvement | +| --- | --- | --- | +| Region merging duration (hrs) | 20 | 2 | + +TiKV scaling performance improves by over 40%, and TiKV node scaling out duration gains a 30% reduction. + +| Metric | Without improvement | With improvement | +| --- | --- | --- | +| TiKV scaling out duration (hrs) | 5 | 3.5 | + +## Benchmark + +In addition to the preceding test data, you can refer to the following benchmark results for v8.5.0 performance: + +- [TPC-C performance test report](/tidb-cloud/v8.5-performance-benchmarking-with-tpcc.md) +- [Sysbench performance test report](/tidb-cloud/v8.5-performance-benchmarking-with-sysbench.md) diff --git a/tidb-cloud/vector-search-changelogs.md b/tidb-cloud/vector-search-changelogs.md new file mode 100644 index 0000000000000..7333993d71ea0 --- /dev/null +++ b/tidb-cloud/vector-search-changelogs.md @@ -0,0 +1,14 @@ +--- +title: Vector Search Changelogs +summary: Learn about the new features, compatibility changes, improvements, and bug fixes for the TiDB Vector Search feature. +--- + +# Vector Search Changelogs + +## June 25, 2024 + +- TiDB Vector Search (beta) is now available for TiDB Cloud Serverless clusters in all regions for all users. + +## April 1, 2024 + +- TiDB Vector Search (beta) is now available for TiDB Cloud Serverless clusters in EU regions for invited users. diff --git a/tidb-cloud/vector-search-data-types.md b/tidb-cloud/vector-search-data-types.md new file mode 100644 index 0000000000000..0c390ab6f03e5 --- /dev/null +++ b/tidb-cloud/vector-search-data-types.md @@ -0,0 +1,246 @@ +--- +title: Vector Data Types +summary: Learn about the Vector data types in TiDB. +--- + +# Vector Data Types + +A vector is a sequence of floating-point numbers, such as `[0.3, 0.5, -0.1, ...]`. TiDB offers Vector data types, specifically optimized for efficiently storing and querying vector embeddings widely used in AI applications. + +The following Vector data types are currently available: + +- `VECTOR`: A sequence of single-precision floating-point numbers with any dimension. +- `VECTOR(D)`: A sequence of single-precision floating-point numbers with a fixed dimension `D`. + +Using vector data types provides the following advantages over using the [`JSON`](/data-type-json.md) type: + +- Vector index support: You can build a [vector search index](/tidb-cloud/vector-search-index.md) to speed up vector searching. +- Dimension enforcement: You can specify a dimension to forbid inserting vectors with different dimensions. +- Optimized storage format: Vector data types are optimized for handling vector data, offering better space efficiency and performance compared to `JSON` types. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Syntax + +You can use a string in the following syntax to represent a Vector value: + +```sql +'[, , ...]' +``` + +Example: + +```sql +CREATE TABLE vector_table ( + id INT PRIMARY KEY, + embedding VECTOR(3) +); + +INSERT INTO vector_table VALUES (1, '[0.3, 0.5, -0.1]'); + +INSERT INTO vector_table VALUES (2, NULL); +``` + +Inserting vector values with invalid syntax will result in an error: + +```sql +[tidb]> INSERT INTO vector_table VALUES (3, '[5, ]'); +ERROR 1105 (HY000): Invalid vector text: [5, ] +``` + +In the following example, because dimension `3` is enforced for the `embedding` column when the table is created, inserting a vector with a different dimension will result in an error: + +```sql +[tidb]> INSERT INTO vector_table VALUES (4, '[0.3, 0.5]'); +ERROR 1105 (HY000): vector has 2 dimensions, does not fit VECTOR(3) +``` + +For available functions and operators over the vector data types, see [Vector Functions and Operators](/tidb-cloud/vector-search-functions-and-operators.md). + +For more information about building and using a vector search index, see [Vector Search Index](/tidb-cloud/vector-search-index.md). + +## Store vectors with different dimensions + +You can store vectors with different dimensions in the same column by omitting the dimension parameter in the `VECTOR` type: + +```sql +CREATE TABLE vector_table ( + id INT PRIMARY KEY, + embedding VECTOR +); + +INSERT INTO vector_table VALUES (1, '[0.3, 0.5, -0.1]'); -- 3 dimensions vector, OK +INSERT INTO vector_table VALUES (2, '[0.3, 0.5]'); -- 2 dimensions vector, OK +``` + +However, note that you cannot build a [vector search index](/tidb-cloud/vector-search-index.md) for this column, as vector distances can be only calculated between vectors with the same dimensions. + +## Comparison + +You can compare vector data types using [comparison operators](/functions-and-operators/operators.md) such as `=`, `!=`, `<`, `>`, `<=`, and `>=`. For a complete list of comparison operators and functions for vector data types, see [Vector Functions and Operators](/tidb-cloud/vector-search-functions-and-operators.md). + +Vector data types are compared element-wise numerically. For example: + +- `[1] < [12]` +- `[1,2,3] < [1,2,5]` +- `[1,2,3] = [1,2,3]` +- `[2,2,3] > [1,2,3]` + +Two vectors with different dimensions are compared using lexicographical comparison, with the following rules: + +- Two vectors are compared element by element from the start, and each element is compared numerically. +- The first mismatching element determines which vector is lexicographically _less_ or _greater_ than the other. +- If one vector is a prefix of another, the shorter vector is lexicographically _less_ than the other. For example, `[1,2,3] < [1,2,3,0]`. +- Vectors of the same length with identical elements are lexicographically _equal_. +- An empty vector is lexicographically _less_ than any non-empty vector. For example, `[] < [1]`. +- Two empty vectors are lexicographically _equal_. + +When comparing vector constants, consider performing an [explicit cast](#cast) from string to vector to avoid comparisons based on string values: + +```sql +-- Because string is given, TiDB is comparing strings: +[tidb]> SELECT '[12.0]' < '[4.0]'; ++--------------------+ +| '[12.0]' < '[4.0]' | ++--------------------+ +| 1 | ++--------------------+ +1 row in set (0.01 sec) + +-- Cast to vector explicitly to compare by vectors: +[tidb]> SELECT VEC_FROM_TEXT('[12.0]') < VEC_FROM_TEXT('[4.0]'); ++--------------------------------------------------+ +| VEC_FROM_TEXT('[12.0]') < VEC_FROM_TEXT('[4.0]') | ++--------------------------------------------------+ +| 0 | ++--------------------------------------------------+ +1 row in set (0.01 sec) +``` + +## Arithmetic + +Vector data types support arithmetic operations `+` (addition) and `-` (subtraction). However, arithmetic operations between vectors with different dimensions are not supported and will result in an error. + +Examples: + +```sql +[tidb]> SELECT VEC_FROM_TEXT('[4]') + VEC_FROM_TEXT('[5]'); ++---------------------------------------------+ +| VEC_FROM_TEXT('[4]') + VEC_FROM_TEXT('[5]') | ++---------------------------------------------+ +| [9] | ++---------------------------------------------+ +1 row in set (0.01 sec) + +[tidb]> SELECT VEC_FROM_TEXT('[2,3,4]') - VEC_FROM_TEXT('[1,2,3]'); ++-----------------------------------------------------+ +| VEC_FROM_TEXT('[2,3,4]') - VEC_FROM_TEXT('[1,2,3]') | ++-----------------------------------------------------+ +| [1,1,1] | ++-----------------------------------------------------+ +1 row in set (0.01 sec) + +[tidb]> SELECT VEC_FROM_TEXT('[4]') + VEC_FROM_TEXT('[1,2,3]'); +ERROR 1105 (HY000): vectors have different dimensions: 1 and 3 +``` + +## Cast + +### Cast between Vector ⇔ String + +To cast between Vector and String, use the following functions: + +- `CAST(... AS VECTOR)`: String ⇒ Vector +- `CAST(... AS CHAR)`: Vector ⇒ String +- `VEC_FROM_TEXT`: String ⇒ Vector +- `VEC_AS_TEXT`: Vector ⇒ String + +To improve usability, if you call a function that only supports vector data types, such as a vector correlation distance function, you can also just pass in a format-compliant string. TiDB automatically performs an implicit cast in this case. + +```sql +-- The VEC_DIMS function only accepts VECTOR arguments, so you can directly pass in a string for an implicit cast. +[tidb]> SELECT VEC_DIMS('[0.3, 0.5, -0.1]'); ++------------------------------+ +| VEC_DIMS('[0.3, 0.5, -0.1]') | ++------------------------------+ +| 3 | ++------------------------------+ +1 row in set (0.01 sec) + +-- You can also explicitly cast a string to a vector using VEC_FROM_TEXT and then pass the vector to the VEC_DIMS function. +[tidb]> SELECT VEC_DIMS(VEC_FROM_TEXT('[0.3, 0.5, -0.1]')); ++---------------------------------------------+ +| VEC_DIMS(VEC_FROM_TEXT('[0.3, 0.5, -0.1]')) | ++---------------------------------------------+ +| 3 | ++---------------------------------------------+ +1 row in set (0.01 sec) + +-- You can also cast explicitly using CAST(... AS VECTOR): +[tidb]> SELECT VEC_DIMS(CAST('[0.3, 0.5, -0.1]' AS VECTOR)); ++----------------------------------------------+ +| VEC_DIMS(CAST('[0.3, 0.5, -0.1]' AS VECTOR)) | ++----------------------------------------------+ +| 3 | ++----------------------------------------------+ +1 row in set (0.01 sec) +``` + +When using an operator or function that accepts multiple data types, you need to explicitly cast the string type to the vector type before passing the string to that operator or function, because TiDB does not perform implicit casts in this case. For example, before performing comparison operations, you need to explicitly cast strings to vectors; otherwise, TiDB compares them as string values rather than as vector numeric values: + +```sql +-- Because string is given, TiDB is comparing strings: +[tidb]> SELECT '[12.0]' < '[4.0]'; ++--------------------+ +| '[12.0]' < '[4.0]' | ++--------------------+ +| 1 | ++--------------------+ +1 row in set (0.01 sec) + +-- Cast to vector explicitly to compare by vectors: +[tidb]> SELECT VEC_FROM_TEXT('[12.0]') < VEC_FROM_TEXT('[4.0]'); ++--------------------------------------------------+ +| VEC_FROM_TEXT('[12.0]') < VEC_FROM_TEXT('[4.0]') | ++--------------------------------------------------+ +| 0 | ++--------------------------------------------------+ +1 row in set (0.01 sec) +``` + +You can also explicitly cast a vector to its string representation. Take using the `VEC_AS_TEXT()` function as an example: + +```sql +-- The string is first implicitly cast to a vector, and then the vector is explicitly cast to a string, thus returning a string in the normalized format: +[tidb]> SELECT VEC_AS_TEXT('[0.3, 0.5, -0.1]'); ++--------------------------------------+ +| VEC_AS_TEXT('[0.3, 0.5, -0.1]') | ++--------------------------------------+ +| [0.3,0.5,-0.1] | ++--------------------------------------+ +1 row in set (0.01 sec) +``` + +For additional cast functions, see [Vector Functions and Operators](/tidb-cloud/vector-search-functions-and-operators.md). + +### Cast between Vector ⇔ other data types + +Currently, direct casting between Vector and other data types (such as `JSON`) is not supported. To work around this limitation, use String as an intermediate data type for casting in your SQL statement. + +Note that vector data type columns stored in a table cannot be converted to other data types using `ALTER TABLE ... MODIFY COLUMN ...`. + +## Limitations + +See [Vector data type limitations](/tidb-cloud/vector-search-limitations.md#vector-data-type-limitations). + +## MySQL compatibility + +Vector data types are TiDB specific, and are not supported in MySQL. + +## See also + +- [Vector Functions and Operators](/tidb-cloud/vector-search-functions-and-operators.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) +- [Improve Vector Search Performance](/tidb-cloud/vector-search-improve-performance.md) diff --git a/tidb-cloud/vector-search-full-text-search-python.md b/tidb-cloud/vector-search-full-text-search-python.md new file mode 100644 index 0000000000000..71f4fb81241d1 --- /dev/null +++ b/tidb-cloud/vector-search-full-text-search-python.md @@ -0,0 +1,172 @@ +--- +title: Full-Text Search with Python +summary: Full-text search lets you retrieve documents for exact keywords. In Retrieval-Augmented Generation (RAG) scenarios, you can use full-text search together with vector search to improve the retrieval quality. +--- + +# Full-Text Search with Python + +Unlike [Vector Search](/tidb-cloud/vector-search-overview.md), which focuses on semantic similarity, full-text search lets you retrieve documents for exact keywords. In Retrieval-Augmented Generation (RAG) scenarios, you can use full-text search together with vector search to improve the retrieval quality. + +The full-text search feature in TiDB provides the following capabilities: + +- **Query text data directly**: you can search any string columns directly without the embedding process. + +- **Support for multiple languages**: no need to specify the language for high-quality search. TiDB supports documents in multiple languages stored in the same table and automatically chooses the best text analyzer for each document. + +- **Order by relevance**: the search result can be ordered by relevance using the widely adopted [BM25 ranking](https://en.wikipedia.org/wiki/Okapi_BM25) algorithm. + +- **Fully compatible with SQL**: all SQL features, such as pre-filtering, post-filtering, grouping, and joining, can be used with full-text search. + +> **Tip:** +> +> For SQL usage, see [Full-Text Search with SQL](/tidb-cloud/vector-search-full-text-search-sql.md). +> +> To use full-text search and vector search together in your AI apps, see [Hybrid Search](/tidb-cloud/vector-search-hybrid-search.md). + +## Prerequisites + +Full-text search is still in the early stages, and we are continuously rolling it out to more customers. Currently, Full-text search is only available for the following product option and region: + +- TiDB Cloud Serverless: Frankfurt (eu-central-1) + +To complete this tutorial, make sure you have a TiDB Cloud Serverless cluster in the supported region. If you don't have one, follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create it. + +## Get started + +### Step 1. Install the [pytidb](https://github.com/pingcap/pytidb) Python SDK + +[pytidb](https://github.com/pingcap/pytidb) is the official Python SDK for TiDB, designed to help developers build AI applications efficiently. It includes built-in support for vector search and full-text search. + +To install the SDK, run the following command: + +```shell +pip install pytidb + +# (Alternative) To use the built-in embedding functions and rerankers: +# pip install "pytidb[models]" + +# (Optional) To convert query results into pandas DataFrames: +# pip install pandas +``` + +### Step 2. Connect to TiDB + +```python +from pytidb import TiDBClient + +db = TiDBClient.connect( + host="HOST_HERE", + port=4000, + username="USERNAME_HERE", + password="PASSWORD_HERE", + database="DATABASE_HERE", +) +``` + +You can get these connection parameters from the [TiDB Cloud console](https://tidbcloud.com): + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed, with connection parameters listed. + + For example, if the connection parameters are displayed as follows: + + ```text + HOST: gateway01.us-east-1.prod.shared.aws.tidbcloud.com + PORT: 4000 + USERNAME: 4EfqPF23YKBxaQb.root + PASSWORD: abcd1234 + DATABASE: test + CA: /etc/ssl/cert.pem + ``` + + The corresponding Python code to connect to the TiDB Cloud Serverless cluster would be as follows: + + ```python + db = TiDBClient.connect( + host="gateway01.us-east-1.prod.shared.aws.tidbcloud.com", + port=4000, + username="4EfqPF23YKBxaQb.root", + password="abcd1234", + database="test", + ) + ``` + + Note that the preceding example is for demonstration purposes only. You need to fill in the parameters with your own values and keep them secure. + +### Step 3. Create a table and a full-text index + +As an example, create a table named `chunks` with the following columns: + +- `id` (int): the ID of the chunk. +- `text` (text): the text content of the chunk. +- `user_id` (int): the ID of the user who created the chunk. + +```python +from pytidb.schema import TableModel, Field + +class Chunk(TableModel, table=True): + __tablename__ = "chunks" + + id: int = Field(primary_key=True) + text: str = Field() + user_id: int = Field() + +table = db.create_table(schema=Chunk) + +if not table.has_fts_index("text"): + table.create_fts_index("text") # 👈 Create a fulltext index on the text column. +``` + +### Step 4. Insert data + +```python +table.bulk_insert( + [ + Chunk(id=2, text="the quick brown", user_id=2), + Chunk(id=3, text="fox jumps", user_id=3), + Chunk(id=4, text="over the lazy dog", user_id=4), + ] +) +``` + +### Step 5. Perform a full-text search + +After inserting data, you can perform a full-text search as follows: + +```python +df = ( + table.search("brown fox", search_type="fulltext") + .limit(2) + .to_pandas() # optional +) + +# id text user_id +# 0 3 fox jumps 3 +# 1 2 the quick brown 2 +``` + +For a complete example, see [pytidb full-text search demo](https://github.com/pingcap/pytidb/blob/main/examples/fulltext_search). + +## See also + +- [pytidb Python SDK Documentation](https://github.com/pingcap/pytidb) + +- [Hybrid Search](/tidb-cloud/vector-search-hybrid-search.md) + +## Feedback & Help + +Full-text search is still in the early stages with limited accessibility. If you would like to try full-text search in a region that is not yet available, or if you have feedback or need help, feel free to reach out to us: + + + +- [Join our Discord](https://discord.gg/zcqexutz2R) + + + + + +- [Join our Discord](https://discord.gg/zcqexutz2R) +- [Visit our Support Portal](https://tidb.support.pingcap.com/) + + diff --git a/tidb-cloud/vector-search-full-text-search-sql.md b/tidb-cloud/vector-search-full-text-search-sql.md new file mode 100644 index 0000000000000..83dbfd8ff5390 --- /dev/null +++ b/tidb-cloud/vector-search-full-text-search-sql.md @@ -0,0 +1,219 @@ +--- +title: Full-Text Search with SQL +summary: Full-text search lets you retrieve documents for exact keywords. In Retrieval-Augmented Generation (RAG) scenarios, you can use full-text search together with vector search to improve the retrieval quality. +--- + +# Full-Text Search with SQL + +Unlike [Vector Search](/tidb-cloud/vector-search-overview.md), which focuses on semantic similarity, full-text search lets you retrieve documents for exact keywords. In Retrieval-Augmented Generation (RAG) scenarios, you can use full-text search together with vector search to improve the retrieval quality. + +The full-text search feature in TiDB provides the following capabilities: + +- **Query text data directly**: you can search any string columns directly without the embedding process. + +- **Support for multiple languages**: no need to specify the language for high-quality search. The text analyzer in TiDB supports documents in multiple languages mixed in the same table and automatically chooses the best analyzer for each document. + +- **Order by relevance**: the search result can be ordered by relevance using the widely adopted [BM25 ranking](https://en.wikipedia.org/wiki/Okapi_BM25) algorithm. + +- **Fully compatible with SQL**: all SQL features, such as pre-filtering, post-filtering, grouping, and joining, can be used with full-text search. + +> **Tip:** +> +> For Python usage, see [Full-Text Search with Python](/tidb-cloud/vector-search-full-text-search-python.md). +> +> To use full-text search and vector search together in your AI apps, see [Hybrid Search](/tidb-cloud/vector-search-hybrid-search.md). + +## Get started + +Full-text search is still in the early stages, and we are continuously rolling it out to more customers. Currently, Full-text search is only available for the following product option and region: + +- TiDB Cloud Serverless: Frankfurt (eu-central-1) + +Before using full-text search, make sure your TiDB Cloud Serverless cluster is created in the supported region. If you don't have one, follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create it. + +To perform a full-text search, follow these steps: + +1. [**Create a full-text index**](#create-a-full-text-index): create a table with a full-text index, or add a full-text index to an existing table. + +2. [**Insert text data**](#insert-text-data): insert text data into the table. + +3. [**Perform a full-text search**](#perform-a-full-text-search): perform a full-text search using text queries and full-text search functions. + +### Create a full-text index + +To perform full-text search, a full-text index is required as it provides the necessary data structure for efficient searching and ranking. Full-text indexes can be created on new tables or added to existing tables. + +Create a table with a full-text index: + +```sql +CREATE TABLE stock_items( + id INT, + title TEXT, + FULLTEXT INDEX (title) WITH PARSER MULTILINGUAL +); +``` + +Or add a full-text index to an existing table: + +```sql +CREATE TABLE stock_items( + id INT, + title TEXT +); + +-- You might insert some data here. +-- The full-text index can be created even if data is already in the table. + +ALTER TABLE stock_items ADD FULLTEXT INDEX (title) WITH PARSER MULTILINGUAL ADD_COLUMNAR_REPLICA_ON_DEMAND; +``` + +The following parsers are accepted in the `WITH PARSER ` clause: + +- `STANDARD`: fast, works for English contents, splitting words by spaces and punctuation. + +- `MULTILINGUAL`: supports multiple languages, including English, Chinese, Japanese, and Korean. + +### Insert text data + +Inserting data into a table with a full-text index is identical to inserting data into any other tables. + +For example, you can execute the following SQL statements to insert data in multiple languages. The multilingual parser in TiDB automatically processes the text. + +```sql +INSERT INTO stock_items VALUES (1, "イヤホン bluetooth ワイヤレスイヤホン "); +INSERT INTO stock_items VALUES (2, "完全ワイヤレスイヤホン/ウルトラノイズキャンセリング 2.0 "); +INSERT INTO stock_items VALUES (3, "ワイヤレス ヘッドホン Bluetooth 5.3 65時間再生 ヘッドホン 40mm HD "); +INSERT INTO stock_items VALUES (4, "楽器用 オンイヤーヘッドホン 密閉型【国内正規品】"); +INSERT INTO stock_items VALUES (5, "ワイヤレスイヤホン ハイブリッドANC搭載 40dBまでアクティブノイズキャンセル"); +INSERT INTO stock_items VALUES (6, "Lightweight Bluetooth Earbuds with 48 Hours Playtime"); +INSERT INTO stock_items VALUES (7, "True Wireless Noise Cancelling Earbuds - Compatible with Apple & Android, Built-in Microphone"); +INSERT INTO stock_items VALUES (8, "In-Ear Earbud Headphones with Mic, Black"); +INSERT INTO stock_items VALUES (9, "Wired Headphones, HD Bass Driven Audio, Lightweight Aluminum Wired in Ear Earbud Headphones"); +INSERT INTO stock_items VALUES (10, "LED Light Bar, Music Sync RGB Light Bar, USB Ambient Lamp"); +INSERT INTO stock_items VALUES (11, "无线消噪耳机-黑色 手势触控蓝牙降噪 主动降噪头戴式耳机（智能降噪 长久续航）"); +INSERT INTO stock_items VALUES (12, "专业版USB7.1声道游戏耳机电竞耳麦头戴式电脑网课办公麦克风带线控"); +INSERT INTO stock_items VALUES (13, "投影仪家用智能投影机便携卧室手机投影"); +INSERT INTO stock_items VALUES (14, "无线蓝牙耳机超长续航42小时快速充电 流光金属耳机"); +INSERT INTO stock_items VALUES (15, "皎月银 国家补贴 心率血氧监测 蓝牙通话 智能手表 男女表"); +``` + +### Perform a full-text search + +To perform a full-text search, you can use the `FTS_MATCH_WORD()` function. + +**Example: search for most relevant 10 documents** + +```sql +SELECT * FROM stock_items + WHERE fts_match_word("bluetoothイヤホン", title) + ORDER BY fts_match_word("bluetoothイヤホン", title) + DESC LIMIT 10; + +-- Results are ordered by relevance, with the most relevant documents first. + ++------+-----------------------------------------------------------------------------------------------------------+ +| id | title | ++------+-----------------------------------------------------------------------------------------------------------+ +| 1 | イヤホン bluetooth ワイヤレスイヤホン | +| 6 | Lightweight Bluetooth Earbuds with 48 Hours Playtime | +| 2 | 完全ワイヤレスイヤホン/ウルトラノイズキャンセリング 2.0 | +| 3 | ワイヤレス ヘッドホン Bluetooth 5.3 65時間再生 ヘッドホン 40mm HD | +| 5 | ワイヤレスイヤホン ハイブリッドANC搭載 40dBまでアクティブノイズキャンセル | ++------+-----------------------------------------------------------------------------------------------------------+ + +-- Try searching in another language: +SELECT * FROM stock_items + WHERE fts_match_word("蓝牙耳机", title) + ORDER BY fts_match_word("蓝牙耳机", title) + DESC LIMIT 10; + +-- Results are ordered by relevance, with the most relevant documents first. + ++------+---------------------------------------------------------------------------------------------------------------+ +| id | title | ++------+---------------------------------------------------------------------------------------------------------------+ +| 14 | 无线蓝牙耳机超长续航42小时快速充电 流光金属耳机 | +| 11 | 无线消噪耳机-黑色 手势触控蓝牙降噪 主动降噪头戴式耳机（智能降噪 长久续航） | +| 15 | 皎月银 国家补贴 心率血氧监测 蓝牙通话 智能手表 男女表 | ++------+---------------------------------------------------------------------------------------------------------------+ +``` + +**Example: count the number of documents matching the user query** + +```sql +SELECT COUNT(*) FROM stock_items + WHERE fts_match_word("bluetoothイヤホン", title); + ++----------+ +| COUNT(*) | ++----------+ +| 5 | ++----------+ +``` + +## Advanced example: Join search results with other tables + +You can combine full-text search with other SQL features such as joins and subqueries. + +Assume you have a `users` table and a `tickets` table, and want to find tickets created by authors based on a full-text search of their names: + +```sql +CREATE TABLE users( + id INT, + name TEXT, + FULLTEXT INDEX (name) WITH PARSER STANDARD +); + +INSERT INTO users VALUES (1, "Alice Smith"); +INSERT INTO users VALUES (2, "Bob Johnson"); + +CREATE TABLE tickets( + id INT, + title TEXT, + author_id INT +); + +INSERT INTO tickets VALUES (1, "Ticket 1", 1); +INSERT INTO tickets VALUES (2, "Ticket 2", 1); +INSERT INTO tickets VALUES (3, "Ticket 3", 2); +``` + +You can use a subquery to find matching user IDs based on the author's name, and then use these IDs in the outer query to retrieve and join related ticket information: + +```sql +SELECT t.title AS TICKET_TITLE, u.id AS AUTHOR_ID, u.name AS AUTHOR_NAME FROM tickets t +LEFT JOIN users u ON t.author_id = u.id +WHERE t.author_id IN +( + SELECT id FROM users + WHERE fts_match_word("Alice", name) +); + ++--------------+-----------+-------------+ +| TICKET_TITLE | AUTHOR_ID | AUTHOR_NAME | ++--------------+-----------+-------------+ +| Ticket 1 | 1 | Alice Smith | +| Ticket 2 | 1 | Alice Smith | ++--------------+-----------+-------------+ +``` + +## See also + +- [Hybrid Search](/tidb-cloud/vector-search-hybrid-search.md) + +## Feedback & help + +Full-text search is still in the early stages with limited accessibility. If you would like to try full-text search in a region that is not yet available, or if you have feedback or need help, feel free to reach out to us: + + + +- [Join our Discord](https://discord.gg/zcqexutz2R) + + + + + +- [Join our Discord](https://discord.gg/zcqexutz2R) +- [Visit our Support Portal](https://tidb.support.pingcap.com/) + + diff --git a/tidb-cloud/vector-search-functions-and-operators.md b/tidb-cloud/vector-search-functions-and-operators.md new file mode 100644 index 0000000000000..4081a2f4fcc45 --- /dev/null +++ b/tidb-cloud/vector-search-functions-and-operators.md @@ -0,0 +1,284 @@ +--- +title: Vector Functions and Operators +summary: Learn about functions and operators available for Vector data types. +--- + +# Vector Functions and Operators + +This document lists the functions and operators available for Vector data types. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Vector functions + +The following functions are designed specifically for [Vector data types](/tidb-cloud/vector-search-data-types.md). + +**Vector distance functions:** + +| Function Name | Description | +| ----------------------------------------------------------- | ---------------------------------------------------------------- | +| [`VEC_L2_DISTANCE`](#vec_l2_distance) | Calculates L2 distance (Euclidean distance) between two vectors | +| [`VEC_COSINE_DISTANCE`](#vec_cosine_distance) | Calculates the cosine distance between two vectors | +| [`VEC_NEGATIVE_INNER_PRODUCT`](#vec_negative_inner_product) | Calculates the negative of the inner product between two vectors | +| [`VEC_L1_DISTANCE`](#vec_l1_distance) | Calculates L1 distance (Manhattan distance) between two vectors | + +**Other vector functions:** + +| Function Name | Description | +| --------------------------------- | --------------------------------------------------- | +| [`VEC_DIMS`](#vec_dims) | Returns the dimension of a vector | +| [`VEC_L2_NORM`](#vec_l2_norm) | Calculates the L2 norm (Euclidean norm) of a vector | +| [`VEC_FROM_TEXT`](#vec_from_text) | Converts a string into a vector | +| [`VEC_AS_TEXT`](#vec_as_text) | Converts a vector into a string | + +## Extended built-in functions and operators + +The following built-in functions and operators are extended to support operations on [Vector data types](/tidb-cloud/vector-search-data-types.md). + +**Arithmetic operators:** + +| Name | Description | +| :-------------------------------------------------------------------------------------- | :--------------------------------------- | +| [`+`](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_plus) | Vector element-wise addition operator | +| [`-`](https://dev.mysql.com/doc/refman/8.0/en/arithmetic-functions.html#operator_minus) | Vector element-wise subtraction operator | + +For more information about how vector arithmetic works, see [Vector Data Type | Arithmetic](/tidb-cloud/vector-search-data-types.md#arithmetic). + +**Aggregate (GROUP BY) functions:** + +| Name | Description | +| :------------------------------------------------------------------------------------------------------------ | :----------------------------------------------- | +| [`COUNT()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_count) | Return a count of the number of rows returned | +| [`COUNT(DISTINCT)`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_count-distinct) | Return the count of a number of different values | +| [`MAX()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_max) | Return the maximum value | +| [`MIN()`](https://dev.mysql.com/doc/refman/8.0/en/aggregate-functions.html#function_min) | Return the minimum value | + +**Comparison functions and operators:** + +| Name | Description | +| ------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------- | +| [`BETWEEN ... AND ...`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_between) | Check whether a value is within a range of values | +| [`COALESCE()`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#function_coalesce) | Return the first non-NULL argument | +| [`=`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_equal) | Equal operator | +| [`<=>`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_equal-to) | NULL-safe equal to operator | +| [`>`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_greater-than) | Greater than operator | +| [`>=`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_greater-than-or-equal) | Greater than or equal operator | +| [`GREATEST()`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#function_greatest) | Return the largest argument | +| [`IN()`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_in) | Check whether a value is within a set of values | +| [`IS NULL`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_is-null) | Test whether a value is `NULL` | +| [`ISNULL()`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#function_isnull) | Test whether the argument is `NULL` | +| [`LEAST()`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#function_least) | Return the smallest argument | +| [`<`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_less-than) | Less than operator | +| [`<=`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_less-than-or-equal) | Less than or equal operator | +| [`NOT BETWEEN ... AND ...`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_not-between) | Check whether a value is not within a range of values | +| [`!=`, `<>`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_not-equal) | Not equal operator | +| [`NOT IN()`](https://dev.mysql.com/doc/refman/8.0/en/comparison-operators.html#operator_not-in) | Check whether a value is not within a set of values | + +For more information about how vectors are compared, see [Vector Data Type | Comparison](/tidb-cloud/vector-search-data-types.md#comparison). + +**Control flow functions:** + +| Name | Description | +| :------------------------------------------------------------------------------------------------ | :----------------------------- | +| [`CASE`](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#operator_case) | Case operator | +| [`IF()`](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#function_if) | If/else construct | +| [`IFNULL()`](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#function_ifnull) | Null if/else construct | +| [`NULLIF()`](https://dev.mysql.com/doc/refman/8.0/en/flow-control-functions.html#function_nullif) | Return `NULL` if expr1 = expr2 | + +**Cast functions:** + +| Name | Description | +| :------------------------------------------------------------------------------------------ | :--------------------------------- | +| [`CAST()`](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#function_cast) | Cast a value as a string or vector | +| [`CONVERT()`](https://dev.mysql.com/doc/refman/8.0/en/cast-functions.html#function_convert) | Cast a value as a string | + +For more information about how to use `CAST()`, see [Vector Data Type | Cast](/tidb-cloud/vector-search-data-types.md#cast). + +## Full references + +### VEC_L2_DISTANCE + +```sql +VEC_L2_DISTANCE(vector1, vector2) +``` + +Calculates the [L2 distance](https://en.wikipedia.org/wiki/Euclidean_distance) (Euclidean distance) between two vectors using the following formula: + +$DISTANCE(p,q)=\sqrt {\sum \limits _{i=1}^{n}{(p_{i}-q_{i})^{2}}}$ + +The two vectors must have the same dimension. Otherwise, an error is returned. + +Example: + +```sql +[tidb]> SELECT VEC_L2_DISTANCE('[0,3]', '[4,0]'); ++-----------------------------------+ +| VEC_L2_DISTANCE('[0,3]', '[4,0]') | ++-----------------------------------+ +| 5 | ++-----------------------------------+ +``` + +### VEC_COSINE_DISTANCE + +```sql +VEC_COSINE_DISTANCE(vector1, vector2) +``` + +Calculates the [cosine distance](https://en.wikipedia.org/wiki/Cosine_similarity) between two vectors using the following formula: + +$DISTANCE(p,q)=1.0 - {\frac {\sum \limits _{i=1}^{n}{p_{i}q_{i}}}{{\sqrt {\sum \limits _{i=1}^{n}{p_{i}^{2}}}}\cdot {\sqrt {\sum \limits _{i=1}^{n}{q_{i}^{2}}}}}}$ + +The two vectors must have the same dimension. Otherwise, an error is returned. + +Example: + +```sql +[tidb]> SELECT VEC_COSINE_DISTANCE('[1, 1]', '[-1, -1]'); ++-------------------------------------------+ +| VEC_COSINE_DISTANCE('[1, 1]', '[-1, -1]') | ++-------------------------------------------+ +| 2 | ++-------------------------------------------+ +``` + +### VEC_NEGATIVE_INNER_PRODUCT + +```sql +VEC_NEGATIVE_INNER_PRODUCT(vector1, vector2) +``` + +Calculates the distance by using the negative of the [inner product](https://en.wikipedia.org/wiki/Dot_product) between two vectors, using the following formula: + +$DISTANCE(p,q)=- INNER\_PROD(p,q)=-\sum \limits _{i=1}^{n}{p_{i}q_{i}}$ + +The two vectors must have the same dimension. Otherwise, an error is returned. + +Example: + +```sql +[tidb]> SELECT VEC_NEGATIVE_INNER_PRODUCT('[1,2]', '[3,4]'); ++----------------------------------------------+ +| VEC_NEGATIVE_INNER_PRODUCT('[1,2]', '[3,4]') | ++----------------------------------------------+ +| -11 | ++----------------------------------------------+ +``` + +### VEC_L1_DISTANCE + +```sql +VEC_L1_DISTANCE(vector1, vector2) +``` + +Calculates the [L1 distance](https://en.wikipedia.org/wiki/Taxicab_geometry) (Manhattan distance) between two vectors using the following formula: + +$DISTANCE(p,q)=\sum \limits _{i=1}^{n}{|p_{i}-q_{i}|}$ + +The two vectors must have the same dimension. Otherwise, an error is returned. + +Example: + +```sql +[tidb]> SELECT VEC_L1_DISTANCE('[0,0]', '[3,4]'); ++-----------------------------------+ +| VEC_L1_DISTANCE('[0,0]', '[3,4]') | ++-----------------------------------+ +| 7 | ++-----------------------------------+ +``` + +### VEC_DIMS + +```sql +VEC_DIMS(vector) +``` + +Returns the dimension of a vector. + +Examples: + +```sql +[tidb]> SELECT VEC_DIMS('[1,2,3]'); ++---------------------+ +| VEC_DIMS('[1,2,3]') | ++---------------------+ +| 3 | ++---------------------+ + +[tidb]> SELECT VEC_DIMS('[]'); ++----------------+ +| VEC_DIMS('[]') | ++----------------+ +| 0 | ++----------------+ +``` + +### VEC_L2_NORM + +```sql +VEC_L2_NORM(vector) +``` + +Calculates the [L2 norm]() (Euclidean norm) of a vector using the following formula: + +$NORM(p)=\sqrt {\sum \limits _{i=1}^{n}{p_{i}^{2}}}$ + +Example: + +```sql +[tidb]> SELECT VEC_L2_NORM('[3,4]'); ++----------------------+ +| VEC_L2_NORM('[3,4]') | ++----------------------+ +| 5 | ++----------------------+ +``` + +### VEC_FROM_TEXT + +```sql +VEC_FROM_TEXT(string) +``` + +Converts a string into a vector. + +Example: + +```sql +[tidb]> SELECT VEC_FROM_TEXT('[1,2]') + VEC_FROM_TEXT('[3,4]'); ++-------------------------------------------------+ +| VEC_FROM_TEXT('[1,2]') + VEC_FROM_TEXT('[3,4]') | ++-------------------------------------------------+ +| [4,6] | ++-------------------------------------------------+ +``` + +### VEC_AS_TEXT + +```sql +VEC_AS_TEXT(vector) +``` + +Converts a vector into a string. + +Example: + +```sql +[tidb]> SELECT VEC_AS_TEXT('[1.000, 2.5]'); ++-------------------------------+ +| VEC_AS_TEXT('[1.000, 2.5]') | ++-------------------------------+ +| [1,2.5] | ++-------------------------------+ +``` + +## MySQL compatibility + +The vector functions and the extended usage of built-in functions and operators over vector data types are TiDB specific, and are not supported in MySQL. + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) diff --git a/tidb-cloud/vector-search-get-started-using-python.md b/tidb-cloud/vector-search-get-started-using-python.md new file mode 100644 index 0000000000000..be7f8e0ff236d --- /dev/null +++ b/tidb-cloud/vector-search-get-started-using-python.md @@ -0,0 +1,195 @@ +--- +title: Get Started with TiDB + AI via Python +summary: Learn how to quickly develop an AI application that performs semantic search using Python and TiDB Vector Search. +--- + +# Get Started with TiDB + AI via Python + +This tutorial demonstrates how to develop a simple AI application that provides **semantic search** features. Unlike traditional keyword search, semantic search intelligently understands the meaning behind your query and returns the most relevant result. For example, if you have documents titled "dog", "fish", and "tree", and you search for "a swimming animal", the application would identify "fish" as the most relevant result. + +Throughout this tutorial, you will develop this AI application using [TiDB Vector Search](/tidb-cloud/vector-search-overview.md), Python, [TiDB Vector SDK for Python](https://github.com/pingcap/tidb-vector-python), and AI models. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Prerequisites + +To complete this tutorial, you need: + +- [Python 3.8 or higher](https://www.python.org/downloads/) installed. +- [Git](https://git-scm.com/downloads) installed. +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create your own TiDB Cloud cluster if you don't have one. + +## Get started + +The following steps show how to develop the application from scratch. To run the demo directly, you can check out the sample code in the [pingcap/tidb-vector-python](https://github.com/pingcap/tidb-vector-python/blob/main/examples/python-client-quickstart) repository. + +### Step 1. Create a new Python project + +In your preferred directory, create a new Python project and a file named `example.py`: + +```shell +mkdir python-client-quickstart +cd python-client-quickstart +touch example.py +``` + +### Step 2. Install required dependencies + +In your project directory, run the following command to install the required packages: + +```shell +pip install sqlalchemy pymysql sentence-transformers tidb-vector python-dotenv +``` + +- `tidb-vector`: the Python client for interacting with TiDB Vector Search. +- [`sentence-transformers`](https://sbert.net): a Python library that provides pre-trained models for generating [vector embeddings](/tidb-cloud/vector-search-overview.md#vector-embedding) from text. + +### Step 3. Configure the connection string to the TiDB cluster + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. + - **Connect With** is set to `SQLAlchemy`. + - **Operating System** matches your environment. + + > **Tip:** + > + > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. + +4. Click the **PyMySQL** tab and copy the connection string. + + > **Tip:** + > + > If you have not set a password yet, click **Generate Password** to generate a random password. + +5. In the root directory of your Python project, create a `.env` file and paste the connection string into it. + + The following is an example for macOS: + + ```dotenv + TIDB_DATABASE_URL="mysql+pymysql://.root:@gateway01..prod.aws.tidbcloud.com:4000/test?ssl_ca=/etc/ssl/cert.pem&ssl_verify_cert=true&ssl_verify_identity=true" + ``` + +### Step 4. Initialize the embedding model + +An [embedding model](/tidb-cloud/vector-search-overview.md#embedding-model) transforms data into [vector embeddings](/tidb-cloud/vector-search-overview.md#vector-embedding). This example uses the pre-trained model [**msmarco-MiniLM-L12-cos-v5**](https://huggingface.co/sentence-transformers/msmarco-MiniLM-L12-cos-v5) for text embedding. This lightweight model, provided by the `sentence-transformers` library, transforms text data into 384-dimensional vector embeddings. + +To set up the model, copy the following code into the `example.py` file. This code initializes a `SentenceTransformer` instance and defines a `text_to_embedding()` function for later use. + +```python +from sentence_transformers import SentenceTransformer + +print("Downloading and loading the embedding model...") +embed_model = SentenceTransformer("sentence-transformers/msmarco-MiniLM-L12-cos-v5", trust_remote_code=True) +embed_model_dims = embed_model.get_sentence_embedding_dimension() + +def text_to_embedding(text): + """Generates vector embeddings for the given text.""" + embedding = embed_model.encode(text) + return embedding.tolist() +``` + +### Step 5. Connect to the TiDB cluster + +Use the `TiDBVectorClient` class to connect to your TiDB cluster and create a table `embedded_documents` with a vector column. + +> **Note** +> +> Make sure the dimension of your vector column in the table matches the dimension of the vectors generated by your embedding model. For example, the **msmarco-MiniLM-L12-cos-v5** model generates vectors with 384 dimensions, so the dimension of your vector columns in `embedded_documents` should be 384 as well. + +```python +import os +from tidb_vector.integrations import TiDBVectorClient +from dotenv import load_dotenv + +# Load the connection string from the .env file +load_dotenv() + +vector_store = TiDBVectorClient( + # The 'embedded_documents' table will store the vector data. + table_name='embedded_documents', + # The connection string to the TiDB cluster. + connection_string=os.environ.get('TIDB_DATABASE_URL'), + # The dimension of the vector generated by the embedding model. + vector_dimension=embed_model_dims, + # Recreate the table if it already exists. + drop_existing_table=True, +) +``` + +### Step 6. Embed text data and store the vectors + +In this step, you will prepare sample documents containing single words, such as "dog", "fish", and "tree". The following code uses the `text_to_embedding()` function to transform these text documents into vector embeddings, and then inserts them into the vector store. + +```python +documents = [ + { + "id": "f8e7dee2-63b6-42f1-8b60-2d46710c1971", + "text": "dog", + "embedding": text_to_embedding("dog"), + "metadata": {"category": "animal"}, + }, + { + "id": "8dde1fbc-2522-4ca2-aedf-5dcb2966d1c6", + "text": "fish", + "embedding": text_to_embedding("fish"), + "metadata": {"category": "animal"}, + }, + { + "id": "e4991349-d00b-485c-a481-f61695f2b5ae", + "text": "tree", + "embedding": text_to_embedding("tree"), + "metadata": {"category": "plant"}, + }, +] + +vector_store.insert( + ids=[doc["id"] for doc in documents], + texts=[doc["text"] for doc in documents], + embeddings=[doc["embedding"] for doc in documents], + metadatas=[doc["metadata"] for doc in documents], +) +``` + +### Step 7. Perform semantic search + +In this step, you will search for "a swimming animal", which doesn't directly match any words in existing documents. + +The following code uses the `text_to_embedding()` function again to convert the query text into a vector embedding, and then queries with the embedding to find the top three closest matches. + +```python +def print_result(query, result): + print(f"Search result (\"{query}\"):") + for r in result: + print(f"- text: \"{r.document}\", distance: {r.distance}") + +query = "a swimming animal" +query_embedding = text_to_embedding(query) +search_result = vector_store.query(query_embedding, k=3) +print_result(query, search_result) +``` + +Run the `example.py` file and the output is as follows: + +```plain +Search result ("a swimming animal"): +- text: "fish", distance: 0.4562914811223072 +- text: "dog", distance: 0.6469335836410557 +- text: "tree", distance: 0.798545178640937 +``` + +The three terms in the search results are sorted by their respective distance from the queried vector: the smaller the distance, the more relevant the corresponding `document`. + +Therefore, according to the output, the swimming animal is most likely a fish, or a dog with a gift for swimming. + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) diff --git a/tidb-cloud/vector-search-get-started-using-sql.md b/tidb-cloud/vector-search-get-started-using-sql.md new file mode 100644 index 0000000000000..7546b060a5c79 --- /dev/null +++ b/tidb-cloud/vector-search-get-started-using-sql.md @@ -0,0 +1,150 @@ +--- +title: Get Started with Vector Search via SQL +summary: Learn how to quickly get started with Vector Search in TiDB using SQL statements to power your generative AI applications. +--- + +# Get Started with Vector Search via SQL + +TiDB extends MySQL syntax to support [Vector Search](/tidb-cloud/vector-search-overview.md) and introduce new [Vector data types](/tidb-cloud/vector-search-data-types.md) and several [vector functions](/tidb-cloud/vector-search-functions-and-operators.md). + +This tutorial demonstrates how to get started with TiDB Vector Search just using SQL statements. You will learn how to use the [MySQL command-line client](https://dev.mysql.com/doc/refman/8.4/en/mysql.html) to complete the following operations: + +- Connect to your TiDB cluster. +- Create a vector table. +- Store vector embeddings. +- Perform vector search queries. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Prerequisites + +To complete this tutorial, you need: + +- [MySQL command-line client](https://dev.mysql.com/doc/refman/8.4/en/mysql.html) (MySQL CLI) installed on your machine. +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create your own TiDB Cloud cluster if you don't have one. + +## Get started + +### Step 1. Connect to the TiDB cluster + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. In the connection dialog, select **MySQL CLI** from the **Connect With** drop-down list and keep the default setting of the **Connection Type** as **Public**. + +4. If you have not set a password yet, click **Generate Password** to generate a random password. + +5. Copy the connection command and paste it into your terminal. The following is an example for macOS: + + ```bash + mysql -u '.root' -h '' -P 4000 -D 'test' --ssl-mode=VERIFY_IDENTITY --ssl-ca=/etc/ssl/cert.pem -p'' + ``` + +### Step 2. Create a vector table + +When creating a table, you can define a column as a [vector](/tidb-cloud/vector-search-overview.md#vector-embedding) column by specifying the `VECTOR` data type. + +For example, to create a table `embedded_documents` with a three-dimensional `VECTOR` column, execute the following SQL statements using your MySQL CLI: + +```sql +USE test; +CREATE TABLE embedded_documents ( + id INT PRIMARY KEY, + -- Column to store the original content of the document. + document TEXT, + -- Column to store the vector representation of the document. + embedding VECTOR(3) +); +``` + +The expected output is as follows: + +```text +Query OK, 0 rows affected (0.27 sec) +``` + +### Step 3. Insert vector embeddings to the table + +Insert three documents with their [vector embeddings](/tidb-cloud/vector-search-overview.md#vector-embedding) into the `embedded_documents` table: + +```sql +INSERT INTO embedded_documents +VALUES + (1, 'dog', '[1,2,1]'), + (2, 'fish', '[1,2,4]'), + (3, 'tree', '[1,0,0]'); +``` + +The expected output is as follows: + +``` +Query OK, 3 rows affected (0.15 sec) +Records: 3 Duplicates: 0 Warnings: 0 +``` + +> **Note** +> +> This example simplifies the dimensions of the vector embeddings and uses only 3-dimensional vectors for demonstration purposes. +> +> In real-world applications, [embedding models](/tidb-cloud/vector-search-overview.md#embedding-model) often produce vector embeddings with hundreds or thousands of dimensions. + +### Step 4. Query the vector table + +To verify that the documents have been inserted correctly, query the `embedded_documents` table: + +```sql +SELECT * FROM embedded_documents; +``` + +The expected output is as follows: + +```sql ++----+----------+-----------+ +| id | document | embedding | ++----+----------+-----------+ +| 1 | dog | [1,2,1] | +| 2 | fish | [1,2,4] | +| 3 | tree | [1,0,0] | ++----+----------+-----------+ +3 rows in set (0.15 sec) +``` + +### Step 5. Perform a vector search query + +Similar to full-text search, users provide search terms to the application when using vector search. + +In this example, the search term is "a swimming animal", and its corresponding vector embedding is assumed to be `[1,2,3]`. In practical applications, you need to use an embedding model to convert the user's search term into a vector embedding. + +Execute the following SQL statement, and TiDB will identify the top three documents closest to `[1,2,3]` by calculating and sorting the cosine distances (`vec_cosine_distance`) between the vector embeddings in the table. + +```sql +SELECT id, document, vec_cosine_distance(embedding, '[1,2,3]') AS distance +FROM embedded_documents +ORDER BY distance +LIMIT 3; +``` + +The expected output is as follows: + +```plain ++----+----------+---------------------+ +| id | document | distance | ++----+----------+---------------------+ +| 2 | fish | 0.00853986601633272 | +| 1 | dog | 0.12712843905603044 | +| 3 | tree | 0.7327387580875756 | ++----+----------+---------------------+ +3 rows in set (0.15 sec) +``` + +The three terms in the search results are sorted by their respective distance from the queried vector: the smaller the distance, the more relevant the corresponding `document`. + +Therefore, according to the output, the swimming animal is most likely a fish, or a dog with a gift for swimming. + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) diff --git a/tidb-cloud/vector-search-hybrid-search.md b/tidb-cloud/vector-search-hybrid-search.md new file mode 100644 index 0000000000000..a068cffe5e340 --- /dev/null +++ b/tidb-cloud/vector-search-hybrid-search.md @@ -0,0 +1,166 @@ +--- +title: Hybrid Search +summary: Use full-text search and vector search together to improve the retrieval quality. +--- + +# Hybrid Search + +By using full-text search, you can retrieve documents based on exact keywords. By using vector search, you can retrieve documents based on semantic similarity. Can we combine these two search methods to improve the retrieval quality and handle more scenarios? Yes, this approach is known as hybrid search and is commonly used in AI applications. + +A general workflow of hybrid search in TiDB is as follows: + +1. Use TiDB for **full-text search** and **vector search**. +2. Use a **reranker** to combine the results from both searches. + +![Hybrid Search](/media/vector-search/hybrid-search-overview.svg) + +This tutorial demonstrates how to use hybrid search in TiDB with the [pytidb](https://github.com/pingcap/pytidb) Python SDK, which provides built-in support for embedding and reranking. Using pytidb is completely optional — you can perform a search using SQL directly and use your own reranking model as you like. + +## Prerequisites + +Hybrid search relies on both [full-text search](/tidb-cloud/vector-search-full-text-search-python.md) and vector search. Full-text search is still in the early stages, and we are continuously rolling it out to more customers. Currently, Full-text search is only available for the following product option and region: + +- TiDB Cloud Serverless: Frankfurt (eu-central-1) + +To complete this tutorial, make sure you have a TiDB Cloud Serverless cluster in the supported region. If you don't have one, follow [Creating a TiDB Cloud Serverless cluster](/develop/dev-guide-build-cluster-in-cloud.md) to create it. + +## Get started + +### Step 1. Install the [pytidb](https://github.com/pingcap/pytidb) Python SDK + +```shell +pip install "pytidb[models]" + +# (Alternative) If you don't want to use built-in embedding functions and rerankers: +# pip install pytidb + +# (Optional) To convert query results to pandas DataFrame: +# pip install pandas +``` + +### Step 2. Connect to TiDB + +```python +from pytidb import TiDBClient + +db = TiDBClient.connect( + host="HOST_HERE", + port=4000, + username="USERNAME_HERE", + password="PASSWORD_HERE", + database="DATABASE_HERE", +) +``` + +You can get these connection parameters from the [TiDB Cloud console](https://tidbcloud.com): + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed, with connection parameters listed. + + For example, if the connection parameters are displayed as follows: + + ```text + HOST: gateway01.us-east-1.prod.shared.aws.tidbcloud.com + PORT: 4000 + USERNAME: 4EfqPF23YKBxaQb.root + PASSWORD: abcd1234 + DATABASE: test + CA: /etc/ssl/cert.pem + ``` + + The corresponding Python code to connect to the TiDB Cloud Serverless cluster would be as follows: + + ```python + db = TiDBClient.connect( + host="gateway01.us-east-1.prod.shared.aws.tidbcloud.com", + port=4000, + username="4EfqPF23YKBxaQb.root", + password="abcd1234", + database="test", + ) + ``` + + Note that the preceding example is for demonstration purposes only. You need to fill in the parameters with your own values and keep them secure. + +### Step 3. Create a table + +As an example, create a table named `chunks` with the following columns: + +- `id` (int): the ID of the chunk. +- `text` (text): the text content of the chunk. +- `text_vec` (vector): the vector representation of the text, automatically generated by the embedding model in pytidb. +- `user_id` (int): the ID of the user who created the chunk. + +```python +from pytidb.schema import TableModel, Field +from pytidb.embeddings import EmbeddingFunction + +text_embed = EmbeddingFunction("openai/text-embedding-3-small") + +class Chunk(TableModel, table=True): + __tablename__ = "chunks" + + id: int = Field(primary_key=True) + text: str = Field() + text_vec: list[float] = text_embed.VectorField( + source_field="text" + ) # 👈 Define the vector field. + user_id: int = Field() + +table = db.create_table(schema=Chunk) +``` + +### Step 4. Insert data + +```python +table.bulk_insert( + [ + Chunk(id=2, text="bar", user_id=2), # 👈 The text field will be embedded to a + Chunk(id=3, text="baz", user_id=3), # vector and stored in the "text_vec" field + Chunk(id=4, text="qux", user_id=4), # automatically. + ] +) +``` + +### Step 5. Perform a hybrid search + +In this example, use the [jina-reranker](https://huggingface.co/jinaai/jina-reranker-m0) model to rerank the search results. + +```python +from pytidb.rerankers import Reranker + +jinaai = Reranker(model_name="jina_ai/jina-reranker-m0") + +df = ( + table.search("", search_type="hybrid") + .rerank(jinaai, "text") # 👈 Rerank the query result using the jinaai model. + .limit(2) + .to_pandas() +) +``` + +For a complete example, see [pytidb hybrid search demo](https://github.com/pingcap/pytidb/tree/main/examples/hybrid_search). + +## See also + +- [pytidb Python SDK Documentation](https://github.com/pingcap/pytidb) + +- [Full-Text Search with Python](/tidb-cloud/vector-search-full-text-search-python.md) + +## Feedback & help + +Full-text search is still in the early stages with limited accessibility. If you would like to try full-text search in a region that is not yet available, or if you have feedback or need help, feel free to reach out to us: + + + +- [Join our Discord](https://discord.gg/zcqexutz2R) + + + + + +- [Join our Discord](https://discord.gg/zcqexutz2R) +- [Visit our Support Portal](https://tidb.support.pingcap.com/) + + diff --git a/tidb-cloud/vector-search-improve-performance.md b/tidb-cloud/vector-search-improve-performance.md new file mode 100644 index 0000000000000..70fd40f468e6a --- /dev/null +++ b/tidb-cloud/vector-search-improve-performance.md @@ -0,0 +1,36 @@ +--- +title: Improve Vector Search Performance +summary: Learn best practices for improving the performance of TiDB Vector Search. +--- + +# Improve Vector Search Performance + +TiDB Vector Search enables you to perform Approximate Nearest Neighbor (ANN) queries that search for results similar to an image, document, or other input. To improve the query performance, review the following best practices. + +## Add vector search index for vector columns + +The [vector search index](/tidb-cloud/vector-search-index.md) dramatically improves the performance of vector search queries, usually by 10x or more, with a trade-off of only a small decrease of recall rate. + +## Ensure vector indexes are fully built + +After you insert a large volume of vector data, some of it might be in the Delta layer waiting for persistence. The vector index for such data will be built after the data is persisted. Until all vector data is indexed, vector search performance is suboptimal. To check the index build progress, see [View index build progress](/tidb-cloud/vector-search-index.md#view-index-build-progress). + +## Reduce vector dimensions or shorten embeddings + +The computational complexity of vector search indexing and queries increases significantly as the dimension of vectors grows, requiring more floating-point comparisons. + +To optimize performance, consider reducing vector dimensions whenever feasible. This usually needs switching to another embedding model. When switching models, you need to evaluate the impact of the model change on the accuracy of vector queries. + +Certain embedding models like OpenAI `text-embedding-3-large` support [shortening embeddings](https://openai.com/index/new-embedding-models-and-api-updates/), which removes some numbers from the end of vector sequences without losing the embedding's concept-representing properties. You can also use such an embedding model to reduce the vector dimensions. + +## Exclude vector columns from the results + +Vector embedding data is usually large and only used during the search process. By excluding vector columns from query results, you can greatly reduce the data transferred between the TiDB server and your SQL client, thereby improving query performance. + +To exclude vector columns, explicitly list the columns you want to retrieve in the `SELECT` clause, instead of using `SELECT *` to retrieve all columns. + +## Warm up the index + +When accessing an index that has never been used or has not been accessed for a long time (cold access), TiDB needs to load the entire index from cloud storage or disk (instead of from memory). This process takes time and often results in higher query latency. Additionally, if there are no SQL queries for an extended period (for example, several hours), computing resources are reclaimed, causing subsequent access to become cold access. + +To avoid such query latency, warm up your index before actual workload by running similar vector search queries that hit the vector index. \ No newline at end of file diff --git a/tidb-cloud/vector-search-index.md b/tidb-cloud/vector-search-index.md new file mode 100644 index 0000000000000..2da5b3d5a63fd --- /dev/null +++ b/tidb-cloud/vector-search-index.md @@ -0,0 +1,245 @@ +--- +title: Vector Search Index +summary: Learn how to build and use the vector search index to accelerate K-Nearest neighbors (KNN) queries in TiDB. +--- + +# Vector Search Index + +As described in the [Vector Search](/tidb-cloud/vector-search-overview.md) document, vector search identifies the Top K-Nearest Neighbors (KNN) to a given vector by calculating the distance between the given vector and all vectors stored in the database. While this approach provides accurate results, it can be slow when the table contains a large number of vectors because it involves a full table scan. [^1] + +To improve search efficiency, you can create vector search indexes in TiDB for approximate KNN (ANN) search. When using vector indexes for vector search, TiDB can greatly improve query performance with only a slight reduction in accuracy, generally maintaining a search recall rate above 90%. + +Currently, TiDB supports the [HNSW (Hierarchical Navigable Small World)](https://en.wikipedia.org/wiki/Hierarchical_navigable_small_world) vector search index algorithm. + +## Create the HNSW vector index + +[HNSW](https://en.wikipedia.org/wiki/Hierarchical_navigable_small_world) is one of the most popular vector indexing algorithms. The HNSW index provides good performance with relatively high accuracy, up to 98% in specific cases. + +In TiDB, you can create an HNSW index for a column with a [vector data type](/tidb-cloud/vector-search-data-types.md) in either of the following ways: + +- When creating a table, use the following syntax to specify the vector column for the HNSW index: + + ```sql + CREATE TABLE foo ( + id INT PRIMARY KEY, + embedding VECTOR(5), + VECTOR INDEX idx_embedding ((VEC_COSINE_DISTANCE(embedding))) + ); + ``` + +- For an existing table that already contains a vector column, use the following syntax to create an HNSW index for the vector column: + + ```sql + CREATE VECTOR INDEX idx_embedding ON foo ((VEC_COSINE_DISTANCE(embedding))); + ALTER TABLE foo ADD VECTOR INDEX idx_embedding ((VEC_COSINE_DISTANCE(embedding))); + + -- You can also explicitly specify "USING HNSW" to build the vector search index. + CREATE VECTOR INDEX idx_embedding ON foo ((VEC_COSINE_DISTANCE(embedding))) USING HNSW; + ALTER TABLE foo ADD VECTOR INDEX idx_embedding ((VEC_COSINE_DISTANCE(embedding))) USING HNSW; + ``` + +> **Note:** +> +> The vector search index feature relies on TiFlash replicas for tables. +> +> - If a vector search index is defined when a table is created, TiDB automatically creates a TiFlash replica for the table. +> - If no vector search index is defined when a table is created, and the table currently does not have a TiFlash replica, you need to manually create a TiFlash replica before adding a vector search index to the table. For example: `ALTER TABLE 'table_name' SET TIFLASH REPLICA 1;`. + +When creating an HNSW vector index, you need to specify the distance function for the vector: + +- Cosine Distance: `((VEC_COSINE_DISTANCE(embedding)))` +- L2 Distance: `((VEC_L2_DISTANCE(embedding)))` + +The vector index can only be created for fixed-dimensional vector columns, such as a column defined as `VECTOR(3)`. It cannot be created for non-fixed-dimensional vector columns (such as a column defined as `VECTOR`) because vector distances can only be calculated between vectors with the same dimension. + +For other limitations, see [Vector index limitations](/tidb-cloud/vector-search-limitations.md#vector-index-limitations). + +## Use the vector index + +The vector search index can be used in K-nearest neighbor search queries by using the `ORDER BY ... LIMIT` clause as follows: + +```sql +SELECT * +FROM foo +ORDER BY VEC_COSINE_DISTANCE(embedding, '[1, 2, 3, 4, 5]') +LIMIT 10 +``` + +To use an index in a vector search, make sure that the `ORDER BY ... LIMIT` clause uses the same distance function as the one specified when creating the vector index. + +## Use the vector index with filters + +Queries that contain a pre-filter (using the `WHERE` clause) cannot utilize the vector index because they are not querying for K-Nearest neighbors according to the SQL semantics. For example: + +```sql +-- For the following query, the `WHERE` filter is performed before KNN, so the vector index cannot be used: + +SELECT * FROM vec_table +WHERE category = "document" +ORDER BY VEC_COSINE_DISTANCE(embedding, '[1, 2, 3]') +LIMIT 5; +``` + +To use the vector index with filters, query for the K-Nearest neighbors first using vector search, and then filter out unwanted results: + +```sql +-- For the following query, the `WHERE` filter is performed after KNN, so the vector index cannot be used: + +SELECT * FROM +( + SELECT * FROM vec_table + ORDER BY VEC_COSINE_DISTANCE(embedding, '[1, 2, 3]') + LIMIT 5 +) t +WHERE category = "document"; + +-- Note that this query might return fewer than 5 results if some are filtered out. +``` + +## View index build progress + +After you insert a large volume of data, some of it might not be instantly persisted to TiFlash. For vector data that has already been persisted, the vector search index is built synchronously. For data that has not yet been persisted, the index will be built once the data is persisted. This process does not affect the accuracy and consistency of the data. You can still perform vector searches at any time and get complete results. However, performance will be suboptimal until vector indexes are fully built. + +To view the index build progress, you can query the `INFORMATION_SCHEMA.TIFLASH_INDEXES` table as follows: + +```sql +SELECT * FROM INFORMATION_SCHEMA.TIFLASH_INDEXES; ++---------------+------------+----------+-------------+---------------+-----------+----------+------------+---------------------+-------------------------+--------------------+------------------------+---------------+------------------+ +| TIDB_DATABASE | TIDB_TABLE | TABLE_ID | COLUMN_NAME | INDEX_NAME | COLUMN_ID | INDEX_ID | INDEX_KIND | ROWS_STABLE_INDEXED | ROWS_STABLE_NOT_INDEXED | ROWS_DELTA_INDEXED | ROWS_DELTA_NOT_INDEXED | ERROR_MESSAGE | TIFLASH_INSTANCE | ++---------------+------------+----------+-------------+---------------+-----------+----------+------------+---------------------+-------------------------+--------------------+------------------------+---------------+------------------+ +| test | tcff1d827 | 219 | col1fff | 0a452311 | 7 | 1 | HNSW | 29646 | 0 | 0 | 0 | | 127.0.0.1:3930 | +| test | foo | 717 | embedding | idx_embedding | 2 | 1 | HNSW | 0 | 0 | 0 | 3 | | 127.0.0.1:3930 | ++---------------+------------+----------+-------------+---------------+-----------+----------+------------+---------------------+-------------------------+--------------------+------------------------+---------------+------------------+ +``` + +- You can check the `ROWS_STABLE_INDEXED` and `ROWS_STABLE_NOT_INDEXED` columns for the index build progress. When `ROWS_STABLE_NOT_INDEXED` becomes 0, the index build is complete. + + As a reference, indexing a 500 MiB vector dataset with 768 dimensions might take up to 20 minutes. The indexer can run in parallel for multiple tables. Currently, adjusting the indexer priority or speed is not supported. + +- You can check the `ROWS_DELTA_NOT_INDEXED` column for the number of rows in the Delta layer. Data in the storage layer of TiFlash is stored in two layers: Delta layer and Stable layer. The Delta layer stores recently inserted or updated rows and is periodically merged into the Stable layer according to the write workload. This merge process is called Compaction. + + The Delta layer is always not indexed. To achieve optimal performance, you can force the merge of the Delta layer into the Stable layer so that all data can be indexed: + + ```sql + ALTER TABLE COMPACT; + ``` + + For more information, see [`ALTER TABLE ... COMPACT`](/sql-statements/sql-statement-alter-table-compact.md). + +In addition, you can monitor the execution progress of the DDL job by executing `ADMIN SHOW DDL JOBS;` and checking the `row count`. However, this method is not fully accurate, because the `row count` value is obtained from the `rows_stable_indexed` field in `TIFLASH_INDEXES`. You can use this approach as a reference for tracking the progress of indexing. + +## Check whether the vector index is used + +Use the [`EXPLAIN`](/sql-statements/sql-statement-explain.md) or [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) statement to check whether a query is using the vector index. When `annIndex:` is presented in the `operator info` column for the `TableFullScan` executor, it means this table scan is utilizing the vector index. + +**Example: the vector index is used** + +```sql +[tidb]> EXPLAIN SELECT * FROM vector_table_with_index +ORDER BY VEC_COSINE_DISTANCE(embedding, '[1, 2, 3]') +LIMIT 10; ++-----+-------------------------------------------------------------------------------------+ +| ... | operator info | ++-----+-------------------------------------------------------------------------------------+ +| ... | ... | +| ... | Column#5, offset:0, count:10 | +| ... | ..., vec_cosine_distance(test.vector_table_with_index.embedding, [1,2,3])->Column#5 | +| ... | MppVersion: 1, data:ExchangeSender_16 | +| ... | ExchangeType: PassThrough | +| ... | ... | +| ... | Column#4, offset:0, count:10 | +| ... | ..., vec_cosine_distance(test.vector_table_with_index.embedding, [1,2,3])->Column#4 | +| ... | annIndex:COSINE(test.vector_table_with_index.embedding..[1,2,3], limit:10), ... | ++-----+-------------------------------------------------------------------------------------+ +9 rows in set (0.01 sec) +``` + +**Example: The vector index is not used because of not specifying a Top K** + +```sql +[tidb]> EXPLAIN SELECT * FROM vector_table_with_index + -> ORDER BY VEC_COSINE_DISTANCE(embedding, '[1, 2, 3]'); ++--------------------------------+-----+--------------------------------------------------+ +| id | ... | operator info | ++--------------------------------+-----+--------------------------------------------------+ +| Projection_15 | ... | ... | +| └─Sort_4 | ... | Column#4 | +| └─Projection_16 | ... | ..., vec_cosine_distance(..., [1,2,3])->Column#4 | +| └─TableReader_14 | ... | MppVersion: 1, data:ExchangeSender_13 | +| └─ExchangeSender_13 | ... | ExchangeType: PassThrough | +| └─TableFullScan_12 | ... | keep order:false, stats:pseudo | ++--------------------------------+-----+--------------------------------------------------+ +6 rows in set, 1 warning (0.01 sec) +``` + +When the vector index cannot be used, a warning occurs in some cases to help you learn the cause: + +```sql +-- Using a wrong distance function: +[tidb]> EXPLAIN SELECT * FROM vector_table_with_index +ORDER BY VEC_L2_DISTANCE(embedding, '[1, 2, 3]') +LIMIT 10; + +[tidb]> SHOW WARNINGS; +ANN index not used: not ordering by COSINE distance + +-- Using a wrong order: +[tidb]> EXPLAIN SELECT * FROM vector_table_with_index +ORDER BY VEC_COSINE_DISTANCE(embedding, '[1, 2, 3]') DESC +LIMIT 10; + +[tidb]> SHOW WARNINGS; +ANN index not used: index can be used only when ordering by vec_cosine_distance() in ASC order +``` + +## Analyze vector search performance + +To learn detailed information about how a vector index is used, you can execute the [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) statement and check the `execution info` column in the output: + +```sql +[tidb]> EXPLAIN ANALYZE SELECT * FROM vector_table_with_index +ORDER BY VEC_COSINE_DISTANCE(embedding, '[1, 2, 3]') +LIMIT 10; ++-----+--------------------------------------------------------+-----+ +| | execution info | | ++-----+--------------------------------------------------------+-----+ +| ... | time:339.1ms, loops:2, RU:0.000000, Concurrency:OFF | ... | +| ... | time:339ms, loops:2 | ... | +| ... | time:339ms, loops:3, Concurrency:OFF | ... | +| ... | time:339ms, loops:3, cop_task: {...} | ... | +| ... | tiflash_task:{time:327.5ms, loops:1, threads:4} | ... | +| ... | tiflash_task:{time:327.5ms, loops:1, threads:4} | ... | +| ... | tiflash_task:{time:327.5ms, loops:1, threads:4} | ... | +| ... | tiflash_task:{time:327.5ms, loops:1, threads:4} | ... | +| ... | tiflash_task:{...}, vector_idx:{ | ... | +| | load:{total:68ms,from_s3:1,from_disk:0,from_cache:0},| | +| | search:{total:0ms,visited_nodes:2,discarded_nodes:0},| | +| | read:{vec_total:0ms,others_total:0ms}},...} | | ++-----+--------------------------------------------------------+-----+ +``` + +> **Note:** +> +> The execution information is internal. Fields and formats are subject to change without any notification. Do not rely on them. + +Explanation of some important fields: + +- `vector_index.load.total`: The total duration of loading index. This field might be larger than the actual query time because multiple vector indexes might be loaded in parallel. +- `vector_index.load.from_s3`: Number of indexes loaded from S3. +- `vector_index.load.from_disk`: Number of indexes loaded from disk. The index was already downloaded from S3 previously. +- `vector_index.load.from_cache`: Number of indexes loaded from cache. The index was already downloaded from S3 previously. +- `vector_index.search.total`: The total duration of searching in the index. Large latency usually means the index is cold (never accessed before, or accessed long ago) so that there are heavy I/O operations when searching through the index. This field might be larger than the actual query time because multiple vector indexes might be searched in parallel. +- `vector_index.search.discarded_nodes`: Number of vector rows visited but discarded during the search. These discarded vectors are not considered in the search result. Large values usually indicate that there are many stale rows caused by `UPDATE` or `DELETE` statements. + +See [`EXPLAIN`](/sql-statements/sql-statement-explain.md), [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md), and [EXPLAIN Walkthrough](/explain-walkthrough.md) for interpreting the output. + +## Limitations + +See [Vector index limitations](/tidb-cloud/vector-search-limitations.md#vector-index-limitations). + +## See also + +- [Improve Vector Search Performance](/tidb-cloud/vector-search-improve-performance.md) +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) + +[^1]: The explanation of KNN search is adapted from the [Approximate Nearest Neighbor Search Indexes](https://github.com/ClickHouse/ClickHouse/pull/50661/files#diff-7ebd9e71df96e74230c9a7e604fa7cb443be69ba5e23bf733fcecd4cc51b7576) document authored by [rschu1ze](https://github.com/rschu1ze) in ClickHouse documentation, licensed under the Apache License 2.0. diff --git a/tidb-cloud/vector-search-integrate-with-amazon-bedrock.md b/tidb-cloud/vector-search-integrate-with-amazon-bedrock.md new file mode 100644 index 0000000000000..251e54fcd920d --- /dev/null +++ b/tidb-cloud/vector-search-integrate-with-amazon-bedrock.md @@ -0,0 +1,313 @@ +--- +title: Integrate TiDB Vector Search with Amazon Bedrock +summary: Learn how to integrate TiDB Vector Search with Amazon Bedrock to build a Retrieval-Augmented Generation (RAG) Q&A bot. +--- + +# Integrate TiDB Vector Search with Amazon Bedrock + +This tutorial demonstrates how to integrate the [vector search](/tidb-cloud/vector-search-overview.md) feature of TiDB with [Amazon Bedrock](https://aws.amazon.com/bedrock/) to build a Retrieval-Augmented Generation (RAG) Q&A bot. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +> **Tip** +> +> You can view the complete [sample code](https://github.com/aws-samples/aws-generativeai-partner-samples/blob/main/tidb/samples/tidb-bedrock-boto3-rag.ipynb) in Notebook format. + +## Prerequisites + +To complete this tutorial, you need: + +- [Python 3.11 or later](https://www.python.org/downloads/) installed +- [Pip](https://pypi.org/project/pip/) installed +- [AWS CLI](https://aws.amazon.com/cli/) installed + + Ensure your AWS CLI profile is configured to a supported [Amazon Bedrock](https://aws.amazon.com/bedrock/) region for this tutorial. You can find the list of supported regions at [Amazon Bedrock Regions](https://docs.aws.amazon.com/bedrock/latest/userguide/models-regions.html). To switch to a supported region, run the following command: + + ```shell + aws configure set region + ``` + +- A TiDB Cloud Serverless cluster + + Follow [creating a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create your own TiDB Cloud cluster if you don't have one. + +- An AWS account with the [required permissions for Amazon Bedrock](https://docs.aws.amazon.com/bedrock/latest/userguide/security_iam_id-based-policy-examples.html) and access to the following models: + + - **Amazon Titan Embeddings** (`amazon.titan-embed-text-v2:0`), used for generating text embeddings + - **Meta Llama 3** (`us.meta.llama3-2-3b-instruct-v1:0`), used for text generation + + If you don't have access, follow the instructions in [Request access to an Amazon Bedrock foundation model](https://docs.aws.amazon.com/bedrock/latest/userguide/getting-started.html#getting-started-model-access). + +## Get started + +This section provides step-by-step instructions for integrating TiDB Vector Search with Amazon Bedrock to build a RAG-based Q&A bot. + +### Step 1. Set the environment variables + +Get the TiDB connection information from the [TiDB Cloud console](https://tidbcloud.com/) and set the environment variables in your development environment as follows: + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public` + - **Branch** is set to `main` + - **Connect With** is set to `General` + - **Operating System** matches your environment. + + > **Tip:** + > + > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. + +4. Click **Generate Password** to create a random password. + + > **Tip:** + > + > If you have created a password before, you can either use the original password or click **Reset Password** to generate a new one. + +5. Run the following commands in your terminal to set the environment variables. You need to replace the placeholders in the commands with the corresponding connection parameters obtained from the connection dialog. + + ```shell + export TIDB_HOST= + export TIDB_PORT=4000 + export TIDB_USER= + export TIDB_PASSWORD= + export TIDB_DB_NAME=test + ``` + +### Step 2. Set up the Python virtual environment + +1. Create a Python file named `demo.py`: + + ```shell + touch demo.py + ``` + +2. Create and activate a virtual environment to manage dependencies: + + ```shell + python3 -m venv env + source env/bin/activate # On Windows, use env\Scripts\activate + ``` + +3. Install the required dependencies: + + ```shell + pip install SQLAlchemy==2.0.30 PyMySQL==1.1.0 tidb-vector==0.0.9 pydantic==2.7.1 boto3 + ``` + +### Step 3. Import required libraries + +Add the following code to the beginning of `demo.py` to import the required libraries: + +```python +import os +import json +import boto3 +from sqlalchemy import Column, Integer, Text, create_engine +from sqlalchemy.orm import declarative_base, Session +from tidb_vector.sqlalchemy import VectorType +``` + +### Step 4. Configure the database connection + +In `demo.py`, add the following code to configure the database connection: + +```python +# ---- Configuration Setup ---- +# Set environment variables: TIDB_HOST, TIDB_PORT, TIDB_USER, TIDB_PASSWORD, TIDB_DB_NAME +TIDB_HOST = os.environ.get("TIDB_HOST") +TIDB_PORT = os.environ.get("TIDB_PORT") +TIDB_USER = os.environ.get("TIDB_USER") +TIDB_PASSWORD = os.environ.get("TIDB_PASSWORD") +TIDB_DB_NAME = os.environ.get("TIDB_DB_NAME") + +# ---- Database Setup ---- +def get_db_url(): + """Build the database connection URL.""" + return f"mysql+pymysql://{TIDB_USER}:{TIDB_PASSWORD}@{TIDB_HOST}:{TIDB_PORT}/{TIDB_DB_NAME}?ssl_verify_cert=True&ssl_verify_identity=True" + +# Create engine +engine = create_engine(get_db_url(), pool_recycle=300) +Base = declarative_base() +``` + +### Step 5. Invoke the Amazon Titan Text Embeddings V2 model using the Bedrock runtime client + +The Amazon Bedrock runtime client provides you with an `invoke_model` API that accepts the following parameters: + +- `modelId`: the model ID of the foundation model available in Amazon Bedrock. +- `accept`: the type of the input request. +- `contentType`: the content type of the input. +- `body`: a JSON string payload consisting of the prompt and the configurations. + +In `demo.py`, add the following code to invoke the `invoke_model` API to generate text embeddings using Amazon Titan Text Embeddings and get responses from Meta Llama 3: + +```python +# Bedrock Runtime Client Setup +bedrock_runtime = boto3.client('bedrock-runtime') + +# ---- Model Invocation ---- +embedding_model_name = "amazon.titan-embed-text-v2:0" +dim_of_embedding_model = 512 +llm_name = "us.meta.llama3-2-3b-instruct-v1:0" + + +def embedding(content): + """Invoke Amazon Bedrock to get text embeddings.""" + payload = { + "modelId": embedding_model_name, + "contentType": "application/json", + "accept": "*/*", + "body": { + "inputText": content, + "dimensions": dim_of_embedding_model, + "normalize": True, + } + } + + body_bytes = json.dumps(payload['body']).encode('utf-8') + + response = bedrock_runtime.invoke_model( + body=body_bytes, + contentType=payload['contentType'], + accept=payload['accept'], + modelId=payload['modelId'] + ) + + result_body = json.loads(response.get("body").read()) + return result_body.get("embedding") + + +def generate_result(query: str, info_str: str): + """Generate answer using Meta Llama 3 model.""" + prompt = f""" + ONLY use the content below to generate an answer: + {info_str} + + ---- + Please carefully think about the question: {query} + """ + + payload = { + "modelId": llm_name, + "contentType": "application/json", + "accept": "application/json", + "body": { + "prompt": prompt, + "temperature": 0 + } + } + + body_bytes = json.dumps(payload['body']).encode('utf-8') + + response = bedrock_runtime.invoke_model( + body=body_bytes, + contentType=payload['contentType'], + accept=payload['accept'], + modelId=payload['modelId'] + ) + + result_body = json.loads(response.get("body").read()) + completion = result_body["generation"] + return completion +``` + +### Step 6. Create a vector table + +In `demo.py`, add the following code to create a vector table to store text and vector embeddings: + +```python +# ---- TiDB Setup and Vector Index Creation ---- +class Entity(Base): + """Define the Entity table with a vector index.""" + __tablename__ = "entity" + id = Column(Integer, primary_key=True) + content = Column(Text) + content_vec = Column(VectorType(dim=dim_of_embedding_model), comment="hnsw(distance=l2)") + +# Create the table in TiDB +Base.metadata.create_all(engine) +``` + +### Step 7. Save the vector data to TiDB Cloud Serverless + +In `demo.py`, add the following code to save the vector data to your TiDB Cloud Serverless cluster: + +```python +# ---- Saving Vectors to TiDB ---- +def save_entities_with_embedding(session, contents): + """Save multiple entities with their embeddings to the TiDB Serverless database.""" + for content in contents: + entity = Entity(content=content, content_vec=embedding(content)) + session.add(entity) + session.commit() +``` + +### Step 8. Run the application + +1. In `demo.py`, add the following code to establish a database session, save embeddings to TiDB, ask an example question (such as "What is TiDB?"), and generate results from the model: + + ```python + if __name__ == "__main__": + # Establish a database session + with Session(engine) as session: + # Example data + contents = [ + "TiDB is a distributed SQL database compatible with MySQL.", + "TiDB supports Hybrid Transactional and Analytical Processing (HTAP).", + "TiDB can scale horizontally and provides high availability.", + "Amazon Bedrock allows seamless integration with foundation models.", + "Meta Llama 3 is a powerful model for text generation." + ] + + # Save embeddings to TiDB + save_entities_with_embedding(session, contents) + + # Example query + query = "What is TiDB?" + info_str = " ".join(contents) + + # Generate result from Meta Llama 3 + result = generate_result(query, info_str) + print(f"Generated answer: {result}") + ``` + +2. Save all changes to `demo.py` and run the script: + + ```shell + python3 demo.py + ``` + + The expected output is similar to the following: + + ``` + Generated answer: What is the main purpose of TiDB? + What are the key features of TiDB? + What are the key benefits of TiDB? + + ---- + Based on the provided text, here is the answer to the question: + What is TiDB? + TiDB is a distributed SQL database compatible with MySQL. + + ## Step 1: Understand the question + The question asks for the definition of TiDB. + + ## Step 2: Identify the key information + The key information provided in the text is that TiDB is a distributed SQL database compatible with MySQL. + + ## Step 3: Provide the answer + Based on the provided text, TiDB is a distributed SQL database compatible with MySQL. + + The final answer is: TiDB is a distributed SQL database compatible with MySQL. + ``` + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) \ No newline at end of file diff --git a/tidb-cloud/vector-search-integrate-with-django-orm.md b/tidb-cloud/vector-search-integrate-with-django-orm.md new file mode 100644 index 0000000000000..61c9012ca54d8 --- /dev/null +++ b/tidb-cloud/vector-search-integrate-with-django-orm.md @@ -0,0 +1,221 @@ +--- +title: Integrate TiDB Vector Search with Django ORM +summary: Learn how to integrate TiDB Vector Search with Django ORM to store embeddings and perform semantic search. +--- + +# Integrate TiDB Vector Search with Django ORM + +This tutorial walks you through how to use [Django](https://www.djangoproject.com/) ORM to interact with the [TiDB Vector Search](/tidb-cloud/vector-search-overview.md), store embeddings, and perform vector search queries. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Prerequisites + +To complete this tutorial, you need: + +- [Python 3.8 or higher](https://www.python.org/downloads/) installed. +- [Git](https://git-scm.com/downloads) installed. +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create your own TiDB Cloud cluster if you don't have one. + +## Run the sample app + +You can quickly learn about how to integrate TiDB Vector Search with Django ORM by following the steps below. + +### Step 1. Clone the repository + +Clone the `tidb-vector-python` repository to your local machine: + +```shell +git clone https://github.com/pingcap/tidb-vector-python.git +``` + +### Step 2. Create a virtual environment + +Create a virtual environment for your project: + +```bash +cd tidb-vector-python/examples/orm-django-quickstart +python3 -m venv .venv +source .venv/bin/activate +``` + +### Step 3. Install required dependencies + +Install the required dependencies for the demo project: + +```bash +pip install -r requirements.txt +``` + +Alternatively, you can install the following packages for your project: + +```bash +pip install Django django-tidb mysqlclient numpy python-dotenv +``` + +If you encounter installation issues with mysqlclient, refer to the mysqlclient official documentation. + +#### What is `django-tidb` + +`django-tidb` is a TiDB dialect for Django, which enhances the Django ORM to support TiDB-specific features (for example, TiDB Vector Search) and resolves compatibility issues between TiDB and Django. + +To install `django-tidb`, choose a version that matches your Django version. For example, if you are using `django==4.2.*`, install `django-tidb==4.2.*`. The minor version does not need to be the same. It is recommended to use the latest minor version. + +For more information, refer to [django-tidb repository](https://github.com/pingcap/django-tidb). + +### Step 4. Configure the environment variables + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public` + - **Branch** is set to `main` + - **Connect With** is set to `General` + - **Operating System** matches your environment. + + > **Tip:** + > + > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. + +4. Copy the connection parameters from the connection dialog. + + > **Tip:** + > + > If you have not set a password yet, click **Generate Password** to generate a random password. + +5. In the root directory of your Python project, create a `.env` file and paste the connection parameters to the corresponding environment variables. + + - `TIDB_HOST`: The host of the TiDB cluster. + - `TIDB_PORT`: The port of the TiDB cluster. + - `TIDB_USERNAME`: The username to connect to the TiDB cluster. + - `TIDB_PASSWORD`: The password to connect to the TiDB cluster. + - `TIDB_DATABASE`: The database name to connect to. + - `TIDB_CA_PATH`: The path to the root certificate file. + + The following is an example for macOS: + + ```dotenv + TIDB_HOST=gateway01.****.prod.aws.tidbcloud.com + TIDB_PORT=4000 + TIDB_USERNAME=********.root + TIDB_PASSWORD=******** + TIDB_DATABASE=test + TIDB_CA_PATH=/etc/ssl/cert.pem + ``` + +### Step 5. Run the demo + +Migrate the database schema: + +```bash +python manage.py migrate +``` + +Run the Django development server: + +```bash +python manage.py runserver +``` + +Open your browser and visit `http://127.0.0.1:8000` to try the demo application. Here are the available API paths: + +| API Path | Description | +| --------------------------------------- | ---------------------------------------- | +| `POST: /insert_documents` | Insert documents with embeddings. | +| `GET: /get_nearest_neighbors_documents` | Get the 3-nearest neighbor documents. | +| `GET: /get_documents_within_distance` | Get documents within a certain distance. | + +## Sample code snippets + +You can refer to the following sample code snippets to complete your own application development. + +### Connect to the TiDB cluster + +In the file `sample_project/settings.py`, add the following configurations: + +```python +dotenv.load_dotenv() + +DATABASES = { + "default": { + # https://github.com/pingcap/django-tidb + "ENGINE": "django_tidb", + "HOST": os.environ.get("TIDB_HOST", "127.0.0.1"), + "PORT": int(os.environ.get("TIDB_PORT", 4000)), + "USER": os.environ.get("TIDB_USERNAME", "root"), + "PASSWORD": os.environ.get("TIDB_PASSWORD", ""), + "NAME": os.environ.get("TIDB_DATABASE", "test"), + "OPTIONS": { + "charset": "utf8mb4", + }, + } +} + +TIDB_CA_PATH = os.environ.get("TIDB_CA_PATH", "") +if TIDB_CA_PATH: + DATABASES["default"]["OPTIONS"]["ssl_mode"] = "VERIFY_IDENTITY" + DATABASES["default"]["OPTIONS"]["ssl"] = { + "ca": TIDB_CA_PATH, + } +``` + +You can create a `.env` file in the root directory of your project and set up the environment variables `TIDB_HOST`, `TIDB_PORT`, `TIDB_USERNAME`, `TIDB_PASSWORD`, `TIDB_DATABASE`, and `TIDB_CA_PATH` with the actual values of your TiDB cluster. + +### Create vector tables + +#### Define a vector column + +`tidb-django` provides a `VectorField` to store vector embeddings in a table. + +Create a table with a column named `embedding` that stores a 3-dimensional vector. + +```python +class Document(models.Model): + content = models.TextField() + embedding = VectorField(dimensions=3) +``` + +### Store documents with embeddings + +```python +Document.objects.create(content="dog", embedding=[1, 2, 1]) +Document.objects.create(content="fish", embedding=[1, 2, 4]) +Document.objects.create(content="tree", embedding=[1, 0, 0]) +``` + +### Search the nearest neighbor documents + +TiDB Vector support the following distance functions: + +- `L1Distance` +- `L2Distance` +- `CosineDistance` +- `NegativeInnerProduct` + +Search for the top-3 documents that are semantically closest to the query vector `[1, 2, 3]` based on the cosine distance function. + +```python +results = Document.objects.annotate( + distance=CosineDistance('embedding', [1, 2, 3]) +).order_by('distance')[:3] +``` + +### Search documents within a certain distance + +Search for the documents whose cosine distance from the query vector `[1, 2, 3]` is less than 0.2. + +```python +results = Document.objects.annotate( + distance=CosineDistance('embedding', [1, 2, 3]) +).filter(distance__lt=0.2).order_by('distance')[:3] +``` + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) diff --git a/tidb-cloud/vector-search-integrate-with-jinaai-embedding.md b/tidb-cloud/vector-search-integrate-with-jinaai-embedding.md new file mode 100644 index 0000000000000..71e79db40fd1d --- /dev/null +++ b/tidb-cloud/vector-search-integrate-with-jinaai-embedding.md @@ -0,0 +1,240 @@ +--- +title: Integrate TiDB Vector Search with Jina AI Embeddings API +summary: Learn how to integrate TiDB Vector Search with Jina AI Embeddings API to store embeddings and perform semantic search. +--- + +# Integrate TiDB Vector Search with Jina AI Embeddings API + +This tutorial walks you through how to use [Jina AI](https://jina.ai/) to generate embeddings for text data, and then store the embeddings in TiDB vector storage and search similar texts based on embeddings. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Prerequisites + +To complete this tutorial, you need: + +- [Python 3.8 or higher](https://www.python.org/downloads/) installed. +- [Git](https://git-scm.com/downloads) installed. +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create your own TiDB Cloud cluster if you don't have one. + +## Run the sample app + +You can quickly learn about how to integrate TiDB Vector Search with JinaAI Embedding by following the steps below. + +### Step 1. Clone the repository + +Clone the `tidb-vector-python` repository to your local machine: + +```shell +git clone https://github.com/pingcap/tidb-vector-python.git +``` + +### Step 2. Create a virtual environment + +Create a virtual environment for your project: + +```bash +cd tidb-vector-python/examples/jina-ai-embeddings-demo +python3 -m venv .venv +source .venv/bin/activate +``` + +### Step 3. Install required dependencies + +Install the required dependencies for the demo project: + +```bash +pip install -r requirements.txt +``` + +### Step 4. Configure the environment variables + +Get the Jina AI API key from the [Jina AI Embeddings API](https://jina.ai/embeddings/) page. Then, obtain the cluster connection string and configure environment variables as follows: + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public` + - **Branch** is set to `main` + - **Connect With** is set to `SQLAlchemy` + - **Operating System** matches your environment. + + > **Tip:** + > + > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. + +4. Switch to the **PyMySQL** tab and click the **Copy** icon to copy the connection string. + + > **Tip:** + > + > If you have not set a password yet, click **Create password** to generate a random password. + +5. Set the Jina AI API key and the TiDB connection string as environment variables in your terminal, or create a `.env` file with the following environment variables: + + ```dotenv + JINAAI_API_KEY="****" + TIDB_DATABASE_URL="{tidb_connection_string}" + ``` + + The following is an example connection string for macOS: + + ```dotenv + TIDB_DATABASE_URL="mysql+pymysql://.root:@gateway01..prod.aws.tidbcloud.com:4000/test?ssl_ca=/etc/ssl/cert.pem&ssl_verify_cert=true&ssl_verify_identity=true" + ``` + +### Step 5. Run the demo + +```bash +python jina-ai-embeddings-demo.py +``` + +Example output: + +```text +- Inserting Data to TiDB... + - Inserting: Jina AI offers best-in-class embeddings, reranker and prompt optimizer, enabling advanced multimodal AI. + - Inserting: TiDB is an open-source MySQL-compatible database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. +- List All Documents and Their Distances to the Query: + - distance: 0.3585317326132522 + content: Jina AI offers best-in-class embeddings, reranker and prompt optimizer, enabling advanced multimodal AI. + - distance: 0.10858102967720984 + content: TiDB is an open-source MySQL-compatible database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. +- The Most Relevant Document and Its Distance to the Query: + - distance: 0.10858102967720984 + content: TiDB is an open-source MySQL-compatible database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads. +``` + +## Sample code snippets + +### Get embeddings from Jina AI + +Define a `generate_embeddings` helper function to call Jina AI embeddings API: + +```python +import os +import requests +import dotenv + +dotenv.load_dotenv() + +JINAAI_API_KEY = os.getenv('JINAAI_API_KEY') + +def generate_embeddings(text: str): + JINAAI_API_URL = 'https://api.jina.ai/v1/embeddings' + JINAAI_HEADERS = { + 'Content-Type': 'application/json', + 'Authorization': f'Bearer {JINAAI_API_KEY}' + } + JINAAI_REQUEST_DATA = { + 'input': [text], + 'model': 'jina-embeddings-v2-base-en' # with dimension 768. + } + response = requests.post(JINAAI_API_URL, headers=JINAAI_HEADERS, json=JINAAI_REQUEST_DATA) + return response.json()['data'][0]['embedding'] +``` + +### Connect to the TiDB cluster + +Connect to the TiDB cluster through SQLAlchemy: + +```python +import os +import dotenv + +from tidb_vector.sqlalchemy import VectorType +from sqlalchemy.orm import Session, declarative_base + +dotenv.load_dotenv() + +TIDB_DATABASE_URL = os.getenv('TIDB_DATABASE_URL') +assert TIDB_DATABASE_URL is not None +engine = create_engine(url=TIDB_DATABASE_URL, pool_recycle=300) +``` + +### Define the vector table schema + +Create a table named `jinaai_tidb_demo_documents` with a `content` column for storing texts and a vector column named `content_vec` for storing embeddings: + +```python +from sqlalchemy import Column, Integer, String, create_engine +from sqlalchemy.orm import declarative_base + +Base = declarative_base() + +class Document(Base): + __tablename__ = "jinaai_tidb_demo_documents" + + id = Column(Integer, primary_key=True) + content = Column(String(255), nullable=False) + content_vec = Column( + # DIMENSIONS is determined by the embedding model, + # for Jina AI's jina-embeddings-v2-base-en model it's 768. + VectorType(dim=768), + comment="hnsw(distance=cosine)" +``` + +> **Note:** +> +> - The dimension of the vector column must match the dimension of the embeddings generated by the embedding model. +> - In this example, the dimension of embeddings generated by the `jina-embeddings-v2-base-en` model is `768`. + +### Create embeddings with Jina AI and store in TiDB + +Use the Jina AI Embeddings API to generate embeddings for each piece of text and store the embeddings in TiDB: + +```python +TEXTS = [ + 'Jina AI offers best-in-class embeddings, reranker and prompt optimizer, enabling advanced multimodal AI.', + 'TiDB is an open-source MySQL-compatible database that supports Hybrid Transactional and Analytical Processing (HTAP) workloads.', +] +data = [] + +for text in TEXTS: + # Generate embeddings for the texts via Jina AI API. + embedding = generate_embeddings(text) + data.append({ + 'text': text, + 'embedding': embedding + }) + +with Session(engine) as session: + print('- Inserting Data to TiDB...') + for item in data: + print(f' - Inserting: {item["text"]}') + session.add(Document( + content=item['text'], + content_vec=item['embedding'] + )) + session.commit() +``` + +### Perform semantic search with Jina AI embeddings in TiDB + +Generate the embedding for the query text via Jina AI embeddings API, and then search for the most relevant document based on the cosine distance between **the embedding of the query text** and **each embedding in the vector table**: + +```python +query = 'What is TiDB?' +# Generate the embedding for the query via Jina AI API. +query_embedding = generate_embeddings(query) + +with Session(engine) as session: + print('- The Most Relevant Document and Its Distance to the Query:') + doc, distance = session.query( + Document, + Document.content_vec.cosine_distance(query_embedding).label('distance') + ).order_by( + 'distance' + ).limit(1).first() + print(f' - distance: {distance}\n' + f' content: {doc.content}') +``` + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) diff --git a/tidb-cloud/vector-search-integrate-with-langchain.md b/tidb-cloud/vector-search-integrate-with-langchain.md new file mode 100644 index 0000000000000..04b518a82b31d --- /dev/null +++ b/tidb-cloud/vector-search-integrate-with-langchain.md @@ -0,0 +1,586 @@ +--- +title: Integrate Vector Search with LangChain +summary: Learn how to integrate Vector Search in TiDB Cloud with LangChain. +--- + +# Integrate Vector Search with LangChain + +This tutorial demonstrates how to integrate the [vector search](/tidb-cloud/vector-search-overview.md) feature in TiDB Cloud with [LangChain](https://python.langchain.com/). + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +> **Tip** +> +> You can view the complete [sample code](https://github.com/langchain-ai/langchain/blob/master/docs/docs/integrations/vectorstores/tidb_vector.ipynb) on Jupyter Notebook, or run the sample code directly in the [Colab](https://colab.research.google.com/github/langchain-ai/langchain/blob/master/docs/docs/integrations/vectorstores/tidb_vector.ipynb) online environment. + +## Prerequisites + +To complete this tutorial, you need: + +- [Python 3.8 or higher](https://www.python.org/downloads/) installed. +- [Jupyter Notebook](https://jupyter.org/install) installed. +- [Git](https://git-scm.com/downloads) installed. +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create your own TiDB Cloud cluster if you don't have one. + +## Get started + +This section provides step-by-step instructions for integrating TiDB Vector Search with LangChain to perform semantic searches. + +### Step 1. Create a new Jupyter Notebook file + +In your preferred directory, create a new Jupyter Notebook file named `integrate_with_langchain.ipynb`: + +```shell +touch integrate_with_langchain.ipynb +``` + +### Step 2. Install required dependencies + +In your project directory, run the following command to install the required packages: + +```shell +!pip install langchain langchain-community +!pip install langchain-openai +!pip install pymysql +!pip install tidb-vector +``` + +Open the `integrate_with_langchain.ipynb` file in Jupyter Notebook, and then add the following code to import the required packages: + +```python +from langchain_community.document_loaders import TextLoader +from langchain_community.vectorstores import TiDBVectorStore +from langchain_openai import OpenAIEmbeddings +from langchain_text_splitters import CharacterTextSplitter +``` + +### Step 3. Set up your environment + +Take the following steps to obtain the cluster connection string and configure environment variables: + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. + - **Connect With** is set to `SQLAlchemy`. + - **Operating System** matches your environment. + +4. Click the **PyMySQL** tab and copy the connection string. + + > **Tip:** + > + > If you have not set a password yet, click **Generate Password** to generate a random password. + +5. Configure environment variables. + + This document uses [OpenAI](https://platform.openai.com/docs/introduction) as the embedding model provider. In this step, you need to provide the connection string obtained from the previous step and your [OpenAI API key](https://platform.openai.com/docs/quickstart/step-2-set-up-your-api-key). + + To configure the environment variables, run the following code. You will be prompted to enter your connection string and OpenAI API key: + + ```python + # Use getpass to securely prompt for environment variables in your terminal. + import getpass + import os + + # Copy your connection string from the TiDB Cloud console. + # Connection string format: "mysql+pymysql://:@:4000/?ssl_ca=/etc/ssl/cert.pem&ssl_verify_cert=true&ssl_verify_identity=true" + tidb_connection_string = getpass.getpass("TiDB Connection String:") + os.environ["OPENAI_API_KEY"] = getpass.getpass("OpenAI API Key:") + ``` + +### Step 4. Load the sample document + +#### Step 4.1 Download the sample document + +In your project directory, create a directory named `data/how_to/` and download the sample document [`state_of_the_union.txt`](https://github.com/langchain-ai/langchain/blob/master/docs/docs/how_to/state_of_the_union.txt) from the [langchain-ai/langchain](https://github.com/langchain-ai/langchain) GitHub repository. + +```shell +!mkdir -p 'data/how_to/' +!wget 'https://raw.githubusercontent.com/langchain-ai/langchain/master/docs/docs/how_to/state_of_the_union.txt' -O 'data/how_to/state_of_the_union.txt' +``` + +#### Step 4.2 Load and split the document + +Load the sample document from `data/how_to/state_of_the_union.txt` and split it into chunks of approximately 1,000 characters each using a `CharacterTextSplitter`. + +```python +loader = TextLoader("data/how_to/state_of_the_union.txt") +documents = loader.load() +text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=0) +docs = text_splitter.split_documents(documents) +``` + +### Step 5. Embed and store document vectors + +TiDB vector store supports both cosine distance (`consine`) and Euclidean distance (`l2`) for measuring similarity between vectors. The default strategy is cosine distance. + +The following code creates a table named `embedded_documents` in TiDB, which is optimized for vector search. + +```python +embeddings = OpenAIEmbeddings() +vector_store = TiDBVectorStore.from_documents( + documents=docs, + embedding=embeddings, + table_name="embedded_documents", + connection_string=tidb_connection_string, + distance_strategy="cosine", # default, another option is "l2" +) +``` + +Upon successful execution, you can directly view and access the `embedded_documents` table in your TiDB database. + +### Step 6. Perform a vector search + +This step demonstrates how to query "What did the president say about Ketanji Brown Jackson" from the document `state_of_the_union.txt`. + +```python +query = "What did the president say about Ketanji Brown Jackson" +``` + +#### Option 1: Use `similarity_search_with_score()` + +The `similarity_search_with_score()` method calculates the vector space distance between the documents and the query. This distance serves as a similarity score, determined by the chosen `distance_strategy`. The method returns the top `k` documents with the lowest scores. A lower score indicates a higher similarity between a document and your query. + +```python +docs_with_score = vector_store.similarity_search_with_score(query, k=3) +for doc, score in docs_with_score: + print("-" * 80) + print("Score: ", score) + print(doc.page_content) + print("-" * 80) +``` + +
+ Expected output + +```plain +-------------------------------------------------------------------------------- +Score: 0.18472413652518527 +Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. + +Tonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. + +One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. + +And I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence. +-------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- +Score: 0.21757513022785557 +A former top litigator in private practice. A former federal public defender. And from a family of public school educators and police officers. A consensus builder. Since she’s been nominated, she’s received a broad range of support—from the Fraternal Order of Police to former judges appointed by Democrats and Republicans. + +And if we are to advance liberty and justice, we need to secure the Border and fix the immigration system. + +We can do both. At our border, we’ve installed new technology like cutting-edge scanners to better detect drug smuggling. + +We’ve set up joint patrols with Mexico and Guatemala to catch more human traffickers. + +We’re putting in place dedicated immigration judges so families fleeing persecution and violence can have their cases heard faster. + +We’re securing commitments and supporting partners in South and Central America to host more refugees and secure their own borders. +-------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- +Score: 0.22676987253721725 +And for our LGBTQ+ Americans, let’s finally get the bipartisan Equality Act to my desk. The onslaught of state laws targeting transgender Americans and their families is wrong. + +As I said last year, especially to our younger transgender Americans, I will always have your back as your President, so you can be yourself and reach your God-given potential. + +While it often appears that we never agree, that isn’t true. I signed 80 bipartisan bills into law last year. From preventing government shutdowns to protecting Asian-Americans from still-too-common hate crimes to reforming military justice. + +And soon, we’ll strengthen the Violence Against Women Act that I first wrote three decades ago. It is important for us to show the nation that we can come together and do big things. + +So tonight I’m offering a Unity Agenda for the Nation. Four big things we can do together. + +First, beat the opioid epidemic. +-------------------------------------------------------------------------------- +``` + +
+ +#### Option 2: Use `similarity_search_with_relevance_scores()` + +The `similarity_search_with_relevance_scores()` method returns the top `k` documents with the highest relevance scores. A higher score indicates a higher degree of similarity between a document and your query. + +```python +docs_with_relevance_score = vector_store.similarity_search_with_relevance_scores(query, k=2) +for doc, score in docs_with_relevance_score: + print("-" * 80) + print("Score: ", score) + print(doc.page_content) + print("-" * 80) +``` + +
+ Expected output + +```plain +-------------------------------------------------------------------------------- +Score: 0.8152758634748147 +Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. + +Tonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. + +One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. + +And I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence. +-------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- +Score: 0.7824248697721444 +A former top litigator in private practice. A former federal public defender. And from a family of public school educators and police officers. A consensus builder. Since she’s been nominated, she’s received a broad range of support—from the Fraternal Order of Police to former judges appointed by Democrats and Republicans. + +And if we are to advance liberty and justice, we need to secure the Border and fix the immigration system. + +We can do both. At our border, we’ve installed new technology like cutting-edge scanners to better detect drug smuggling. + +We’ve set up joint patrols with Mexico and Guatemala to catch more human traffickers. + +We’re putting in place dedicated immigration judges so families fleeing persecution and violence can have their cases heard faster. + +We’re securing commitments and supporting partners in South and Central America to host more refugees and secure their own borders. +-------------------------------------------------------------------------------- +``` + +
+ +### Use as a retriever + +In Langchain, a [retriever](https://python.langchain.com/v0.2/docs/concepts/#retrievers) is an interface that retrieves documents in response to an unstructured query, providing more functionality than a vector store. The following code demonstrates how to use TiDB vector store as a retriever. + +```python +retriever = vector_store.as_retriever( + search_type="similarity_score_threshold", + search_kwargs={"k": 3, "score_threshold": 0.8}, +) +docs_retrieved = retriever.invoke(query) +for doc in docs_retrieved: + print("-" * 80) + print(doc.page_content) + print("-" * 80) +``` + +The expected output is as follows: + +``` +-------------------------------------------------------------------------------- +Tonight. I call on the Senate to: Pass the Freedom to Vote Act. Pass the John Lewis Voting Rights Act. And while you’re at it, pass the Disclose Act so Americans can know who is funding our elections. + +Tonight, I’d like to honor someone who has dedicated his life to serve this country: Justice Stephen Breyer—an Army veteran, Constitutional scholar, and retiring Justice of the United States Supreme Court. Justice Breyer, thank you for your service. + +One of the most serious constitutional responsibilities a President has is nominating someone to serve on the United States Supreme Court. + +And I did that 4 days ago, when I nominated Circuit Court of Appeals Judge Ketanji Brown Jackson. One of our nation’s top legal minds, who will continue Justice Breyer’s legacy of excellence. +-------------------------------------------------------------------------------- +``` + +### Remove the vector store + +To remove an existing TiDB vector store, use the `drop_vectorstore()` method: + +```python +vector_store.drop_vectorstore() +``` + +## Search with metadata filters + +To refine your searches, you can use metadata filters to retrieve specific nearest-neighbor results that match the applied filters. + +### Supported metadata types + +Each document in the TiDB vector store can be paired with metadata, structured as key-value pairs within a JSON object. Keys are always strings, while values can be any of the following types: + +- String +- Number: integer or floating point +- Boolean: `true` or `false` + +For example, the following is a valid metadata payload: + +```json +{ + "page": 12, + "book_title": "Siddhartha" +} +``` + +### Metadata filter syntax + +Available filters include the following: + +- `$or`: Selects vectors that match any one of the specified conditions. +- `$and`: Selects vectors that match all the specified conditions. +- `$eq`: Equal to the specified value. +- `$ne`: Not equal to the specified value. +- `$gt`: Greater than the specified value. +- `$gte`: Greater than or equal to the specified value. +- `$lt`: Less than the specified value. +- `$lte`: Less than or equal to the specified value. +- `$in`: In the specified array of values. +- `$nin`: Not in the specified array of values. + +If the metadata of a document is as follows: + +```json +{ + "page": 12, + "book_title": "Siddhartha" +} +``` + +The following metadata filters can match this document: + +```json +{ "page": 12 } +``` + +```json +{ "page": { "$eq": 12 } } +``` + +```json +{ + "page": { + "$in": [11, 12, 13] + } +} +``` + +```json +{ "page": { "$nin": [13] } } +``` + +```json +{ "page": { "$lt": 11 } } +``` + +```json +{ + "$or": [{ "page": 11 }, { "page": 12 }], + "$and": [{ "page": 12 }, { "page": 13 }] +} +``` + +In a metadata filter, TiDB treats each key-value pair as a separate filter clause and combines these clauses using the `AND` logical operator. + +### Example + +The following example adds two documents to `TiDBVectorStore` and adds a `title` field to each document as the metadata: + +```python +vector_store.add_texts( + texts=[ + "TiDB Vector offers advanced, high-speed vector processing capabilities, enhancing AI workflows with efficient data handling and analytics support.", + "TiDB Vector, starting as low as $10 per month for basic usage", + ], + metadatas=[ + {"title": "TiDB Vector functionality"}, + {"title": "TiDB Vector Pricing"}, + ], +) +``` + +The expected output is as follows: + +```plain +[UUID('c782cb02-8eec-45be-a31f-fdb78914f0a7'), + UUID('08dcd2ba-9f16-4f29-a9b7-18141f8edae3')] +``` + +Perform a similarity search with metadata filters: + +```python +docs_with_score = vector_store.similarity_search_with_score( + "Introduction to TiDB Vector", filter={"title": "TiDB Vector functionality"}, k=4 +) +for doc, score in docs_with_score: + print("-" * 80) + print("Score: ", score) + print(doc.page_content) + print("-" * 80) +``` + +The expected output is as follows: + +```plain +-------------------------------------------------------------------------------- +Score: 0.12761409169211535 +TiDB Vector offers advanced, high-speed vector processing capabilities, enhancing AI workflows with efficient data handling and analytics support. +-------------------------------------------------------------------------------- +``` + +## Advanced usage example: travel agent + +This section demonstrates a use case of integrating vector search with Langchain for a travel agent. The goal is to create personalized travel reports for clients, helping them find airports with specific amenities, such as clean lounges and vegetarian options. + +The process involves two main steps: + +1. Perform a semantic search across airport reviews to identify airport codes that match the desired amenities. +2. Execute a SQL query to merge these codes with route information, highlighting airlines and destinations that align with user's preferences. + +### Prepare data + +First, create a table to store airport route data: + +```python +# Create a table to store flight plan data. +vector_store.tidb_vector_client.execute( + """CREATE TABLE airplan_routes ( + id INT AUTO_INCREMENT PRIMARY KEY, + airport_code VARCHAR(10), + airline_code VARCHAR(10), + destination_code VARCHAR(10), + route_details TEXT, + duration TIME, + frequency INT, + airplane_type VARCHAR(50), + price DECIMAL(10, 2), + layover TEXT + );""" +) + +# Insert some sample data into airplan_routes and the vector table. +vector_store.tidb_vector_client.execute( + """INSERT INTO airplan_routes ( + airport_code, + airline_code, + destination_code, + route_details, + duration, + frequency, + airplane_type, + price, + layover + ) VALUES + ('JFK', 'DL', 'LAX', 'Non-stop from JFK to LAX.', '06:00:00', 5, 'Boeing 777', 299.99, 'None'), + ('LAX', 'AA', 'ORD', 'Direct LAX to ORD route.', '04:00:00', 3, 'Airbus A320', 149.99, 'None'), + ('EFGH', 'UA', 'SEA', 'Daily flights from SFO to SEA.', '02:30:00', 7, 'Boeing 737', 129.99, 'None'); + """ +) +vector_store.add_texts( + texts=[ + "Clean lounges and excellent vegetarian dining options. Highly recommended.", + "Comfortable seating in lounge areas and diverse food selections, including vegetarian.", + "Small airport with basic facilities.", + ], + metadatas=[ + {"airport_code": "JFK"}, + {"airport_code": "LAX"}, + {"airport_code": "EFGH"}, + ], +) +``` + +The expected output is as follows: + +```plain +[UUID('6dab390f-acd9-4c7d-b252-616606fbc89b'), + UUID('9e811801-0e6b-4893-8886-60f4fb67ce69'), + UUID('f426747c-0f7b-4c62-97ed-3eeb7c8dd76e')] +``` + +### Perform a semantic search + +The following code searches for airports with clean facilities and vegetarian options: + +```python +retriever = vector_store.as_retriever( + search_type="similarity_score_threshold", + search_kwargs={"k": 3, "score_threshold": 0.85}, +) +semantic_query = "Could you recommend a US airport with clean lounges and good vegetarian dining options?" +reviews = retriever.invoke(semantic_query) +for r in reviews: + print("-" * 80) + print(r.page_content) + print(r.metadata) + print("-" * 80) +``` + +The expected output is as follows: + +```plain +-------------------------------------------------------------------------------- +Clean lounges and excellent vegetarian dining options. Highly recommended. +{'airport_code': 'JFK'} +-------------------------------------------------------------------------------- +-------------------------------------------------------------------------------- +Comfortable seating in lounge areas and diverse food selections, including vegetarian. +{'airport_code': 'LAX'} +-------------------------------------------------------------------------------- +``` + +### Retrieve detailed airport information + +Extract airport codes from the search results and query the database for detailed route information: + +```python +# Extracting airport codes from the metadata +airport_codes = [review.metadata["airport_code"] for review in reviews] + +# Executing a query to get the airport details +search_query = "SELECT * FROM airplan_routes WHERE airport_code IN :codes" +params = {"codes": tuple(airport_codes)} + +airport_details = vector_store.tidb_vector_client.execute(search_query, params) +airport_details.get("result") +``` + +The expected output is as follows: + +```plain +[(1, 'JFK', 'DL', 'LAX', 'Non-stop from JFK to LAX.', datetime.timedelta(seconds=21600), 5, 'Boeing 777', Decimal('299.99'), 'None'), + (2, 'LAX', 'AA', 'ORD', 'Direct LAX to ORD route.', datetime.timedelta(seconds=14400), 3, 'Airbus A320', Decimal('149.99'), 'None')] +``` + +### Streamline the process + +Alternatively, you can streamline the entire process using a single SQL query: + +```python +search_query = f""" + SELECT + VEC_Cosine_Distance(se.embedding, :query_vector) as distance, + ar.*, + se.document as airport_review + FROM + airplan_routes ar + JOIN + {TABLE_NAME} se ON ar.airport_code = JSON_UNQUOTE(JSON_EXTRACT(se.meta, '$.airport_code')) + ORDER BY distance ASC + LIMIT 5; +""" +query_vector = embeddings.embed_query(semantic_query) +params = {"query_vector": str(query_vector)} +airport_details = vector_store.tidb_vector_client.execute(search_query, params) +airport_details.get("result") +``` + +The expected output is as follows: + +```plain +[(0.1219207353407008, 1, 'JFK', 'DL', 'LAX', 'Non-stop from JFK to LAX.', datetime.timedelta(seconds=21600), 5, 'Boeing 777', Decimal('299.99'), 'None', 'Clean lounges and excellent vegetarian dining options. Highly recommended.'), + (0.14613754359804654, 2, 'LAX', 'AA', 'ORD', 'Direct LAX to ORD route.', datetime.timedelta(seconds=14400), 3, 'Airbus A320', Decimal('149.99'), 'None', 'Comfortable seating in lounge areas and diverse food selections, including vegetarian.'), + (0.19840519342700513, 3, 'EFGH', 'UA', 'SEA', 'Daily flights from SFO to SEA.', datetime.timedelta(seconds=9000), 7, 'Boeing 737', Decimal('129.99'), 'None', 'Small airport with basic facilities.')] +``` + +### Clean up data + +Finally, clean up the resources by dropping the created table: + +```python +vector_store.tidb_vector_client.execute("DROP TABLE airplan_routes") +``` + +The expected output is as follows: + +```plain +{'success': True, 'result': 0, 'error': None} +``` + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) diff --git a/tidb-cloud/vector-search-integrate-with-llamaindex.md b/tidb-cloud/vector-search-integrate-with-llamaindex.md new file mode 100644 index 0000000000000..649d9381af121 --- /dev/null +++ b/tidb-cloud/vector-search-integrate-with-llamaindex.md @@ -0,0 +1,266 @@ +--- +title: Integrate Vector Search with LlamaIndex +summary: Learn how to integrate TiDB Vector Search with LlamaIndex. +--- + +# Integrate Vector Search with LlamaIndex + +This tutorial demonstrates how to integrate the [vector search](/tidb-cloud/vector-search-overview.md) feature of TiDB with [LlamaIndex](https://www.llamaindex.ai). + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +> **Tip** +> +> You can view the complete [sample code](https://github.com/run-llama/llama_index/blob/main/docs/docs/examples/vector_stores/TiDBVector.ipynb) on Jupyter Notebook, or run the sample code directly in the [Colab](https://colab.research.google.com/github/run-llama/llama_index/blob/main/docs/docs/examples/vector_stores/TiDBVector.ipynb) online environment. + +## Prerequisites + +To complete this tutorial, you need: + +- [Python 3.8 or higher](https://www.python.org/downloads/) installed. +- [Jupyter Notebook](https://jupyter.org/install) installed. +- [Git](https://git-scm.com/downloads) installed. +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create your own TiDB Cloud cluster if you don't have one. + +## Get started + +This section provides step-by-step instructions for integrating TiDB Vector Search with LlamaIndex to perform semantic searches. + +### Step 1. Create a new Jupyter Notebook file + +In the root directory, create a new Jupyter Notebook file named `integrate_with_llamaindex.ipynb`: + +```shell +touch integrate_with_llamaindex.ipynb +``` + +### Step 2. Install required dependencies + +In your project directory, run the following command to install the required packages: + +```shell +pip install llama-index-vector-stores-tidbvector +pip install llama-index +``` + +Open the `integrate_with_llamaindex.ipynb` file in Jupyter Notebook and add the following code to import the required packages: + +```python +import textwrap + +from llama_index.core import SimpleDirectoryReader, StorageContext +from llama_index.core import VectorStoreIndex +from llama_index.vector_stores.tidbvector import TiDBVectorStore +``` + +### Step 3. Configure environment variables + +Take the following steps to obtain the cluster connection string and configure environment variables: + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. + - **Connect With** is set to `SQLAlchemy`. + - **Operating System** matches your environment. + +4. Click the **PyMySQL** tab and copy the connection string. + + > **Tip:** + > + > If you have not set a password yet, click **Generate Password** to generate a random password. + +5. Configure environment variables. + + This document uses [OpenAI](https://platform.openai.com/docs/introduction) as the embedding model provider. In this step, you need to provide the connection string obtained from the previous step and your [OpenAI API key](https://platform.openai.com/docs/quickstart/step-2-set-up-your-api-key). + + To configure the environment variables, run the following code. You will be prompted to enter your connection string and OpenAI API key: + + ```python + # Use getpass to securely prompt for environment variables in your terminal. + import getpass + import os + + # Copy your connection string from the TiDB Cloud console. + # Connection string format: "mysql+pymysql://:@:4000/?ssl_ca=/etc/ssl/cert.pem&ssl_verify_cert=true&ssl_verify_identity=true" + tidb_connection_string = getpass.getpass("TiDB Connection String:") + os.environ["OPENAI_API_KEY"] = getpass.getpass("OpenAI API Key:") + ``` + +### Step 4. Load the sample document + +#### Step 4.1 Download the sample document + +In your project directory, create a directory named `data/paul_graham/` and download the sample document [`paul_graham_essay.txt`](https://github.com/run-llama/llama_index/blob/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt) from the [run-llama/llama_index](https://github.com/run-llama/llama_index) GitHub repository. + +```shell +!mkdir -p 'data/paul_graham/' +!wget 'https://raw.githubusercontent.com/run-llama/llama_index/main/docs/docs/examples/data/paul_graham/paul_graham_essay.txt' -O 'data/paul_graham/paul_graham_essay.txt' +``` + +#### Step 4.2 Load the document + +Load the sample document from `data/paul_graham/paul_graham_essay.txt` using the `SimpleDirectoryReader` class. + +```python +documents = SimpleDirectoryReader("./data/paul_graham").load_data() +print("Document ID:", documents[0].doc_id) + +for index, document in enumerate(documents): + document.metadata = {"book": "paul_graham"} +``` + +### Step 5. Embed and store document vectors + +#### Step 5.1 Initialize the TiDB vector store + +The following code creates a table named `paul_graham_test` in TiDB, which is optimized for vector search. + +```python +tidbvec = TiDBVectorStore( + connection_string=tidb_connection_url, + table_name="paul_graham_test", + distance_strategy="cosine", + vector_dimension=1536, + drop_existing_table=False, +) +``` + +Upon successful execution, you can directly view and access the `paul_graham_test` table in your TiDB database. + +#### Step 5.2 Generate and store embeddings + +The following code parses the documents, generates embeddings, and stores them in the TiDB vector store. + +```python +storage_context = StorageContext.from_defaults(vector_store=tidbvec) +index = VectorStoreIndex.from_documents( + documents, storage_context=storage_context, show_progress=True +) +``` + +The expected output is as follows: + +```plain +Parsing nodes: 100%|██████████| 1/1 [00:00<00:00, 8.76it/s] +Generating embeddings: 100%|██████████| 21/21 [00:02<00:00, 8.22it/s] +``` + +### Step 6. Perform a vector search + +The following creates a query engine based on the TiDB vector store and performs a semantic similarity search. + +```python +query_engine = index.as_query_engine() +response = query_engine.query("What did the author do?") +print(textwrap.fill(str(response), 100)) +``` + +> **Note** +> +> `TiDBVectorStore` only supports the [`default`](https://docs.llamaindex.ai/en/stable/api_reference/storage/vector_store/?h=vectorstorequerymode#llama_index.core.vector_stores.types.VectorStoreQueryMode) query mode. + +The expected output is as follows: + +```plain +The author worked on writing, programming, building microcomputers, giving talks at conferences, +publishing essays online, developing spam filters, painting, hosting dinner parties, and purchasing +a building for office use. +``` + +### Step 7. Search with metadata filters + +To refine your searches, you can use metadata filters to retrieve specific nearest-neighbor results that match the applied filters. + +#### Query with `book != "paul_graham"` filter + +The following example excludes results where the `book` metadata field is `"paul_graham"`: + +```python +from llama_index.core.vector_stores.types import ( + MetadataFilter, + MetadataFilters, +) + +query_engine = index.as_query_engine( + filters=MetadataFilters( + filters=[ + MetadataFilter(key="book", value="paul_graham", operator="!="), + ] + ), + similarity_top_k=2, +) +response = query_engine.query("What did the author learn?") +print(textwrap.fill(str(response), 100)) +``` + +The expected output is as follows: + +```plain +Empty Response +``` + +#### Query with `book == "paul_graham"` filter + +The following example filters results to include only documents where the `book` metadata field is `"paul_graham"`: + +```python +from llama_index.core.vector_stores.types import ( + MetadataFilter, + MetadataFilters, +) + +query_engine = index.as_query_engine( + filters=MetadataFilters( + filters=[ + MetadataFilter(key="book", value="paul_graham", operator="=="), + ] + ), + similarity_top_k=2, +) +response = query_engine.query("What did the author learn?") +print(textwrap.fill(str(response), 100)) +``` + +The expected output is as follows: + +```plain +The author learned programming on an IBM 1401 using an early version of Fortran in 9th grade, then +later transitioned to working with microcomputers like the TRS-80 and Apple II. Additionally, the +author studied philosophy in college but found it unfulfilling, leading to a switch to studying AI. +Later on, the author attended art school in both the US and Italy, where they observed a lack of +substantial teaching in the painting department. +``` + +### Step 8. Delete documents + +Delete the first document from the index: + +```python +tidbvec.delete(documents[0].doc_id) +``` + +Check whether the documents had been deleted: + +```python +query_engine = index.as_query_engine() +response = query_engine.query("What did the author learn?") +print(textwrap.fill(str(response), 100)) +``` + +The expected output is as follows: + +```plain +Empty Response +``` + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) diff --git a/tidb-cloud/vector-search-integrate-with-peewee.md b/tidb-cloud/vector-search-integrate-with-peewee.md new file mode 100644 index 0000000000000..0a72a34135ef8 --- /dev/null +++ b/tidb-cloud/vector-search-integrate-with-peewee.md @@ -0,0 +1,211 @@ +--- +title: Integrate TiDB Vector Search with peewee +summary: Learn how to integrate TiDB Vector Search with peewee to store embeddings and perform semantic searches. +--- + +# Integrate TiDB Vector Search with peewee + +This tutorial walks you through how to use [peewee](https://docs.peewee-orm.com/) to interact with the [TiDB Vector Search](/tidb-cloud/vector-search-overview.md), store embeddings, and perform vector search queries. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Prerequisites + +To complete this tutorial, you need: + +- [Python 3.8 or higher](https://www.python.org/downloads/) installed. +- [Git](https://git-scm.com/downloads) installed. +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create your own TiDB Cloud cluster if you don't have one. + +## Run the sample app + +You can quickly learn about how to integrate TiDB Vector Search with peewee by following the steps below. + +### Step 1. Clone the repository + +Clone the [`tidb-vector-python`](https://github.com/pingcap/tidb-vector-python) repository to your local machine: + +```shell +git clone https://github.com/pingcap/tidb-vector-python.git +``` + +### Step 2. Create a virtual environment + +Create a virtual environment for your project: + +```bash +cd tidb-vector-python/examples/orm-peewee-quickstart +python3 -m venv .venv +source .venv/bin/activate +``` + +### Step 3. Install required dependencies + +Install the required dependencies for the demo project: + +```bash +pip install -r requirements.txt +``` + +Alternatively, you can install the following packages for your project: + +```bash +pip install peewee pymysql python-dotenv tidb-vector +``` + +### Step 4. Configure the environment variables + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your operating environment. + + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. + - **Connect With** is set to `General`. + - **Operating System** matches your environment. + + > **Tip:** + > + > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. + +4. Copy the connection parameters from the connection dialog. + + > **Tip:** + > + > If you have not set a password yet, click **Generate Password** to generate a random password. + +5. In the root directory of your Python project, create a `.env` file and paste the connection parameters to the corresponding environment variables. + + - `TIDB_HOST`: The host of the TiDB cluster. + - `TIDB_PORT`: The port of the TiDB cluster. + - `TIDB_USERNAME`: The username to connect to the TiDB cluster. + - `TIDB_PASSWORD`: The password to connect to the TiDB cluster. + - `TIDB_DATABASE`: The database name to connect to. + - `TIDB_CA_PATH`: The path to the root certificate file. + + The following is an example for macOS: + + ```dotenv + TIDB_HOST=gateway01.****.prod.aws.tidbcloud.com + TIDB_PORT=4000 + TIDB_USERNAME=********.root + TIDB_PASSWORD=******** + TIDB_DATABASE=test + TIDB_CA_PATH=/etc/ssl/cert.pem + ``` + +### Step 5. Run the demo + +```bash +python peewee-quickstart.py +``` + +Example output: + +```text +Get 3-nearest neighbor documents: + - distance: 0.00853986601633272 + document: fish + - distance: 0.12712843905603044 + document: dog + - distance: 0.7327387580875756 + document: tree +Get documents within a certain distance: + - distance: 0.00853986601633272 + document: fish + - distance: 0.12712843905603044 + document: dog +``` + +## Sample code snippets + +You can refer to the following sample code snippets to develop your application. + +### Create vector tables + +#### Connect to TiDB cluster + +```python +import os +import dotenv + +from peewee import Model, MySQLDatabase, SQL, TextField +from tidb_vector.peewee import VectorField + +dotenv.load_dotenv() + +# Using `pymysql` as the driver. +connect_kwargs = { + 'ssl_verify_cert': True, + 'ssl_verify_identity': True, +} + +# Using `mysqlclient` as the driver. +# connect_kwargs = { +# 'ssl_mode': 'VERIFY_IDENTITY', +# 'ssl': { +# # Root certificate default path +# # https://docs.pingcap.com/tidbcloud/secure-connections-to-serverless-clusters/#root-certificate-default-path +# 'ca': os.environ.get('TIDB_CA_PATH', '/path/to/ca.pem'), +# }, +# } + +db = MySQLDatabase( + database=os.environ.get('TIDB_DATABASE', 'test'), + user=os.environ.get('TIDB_USERNAME', 'root'), + password=os.environ.get('TIDB_PASSWORD', ''), + host=os.environ.get('TIDB_HOST', 'localhost'), + port=int(os.environ.get('TIDB_PORT', '4000')), + **connect_kwargs, +) +``` + +#### Define a vector column + +Create a table with a column named `peewee_demo_documents` that stores a 3-dimensional vector. + +```python +class Document(Model): + class Meta: + database = db + table_name = 'peewee_demo_documents' + + content = TextField() + embedding = VectorField(3) +``` + +### Store documents with embeddings + +```python +Document.create(content='dog', embedding=[1, 2, 1]) +Document.create(content='fish', embedding=[1, 2, 4]) +Document.create(content='tree', embedding=[1, 0, 0]) +``` + +### Search the nearest neighbor documents + +Search for the top-3 documents that are semantically closest to the query vector `[1, 2, 3]` based on the cosine distance function. + +```python +distance = Document.embedding.cosine_distance([1, 2, 3]).alias('distance') +results = Document.select(Document, distance).order_by(distance).limit(3) +``` + +### Search documents within a certain distance + +Search for the documents whose cosine distance from the query vector `[1, 2, 3]` is less than 0.2. + +```python +distance_expression = Document.embedding.cosine_distance([1, 2, 3]) +distance = distance_expression.alias('distance') +results = Document.select(Document, distance).where(distance_expression < 0.2).order_by(distance).limit(3) +``` + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) diff --git a/tidb-cloud/vector-search-integrate-with-sqlalchemy.md b/tidb-cloud/vector-search-integrate-with-sqlalchemy.md new file mode 100644 index 0000000000000..f9fea8e3f11cd --- /dev/null +++ b/tidb-cloud/vector-search-integrate-with-sqlalchemy.md @@ -0,0 +1,185 @@ +--- +title: Integrate TiDB Vector Search with SQLAlchemy +summary: Learn how to integrate TiDB Vector Search with SQLAlchemy to store embeddings and perform semantic searches. +--- + +# Integrate TiDB Vector Search with SQLAlchemy + +This tutorial walks you through how to use [SQLAlchemy](https://www.sqlalchemy.org/) to interact with [TiDB Vector Search](/tidb-cloud/vector-search-overview.md), store embeddings, and perform vector search queries. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Prerequisites + +To complete this tutorial, you need: + +- [Python 3.8 or higher](https://www.python.org/downloads/) installed. +- [Git](https://git-scm.com/downloads) installed. +- A TiDB Cloud Serverless cluster. Follow [creating a TiDB Cloud Serverless cluster](/tidb-cloud/create-tidb-cluster-serverless.md) to create your own TiDB Cloud cluster if you don't have one. + +## Run the sample app + +You can quickly learn about how to integrate TiDB Vector Search with SQLAlchemy by following the steps below. + +### Step 1. Clone the repository + +Clone the `tidb-vector-python` repository to your local machine: + +```shell +git clone https://github.com/pingcap/tidb-vector-python.git +``` + +### Step 2. Create a virtual environment + +Create a virtual environment for your project: + +```bash +cd tidb-vector-python/examples/orm-sqlalchemy-quickstart +python3 -m venv .venv +source .venv/bin/activate +``` + +### Step 3. Install the required dependencies + +Install the required dependencies for the demo project: + +```bash +pip install -r requirements.txt +``` + +Alternatively, you can install the following packages for your project: + +```bash +pip install pymysql python-dotenv sqlalchemy tidb-vector +``` + +### Step 4. Configure the environment variables + +1. Navigate to the [**Clusters**](https://tidbcloud.com/console/clusters) page, and then click the name of your target cluster to go to its overview page. + +2. Click **Connect** in the upper-right corner. A connection dialog is displayed. + +3. Ensure the configurations in the connection dialog match your environment. + + - **Connection Type** is set to `Public`. + - **Branch** is set to `main`. + - **Connect With** is set to `SQLAlchemy`. + - **Operating System** matches your environment. + + > **Tip:** + > + > If your program is running in Windows Subsystem for Linux (WSL), switch to the corresponding Linux distribution. + +4. Click the **PyMySQL** tab and copy the connection string. + + > **Tip:** + > + > If you have not set a password yet, click **Generate Password** to generate a random password. + +5. In the root directory of your Python project, create a `.env` file and paste the connection string into it. + + The following is an example for macOS: + + ```dotenv + TIDB_DATABASE_URL="mysql+pymysql://.root:@gateway01..prod.aws.tidbcloud.com:4000/test?ssl_ca=/etc/ssl/cert.pem&ssl_verify_cert=true&ssl_verify_identity=true" + ``` + +### Step 5. Run the demo + +```bash +python sqlalchemy-quickstart.py +``` + +Example output: + +```text +Get 3-nearest neighbor documents: + - distance: 0.00853986601633272 + document: fish + - distance: 0.12712843905603044 + document: dog + - distance: 0.7327387580875756 + document: tree +Get documents within a certain distance: + - distance: 0.00853986601633272 + document: fish + - distance: 0.12712843905603044 + document: dog +``` + +## Sample code snippets + +You can refer to the following sample code snippets to develop your application. + +### Create vector tables + +#### Connect to TiDB cluster + +```python +import os +import dotenv + +from sqlalchemy import Column, Integer, create_engine, Text +from sqlalchemy.orm import declarative_base, Session +from tidb_vector.sqlalchemy import VectorType + +dotenv.load_dotenv() + +tidb_connection_string = os.environ['TIDB_DATABASE_URL'] +engine = create_engine(tidb_connection_string) +``` + +#### Define a vector column + +Create a table with a column named `embedding` that stores a 3-dimensional vector. + +```python +Base = declarative_base() + +class Document(Base): + __tablename__ = 'sqlalchemy_demo_documents' + id = Column(Integer, primary_key=True) + content = Column(Text) + embedding = Column(VectorType(3)) +``` + +### Store documents with embeddings + +```python +with Session(engine) as session: + session.add(Document(content="dog", embedding=[1, 2, 1])) + session.add(Document(content="fish", embedding=[1, 2, 4])) + session.add(Document(content="tree", embedding=[1, 0, 0])) + session.commit() +``` + +### Search the nearest neighbor documents + +Search for the top-3 documents that are semantically closest to the query vector `[1, 2, 3]` based on the cosine distance function. + +```python +with Session(engine) as session: + distance = Document.embedding.cosine_distance([1, 2, 3]).label('distance') + results = session.query( + Document, distance + ).order_by(distance).limit(3).all() +``` + +### Search documents within a certain distance + +Search for documents whose cosine distance from the query vector `[1, 2, 3]` is less than 0.2. + +```python +with Session(engine) as session: + distance = Document.embedding.cosine_distance([1, 2, 3]).label('distance') + results = session.query( + Document, distance + ).filter(distance < 0.2).order_by(distance).limit(3).all() +``` + +## See also + +- [Vector Data Types](/tidb-cloud/vector-search-data-types.md) +- [Vector Search Index](/tidb-cloud/vector-search-index.md) diff --git a/tidb-cloud/vector-search-integration-overview.md b/tidb-cloud/vector-search-integration-overview.md new file mode 100644 index 0000000000000..0a7b1dea242ba --- /dev/null +++ b/tidb-cloud/vector-search-integration-overview.md @@ -0,0 +1,77 @@ +--- +title: Vector Search Integration Overview +summary: An overview of TiDB Vector Search integration, including supported AI frameworks, embedding models, and ORM libraries. +--- + +# Vector Search Integration Overview + +This document provides an overview of TiDB Vector Search integration, including supported AI frameworks, embedding models, and Object Relational Mapping (ORM) libraries. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## AI frameworks + +TiDB provides official support for the following AI frameworks, enabling you to easily integrate AI applications developed based on these frameworks with TiDB Vector Search. + +| AI frameworks | Tutorial | +| ------------- | ------------------------------------------------------------------------------------------------- | +| Langchain | [Integrate Vector Search with LangChain](/tidb-cloud/vector-search-integrate-with-langchain.md) | +| LlamaIndex | [Integrate Vector Search with LlamaIndex](/tidb-cloud/vector-search-integrate-with-llamaindex.md) | + +Moreover, you can also use TiDB for various purposes, such as document storage and knowledge graph storage for AI applications. + +## AI services + +TiDB Vector Search supports integration with the following AI service, enabling you to easily build Retrieval-Augmented Generation (RAG) based applications. + +- [Amazon Bedrock](/tidb-cloud/vector-search-integrate-with-amazon-bedrock.md) + +## Embedding models and services + +TiDB Vector Search supports storing vectors of up to 16383 dimensions, which accommodates most embedding models. + +You can either use self-deployed open-source embedding models or third-party embedding APIs provided by third-party embedding providers to generate vectors. + +The following table lists some mainstream embedding service providers and the corresponding integration tutorials. + +| Embedding service providers | Tutorial | +| --------------------------- | ------------------------------------------------------------------------------------------------------------------- | +| Jina AI | [Integrate Vector Search with Jina AI Embeddings API](/tidb-cloud/vector-search-integrate-with-jinaai-embedding.md) | + +## Object Relational Mapping (ORM) libraries + +You can integrate TiDB Vector Search with your ORM library to interact with the TiDB database. + +The following table lists the supported ORM libraries and the corresponding integration tutorials: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
LanguageORM/ClientHow to installTutorial
PythonTiDB Vector Clientpip install tidb-vector[client]Get Started with Vector Search Using Python
SQLAlchemypip install tidb-vectorIntegrate TiDB Vector Search with SQLAlchemy
peeweepip install tidb-vectorIntegrate TiDB Vector Search with peewee
Djangopip install django-tidb[vector]Integrate TiDB Vector Search with Django
diff --git a/tidb-cloud/vector-search-limitations.md b/tidb-cloud/vector-search-limitations.md new file mode 100644 index 0000000000000..a3b72c488cd77 --- /dev/null +++ b/tidb-cloud/vector-search-limitations.md @@ -0,0 +1,42 @@ +--- +title: Vector Search Limitations +summary: Learn the limitations of the TiDB Vector Search. +--- + +# Vector Search Limitations + +This document describes the known limitations of TiDB Vector Search. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Vector data type limitations + +- Each [vector](/tidb-cloud/vector-search-data-types.md) supports up to 16383 dimensions. +- Vector data types cannot store `NaN`, `Infinity`, or `-Infinity` values. +- Vector data types cannot store double-precision floating-point numbers. If you insert or store double-precision floating-point numbers in vector columns, TiDB converts them to single-precision floating-point numbers. +- Vector columns cannot be used in primary keys, unique indexes, or partition keys. To accelerate the vector search performance, use [Vector Search Index](/tidb-cloud/vector-search-index.md). +- A table can have multiple vector columns. However, there is [a limit on the total number of columns in a table](/tidb-limitations.md#limitations-on-a-single-table). +- Currently, TiDB does not support dropping a vector column with a vector index. To drop such a column, drop the vector index first, then drop the vector column. +- Currently, TiDB does not support modifying a vector column to other data types such as `JSON` and `VARCHAR`. + +## Vector index limitations + +- Vector index is used for vector search. It cannot accelerate other queries like range queries or equality queries. Thus, it is not possible to create a vector index on a non-vector column, or on multiple vector columns. +- A table can have multiple vector indexes. However, there is [a limit on the total number of indexes in a table](/tidb-limitations.md#limitations-on-a-single-table). +- Creating multiple vector indexes on the same column is allowed only if they use different distance functions. +- Currently, only `VEC_COSINE_DISTANCE()` and `VEC_L2_DISTANCE()` are supported as the distance functions for vector indexes. +- Currently, TiDB does not support dropping a vector column with a vector index. To drop such a column, drop the vector index first, then drop the vector column. +- Currently, TiDB does not support setting vector index as [invisible](/sql-statements/sql-statement-alter-index.md). + +## Compatibility with TiDB tools + +- The Data Migration feature in the TiDB Cloud console does not support migrating or replicating MySQL 9.0 vector data types to TiDB Cloud. + +## Feedback + +We value your feedback and are always here to help: + +- [Join our Discord](https://discord.gg/zcqexutz2R) +- [Visit our Support Portal](https://tidb.support.pingcap.com/) diff --git a/tidb-cloud/vector-search-overview.md b/tidb-cloud/vector-search-overview.md new file mode 100644 index 0000000000000..612022eae176a --- /dev/null +++ b/tidb-cloud/vector-search-overview.md @@ -0,0 +1,72 @@ +--- +title: Vector Search (Beta) Overview +summary: Learn about Vector Search in TiDB. This feature provides an advanced search solution for performing semantic similarity searches across various data types, including documents, images, audio, and video. +--- + +# Vector Search (Beta) Overview + +TiDB Vector Search (beta) provides an advanced search solution for performing semantic similarity searches across various data types, including documents, images, audio, and video. This feature enables developers to easily build scalable applications with generative artificial intelligence (AI) capabilities using familiar MySQL skills. + +> **Note** +> +> TiDB Vector Search is only available for TiDB Self-Managed (TiDB >= v8.4) and [TiDB Cloud Serverless](/tidb-cloud/select-cluster-tier.md#tidb-cloud-serverless). It is not available for [TiDB Cloud Dedicated](/tidb-cloud/select-cluster-tier.md#tidb-cloud-dedicated). + +## Concepts + +Vector search is a search method that prioritizes the meaning of your data to deliver relevant results. + +Unlike traditional full-text search, which relies on exact keyword matching and word frequency, vector search converts various data types (such as text, images, or audio) into high-dimensional vectors and queries based on the similarity between these vectors. This search method captures the semantic meaning and contextual information of the data, leading to a more precise understanding of user intent. + +Even when the search terms do not exactly match the content in the database, vector search can still provide results that align with the user's intent by analyzing the semantics of the data. + +For example, a full-text search for "a swimming animal" only returns results containing these exact keywords. In contrast, vector search can return results for other swimming animals, such as fish or ducks, even if these results do not contain the exact keywords. + +### Vector embedding + +A vector embedding, also known as an embedding, is a sequence of numbers that represents real-world objects in a high-dimensional space. It captures the meaning and context of unstructured data, such as documents, images, audio, and videos. + +Vector embeddings are essential in machine learning and serve as the foundation for semantic similarity searches. + +TiDB introduces [Vector data types](/tidb-cloud/vector-search-data-types.md) and [Vector search index](/tidb-cloud/vector-search-index.md) designed to optimize the storage and retrieval of vector embeddings, enhancing their use in AI applications. You can store vector embeddings in TiDB and perform vector search queries to find the most relevant data using these data types. + +### Embedding model + +Embedding models are algorithms that transform data into [vector embeddings](#vector-embedding). + +Choosing an appropriate embedding model is crucial for ensuring the accuracy and relevance of semantic search results. For unstructured text data, you can find top-performing text embedding models on the [Massive Text Embedding Benchmark (MTEB) Leaderboard](https://huggingface.co/spaces/mteb/leaderboard). + +To learn how to generate vector embeddings for your specific data types, refer to integration tutorials or examples of embedding models. + +## How vector search works + +After converting raw data into vector embeddings and storing them in TiDB, your application can execute vector search queries to find the data most semantically or contextually relevant to a user's query. + +TiDB Vector Search identifies the top-k nearest neighbor (KNN) vectors by using a [distance function](/tidb-cloud/vector-search-functions-and-operators.md) to calculate the distance between the given vector and vectors stored in the database. The vectors closest to the given vector in the query represent the most similar data in meaning. + +![The Schematic TiDB Vector Search](/media/vector-search/embedding-search.png) + +As a relational database with integrated vector search capabilities, TiDB enables you to store data and their corresponding vector representations (that is, vector embeddings) together in one database. You can choose any of the following ways for storage: + +- Store data and their corresponding vector representations in different columns of the same table. +- Store data and their corresponding vector representation in different tables. In this way, you need to use `JOIN` queries to combine the tables when retrieving data. + +## Use cases + +### Retrieval-Augmented Generation (RAG) + +Retrieval-Augmented Generation (RAG) is an architecture designed to optimize the output of Large Language Models (LLMs). By using vector search, RAG applications can store vector embeddings in the database and retrieve relevant documents as additional context when the LLM generates responses, thereby improving the quality and relevance of the answers. + +### Semantic search + +Semantic search is a search technology that returns results based on the meaning of a query, rather than simply matching keywords. It interprets the meaning across different languages and various types of data (such as text, images, and audio) using embeddings. Vector search algorithms then use these embeddings to find the most relevant data that satisfies the user's query. + +### Recommendation engine + +A recommendation engine is a system that proactively suggests content, products, or services that are relevant and personalized to users. It accomplishes this by creating embeddings that represent user behavior and preferences. These embeddings help the system identify similar items that other users have interacted with or shown interest in. This increases the likelihood that the recommendations will be both relevant and appealing to the user. + +## See also + +To get started with TiDB Vector Search, see the following documents: + +- [Get started with vector search using Python](/tidb-cloud/vector-search-get-started-using-python.md) +- [Get started with vector search using SQL](/tidb-cloud/vector-search-get-started-using-sql.md) diff --git a/tidb-configuration-file.md b/tidb-configuration-file.md index 38ca4f0acd927..73faa832bfef1 100644 --- a/tidb-configuration-file.md +++ b/tidb-configuration-file.md @@ -1,7 +1,6 @@ --- title: TiDB Configuration File summary: Learn the TiDB configuration file options that are not involved in command line options. -aliases: ['/docs/dev/tidb-configuration-file/','/docs/dev/reference/configuration/tidb-server/configuration-file/'] --- @@ -9,7 +8,7 @@ aliases: ['/docs/dev/tidb-configuration-file/','/docs/dev/reference/configuratio # TiDB Configuration File -The TiDB configuration file supports more options than command-line parameters. You can download the default configuration file [`config.toml.example`](https://github.com/pingcap/tidb/blob/master/pkg/config/config.toml.example) and rename it to `config.toml`. This document describes only the options that are not involved in [command line options](/command-line-flags-for-tidb-configuration.md). +The TiDB configuration file supports more options than command-line parameters. You can download the default configuration file [`config.toml.example`](https://github.com/pingcap/tidb/blob/release-8.1/pkg/config/config.toml.example) and rename it to `config.toml`. This document describes only the options that are not involved in [command line options](/command-line-flags-for-tidb-configuration.md). > **Tip:** > @@ -92,7 +91,7 @@ The TiDB configuration file supports more options than command-line parameters. + When [`enable-global-kill`](#enable-global-kill-new-in-v610) is `false`, `compatible-kill-query` controls whether you need to append the `TIDB` keyword when killing a query. - When `compatible-kill-query` is `false`, the behavior of `KILL xxx` in TiDB is different from that in MySQL. To kill a query in TiDB, you need to append the `TIDB` keyword, such as `KILL TIDB xxx`. - When `compatible-kill-query` is `true`, to kill a query in TiDB, there is no need to append the `TIDB` keyword. It is **STRONGLY NOT RECOMMENDED** to set `compatible-kill-query` to `true` in your configuration file UNLESS you are certain that clients will be always connected to the same TiDB instance. This is because pressing Control+C in the default MySQL client opens a new connection in which `KILL` is executed. If there is a proxy between the client and the TiDB cluster, the new connection might be routed to a different TiDB instance, which possibly kills a different session by mistake. -+ When [`enable-global-kill`](#enable-global-kill-new-in-v610) is `true`, `KILL xxx` and `KILL TIDB xxx` have the same effect, but using Control+C to kill a query is not supported. ++ When [`enable-global-kill`](#enable-global-kill-new-in-v610) is `true`, `KILL xxx` and `KILL TIDB xxx` have the same effect. + For more information about the `KILL` statement, see [KILL [TIDB]](/sql-statements/sql-statement-kill.md). ### `check-mb4-value-in-utf8` @@ -172,12 +171,14 @@ The TiDB configuration file supports more options than command-line parameters. - Default value: `64` - Currently, the valid value range is `[64, 512]`. -### `enable-telemetry` New in v4.0.2 +### `enable-telemetry` New in v4.0.2 and deprecated in v8.1.0 -- Enables or disables the telemetry collection in TiDB. +> **Warning:** +> +> Starting from v8.1.0, the telemetry feature in TiDB is removed, and this configuration item is no longer functional. It is retained solely for compatibility with earlier versions. + +- Before v8.1.0, this configuration item controls whether to enable telemetry collection in a TiDB instance. - Default value: `false` -- When this configuration is set to `true` on a TiDB instance, the telemetry collection in this TiDB instance is enabled and the [`tidb_enable_telemetry`](/system-variables.md#tidb_enable_telemetry-new-in-v402) system variable takes effect. -- When this configuration is set to `false` on all TiDB instances, the telemetry collection in TiDB is disabled and the [`tidb_enable_telemetry`](/system-variables.md#tidb_enable_telemetry-new-in-v402) system variable does not take effect. See [Telemetry](/telemetry.md) for details. ### `deprecate-integer-display-length` @@ -202,6 +203,16 @@ The TiDB configuration file supports more options than command-line parameters. - Default value: `0` - When TiDB is waiting for shutdown (in the grace period), the HTTP status will indicate a failure, which allows the load balancers to reroute traffic. +> **Note:** +> +> The duration that TiDB waits before shutting down the server is also affected by the following parameters: +> +> - When you use a platform that employs SystemD, the default stop timeout is 90 seconds. If you need a longer timeout, you can set [`TimeoutStopSec=`](https://www.freedesktop.org/software/systemd/man/latest/systemd.service.html#TimeoutStopSec=). +> +> - When you use the TiUP Cluster component, the default [`--wait-timeout`](/tiup/tiup-component-cluster.md#--wait-timeout) is 120 seconds. +> +> - When you use Kubernetes, the default [`terminationGracePeriodSeconds`](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#lifecycle) is 30 seconds. + ### `enable-global-kill` New in v6.1.0 + Controls whether to enable the Global Kill (terminating queries or connections across instances) feature. @@ -253,7 +264,7 @@ The TiDB configuration file supports more options than command-line parameters. > - In TiDB, the `zone` label is specially used to specify the zone where a server is located. If `zone` is set to a non-null value, the corresponding value is automatically used by features such as [`txn-score`](/system-variables.md#txn_scope) and [`Follower read`](/follower-read.md). > - The `group` label has a special use in TiDB Operator. For clusters deployed using [TiDB Operator](/tidb-operator-overview.md), it is **NOT** recommended that you specify the `group` label manually. -## Log +## log Configuration items related to log. @@ -300,7 +311,7 @@ Configuration items related to log. - Outputs the threshold value of consumed time in the slow log. - Default value: `300` - Unit: Milliseconds -- If the value in a query is larger than the default value, it is a slow query and is output to the slow log. +- When the time consumed by a query is larger than this value, this query is considered as a slow query and its log is output to the slow query log. Note that when the output level of [`log.level`](#level) is `"debug"`, all queries are recorded in the slow query log, regardless of the setting of this parameter. - Since v6.1.0, the threshold value of consumed time in the slow log is specified by the TiDB configuration item [`instance.tidb_slow_log_threshold`](/tidb-configuration-file.md#tidb_slow_log_threshold) or the system variable [`tidb_slow_log_threshold`](/system-variables.md#tidb_slow_log_threshold). `slow-threshold` still takes effect. But if `slow-threshold` and `instance.tidb_slow_log_threshold` are set at the same time, the latter takes effect. ### `record-plan-in-slow-log` @@ -368,7 +379,7 @@ Configuration items related to log files. + Value options: `""`, `"gzip"` + The default value is `""`, which means no compression. To enable the gzip compression, set this value to `"gzip"`. After compression is enabled, all log files are affected, such as [`slow-query-file`](#slow-query-file) and [`general-log-file`](#general-log-file-new-in-v800). -## Security +## security Configuration items related to security. @@ -413,6 +424,11 @@ Configuration items related to security. - The path of the SSL private key file used to connect TiKV or PD with TLS. - Default value: "" +### `cluster-verify-cn` + +- A list of acceptable X.509 Common Names in certificates presented by clients. Requests are permitted only when the presented Common Name is an exact match with one of the entries in the list. +- Default value: [], which means that the client certificate CN check is disabled. + ### `spilled-file-encryption-method` + Determines the encryption method used for saving the spilled files to disk. @@ -463,7 +479,7 @@ Configuration items related to security. + Default value: "" + Refer to the descriptions of [`session-token-signing-cert`](#session-token-signing-cert-new-in-v640). -## Performance +## performance Configuration items related to performance. @@ -496,7 +512,7 @@ Configuration items related to performance. - Default value: `5000` - If a transaction does not roll back or commit after the number of statements exceeds `stmt-count-limit`, TiDB returns the `statement count 5001 exceeds the transaction limitation, autocommit = false` error. This configuration takes effect **only** in the retryable optimistic transaction. If you use the pessimistic transaction or have disabled the transaction retry, the number of statements in a transaction is not limited by this configuration. -### `txn-entry-size-limit` New in v5.0 +### `txn-entry-size-limit` New in v4.0.10 and v5.0.0 - The size limit of a single row of data in TiDB. - Default value: `6291456` (in bytes) @@ -510,10 +526,11 @@ Configuration items related to performance. - The size limit of a single transaction in TiDB. - Default value: `104857600` (in bytes) -- In a single transaction, the total size of key-value records cannot exceed this value. The maximum value of this parameter is `1099511627776` (1 TB). Note that if you have used the binlog to serve the downstream consumer Kafka (such as the `arbiter` cluster), the value of this parameter must be no more than `1073741824` (1 GB). This is because 1 GB is the upper limit of a single message size that Kafka can process. Otherwise, an error is returned if this limit is exceeded. +- In a single transaction, the total size of key-value records cannot exceed this value. The maximum value of this parameter is `1099511627776` (1 TB). Note that if you have used the binlog to serve the downstream consumer Kafka, the value of this parameter must be no more than `1073741824` (1 GB). This is because 1 GB is the upper limit of a single message size that Kafka can process. Otherwise, an error is returned if this limit is exceeded. - In TiDB v6.5.0 and later versions, this configuration is no longer recommended. The memory size of a transaction will be accumulated into the memory usage of the session, and the [`tidb_mem_quota_query`](/system-variables.md#tidb_mem_quota_query) variable will take effect when the session memory threshold is exceeded. To be compatible with previous versions, this configuration works as follows when you upgrade from an earlier version to TiDB v6.5.0 or later: - If this configuration is not set or is set to the default value (`104857600`), after an upgrade, the memory size of a transaction will be accumulated into the memory usage of the session, and the `tidb_mem_quota_query` variable will take effect. - If this configuration is not defaulted (`104857600`), it still takes effect and its behavior on controlling the size of a single transaction remains unchanged before and after the upgrade. This means that the memory size of the transaction is not controlled by the `tidb_mem_quota_query` variable. +- When TiDB executes transactions in the [`tidb_dml_type`](/system-variables.md#tidb_dml_type-new-in-v800) `"bulk"` mode, transaction size is not limited by the TiDB configuration item [`txn-total-size-limit`](https://docs.pingcap.com/tidb/stable/tidb-configuration-file#txn-total-size-limit). ### `tcp-keep-alive` @@ -530,6 +547,10 @@ Configuration items related to performance. - Default value: `true` - TiDB supports executing the `JOIN` statement without any condition (the `WHERE` field) of both sides tables by default; if you set the value to `false`, the server refuses to execute when such a `JOIN` statement appears. +> **Note:** +> +> When creating a cluster, **DO NOT** set `cross-join` to false. Otherwise, the cluster will fail to start up. + ### `stats-lease` - The time interval of reloading statistics, updating the number of table rows, checking whether it is needed to perform the automatic analysis, using feedback to update statistics and loading statistics of columns. @@ -756,7 +777,7 @@ Configuration items related to opentracing.reporter. ## tikv-client.copr-cache New in v4.0.0 -This section introduces configuration items related to the Coprocessor Cache feature. +This section introduces configuration items related to the [Coprocessor Cache](/coprocessor-cache.md) feature. ### `capacity-mb` @@ -827,7 +848,7 @@ Configuration related to the status of TiDB service. ### `record-db-label` - Determines whether to transmit the database-related QPS metrics to Prometheus. -- Supports more metircs types than `record-db-qps`, for example, duration and statements. +- Supports more metrics types than `record-db-qps`, for example, duration and statements. - Default value: `false` ## pessimistic-txn @@ -891,10 +912,11 @@ Configuration items related to read isolation. ### `tidb_slow_log_threshold` -- This configuration is used to output the threshold value of the time consumed by the slow log. When the time consumed by a query is larger than this value, this query is considered as a slow log and its log is output to the slow query log. +- Outputs the threshold value of the time consumed by the slow log. - Default value: `300` - Range: `[-1, 9223372036854775807]` - Unit: Milliseconds +- When the time consumed by a query is larger than this value, this query is considered as a slow query and its log is output to the slow query log. Note that when the output level of [`log.level`](#level) is `"debug"`, all queries are recorded in the slow query log, regardless of the setting of this parameter. - Before v6.1.0, this configuration is set by `slow-threshold`. ### `in-mem-slow-query-topn-num` New in v7.3.0 diff --git a/tidb-control.md b/tidb-control.md index 2c79e6ff347f4..c63d2eda2a742 100644 --- a/tidb-control.md +++ b/tidb-control.md @@ -1,7 +1,6 @@ --- title: TiDB Control User Guide summary: Use TiDB Control to obtain TiDB status information for debugging. -aliases: ['/docs/dev/tidb-control/','/docs/dev/reference/tools/tidb-control/'] --- # TiDB Control User Guide @@ -179,7 +178,7 @@ tidb-ctl base64decode [table_id] [base64_data] alter table t add column e varchar(20); ``` -2. Obtian MVCC data using the HTTP API interface: +2. Obtain MVCC data using the HTTP API interface: ```shell $ curl "http://$IP:10080/mvcc/index/test/t/a/1?a=1" diff --git a/tidb-distributed-execution-framework.md b/tidb-distributed-execution-framework.md index d733ed731e2f3..5a328041ac189 100644 --- a/tidb-distributed-execution-framework.md +++ b/tidb-distributed-execution-framework.md @@ -7,15 +7,15 @@ summary: Learn the use cases, limitations, usage, and implementation principles > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. -TiDB adopts a computing-storage separation architecture with excellent scalability and elasticity. Starting from v7.1.0, TiDB introduces a Distributed eXecution Framework (DXF) to further leverage the resource advantages of the distributed architecture. The goal of the DXF is to implement unified scheduling and distributed execution of tasks, and to provide unified resource management capabilities for both overall and individual tasks, which better meets users' expectations for resource usage. +TiDB adopts a computing-storage separation architecture with excellent scalability and elasticity. Starting from v7.1.0, TiDB introduces a **Distributed eXecution Framework (DXF)** to further leverage the resource advantages of the distributed architecture. The goal of the DXF is to implement unified scheduling and distributed execution of tasks, and to provide unified resource management capabilities for both overall and individual tasks, which better meets users' expectations for resource usage. This document describes the use cases, limitations, usage, and implementation principles of the DXF. ## Use cases -In a database management system, in addition to the core transactional processing (TP) and analytical processing (AP) workloads, there are other important tasks, such as DDL operations, IMPORT INTO, TTL, Analyze, and Backup/Restore. These tasks need to process a large amount of data in database objects (tables), so they typically have the following characteristics: +In a database management system, in addition to the core transactional processing (TP) and analytical processing (AP) workloads, there are other important tasks, such as DDL operations, [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md), [TTL](/time-to-live.md), [`ANALYZE`](/sql-statements/sql-statement-analyze-table.md), and Backup/Restore. These tasks need to process a large amount of data in database objects (tables), so they typically have the following characteristics: - Need to process all data in a schema or a database object (table). - Might need to be executed periodically, but at a low frequency. @@ -27,29 +27,24 @@ Enabling the DXF can solve the above problems and has the following three advant - The DXF supports distributed execution of tasks, which can flexibly schedule the available computing resources of the entire TiDB cluster, thereby better utilizing the computing resources in a TiDB cluster. - The DXF provides unified resource usage and management capabilities for both overall and individual tasks. -Currently, the DXF supports the distributed execution of the `ADD INDEX` and `IMPORT INTO` statements. +Currently, the DXF supports the distributed execution of the [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) and [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) statements. -- `ADD INDEX` is a DDL statement used to create indexes. For example: +- [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) is a DDL statement used to create indexes. For example: ```sql ALTER TABLE t1 ADD INDEX idx1(c1); CREATE INDEX idx1 ON table t1(c1); ``` -- `IMPORT INTO` is used to import data in formats such as `CSV`, `SQL`, and `PARQUET` into an empty table. For more information, see [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md). +- [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) is used to import data in formats such as CSV, SQL, and Parquet into an empty table. ## Limitation -The DXF can only schedule up to 16 tasks (including `ADD INDEX` tasks and `IMPORT INTO` tasks) simultaneously. - -## `ADD INDEX` limitation - -- For each cluster, only one `ADD INDEX` task is allowed for distributed execution at a time. If a new `ADD INDEX` task is submitted before the current `ADD INDEX` distributed task has finished, the new `ADD INDEX` task is executed through a transaction instead of being scheduled by DXF. -- Adding indexes on columns with the `TIMESTAMP` data type through the DXF is not supported, because it might lead to inconsistency between the index and the data. +The DXF can only schedule up to 16 tasks (including [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) tasks and [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) tasks) simultaneously. ## Prerequisites -Before using the DXF to execute `ADD INDEX` tasks, you need to enable the [Fast Online DDL](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) mode. +Before using the DXF to execute [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) tasks, you need to enable the [Fast Online DDL](/system-variables.md#tidb_ddl_enable_fast_reorg-new-in-v630) mode. @@ -96,18 +91,18 @@ Adjust the following system variables related to Fast Online DDL: ## Task scheduling -By default, the DXF schedules all TiDB nodes to execute distributed tasks. Starting from v7.4.0, for TiDB Self-Hosted clusters, you can control which TiDB nodes can be scheduled by the DXF to execute distributed tasks by configuring [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740). +By default, the DXF schedules all TiDB nodes to execute distributed tasks. Starting from v7.4.0, for TiDB Self-Managed clusters, you can control which TiDB nodes can be scheduled by the DXF to execute distributed tasks by configuring [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740). - For versions from v7.4.0 to v8.0.0, the optional values of [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) are `''` or `background`. If the current cluster has TiDB nodes with `tidb_service_scope = 'background'`, the DXF schedules tasks to these nodes for execution. If the current cluster does not have TiDB nodes with `tidb_service_scope = 'background'`, whether due to faults or normal scaling in, the DXF schedules tasks to nodes with `tidb_service_scope = ''` for execution. -- Starting from v8.1.0, you can set [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) to any valid value. When a distributed task is submitted, the task binds to the `tidb_service_scope` value of the currently connected TiDB node, and the DXF only schedules the task to the TiDB nodes with the same `tidb_service_scope` value for execution. However, for configuration compatibility with earlier versions, if a distributed task is submitted on a node with `tidb_service_scope = ''` and the current cluster has TiDB nodes with `tidb_service_scope = 'background'`, the DXF schedules the task to TiDB nodes with `tidb_service_scope = 'background'` for execution. +- Starting from v8.1.0, you can set [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) to any valid value. When a distributed task is submitted, the task binds to the [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) value of the currently connected TiDB node, and the DXF only schedules the task to the TiDB nodes with the same [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) value for execution. However, for configuration compatibility with earlier versions, if a distributed task is submitted on a node with `tidb_service_scope = ''` and the current cluster has TiDB nodes with `tidb_service_scope = 'background'`, the DXF schedules the task to TiDB nodes with `tidb_service_scope = 'background'` for execution. -If new nodes are added during task execution, the DXF determines whether to schedule tasks to the new nodes for execution based on the preceding rules. If you do not want newly added nodes to execute tasks, it is recommended to set `tidb_service_scope` for these nodes in advance. +Starting from v8.1.0, if new nodes are added during task execution, the DXF determines whether to schedule tasks to the new nodes for execution based on the preceding rules. If you do not want newly added nodes to execute tasks, it is recommended to set a different [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) for those newly added nodes in advance. > **Note:** > -> - For versions from v7.4.0 to v8.0.0, in clusters with multiple TiDB nodes, it is strongly recommended to set `tidb_service_scope` to `background` on two or more TiDB nodes. If this variable is set only on a single TiDB node, when that node restarts or fails, tasks will be rescheduled to TiDB nodes with `tidb_service_scope = ''`, which affects applications running on these TiDB nodes. -> - During the execution of a distributed task, changes to the `tidb_service_scope` configuration do not take effect for the current task, but take effect from the next task. +> - For versions from v7.4.0 to v8.0.0, in clusters with multiple TiDB nodes, it is strongly recommended to set [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) to `background` on two or more TiDB nodes. If this variable is set only on a single TiDB node, when that node restarts or fails, tasks will be rescheduled to TiDB nodes with `tidb_service_scope = ''`, which affects applications running on these TiDB nodes. +> - During the execution of a distributed task, changes to the [`tidb_service_scope`](/system-variables.md#tidb_service_scope-new-in-v740) configuration do not take effect for the current task, but take effect from the next task. ## Implementation principles @@ -133,4 +128,4 @@ As shown in the preceding diagram, the execution of tasks in the DXF is mainly h * [Execution Principles and Best Practices of DDL Statements](https://docs.pingcap.com/tidb/stable/ddl-introduction) - + \ No newline at end of file diff --git a/tidb-global-sort.md b/tidb-global-sort.md index c80f57af664c5..7043e787dd5bd 100644 --- a/tidb-global-sort.md +++ b/tidb-global-sort.md @@ -15,7 +15,7 @@ summary: Learn the use cases, limitations, usage, and implementation principles > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. ## Overview diff --git a/tidb-in-kubernetes.md b/tidb-in-kubernetes.md index f85259fb9bd09..1963d64c80bcd 100644 --- a/tidb-in-kubernetes.md +++ b/tidb-in-kubernetes.md @@ -1,7 +1,6 @@ --- title: Deploy a TiDB Cluster on Kubernetes summary: Learn how to deploy a TiDB cluster on Kubernetes. -aliases: ['/docs/tidb-in-kubernetes/dev/'] --- # Deploy a TiDB Cluster on Kubernetes diff --git a/tidb-lightning/deploy-tidb-lightning.md b/tidb-lightning/deploy-tidb-lightning.md index b2c57bbc0d3c8..f2afceff11ebc 100644 --- a/tidb-lightning/deploy-tidb-lightning.md +++ b/tidb-lightning/deploy-tidb-lightning.md @@ -1,7 +1,6 @@ --- title: Deploy TiDB Lightning summary: Deploy TiDB Lightning to quickly import large amounts of new data. -aliases: ['/docs/dev/tidb-lightning/deploy-tidb-lightning/','/docs/dev/reference/tools/tidb-lightning/deployment/'] --- # Deploy TiDB Lightning diff --git a/tidb-lightning/import-into-vs-tidb-lightning.md b/tidb-lightning/import-into-vs-tidb-lightning.md new file mode 100644 index 0000000000000..69e9cadd52927 --- /dev/null +++ b/tidb-lightning/import-into-vs-tidb-lightning.md @@ -0,0 +1,133 @@ +--- +title: IMPORT INTO vs. TiDB Lightning +summary: Learn about the differences between `IMPORT INTO` and TiDB Lightning. +--- + +# IMPORT INTO vs. TiDB Lightning + +Many users have provided feedback that the deployment, configuration, and maintenance of [TiDB Lightning](/tidb-lightning/tidb-lightning-configuration.md) are complex, especially in scenarios involving [parallel importing](/tidb-lightning/tidb-lightning-distributed-import.md) of large datasets. + +Based on the feedback, TiDB has gradually integrated some functionalities of TiDB Lightning into the [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) SQL statement. You can directly import data by executing `IMPORT INTO`, thereby improving the efficiency of data import. In addition, `IMPORT INTO` supports some functionalities that TiDB Lightning does not, such as automatic distributed task scheduling and [TiDB Global Sort](/tidb-global-sort.md). + +`IMPORT INTO` is introduced in v7.2.0 and becomes generally available (GA) in v7.5.0. It will continue to be improved and optimized in future versions. Once the `IMPORT INTO` capability can fully replace TiDB Lightning, TiDB Lightning will be deprecated. At that time, relevant notification will be provided in advance in TiDB Release Notes and documentation. + +## Comparison between `IMPORT INTO` and TiDB Lightning + +The following sections describe the differences between `IMPORT INTO` and TiDB Lightning in multiple dimensions. + +### Deployment cost + +#### `IMPORT INTO` + +`IMPORT INTO` does not require separate deployment. You can run it directly on TiDB nodes, which eliminates additional deployment work. + +#### TiDB Lightning + +By contrast, TiDB Lightning requires [separate server deployment](/tidb-lightning/deploy-tidb-lightning.md). + +### Resource utilization + +#### `IMPORT INTO` + +The `IMPORT INTO` task and other business workloads can share TiDB resources or utilize them at different times to fully leverage the TiDB resources. To ensure stable operation of business workloads while maintaining the performance and stability of the `IMPORT INTO` task, you can specify [specific TiDB nodes](/system-variables.md#tidb_service_scope-new-in-v740) dedicated to `IMPORT INTO` for data import. + +When you use [TiDB Global Sort](/tidb-global-sort.md), there is no need to mount large local disks. TiDB Global Sort can use Amazon S3 as the storage. Once the import task is completed, the temporary data stored on Amazon S3 for global sorting will be automatically deleted to save storage costs. + +#### TiDB Lightning + +You need separate servers to deploy and run TiDB Lightning. When no import tasks are executed, these resources remain idle. The total idle time is even longer in scenarios where import tasks are executed periodically, resulting in a waste of resources. + +If the dataset to be imported is large, you also need to prepare large local disks to sort the data to be imported. + +### Task configuration and integration + +#### `IMPORT INTO` + +You can directly write SQL statements to submit import tasks, which are easy to call and integrate. + +#### TiDB Lightning + +By contrast, TiDB Lightning requires you to write [task configuration files](/tidb-lightning/tidb-lightning-configuration.md). These configuration files are complex and not easily called by third parties. + +### Task scheduling + +#### `IMPORT INTO` + +`IMPORT INTO` supports distributed execution. For example, when you import 40 TiB of source data files into one target table, after submitting the SQL statement, TiDB will automatically split the import task into multiple sub-tasks and schedule different TiDB nodes to execute these sub-tasks. + +#### TiDB Lightning + +By contrast, the configuration for TiDB Lightning is complex, inefficient, and prone to errors. + +Assume that you start 10 TiDB Lightning instances to import data in parallel, then you need to create 10 TiDB Lightning configuration files. In each file, you need to configure the range of source files to be read by the corresponding TiDB Lightning instance. For example, TiDB Lightning instance 1 reads the first 100 files, instance 2 reads the next 100 files, and so on. + +In addition, you need to configure the shared metadata table and other configuration information for these 10 TiDB Lightning instances, which is complex. + +### Global Sort vs. local sort + +#### `IMPORT INTO` + +With TiDB Global Sort, `IMPORT INTO` can transmit tens of TiB of source data to multiple TiDB nodes, encode the data KV pairs and index KV pairs, and then transfer these pairs to Amazon S3 for global sorting before writing them into TiKV. + +Because these KV pairs are globally sorted, data imported from various TiDB nodes into TiKV does not overlap, allowing it to be written directly into the RocksDB. This eliminates the need for TiKV to perform compaction operations, resulting in significant improvement in both write performance and stability of TiKV. + +After the import is completed, the data used for global sorting on Amazon S3 will be automatically deleted, saving storage costs. + +#### TiDB Lightning + +TiDB Lightning only supports local sort. For example, for tens of TiB of source data, if TiDB Lightning does not have large local disks configured, or if multiple TiDB Lightning instances are used for parallel import, each TiDB Lightning instance can only use local disks to sort the data to be imported. Due to the inability to perform global sort, there will be an overlap between the data imported into TiKV by multiple TiDB Lightning instances, especially in scenarios where index data is more prevalent, triggering TiKV to perform compaction operations. Compaction operations are resource-intensive, which will lead to a decrease in TiKV's write performance and stability. + +If you want to continue importing data later, you will need to keep the TiDB Lightning server and the disks on the server for the next import. The cost of using preallocated disks is relatively high, compared with `IMPORT INTO` using Amazon S3 on a pay-as-you-go basis. + +### Performance + +Currently, there are no performance test comparison results under equivalent test environments between `IMPORT INTO` and TiDB Lightning. + +When Amazon S3 is used as the storage for global sorting, the performance test results for `IMPORT INTO` are as follows: + +| Source dataset | Node configuration | Average import speed per TiDB node | +|------------------------------------|--------------------------------------------------------------|------------------------------------| +| 40 TiB data (2.26 billion rows, 19 KiB per row) | 10 TiDB (16C32G) nodes and 20 TiKV (16C27G) nodes | 222 GiB/h | +| 10 TiB data (565 million rows, 19 KiB per row) | 5 TiDB (16C32G) nodes and 10 TiKV (16C27G) nodes | 307 GiB/h | + +### High availability + +#### `IMPORT INTO` + +After a TiDB node fails, tasks on that node are automatically transferred to the remaining TiDB nodes to continue running. + +#### TiDB Lightning + +After a TiDB Lightning instance node fails, you need to perform manual recovery of tasks on a new node based on previously recorded checkpoints. + +### Scalability + +#### `IMPORT INTO` + +Due to the use of Global Sort, data imported into TiKV does not overlap, resulting in better scalability compared with TiDB Lightning. + +#### TiDB Lightning + +Due to only supporting local sort, data imported into TiKV might overlap when new TiDB Lightning instances are added, resulting in more compaction operations for TiKV and limiting scalability relative to `IMPORT INTO`. + +## Functionalities not supported by `IMPORT INTO` + +Currently, `IMPORT INTO` still lacks some functionalities and cannot fully replace TiDB Lightning in some scenarios, such as: + +- Logical import + + Before importing data with `IMPORT INTO`, the target table must be empty. If you need to import data into a table that already contains data, it is recommended to use methods such as [`LOAD DATA`](/sql-statements/sql-statement-load-data.md) or direct insertion. Starting from v8.0, TiDB supports [bulk DML](/system-variables.md#tidb_dml_type-new-in-v800) for executing large transactions. + +- Conflict data handling + + `IMPORT INTO` currently does not support conflict data handling. Before the data import, you need to define the table schema properly to ensure that the data to be imported does not conflict with primary keys (PK) or unique keys (UK). Otherwise, it might cause task failures. + +- Importing data into multiple target tables + + Currently, only one target table is allowed for one `IMPORT INTO` SQL statement. If you want to import data into multiple target tables, you need to submit multiple `IMPORT INTO` SQL statements. + +In future versions, these functionalities will be supported by `IMPORT INTO`, along with additional enhancements to its capabilities, such as allowing modification of concurrency during task execution and adjusting throughput for writing to TiKV. This will make it more convenient for you to manage tasks. + +## Summary + +Compared with TiDB Lightning, `IMPORT INTO`can be directly executed on TiDB nodes, supports automated distributed task scheduling and [TiDB Global Sort](/tidb-global-sort.md), and offers significant improvements in deployment, resource utilization, task configuration convenience, ease of invocation and integration, high availability, and scalability. It is recommended that you consider using `IMPORT INTO` instead of TiDB Lightning in appropriate scenarios. diff --git a/tidb-lightning/monitor-tidb-lightning.md b/tidb-lightning/monitor-tidb-lightning.md index 6b922b4445c37..01b9c6e823eb8 100644 --- a/tidb-lightning/monitor-tidb-lightning.md +++ b/tidb-lightning/monitor-tidb-lightning.md @@ -1,7 +1,6 @@ --- title: TiDB Lightning Monitoring summary: Learn about the monitor configuration and monitoring metrics of TiDB Lightning. -aliases: ['/docs/dev/tidb-lightning/monitor-tidb-lightning/','/docs/dev/reference/tools/tidb-lightning/monitor/'] --- # TiDB Lightning Monitoring diff --git a/tidb-lightning/tidb-lightning-checkpoints.md b/tidb-lightning/tidb-lightning-checkpoints.md index c68bc8201496d..e874f6290efb3 100644 --- a/tidb-lightning/tidb-lightning-checkpoints.md +++ b/tidb-lightning/tidb-lightning-checkpoints.md @@ -1,7 +1,6 @@ --- title: TiDB Lightning Checkpoints summary: Use checkpoints to avoid redoing the previously completed tasks before the crash. -aliases: ['/docs/dev/tidb-lightning/tidb-lightning-checkpoints/','/docs/dev/reference/tools/tidb-lightning/checkpoints/'] --- # TiDB Lightning Checkpoints diff --git a/tidb-lightning/tidb-lightning-compatibility-and-scenarios.md b/tidb-lightning/tidb-lightning-compatibility-and-scenarios.md new file mode 100644 index 0000000000000..6130dc96780ba --- /dev/null +++ b/tidb-lightning/tidb-lightning-compatibility-and-scenarios.md @@ -0,0 +1,102 @@ +--- +title: Compatibility of TiDB Lightning and IMPORT INTO with TiCDC and Log Backup +summary: Learn about compatibility of IMPORT INTO and TiDB Lightning with log backup and TiCDC. +--- + +# Compatibility of TiDB Lightning and IMPORT INTO with TiCDC and Log Backup + +This document describes TiDB Lightning and [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) compatibility with [log backup](/br/br-pitr-guide.md) and [TiCDC](/ticdc/ticdc-overview.md), as well as some special usage scenarios. + +## `IMPORT INTO` vs. TiDB Lightning + +[`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) integrates with the physical import mode of TiDB Lightning, but there are some differences. See [`IMPORT INTO` vs. TiDB Lightning](/tidb-lightning/import-into-vs-tidb-lightning.md) for details. + +## Compatibility with log backup and TiCDC + +- TiDB Lightning [logical import mode](/tidb-lightning/tidb-lightning-logical-import-mode.md) is compatible with log backup and TiCDC. + +- TiDB Lightning [physical import mode](/tidb-lightning/tidb-lightning-physical-import-mode.md) is not compatible with log backup and TiCDC. The reason is that physical import mode directly ingests encoded KV pairs of the source data to TiKV, causing TiKV not to generate corresponding change logs during this process. Without such change logs, the relevant data cannot be backed up via log backup and cannot be replicated by TiCDC. + +- To use TiDB Lightning and TiCDC together in a cluster, see [Compatibility with TiDB Lightning](/ticdc/ticdc-compatibility.md#compatibility-with-tidb-lightning). + +- `IMPORT INTO` is not compatible with log backup and TiCDC. The reason is that `IMPORT INTO` also ingests encoded KV pairs of the source data directly to TiKV. + +## Scenarios for TiDB Lightning logical import mode + +If TiDB Lightning logical import mode can meet your application's performance requirements and your application requires imported tables to be backed up or replicated downstream using TiCDC, it is recommended to use TiDB Lightning logical import mode. + +## Scenarios for TiDB Lightning physical import mode + +This section describes how to use TiDB Lightning together with [log backup](/br/br-pitr-guide.md) and [TiCDC](/ticdc/ticdc-overview.md). + +If TiDB Lightning logical import mode does not meet your application's performance requirements, you have to use TiDB Lightning physical import mode, and imported tables need to be backed up or replicated downstream using TiCDC, then the following scenarios are recommended. + +### Used with log backup + +You can perform in different scenarios as follows: + +- Scenario 1: tables in physical import mode do not need to be backed up + + In this scenario, if [PITR](/br/br-log-architecture.md#process-of-pitr) is enabled, the compatibility check will report an error after TiDB Lightning starts. If you are sure that these tables do not need backup or [log backup](/br/br-pitr-guide.md), you can change the `Lightning.check-requirements` parameter in the [TiDB Lightning configuration file](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) to `false` and restart the import task. + +- Scenario 2: after the physical import is finished, there will be no new DML operations on the table + + This scenario does not involve incremental data writes, so it is sufficient to perform a table-level snapshot backup of the table after completing the data import in TiDB Lightning physical import mode, as described in [Back up a table](/br/br-snapshot-manual.md#back-up-a-table). + + During data recovery, the snapshot data of the table is restored. See [Restore a table](/br/br-snapshot-manual.md#restore-a-table) for the procedure. + +- Scenario 3: after the physical import is finished, there will be new DML operations on the table (not supported) + + In this scenario, you can only choose either [full snapshot backup](/br/br-snapshot-guide.md) or [log backup](/br/br-pitr-guide.md) for this table. You cannot back up and restore both the full snapshot data and log backup data of this table. + +### Used with TiCDC + +Using TiCDC with physical import mode is not compatible in the short term, because TiCDC cannot keep up with the write speed of TiDB Lightning physical import mode, which might result in increasing cluster replication latency. + +You can perform in different scenarios as follows: + +- Scenario 1: the table does not need to be replicated downstream by TiCDC. + + In this scenario, if TiCDC changefeed is enabled, the compatibility check will report an error after TiDB Lightning starts. If you are sure that these tables do not need backup or [log backup](/br/br-pitr-guide.md), you can change the `Lightning.check-requirements` parameter in the [TiDB Lightning configuration file](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) to `false` and restart the import task. + +- Scenario 2: the table needs to be replicated downstream by TiCDC. + + In this scenario, if TiCDC changefeed is enabled, the compatibility check will report an error after TiDB Lightning starts. You need to change the `Lightning.check-requirements` parameter in the [TiDB Lightning configuration file](/tidb-lightning/tidb-lightning-configuration.md#tidb-lightning-task) in the upstream TiDB cluster to `false` and restart the import task. + + After the import task for the upstream TiDB cluster is finished, use TiDB Lightning to import the same data in the downstream TiDB cluster. If you have databases such as Redshift and Snowflake in the downstream, you can configure them to read CSV, SQL, or Parquet files from a cloud storage service and then write the data to the database. + +## Scenarios for `IMPORT INTO` + +This section describes how to use `IMPORT INTO` together with [log backup](/br/br-pitr-guide.md) and [TiCDC](/ticdc/ticdc-overview.md). + +### Used with log backup + +You can perform in different scenarios as follows: + +- Scenario 1: tables do not need to be backed up + + In this scenario, if [PITR](/br/br-log-architecture.md#process-of-pitr) is enabled, the compatibility check will report an error after you submit the `IMPORT INTO` statement. If you are sure that these tables do not need backup or [log backup](/br/br-pitr-guide.md), you can include `DISABLE_PRECHECK` (introduced in v8.0.0) in [`WithOptions`](/sql-statements/sql-statement-import-into.md#withoptions) of that statement, and then resubmit it. In this way, the data import task ignores the compatibility check and imports the data directly. + +- Scenario 2: after the import is finished, there will be no new DML operations on the table + + This scenario does not involve incremental data writes, so it is sufficient to perform a table-level snapshot backup of the table after completing the data import, as described in [Back up a table](/br/br-snapshot-manual.md#back-up-a-table). + + During data recovery, the snapshot data of the table is restored. See [Restore a table](/br/br-snapshot-manual.md#restore-a-table) for the procedure. + +- Scenario 3: after the import is finished, there will be new DML operations on the table (not supported) + + In this scenario, you can only choose either [full snapshot backup](/br/br-snapshot-guide.md) or [log backup](/br/br-pitr-guide.md) for this table. You cannot back up and restore both the full snapshot data and log backup data of this table. + +### Used with TiCDC + +You can perform in different scenarios as follows: + +- Scenario 1: the table does not need to be replicated downstream by TiCDC. + + In this scenario, if TiCDC changefeed is enabled, the compatibility check will report an error after you submit the `IMPORT INTO` statement. If you are sure that these tables do not need to be replicated by TiCDC, you can include `DISABLE_PRECHECK` (introduced in v8.0.0) in [`WithOptions`](/sql-statements/sql-statement-import-into.md#withoptions) of that statement, and then resubmit it. In this way, the data import task ignores the compatibility check and imports the data directly. + +- Scenario 2: the table needs to be replicated downstream by TiCDC. + + In this scenario, if TiCDC changefeed is enabled, the compatibility check will report an error after you submit the `IMPORT INTO` statement. You can include `DISABLE_PRECHECK` (introduced in v8.0.0) in [`WithOptions`](/sql-statements/sql-statement-import-into.md#withoptions) of that statement, and then resubmit it. In this way, the data import task ignores the compatibility check and imports the data directly. + + After the import task for the upstream TiDB cluster is finished, use `IMPORT INTO` to import the same data in the downstream TiDB cluster. If you have databases such as Redshift and Snowflake in the downstream, you can configure them to read CSV, SQL, or Parquet files from a cloud storage service and then write the data to the database. diff --git a/tidb-lightning/tidb-lightning-configuration.md b/tidb-lightning/tidb-lightning-configuration.md index 741f565360297..0282833d24f9c 100644 --- a/tidb-lightning/tidb-lightning-configuration.md +++ b/tidb-lightning/tidb-lightning-configuration.md @@ -1,7 +1,6 @@ --- title: TiDB Lightning Configuration summary: Learn about the CLI usage and sample configuration in TiDB Lightning. -aliases: ['/docs/dev/tidb-lightning/tidb-lightning-configuration/','/docs/dev/reference/tools/tidb-lightning/config/'] --- # TiDB Lightning Configuration @@ -129,7 +128,7 @@ driver = "file" # - "": in the physical import mode, TiDB Lightning does not detect or handle conflicting data. If the source file contains conflicting primary or unique key records, the subsequent step reports an error. In the logical import mode, TiDB Lightning converts the "" strategy to the "error" strategy for processing. # - "error": when detecting conflicting primary or unique key records in the imported data, TiDB Lightning terminates the import and reports an error. # - "replace": when encountering conflicting primary or unique key records, TiDB Lightning retains the latest data and overwrites the old data. -# The conflicting data are recorded in the `lightning_task_info.conflict_error_v2` table (recording conflicting data detected by post-import conflict detection in the physical import mode) and the `conflict_records` table (recording conflicting data detected by preprocess conflict detection in both logical and physical import modes) of the target TiDB cluster. +# When you use the physical import mode, the conflicting data are recorded in the `lightning_task_info.conflict_error_v2` table (recording conflicting data detected by post-import conflict detection in the physical import mode) and the `conflict_records` table (recording conflicting data detected by preprocess conflict detection in both logical and physical import modes) of the target TiDB cluster. # If you set `conflict.strategy = "replace"` in physical import mode, the conflicting data can be checked in the `lightning_task_info.conflict_view` view. # You can manually insert the correct records into the target table based on your application requirements. Note that the target TiKV must be v5.2.0 or later versions. # - "ignore": when encountering conflicting primary or unique key records, TiDB Lightning retains the old data and ignores the new data. This option can only be used in the logical import mode. @@ -424,6 +423,10 @@ max-allowed-packet = 67_108_864 # Private key of this service. Default to copy of `security.key-path` # key-path = "/path/to/lightning.key" +# Sets other TiDB session variables +# [tidb.session-vars] +# tidb_enable_clustered_index = "OFF" + # In the physical import mode, when data importing is complete, TiDB Lightning can # automatically perform the Checksum and Analyze operations. It is recommended # to leave these as true in the production environment. @@ -463,54 +466,3 @@ log-progress = "5m" # The default value is 60 seconds. # check-disk-quota = "60s" ``` - -## Command line parameters - -### Usage of `tidb-lightning` - -| Parameter | Explanation | Corresponding setting | -|:----|:----|:----| -| --config *file* | Reads global configuration from *file*. If not specified, the default configuration would be used. | | -| -V | Prints program version | | -| -d *directory* | Directory or [external storage URI](/external-storage-uri.md) of the data dump to read from | `mydumper.data-source-dir` | -| -L *level* | Log level: debug, info, warn, error, fatal (default = info) | `lightning.log-level` | -| -f *rule* | [Table filter rules](/table-filter.md) (can be specified multiple times) | `mydumper.filter` | -| --backend *[backend](/tidb-lightning/tidb-lightning-overview.md)* | Select an import mode. `local` refers to the physical import mode; `tidb` refers to the logical import mode. | `local` | -| --log-file *file* | Log file path. By default, it is `/tmp/lightning.log.{timestamp}`. If set to '-', it means that the log files will be output to stdout. | `lightning.log-file` | -| --status-addr *ip:port* | Listening address of the TiDB Lightning server | `lightning.status-port` | -| --pd-urls *host:port* | PD endpoint address | `tidb.pd-addr` | -| --tidb-host *host* | TiDB server host | `tidb.host` | -| --tidb-port *port* | TiDB server port (default = 4000) | `tidb.port` | -| --tidb-status *port* | TiDB status port (default = 10080) | `tidb.status-port` | -| --tidb-user *user* | User name to connect to TiDB | `tidb.user` | -| --tidb-password *password* | Password to connect to TiDB. The password can either be plaintext or Base64 encoded. | `tidb.password` | -| --enable-checkpoint *bool* | Whether to enable checkpoints (default = true) | `checkpoint.enable` | -| --analyze *level* | Analyze tables after importing. Available values are "required", "optional" (default value), and "off" | `post-restore.analyze` | -| --checksum *level* | Compare checksum after importing. Available values are "required" (default value), "optional", and "off" | `post-restore.checksum` | -| --check-requirements *bool* | Check cluster version compatibility before starting the task, and check whether TiKV has more than 10% free space left during running time. (default = true) | `lightning.check-requirements` | -| --ca *file* | CA certificate path for TLS connection | `security.ca-path` | -| --cert *file* | Certificate path for TLS connection | `security.cert-path` | -| --key *file* | Private key path for TLS connection | `security.key-path` | -| --server-mode | Start TiDB Lightning in server mode | `lightning.server-mode` | - -If a command line parameter and the corresponding setting in the configuration file are both provided, the command line parameter will be used. For example, running `tiup tidb-lightning -L debug --config cfg.toml` would always set the log level to "debug" regardless of the content of `cfg.toml`. - -## Usage of `tidb-lightning-ctl` - -This tool can execute various actions given one of the following parameters: - -| Parameter | Explanation | -|:----|:----| -| --compact | Performs a full compaction | -| --switch-mode *mode* | Switches every TiKV store to the given mode: normal, import | -| --fetch-mode | Prints the current mode of every TiKV store | -| --import-engine *uuid* | Imports the closed engine file from TiKV Importer into the TiKV cluster | -| --cleanup-engine *uuid* | Deletes the engine file from TiKV Importer | -| --checkpoint-dump *folder* | Dumps current checkpoint as CSVs into the folder | -| --checkpoint-error-destroy *tablename* | Removes the checkpoint and drops the table if it caused error | -| --checkpoint-error-ignore *tablename* | Ignores any error recorded in the checkpoint involving the given table | -| --checkpoint-remove *tablename* | Unconditionally removes the checkpoint of the table | - -The *tablename* must either be a qualified table name in the form `` `db`.`tbl` `` (including the backquotes), or the keyword "all". - -Additionally, all parameters of `tidb-lightning` described in the section above are valid in `tidb-lightning-ctl`. diff --git a/tidb-lightning/tidb-lightning-data-source.md b/tidb-lightning/tidb-lightning-data-source.md index ff7722a070816..4abd184210b34 100644 --- a/tidb-lightning/tidb-lightning-data-source.md +++ b/tidb-lightning/tidb-lightning-data-source.md @@ -1,7 +1,6 @@ --- title: TiDB Lightning Data Sources summary: Learn all the data sources supported by TiDB Lightning. -aliases: ['/docs/dev/tidb-lightning/migrate-from-csv-using-tidb-lightning/','/docs/dev/reference/tools/tidb-lightning/csv/','/tidb/dev/migrate-from-csv-using-tidb-lightning/'] --- # TiDB Lightning Data Sources @@ -12,7 +11,7 @@ To specify the data source for TiDB Lightning, use the following configuration: ```toml [mydumper] -# Local source data directory or the URI of the external storage such as S3. For more information about the URI of the external storage, see https://docs.pingcap.com/tidb/v6.6/backup-and-restore-storages#uri-format. +# Local source data directory or the URI of the external storage such as S3. For more information about the URI of the external storage, see https://docs.pingcap.com/tidb/v8.1/backup-and-restore-storages#uri-format. data-source-dir = "/data/my_database" ``` @@ -342,7 +341,7 @@ When TiDB Lightning processes a SQL file, because TiDB Lightning cannot quickly ## Parquet -TiDB Lightning currently only supports Parquet files generated by Amazon Aurora or Apache Hive. To identify the file structure in S3, use the following configuration to match all data files: +TiDB Lightning currently only supports Parquet files generated by Amazon Aurora, Apache Hive, and Snowflake. To identify the file structure in S3, use the following configuration to match all data files: ``` [[mydumper.files]] diff --git a/tidb-lightning/tidb-lightning-distributed-import.md b/tidb-lightning/tidb-lightning-distributed-import.md index b80dbc82f3ccf..04161047836c3 100644 --- a/tidb-lightning/tidb-lightning-distributed-import.md +++ b/tidb-lightning/tidb-lightning-distributed-import.md @@ -5,7 +5,7 @@ summary: Learn the concept, user scenarios, usages, and limitations of importing # Use TiDB Lightning to Import Data in Parallel -Since v5.3.0, the [physical import mode](/tidb-lightning/tidb-lightning-physical-import-mode.md) of TiDB Lightning supports the parallel import of a single table or multiple tables. By simultaneously running multiple TiDB Lightning instances, you can import data in parallel from different single tables or multiple tables. In this way, TiDB Lightning provides the ability to scale horizontally, which greatly reduces the time required to import large amount of data. +Since v5.3.0, the [physical import mode](/tidb-lightning/tidb-lightning-physical-import-mode.md) of TiDB Lightning supports the parallel import of a single table or multiple tables. By simultaneously running multiple TiDB Lightning instances, you can import data from single or multiple tables in parallel. In this way, TiDB Lightning provides the ability to scale horizontally, which greatly reduces the time required to import large amount of data. In technical implementation, TiDB Lightning records the meta data of each instance and the data of each imported table in the target TiDB, and coordinates the Row ID allocation range of different instances, the record of global Checksum, and the configuration changes and recovery of TiKV and PD. diff --git a/tidb-lightning/tidb-lightning-faq.md b/tidb-lightning/tidb-lightning-faq.md index 5aaf8b97c702c..b37f531a9cb45 100644 --- a/tidb-lightning/tidb-lightning-faq.md +++ b/tidb-lightning/tidb-lightning-faq.md @@ -1,7 +1,6 @@ --- title: TiDB Lightning FAQs summary: Learn about the frequently asked questions (FAQs) and answers about TiDB Lightning. -aliases: ['/docs/dev/tidb-lightning/tidb-lightning-faq/','/docs/dev/faq/tidb-lightning/'] --- # TiDB Lightning FAQs @@ -52,7 +51,7 @@ ADMIN CHECKSUM TABLE `schema`.`table`; TiDB Lightning supports: -- Importing files exported by [Dumpling](/dumpling-overview.md), CSV files, and [Apache Parquet files generated by Amazon Aurora](/migrate-aurora-to-tidb.md). +- Importing files exported by [Dumpling](/dumpling-overview.md), CSV files, and [Apache Parquet files generated by Amazon Aurora](/migrate-aurora-to-tidb.md), Apache Hive, and Snowflake. - Reading data from a local disk or from the Amazon S3 storage. ## Could TiDB Lightning skip creating schema and tables? diff --git a/tidb-lightning/tidb-lightning-glossary.md b/tidb-lightning/tidb-lightning-glossary.md index 3645dc3997970..0fc8056071e60 100644 --- a/tidb-lightning/tidb-lightning-glossary.md +++ b/tidb-lightning/tidb-lightning-glossary.md @@ -1,7 +1,6 @@ --- title: TiDB Lightning Glossary summary: List of special terms used in TiDB Lightning. -aliases: ['/docs/dev/tidb-lightning/tidb-lightning-glossary/','/docs/dev/reference/tools/tidb-lightning/glossary/'] --- # TiDB Lightning Glossary diff --git a/tidb-lightning/tidb-lightning-logical-import-mode-usage.md b/tidb-lightning/tidb-lightning-logical-import-mode-usage.md index 419cc529c832f..f9f2498c7e30e 100644 --- a/tidb-lightning/tidb-lightning-logical-import-mode-usage.md +++ b/tidb-lightning/tidb-lightning-logical-import-mode-usage.md @@ -52,7 +52,7 @@ Conflicting data refers to two or more records with the same data in the PK or U | Strategy | Default behavior of conflicting data | The corresponding SQL statement | | :-- | :-- | :-- | | `"replace"` | Replacing existing data with new data. | `REPLACE INTO ...` | -| `"ignore"` | Keeping existing data and ignoring new data. | `INSERT IGNORE INTO ...` | +| `"ignore"` | Keeping existing data and ignoring new data. | If `conflict.threshold` is greater than 0, `INSERT INTO` is used; if `conflict.threshold` is `0`, `INSERT IGNORE INTO ...` is used | | `"error"` | Terminating the import when conflicting data is detected. | `INSERT INTO ...` | | `""` | Converted to `"error"`, which means terminating the import when conflicting data is detected. | None | diff --git a/tidb-lightning/tidb-lightning-overview.md b/tidb-lightning/tidb-lightning-overview.md index 9e1b8eed49ed9..a7d1d2e41e8c1 100644 --- a/tidb-lightning/tidb-lightning-overview.md +++ b/tidb-lightning/tidb-lightning-overview.md @@ -1,18 +1,17 @@ --- title: TiDB Lightning Overview summary: Learn about Lightning and the whole architecture. -aliases: ['/docs/dev/tidb-lightning/tidb-lightning-overview/','/docs/dev/reference/tools/tidb-lightning/overview/','/docs/dev/tidb-lightning/tidb-lightning-tidb-backend/','/docs/dev/reference/tools/tidb-lightning/tidb-backend/','/tidb/dev/tidb-lightning-tidb-backend','/docs/dev/loader-overview/','/docs/dev/reference/tools/loader/','/docs/dev/load-misuse-handling/','/docs/dev/reference/tools/error-case-handling/load-misuse-handling/','/tidb/dev/load-misuse-handling','/tidb/dev/loader-overview/','/tidb/dev/tidb-lightning-backends'] --- # TiDB Lightning Overview -[TiDB Lightning](https://github.com/pingcap/tidb/tree/master/lightning) is a tool used for importing data at TB scale to TiDB clusters. It is often used for initial data import to TiDB clusters. +[TiDB Lightning](https://github.com/pingcap/tidb/tree/release-8.1/lightning) is a tool used for importing data at TB scale to TiDB clusters. It is often used for initial data import to TiDB clusters. TiDB Lightning supports the following file formats: - Files exported by [Dumpling](/dumpling-overview.md) - CSV files -- [Apache Parquet files generated by Amazon Aurora](/migrate-aurora-to-tidb.md) +- [Apache Parquet files generated by Amazon Aurora](/migrate-aurora-to-tidb.md), Apache Hive, or Snowflake TiDB Lightning can read data from the following sources: @@ -20,6 +19,10 @@ TiDB Lightning can read data from the following sources: - [Amazon S3](/external-storage-uri.md#amazon-s3-uri-format) - [Google Cloud Storage](/external-storage-uri.md#gcs-uri-format) +> **Note:** +> +> Compared with TiDB Lightning, the [`IMPORT INTO`](/sql-statements/sql-statement-import-into.md) statement can be directly executed on TiDB nodes, supports automated distributed task scheduling and [TiDB Global Sort](/tidb-global-sort.md), and offers significant improvements in deployment, resource utilization, task configuration convenience, ease of invocation and integration, high availability, and scalability. It is recommended that you consider using `IMPORT INTO` instead of TiDB Lightning in appropriate scenarios. + ## TiDB Lightning architecture ![Architecture of TiDB Lightning tool set](/media/tidb-lightning-architecture.png) diff --git a/tidb-lightning/tidb-lightning-physical-import-mode.md b/tidb-lightning/tidb-lightning-physical-import-mode.md index 6e017e0ba944b..78a0e26d450c3 100644 --- a/tidb-lightning/tidb-lightning-physical-import-mode.md +++ b/tidb-lightning/tidb-lightning-physical-import-mode.md @@ -21,7 +21,7 @@ The backend for the physical import mode is `local`. You can modify it in `tidb- 1. Before importing data, TiDB Lightning automatically switches the TiKV nodes to "import mode", which improves write performance and stops auto-compaction. TiDB Lightning determines whether to pause global scheduling according to the TiDB Lightning version. - - Starting from v7.1.0, you can you can control the scope of pausing scheduling by using the TiDB Lightning parameter [`pause-pd-scheduler-scope`](/tidb-lightning/tidb-lightning-configuration.md). + - Starting from v7.1.0, you can control the scope of pausing scheduling by using the TiDB Lightning parameter [`pause-pd-scheduler-scope`](/tidb-lightning/tidb-lightning-configuration.md). - For TiDB Lightning versions between v6.2.0 and v7.0.0, the behavior of pausing global scheduling depends on the TiDB cluster version. When the TiDB cluster >= v6.1.0, TiDB Lightning pauses scheduling for the Region that stores the target table data. After the import is completed, TiDB Lightning recovers scheduling. For other versions, TiDB Lightning pauses global scheduling. - When TiDB Lightning < v6.2.0, TiDB Lightning pauses global scheduling. @@ -74,7 +74,7 @@ It is recommended that you allocate CPU more than 32 cores and memory greater th - Do not use the physical import mode to directly import data to TiDB clusters in production. It has severe performance implications. If you need to do so, refer to [Pause scheduling on the table level](/tidb-lightning/tidb-lightning-physical-import-mode-usage.md#scope-of-pausing-scheduling-during-import). - If your TiDB cluster has a latency-sensitive application and a low concurrency, it is strongly recommended that you **do not** use the physical import mode to import data into the cluster. This mode might have significant impact on the online application. -- Do not use multiple TiDB Lightning instances to import data to the same TiDB cluster by default. Use [Parallel Import](/tidb-lightning/tidb-lightning-distributed-import.md) instead. +- If you want to run multiple TiDB Lightning instances simultaneously to import data into the same TiDB cluster, you can enable [Parallel Import](/tidb-lightning/tidb-lightning-distributed-import.md) to coordinate the import. If each instance imports data into **different tables**, the Parallel Import option is not required. However, if multiple instances import data into **the same table**, you need to enable Parallel Import to prevent conflicts and ensure data integrity. - When you use multiple TiDB Lightning to import data to the same target cluster, do not mix the import modes. That is, do not use the physical import mode and the logical import mode at the same time. - During the process of importing data, do not perform DDL and DML operations in the target table. Otherwise the import will fail or the data will be inconsistent. At the same time, it is not recommended to perform read operations, because the data you read might be inconsistent. You can perform read and write operations after the import operation is completed. - A single Lightning process can import a single table of 10 TiB at most. Parallel import can use 10 Lightning instances at most. @@ -97,4 +97,4 @@ It is recommended that you allocate CPU more than 32 cores and memory greater th - When BR backs up snapshots of tables that are being imported by TiDB Lightning, it might result in inconsistent backup data for those tables. - When BR backs up data using AWS EBS volume snapshots, TiDB Lightning might fail to import data. - - Point-in-time recovery (PITR) cannot back up data imported by TiDB Lightning. + - Data imported in TiDB Lightning physical import mode does not support [log backup](/br/br-pitr-guide.md#start-log-backup) and thereby cannot be restored by Point-in-Time Recovery (PITR). \ No newline at end of file diff --git a/tidb-lightning/tidb-lightning-prechecks.md b/tidb-lightning/tidb-lightning-prechecks.md index c5ae306246c71..c55455afc9bce 100644 --- a/tidb-lightning/tidb-lightning-prechecks.md +++ b/tidb-lightning/tidb-lightning-prechecks.md @@ -14,7 +14,7 @@ The following table describes each check item and detailed explanation. | Cluster version and status| >= 5.3.0 | Check whether the cluster can be connected in the configuration, and whether the TiKV/PD/TiFlash version supports the physical import mode. | | Permissions | >= 5.3.0 | When the data source is cloud storage (Amazon S3), check whether TiDB Lightning has the necessary permissions and make sure that the import will not fail due to lack of permissions. | | Disk space | >= 5.3.0 | Check whether there is enough space on the local disk and on the TiKV cluster for importing data. TiDB Lightning samples the data sources and estimates the percentage of the index size from the sample result. Because indexes are included in the estimation, there might be cases where the size of the source data is less than the available space on the local disk, but still, the check fails. In the physical import mode, TiDB Lightning also checks whether the local storage is sufficient because external sorting needs to be done locally. For more details about the TiKV cluster space and local storage space (controlled by `sort-kv-dir`), see [Downstream storage space requirements](/tidb-lightning/tidb-lightning-requirements.md#storage-space-of-the-target-database) and [Resource requirements](/tidb-lightning/tidb-lightning-physical-import-mode.md#environment-requirements). | -| Region distribution status | >= 5.3.0 | Check whether the Regions in the TiKV cluster are distributed evenly and whether there are too many empty Regions. If the number of empty Regions exceeds max(1000, number of tables * 3), i.e. greater than the bigger one of "1000" or "3 times the number of tables ", then the import cannot be executed. | +| Region distribution status | >= 5.3.0 | Check whether the Regions in the TiKV cluster are distributed evenly and whether there are too many empty Regions. If the number of empty Regions exceeds max(1000, number of tables * 3), i.e. greater than the bigger one of "1000" or "3 times the number of tables", then the import cannot be executed. | | Exceedingly Large CSV files in the data file | >= 5.3.0 | When there are CSV files larger than 10 GiB in the backup file and auto-slicing is not enabled (StrictFormat=false), it will impact the import performance. The purpose of this check is to remind you to ensure the data is in the right format and to enable auto-slicing. | | Recovery from breakpoints | >= 5.3.0 | This check ensures that no changes are made to the source file or schema in the database during the breakpoint recovery process that would result in importing the wrong data. | | Import into an existing table | >= 5.3.0 | When importing into an already created table, it checks, as much as possible, whether the source file matches the existing table. Check if the number of columns matches. If the source file has column names, check if the column names match. When there are default columns in the source file, it checks if the default columns have Default Value, and if they have, the check passes. | diff --git a/tidb-lightning/tidb-lightning-web-interface.md b/tidb-lightning/tidb-lightning-web-interface.md index ff853bb960ef9..9836257ffbfd3 100644 --- a/tidb-lightning/tidb-lightning-web-interface.md +++ b/tidb-lightning/tidb-lightning-web-interface.md @@ -1,7 +1,6 @@ --- title: TiDB Lightning Web Interface summary: Control TiDB Lightning through the web interface. -aliases: ['/docs/dev/tidb-lightning/tidb-lightning-web-interface/','/docs/dev/reference/tools/tidb-lightning/web/'] --- # TiDB Lightning Web Interface diff --git a/tidb-lightning/troubleshoot-tidb-lightning.md b/tidb-lightning/troubleshoot-tidb-lightning.md index 0748a5de36c54..20880a475fbb3 100644 --- a/tidb-lightning/troubleshoot-tidb-lightning.md +++ b/tidb-lightning/troubleshoot-tidb-lightning.md @@ -1,7 +1,6 @@ --- title: Troubleshoot TiDB Lightning summary: Learn the common problems you might encounter when you use TiDB Lightning and their solutions. -aliases: ['/docs/dev/troubleshoot-tidb-lightning/','/docs/dev/how-to/troubleshoot/tidb-lightning/','/docs/dev/tidb-lightning/tidb-lightning-misuse-handling/','/docs/dev/reference/tools/error-case-handling/lightning-misuse-handling/','/tidb/dev/tidb-lightning-misuse-handling','/tidb/dev/troubleshoot-tidb-lightning'] --- # Troubleshoot TiDB Lightning @@ -47,7 +46,7 @@ It is potentially caused by starting `tidb-lightning` incorrectly, which causes [2018/08/10 07:29:08.310 +08:00] [INFO] [main.go:41] ["got signal to exit"] [signal=hangup] ``` -It is not recommended to directly use `nohup` in the command line to start `tidb-lightning`. You can [start `tidb-lightning`](/tidb-lightning/deploy-tidb-lightning.md) by executing a script. +It is not recommended to directly use `nohup` in the command line to start `tidb-lightning`. You can [start `tidb-lightning`](/get-started-with-tidb-lightning.md#step-4-start-tidb-lightning) by executing a script. In addition, if the last log of TiDB Lightning shows that the error is "Context canceled", you need to search for the first "ERROR" level log. This "ERROR" level log is usually followed by "got signal to exit", which indicates that TiDB Lightning received an interrupt signal and then exited. @@ -156,7 +155,8 @@ See the [Checkpoints control](/tidb-lightning/tidb-lightning-checkpoints.md#chec **Solution**: -Currently, the limitation of TiDB cannot be bypassed. You can only ignore this table to ensure the successful import of other tables. +- Use the [`tidb_txn_entry_size_limit`](/system-variables.md#tidb_txn_entry_size_limit-new-in-v760) system variable to dynamically increase the limit. +- Note that TiKV has a similar limit. If the data size of a single write request exceeds [`raft-entry-max-size`](/tikv-configuration-file.md#raft-entry-max-size) (`8MiB` by default), TiKV refuses to process this request. When a table has a row of a large size, you need to modify both configurations. ### Encounter `rpc error: code = Unimplemented ...` when TiDB Lightning switches the mode diff --git a/tidb-limitations.md b/tidb-limitations.md index 519ad878c201f..1369fcc929dee 100644 --- a/tidb-limitations.md +++ b/tidb-limitations.md @@ -1,13 +1,16 @@ --- title: TiDB Limitations summary: Learn the usage limitations of TiDB. -aliases: ['/docs/dev/tidb-limitations/'] --- # TiDB Limitations This document describes the common usage limitations of TiDB, including the maximum identifier length and the maximum number of supported databases, tables, indexes, partitioned tables, and sequences. +> **Note:** +> +> TiDB offers high compatibility with the MySQL protocol and syntax, including many MySQL limitations. For example, a single index can include a maximum of 16 columns. For more information, see [MySQL Compatibility](/mysql-compatibility.md) and the official MySQL documentation. + ## Limitations on identifier length | Identifier type | Maximum length (number of characters allowed) | @@ -59,7 +62,7 @@ This document describes the common usage limitations of TiDB, including the maxi -You can adjust the size limit via the [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v50) configuration item. +You can adjust the size limit via the [`txn-entry-size-limit`](/tidb-configuration-file.md#txn-entry-size-limit-new-in-v4010-and-v500) configuration item. @@ -67,8 +70,8 @@ You can adjust the size limit via the [`txn-entry-size-limit`](/tidb-configurati | Type | Upper limit | |:----------|:----------| -| CHAR | 256 characters | -| BINARY | 256 characters | +| CHAR | 255 characters | +| BINARY | 255 characters | | VARBINARY | 65535 characters | | VARCHAR | 16383 characters | | TEXT | Defaults to 6 MiB and can be adjusted to 120 MiB | diff --git a/tidb-monitoring-api.md b/tidb-monitoring-api.md index 0179b1d4de566..1bdd5aad92734 100644 --- a/tidb-monitoring-api.md +++ b/tidb-monitoring-api.md @@ -1,7 +1,6 @@ --- title: TiDB Monitoring API summary: Learn the API of TiDB monitoring services. -aliases: ['/docs/dev/tidb-monitoring-api/'] --- # TiDB Monitoring API @@ -28,7 +27,7 @@ The following example uses `http://${host}:${port}/status` to get the current st curl http://127.0.0.1:10080/status { connections: 0, # The current number of clients connected to the TiDB server. - version: "8.0.11-TiDB-v8.0.0", # The TiDB version number. + version: "8.0.11-TiDB-v8.1.2", # The TiDB version number. git_hash: "778c3f4a5a716880bcd1d71b257c8165685f0d70" # The Git Hash of the current TiDB code. } ``` @@ -79,7 +78,7 @@ curl http://127.0.0.1:10080/schema_storage/test - PD API address: `http://${host}:${port}/pd/api/v1/${api_name}` - Default port: `2379` -- Details about API names: see [PD API doc](https://download.pingcap.com/pd-api-v1.html) +- Details about API names: see [PD API doc](https://docs-download.pingcap.com/api/pd-api/pd-api-v1.html) The PD interface provides the status of all the TiKV servers and the information about load balancing. See the following example for the information about a single-node TiKV cluster: diff --git a/tidb-monitoring-framework.md b/tidb-monitoring-framework.md index d178b1bbb5599..568c6c0847232 100644 --- a/tidb-monitoring-framework.md +++ b/tidb-monitoring-framework.md @@ -1,12 +1,11 @@ --- title: TiDB Monitoring Framework Overview -summary: Use Prometheus and Grafana to build the TiDB monitoring framework. -aliases: ['/docs/dev/tidb-monitoring-framework/','/docs/dev/how-to/monitor/overview/'] +summary: Use Prometheus, Grafana, and TiDB Dashboard to build the TiDB monitoring framework. --- # TiDB Monitoring Framework Overview -The TiDB monitoring framework adopts two open source projects: Prometheus and Grafana. TiDB uses [Prometheus](https://prometheus.io) to store the monitoring and performance metrics and [Grafana](https://grafana.com/grafana) to visualize these metrics. +The TiDB monitoring framework adopts two open source projects: Prometheus and Grafana. TiDB uses [Prometheus](https://prometheus.io) to store the monitoring and performance metrics and [Grafana](https://grafana.com/grafana) to visualize these metrics. TiDB also provides a built-in [TiDB Dashboard](/dashboard/dashboard-intro.md) for monitoring and diagnosing TiDB clusters. ## About Prometheus in TiDB @@ -52,3 +51,7 @@ Grafana is an open source project for analyzing and visualizing metrics. TiDB us Each group has multiple panel labels of monitoring metrics, and each panel contains detailed information of multiple monitoring metrics. For example, the **Overview** monitoring group has five panel labels, and each labels corresponds to a monitoring panel. See the following UI: ![Grafana Overview](/media/grafana-monitor-overview.png) + +## TiDB Dashboard + +TiDB Dashboard is a web UI for monitoring, diagnosing, and managing the TiDB cluster, which is introduced in v4.0. It is built into the PD component and does not require an independent deployment. For more information, see [TiDB Dashboard introduction](/dashboard/dashboard-intro.md). diff --git a/tidb-operator-overview.md b/tidb-operator-overview.md index e03a0119e2c6b..901e5e2c0b0e2 100644 --- a/tidb-operator-overview.md +++ b/tidb-operator-overview.md @@ -1,7 +1,6 @@ --- title: TiDB Operator summary: Learn about TiDB Operator, the automatic operation system for TiDB clusters on Kubernetes. -aliases: ['/docs/tidb-in-kubernetes/dev/'] --- # TiDB Operator diff --git a/tidb-performance-tuning-config.md b/tidb-performance-tuning-config.md new file mode 100644 index 0000000000000..e15342944f0fd --- /dev/null +++ b/tidb-performance-tuning-config.md @@ -0,0 +1,391 @@ +--- +title: Configure TiDB for Optimal Performance +summary: Learn how to optimize TiDB performance by configuring key settings and addressing edge cases. +--- + +## Configure TiDB for Optimal Performance + +This guide describes how to optimize the performance of TiDB, including: + +- Best practices for common workloads. +- Strategies for handling challenging performance scenarios. + +> **Note:** +> +> The optimization techniques in this guide can help achieve optimal performance in TiDB. However, performance tuning often involves balancing multiple factors, and no single solution addresses all performance needs. Some techniques in this guide use experimental features, which are marked accordingly. Although these optimizations can significantly improve performance, they might not be suitable for production environments and require careful evaluation before implementation. + +## Overview + +Optimizing TiDB for peak performance requires careful tuning of various settings. In many cases, achieving optimal performance involves adjusting configurations beyond their default values. + +The default settings prioritize stability over performance. To maximize performance, you might need to use more aggressive configurations and, in some cases, experimental features. These recommendations are based on production deployment experience and performance optimization research. + +This guide describes the non-default settings, including their benefits and potential trade-offs. Use this information to optimize TiDB settings for your workload requirements. + +## Key settings for common workloads + +The following settings are commonly used to optimize TiDB performance: + +- Enhance execution plan cache, such as [SQL Prepared Execution Plan Cache](/sql-prepared-plan-cache.md) and [Non-prepared plan cache](/sql-non-prepared-plan-cache.md). +- Optimize the behavior of the TiDB optimizer by using [Optimizer Fix Controls](/optimizer-fix-controls.md). +- Use the [Titan](/storage-engine/titan-overview.md) storage engine more aggressively. + +These settings can significantly improve performance for many workloads. However, as with any optimization, thoroughly test them in your environment before deploying to production. + +### System variables + +Execute the following SQL commands to apply the recommended settings: + +```sql +SET GLOBAL tidb_session_plan_cache_size=200; +SET GLOBAL tidb_enable_non_prepared_plan_cache=on; +SET GLOBAL tidb_ignore_prepared_cache_close_stmt=on; +SET GLOBAL tidb_stats_load_sync_wait=2000; +SET GLOBAL tidb_enable_inl_join_inner_multi_pattern=on; +SET GLOBAL tidb_opt_derive_topn=on; +SET GLOBAL tidb_runtime_filter_mode=LOCAL; +SET GLOBAL tidb_opt_enable_mpp_shared_cte_execution=on; +SET GLOBAL tidb_rc_read_check_ts=on; +SET GLOBAL tidb_guarantee_linearizability=off; +SET GLOBAL pd_enable_follower_handle_region=on; +SET GLOBAL tidb_opt_fix_control = '44262:ON,44389:ON,44823:10000,44830:ON,44855:ON,52869:ON'; +``` + +The following table outlines the impact of specific system variable configurations: + +| System variable | Description | Note | +| ---------| ---- | ----| +| [`tidb_session_plan_cache_size`](/system-variables.md#tidb_session_plan_cache_size-new-in-v710) | Increase the maximum number of plans that can be cached from the default `100` to `200`. This improves performance for workloads with many prepared statement patterns. | Increasing this value can lead to higher memory usage for the session plan cache. | +| [`tidb_enable_non_prepared_plan_cache`](/system-variables.md#tidb_enable_non_prepared_plan_cache)| Enable the [Non-prepared plan cache](/sql-non-prepared-plan-cache.md) feature to reduce compile costs for applications that do not use prepared statements. | N/A | +| [`tidb_ignore_prepared_cache_close_stmt`](/system-variables.md#tidb_ignore_prepared_cache_close_stmt-new-in-v600)| Cache plans for applications that use prepared statements but close the plan after each execution. | N/A | +| [`tidb_stats_load_sync_wait`](/system-variables.md#tidb_stats_load_sync_wait-new-in-v540)| Increase the timeout for synchronously loading statistics from the default 100 milliseconds to 2 seconds. This ensures TiDB loads the necessary statistics before query compilation. | Increasing this value leads to a longer synchronization wait time before query compilation. | +| [`tidb_enable_inl_join_inner_multi_pattern`](/system-variables.md#tidb_enable_inl_join_inner_multi_pattern-new-in-v700) | Enable Index Join support when the inner table has `Selection` or `Projection` operators on it. | N/A | +| [`tidb_opt_derive_topn`](/system-variables.md#tidb_opt_derive_topn-new-in-v700)| Enable the optimization rule of [Deriving TopN or Limit from window functions](/derive-topn-from-window.md). | This is limited to the `ROW_NUMBER()` window function. | +| [`tidb_runtime_filter_mode`](/system-variables.md#tidb_runtime_filter_mode-new-in-v720)| Enable [Runtime Filter](/runtime-filter.md#runtime-filter-mode) in the local mode to improve hash join efficiency. | The variable is introduced in v7.2.0 and is disabled by default for safety. | +| [`tidb_opt_enable_mpp_shared_cte_execution`](/system-variables.md#tidb_opt_enable_mpp_shared_cte_execution-new-in-v720)| Enable non-recursive [Common Table Expressions (CTE)](/sql-statements/sql-statement-with.md) pushdown to TiFlash. | This is an experimental feature. | +| [`tidb_rc_read_check_ts`](/system-variables.md#tidb_rc_read_check_ts-new-in-v600)| For the read-committed isolation level, enabling this variable avoids the latency and cost of getting the global timestamp and optimizes transaction-level read latency. | This feature is incompatible with the Repeatable Read isolation level. | +| [`tidb_guarantee_linearizability`](/system-variables.md#tidb_guarantee_linearizability-new-in-v50)| Improve performance by skipping the commit timestamp fetch from the PD server. | This sacrifices linearizability in favor of performance. Only causal consistency is guaranteed. It is not suitable for scenarios requiring strict linearizability. | +| [`pd_enable_follower_handle_region`](/system-variables.md#pd_enable_follower_handle_region-new-in-v760)| Activate the PD Follower feature, allowing PD followers to process Region requests. This helps distribute load evenly across all PD servers and reduces CPU pressure on the PD leader. | This is an experimental feature. Test in non-production environments. | +| [`tidb_opt_fix_control`](/system-variables.md#tidb_opt_fix_control-new-in-v653-and-v710) | Enable advanced query optimization strategies to improve performance through additional optimization rules and heuristics. | Test thoroughly in your environment, as performance improvements vary by workload. | + +The following describes the optimizer control configurations that enable additional optimizations: + +- [`44262:ON`](/optimizer-fix-controls.md#44262-new-in-v653-and-v720): Use [Dynamic pruning mode](/partitioned-table.md#dynamic-pruning-mode) to access the partitioned table when the [GlobalStats](/statistics.md#collect-statistics-of-partitioned-tables-in-dynamic-pruning-mode) are missing. +- [`44389:ON`](/optimizer-fix-controls.md#44389-new-in-v653-and-v720): For filters such as `c = 10 and (a = 'xx' or (a = 'kk' and b = 1))`, build more comprehensive scan ranges for `IndexRangeScan`. +- [`44823:10000`](/optimizer-fix-controls.md#44823-new-in-v730): To save memory, plan cache does not cache queries with parameters exceeding the specified number of this variable. Increase plan cache parameter limit from `200` to `10000` to make plan cache available for query with long in-lists. +- [`44830:ON`](/optimizer-fix-controls.md#44830-new-in-v657-and-v730): Plan cache is allowed to cache execution plans with the `PointGet` operator generated during physical optimization. +- [`44855:ON`](/optimizer-fix-controls.md#44855-new-in-v654-and-v730): The optimizer selects `IndexJoin` when the `Probe` side of an `IndexJoin` operator contains a `Selection` operator. +- [`52869:ON`](/optimizer-fix-controls.md#52869-new-in-v810): The optimizer chooses index merge automatically if the optimizer can choose the single index scan method (other than full table scan) for a query plan. + +### TiKV configurations + +Add the following configuration items to the TiKV configuration file: + +```toml +[server] +concurrent-send-snap-limit = 64 +concurrent-recv-snap-limit = 64 +snap-io-max-bytes-per-sec = "400MiB" + +[rocksdb.titan] +enabled = true +[rocksdb.defaultcf.titan] +min-blob-size = "1KB" +blob-file-compression = "zstd" + +[storage.flow-control] +l0-files-threshold = 60 +``` + +| Configuration item | Description | Note | +| ---------| ---- | ----| +| [`concurrent-send-snap-limit`](/tikv-configuration-file.md#concurrent-send-snap-limit), [`concurrent-recv-snap-limit`](/tikv-configuration-file.md#concurrent-recv-snap-limit), and [`snap-io-max-bytes-per-sec`](/tikv-configuration-file.md#snap-io-max-bytes-per-sec) | Set limits for concurrent snapshot transfer and I/O bandwidth during TiKV scaling operations. Higher limits reduce scaling time by allowing faster data migration. | Adjusting these limits affects the trade-off between scaling speed and online transaction performance. | +| [`rocksdb.titan`](/tikv-configuration-file.md#rocksdbtitan), [`rocksdb.defaultcf.titan`](/tikv-configuration-file.md#rocksdbdefaultcftitan), [`min-blob-size`](/tikv-configuration-file.md#min-blob-size), and [`blob-file-compression`](/tikv-configuration-file.md#blob-file-compression) | Enable the Titan storage engine to reduce write amplification and alleviate disk I/O bottlenecks. Particularly useful when RocksDB compaction cannot keep up with write workloads, resulting in accumulated pending compaction bytes. | Enable it when write amplification is the primary bottleneck. Trade-offs include: 1. Potential performance impact on primary key range scans. 2. Increased space amplification (up to 2x in the worst case). 3. Additional memory usage for blob cache. | +| [`storage.flow-control.l0-files-threshold`](/tikv-configuration-file.md#l0-files-threshold) | Control when write flow control is triggered based on the number of kvDB L0 files. Increasing the threshold reduces write stalls during high write workloads. | Higher thresholds might lead to more aggressive compactions when many L0 files exist. | + +### TiFlash configurations + +Add the following configuration items to the TiFlash configuration file: + +```toml +[raftstore-proxy.server] +snap-io-max-bytes-per-sec = "300MiB" +``` + +| Configuration item | Description | Note | +| ---------| ---- | ----| +| [`snap-io-max-bytes-per-sec`](/tikv-configuration-file.md#snap-io-max-bytes-per-sec) | Control the maximum allowable disk bandwidth for data replication from TiKV to TiFlash. Higher limits accelerate initial data loading and catch-up replication. | Higher bandwidth consumption might impact online transaction performance. Balance between replication speed and system stability. | + +## Benchmark + +This section compares performance between default settings (baseline) and optimized settings based on the preceding [key settings for common loads](#key-settings-for-common-workloads). + +### YCSB workloads on large record value + +#### Test environment + +The test environment is as follows: + +- 3 TiDB servers (16 cores, 64 GiB) +- 3 TiKV servers (16 cores, 64 GiB) +- TiDB version: v8.1.0 +- Workload: [go-ycsb workloada](https://github.com/pingcap/go-ycsb/blob/master/workloads/workloada) + +#### Performance comparison + +The following table compares throughput (operations per second) between the baseline and optimized settings. + +| Item | Baseline (OPS) | Optimized (OPS) | Improvement | +| ---------| ---- | ----| ----| +| load data | 2858.5 | 5074.3 | +77.59% | +| workloada | 2243.0 | 12804.3 | +470.86% | + +#### Performance analysis + +Titan is enabled by default starting from v7.6.0 and the default `min-blob-size` of Titan in TiDB v8.1.0 is `32KiB`. The baseline configuration uses a record size of `31KiB` to ensure data is stored in RocksDB. In contrast, for the key settings configuration, set `min-blob-size` to `1KiB`, causing data to be stored in Titan. + +The performance improvement observed in the key settings is primarily attributed to Titan's ability to reduce RocksDB compactions. As shown in the following figures: + +- Baseline: The total throughput of RocksDB compaction exceeds 1 GiB/s, with peaks over 3 GiB/s. +- Key settings: The peak throughput of RocksDB compaction remains below 100 MiB/s. + +This significant reduction in compaction overhead contributes to the overall throughput improvement seen in the key settings configuration. + +![Titan RocksDB compaction:](/media/performance/titan-rocksdb-compactions.png) + +#### Test workload + +The following `go-ycsb load` command loads data: + +```bash +go-ycsb load mysql -P /ycsb/workloads/workloada -p {host} -p mysql.port={port} -p threadcount=100 -p recordcount=5000000 -p operationcount=5000000 -p workload=core -p requestdistribution=uniform -pfieldcount=31 -p fieldlength=1024 +``` + +The following `go-ycsb run` command runs workload: + +```bash +go-ycsb run mysql -P /ycsb/workloads/workloada -p {host} -p mysql.port={port} -p mysql.db=test -p threadcount=100 -p recordcount=5000000 -p operationcount=5000000 -p workload=core -prequestdistribution=uniform -p fieldcount=31 -p fieldlength=1024 +``` + +## Edge cases and optimizations + +This section shows you how to optimize TiDB for specific scenarios that need targeted adjustments beyond basic optimizations. You will learn how to tune TiDB for your particular use cases. + +### Identify edge cases + +To identify edge cases, perform the following steps: + +1. Analyze query patterns and workload characteristics. +2. Monitor system metrics to identify performance bottlenecks. +3. Gather feedback from application teams about specific issues. + +### Common edge cases + +The following lists some common edge cases: + +- High TSO wait for high-frequency small queries +- Choose the proper max chunk size for different workloads +- Tune coprocessor cache for read-heavy workloads +- Optimize chunk size for workload characteristics +- Optimize transaction mode and DML type for different workloads +- Optimize `GROUP BY` and `DISTINCT` operations with TiKV pushdown +- Optimize statistics collection during batch operations +- Optimize thread pool settings for different instance types + +The following sections explain how to handle each of these cases. You need to adjust different parameters or use specific TiDB features for each scenario. + +> **Note:** +> +> Apply these optimizations carefully and test thoroughly, as their effectiveness might vary based on your use case and data patterns. + +### High TSO wait for high-frequency small queries + +#### Troubleshooting + +If your workload involves frequent small transactions or queries that frequently request timestamps, TSO (Timestamp Oracle) can become a performance bottleneck. To check if TSO wait time is impacting your system, check the [**Performance Overview > SQL Execute Time Overview**](/grafana-performance-overview-dashboard.md#sql-execute-time-overview) panel. If TSO wait time constitutes a large portion of your SQL execution time, consider the following optimizations: + +- Use low-precision TSO (enable [`tidb_low_resolution_tso`](/system-variables.md#tidb_low_resolution_tso)) for read operations that do not need strict consistency. For more information, see [Solution 1: use low-precision TSO](#solution-1-low-precision-tso). +- Enable [`tidb_enable_batch_dml`](/system-variables.md#tidb_enable_batch_dml) to reduce TSO requests for batch operations. + +#### Solution 1: low-precision TSO + +You can reduce TSO wait time by enabling the low-precision TSO feature ([`tidb_low_resolution_tso`](/system-variables.md#tidb_low_resolution_tso)). After this feature is enabled, TiDB uses the cached timestamp to read data, reducing TSO wait time at the expense of potentially stale reads. + +This optimization is particularly effective in the following scenarios: + +- Read-heavy workloads where slight staleness is acceptable. +- Scenarios where reducing query latency is more important than absolute consistency. +- Applications that can tolerate reads that are a few seconds behind the latest committed state. + +Benefits and trade-offs: + +- Reduce query latency by enabling stale reads with a cached TSO, eliminating the need to request new timestamps. +- Balance performance against data consistency: this feature is only suitable for scenarios where stale reads are acceptable. It is not recommended to use it when strict data consistency is required. + +To enable this optimization: + +```sql +SET GLOBAL tidb_low_resolution_tso=ON; +``` + +### Tune coprocessor cache for read-heavy workloads + +You can improve query performance for read-heavy workloads by optimizing the [coprocessor cache](/coprocessor-cache.md). This cache stores the results of coprocessor requests, reducing repeated computations of frequently accessed data. To optimize cache performance, perform the following steps: + +1. Monitor the cache hit ratio using the metrics described in [Coprocessor Cache](/coprocessor-cache.md#view-the-grafana-monitoring-panel). +2. Increase the cache size to improve hit rates for larger working sets. +3. Adjust the admission threshold based on query patterns. + +The following lists some recommended settings for a read-heavy workload: + +```toml +[tikv-client.copr-cache] +capacity-mb = 4096 +admission-max-ranges = 5000 +admission-max-result-mb = 10 +admission-min-process-ms = 0 +``` + +### Optimize chunk size for workload characteristics + +The [`tidb_max_chunk_size`](/system-variables.md#tidb_max_chunk_size) system variable sets the maximum number of rows in a chunk during the execution process. Adjusting this value based on your workload can improve performance. + +- For OLTP workloads with large concurrency and small transactions: + + - Set the value between `128` and `256` rows (the default value is `1024`). + - This reduces memory usage and makes limit queries faster. + - Use case: point queries, small range scans. + + ```sql + SET GLOBAL tidb_max_chunk_size = 128; + ``` + +- For OLAP or analytical workloads with complex queries and large result sets: + + - Set the value between `1024` and `4096` rows. + - This increases throughput when scanning large amounts of data. + - Use case: aggregations, large table scans. + + ```sql + SET GLOBAL tidb_max_chunk_size = 4096; + ``` + +### Optimize transaction mode and DML type for different workloads + +TiDB provides different transaction modes and DML execution types to optimize performance for various workload patterns. + +#### Transaction modes + +You can set the transaction mode using the [`tidb_txn_mode`](/system-variables.md#tidb_txn_mode) system variable. + +- [Pessimistic transaction mode](/pessimistic-transaction.md) (default): + + - Suitable for general workloads with potential write conflicts. + - Provides stronger consistency guarantees. + + ```sql + SET SESSION tidb_txn_mode = "pessimistic"; + ``` + +- [Optimistic transaction mode](/optimistic-transaction.md): + + - Suitable for workloads with minimal write conflicts. + - Better performance for multi-statement transactions. + - Example: `BEGIN; INSERT...; INSERT...; COMMIT;`. + + ```sql + SET SESSION tidb_txn_mode = "optimistic"; + ``` + +#### DML types + +You can control the execution mode of DML statements using the [`tidb_dml_type`](/system-variables.md#tidb_dml_type-new-in-v800) system variable, which is introduced in v8.0.0. + +To use the bulk DML execution mode, set `tidb_dml_type` to `"bulk"`. This mode optimizes bulk data loading without conflicts and reduces memory usage during large write operations. Before using this mode, ensure that: + +- Auto-commit is enabled. +- The [`pessimistic-auto-commit`](/tidb-configuration-file.md#pessimistic-auto-commit-new-in-v600) configuration item is set to `false`. + +```sql +SET SESSION tidb_dml_type = "bulk"; +``` + +### Optimize `GROUP BY` and `DISTINCT` operations with TiKV pushdown + +TiDB pushes down aggregation operations to TiKV to reduce data transfer and processing overhead. The performance improvement varies based on your data characteristics. + +#### Usage scenarios + +- **Ideal scenarios** (high performance gain): + - Columns containing few distinct values (low NDV). + - Data containing frequent duplicate values. + - Example: status columns, category codes, date parts. + +- **Non-ideal scenarios** (potential performance loss): + - Columns containing mostly unique values (high NDV). + - Unique identifiers or timestamps. + - Example: User IDs, transaction IDs. + +#### Configuration + +Enable pushdown optimizations at the session or global level: + +```sql +-- Enable regular aggregation pushdown +SET GLOBAL tidb_opt_agg_push_down = ON; + +-- Enable distinct aggregation pushdown +SET GLOBAL tidb_opt_distinct_agg_push_down = ON; +``` + +### Optimize statistics collection during batch operations + +You can optimize performance during batch operations while maintaining query optimization by managing statistics collection. This section describes how to manage this process effectively. + +#### When to disable auto analyze + +You can disable auto analyze by setting the [`tidb_enable_auto_analyze`](/system-variables.md#tidb_enable_auto_analyze-new-in-v610) system variable to `OFF` in the following scenarios: + +- During large data imports. +- During bulk update operations. +- For time-sensitive batch processing. +- When you need full control over the timing of statistics collection. + +#### Best practices + +- Before the batch operation: + + ```sql + -- Disable auto analyze + SET GLOBAL tidb_enable_auto_analyze = OFF; + ``` + +- After the batch operation: + + ```sql + -- Manually collect statistics + ANALYZE TABLE your_table; + + -- Re-enable auto analyze + SET GLOBAL tidb_enable_auto_analyze = ON; + ``` + +### Optimize thread pool settings for different instance types + +To improve TiKV performance, configure the thread pools based on your instance's CPU resources. The following guidelines help you optimize these settings: + +- For instances with 8 to 16 cores, the default settings are typically sufficient. + +- For instances with 32 or more cores, increase the pool sizes for better resource utilization. Adjust the settings as follows: + + ```toml + [server] + # Increase gRPC thread pool + grpc-concurrency = 10 + + [raftstore] + # Optimize for write-intensive workloads + apply-pool-size = 4 + store-pool-size = 4 + store-io-pool-size = 2 + ``` \ No newline at end of file diff --git a/tidb-resource-control.md b/tidb-resource-control.md index 966d0881b126b..1b50cc2637f41 100644 --- a/tidb-resource-control.md +++ b/tidb-resource-control.md @@ -7,7 +7,7 @@ summary: Learn how to use the resource control feature to control and schedule a > **Note:** > -> This feature is not available on [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless) clusters. +> This feature is not available on [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) clusters. As a cluster administrator, you can use the resource control feature to create resource groups, set quotas for resource groups, and bind users to those groups. @@ -96,7 +96,7 @@ Request Unit (RU) is a unified abstraction unit in TiDB for system resources, wh > **Note:** > > - Each write operation is eventually replicated to all replicas (by default, TiKV has 3 replicas). Each replication operation is considered a different write operation. -> - The preceding table lists only the resources involved in RU calculation for TiDB Self-Hosted clusters, excluding the network and storage consumption. For TiDB Serverless RUs, see [TiDB Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details/). +> - The preceding table lists only the resources involved in RU calculation for TiDB Self-Managed clusters, excluding the network and storage consumption. For TiDB Cloud Serverless RUs, see [TiDB Cloud Serverless Pricing Details](https://www.pingcap.com/tidb-cloud-serverless-pricing-details/). > - Currently, TiFlash resource control only considers SQL CPU, which is the CPU time consumed by the execution of pipeline tasks for queries, and read request payload. ## Parameters for resource control @@ -114,8 +114,8 @@ The resource control feature introduces the following system variables or parame -* TiKV: For TiDB Self-Hosted, you can use the `resource-control.enabled` parameter to control whether to use request scheduling based on resource group quotas. For TiDB Cloud, the value of the `resource-control.enabled` parameter is `true` by default and does not support dynamic modification. -* TiFlash: For TiDB Self-Hosted, you can use the `tidb_enable_resource_control` system variable and the `enable_resource_control` configuration item (introduced in v7.4.0) to control whether to enable TiFlash resource control. +* TiKV: For TiDB Self-Managed, you can use the `resource-control.enabled` parameter to control whether to use request scheduling based on resource group quotas. For TiDB Cloud, the value of the `resource-control.enabled` parameter is `true` by default and does not support dynamic modification. +* TiFlash: For TiDB Self-Managed, you can use the `tidb_enable_resource_control` system variable and the `enable_resource_control` configuration item (introduced in v7.4.0) to control whether to enable TiFlash resource control. @@ -138,7 +138,7 @@ Starting from v7.4.0, the TiFlash configuration item `enable_resource_control` i -For more information about the resource control mechanism and parameters, see [RFC: Global Resource Control in TiDB](https://github.com/pingcap/tidb/blob/master/docs/design/2022-11-25-global-resource-control.md) and [TiFlash Resource Control](https://github.com/pingcap/tiflash/blob/master/docs/design/2023-09-21-tiflash-resource-control.md). +For more information about the resource control mechanism and parameters, see [RFC: Global Resource Control in TiDB](https://github.com/pingcap/tidb/blob/release-8.1/docs/design/2022-11-25-global-resource-control.md) and [TiFlash Resource Control](https://github.com/pingcap/tiflash/blob/release-8.1/docs/design/2023-09-21-tiflash-resource-control.md). ## How to use resource control @@ -159,9 +159,9 @@ You can view the [Resource Manager page](/dashboard/dashboard-resource-manager.m -For TiDB Self-Hosted, you can use the [`CALIBRATE RESOURCE`](https://docs.pingcap.com/zh/tidb/stable/sql-statement-calibrate-resource) statement to estimate the cluster capacity. +For TiDB Self-Managed, you can use the [`CALIBRATE RESOURCE`](https://docs.pingcap.com/tidb/stable/sql-statement-calibrate-resource) statement to estimate the cluster capacity. -For TiDB Cloud, the [`CALIBRATE RESOURCE`](https://docs.pingcap.com/zh/tidb/stable/sql-statement-calibrate-resource) statement is inapplicable. +For TiDB Cloud, the [`CALIBRATE RESOURCE`](https://docs.pingcap.com/tidb/stable/sql-statement-calibrate-resource) statement is inapplicable. @@ -508,9 +508,9 @@ By default, the task types that are marked as background tasks are `""`, and the SET GLOBAL tidb_enable_resource_control = 'OFF'; ``` -2. For TiDB Self-Hosted, you can use the `resource-control.enabled` parameter to control whether to use request scheduling based on resource group quotas. For TiDB Cloud, the value of the `resource-control.enabled` parameter is `true` by default and does not support dynamic modification. If you need to disable it for TiDB Dedicated clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). +2. For TiDB Self-Managed, you can use the `resource-control.enabled` parameter to control whether to use request scheduling based on resource group quotas. For TiDB Cloud, the value of the `resource-control.enabled` parameter is `true` by default and does not support dynamic modification. If you need to disable it for TiDB Cloud Dedicated clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). -3. For TiDB Self-Hosted, you can use the `enable_resource_control` configuration item to control whether to enable TiFlash resource control. For TiDB Cloud, the value of the `enable_resource_control` parameter is `true` by default and does not support dynamic modification. If you need to disable it for TiDB Dedicated clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). +3. For TiDB Self-Managed, you can use the `enable_resource_control` configuration item to control whether to enable TiFlash resource control. For TiDB Cloud, the value of the `enable_resource_control` parameter is `true` by default and does not support dynamic modification. If you need to disable it for TiDB Cloud Dedicated clusters, contact [TiDB Cloud Support](/tidb-cloud/tidb-cloud-support.md). @@ -585,7 +585,7 @@ The system table [`INFORMATION_SCHEMA.statements_summary`](/statement-summary-ta ### View the RU consumption of resource groups -Starting from v7.6.0, TiDB provides the system table [`mysql.request_unit_by_group`](/mysql-schema.md#system-tables-related-to-resource-control) to store the historical records of the RU consumption of each resource group. +Starting from v7.6.0, TiDB provides the system table [`mysql.request_unit_by_group`](/mysql-schema/mysql-schema.md#system-tables-related-to-resource-control) to store the historical records of the RU consumption of each resource group. Example: @@ -626,7 +626,7 @@ You can view the data of resource groups in the current [`RESOURCE_GROUPS`](/inf > **Note:** > -> This section is only applicable to TiDB Self-Hosted. Currently, TiDB Cloud does not provide resource control metrics. +> This section is only applicable to TiDB Self-Managed. Currently, TiDB Cloud does not provide resource control metrics. TiDB regularly collects runtime information about resource control and provides visual charts of the metrics in Grafana's **TiDB** > **Resource Control** dashboard. @@ -657,4 +657,4 @@ The resource control feature does not impact the regular usage of data import, e * [CREATE RESOURCE GROUP](/sql-statements/sql-statement-create-resource-group.md) * [ALTER RESOURCE GROUP](/sql-statements/sql-statement-alter-resource-group.md) * [DROP RESOURCE GROUP](/sql-statements/sql-statement-drop-resource-group.md) -* [RESOURCE GROUP RFC](https://github.com/pingcap/tidb/blob/master/docs/design/2022-11-25-global-resource-control.md) +* [RESOURCE GROUP RFC](https://github.com/pingcap/tidb/blob/release-8.1/docs/design/2022-11-25-global-resource-control.md) diff --git a/tidb-roadmap.md b/tidb-roadmap.md deleted file mode 100644 index 41333efb2b866..0000000000000 --- a/tidb-roadmap.md +++ /dev/null @@ -1,329 +0,0 @@ ---- -title: TiDB Roadmap -summary: Learn about what's coming in the future for TiDB. ---- - -# TiDB Roadmap - -This roadmap provides a look into the proposed future. This will be continually updated as we release long-term stable (LTS) versions. The purpose is to provide visibility into what is coming, so that you can more closely follow the progress, learn about the key milestones on the way, and give feedback as the development work goes on. - -In the course of development, this roadmap is subject to change based on user needs and feedback. **DO NOT** schedule your release plans according to the content of the roadmap. If you have a feature request or want to prioritize a feature, please file an issue on [GitHub](https://github.com/pingcap/tidb/issues). - -## Rolling roadmap highlights - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
CategoryEnd of CY24 LTS releaseMid of CY25 LTS releaseFuture releases
- Scalability and Performance
Enhance horsepower - 		-
    -
  • - TiKV in-memory data cache
    - TiKV maintains recent versions of data in memory to reduce redundant MVCC scans, thus improving performance. -
    • -
    -
  • - Global indexing for partitioned tables
    -
    • -
    -
  • - Adaptive concurrency for statistics collection
    - TiDB automatically adjusts the parallelism and scan concurrency of statistics collection tasks based on the number of deployed nodes and hardware specifications, improving the speed of statistics collection. -
    • -
    -
  • - Rapid database recovery
    - Reduce the time required for full database recovery and point-in-time recovery (PITR). -
    • -
    -
  • - Unlimited-size transactions
    - The volume of data processed by uncommitted transactions is no longer limited by the available memory of TiDB nodes, thus improving the success rate of transactions and batch tasks. -
    • -
    -
  • - Load-based traffic routing by TiProxy
    - TiProxy forwards traffic based on the workload of the target TiDB node, maximizing hardware resource utilization. -
    • -
- 		-
    -
  • - Microservice for PD heartbeat
    - Heartbeat services in PD can be independently deployed and scaled, preventing PD from becoming a bottleneck for the cluster's resources. -
    • -
    -
  • - Less I/O consumption for statistics collection
    - Users can choose to scan only a portion of the data samples on TiKV during statistics collection, reducing time and resource consumption. -
    • -
    -
  • - Remove the limitation for pushing down Limit operator to TiKV
    -
    • -
    -
  • - Cascades optimizer framework
    - Introduce a more mature and powerful optimizer framework to expand the capabilities of the current optimizer. -
    • -
    -
  • - Single DM task reaches 150 MiB/s during full data migration
    -
    • -
    -
  • - Enhanced DDL execution framework
    - Provide a scalable parallel DDL execution framework to improve the performance and stability of DDL operations. -
    • -
- 		-
    -
  • - Table-level load balancing
    - PD determines data scheduling strategies based on the workload of each Region on every table. -
    • -
  • - Improve performance of handling system tables with large data volumes
    - Enhance query performance and reduce query overhead for system tables with large data volumes. -
    • -
-
- Reliability and Availability -
Enhance dependability - 		-
    -
  • - Limit memory consumption for backup tasks
    -
    • -
    -
  • - Limit memory consumption for statistics collection
    -
    • -
    -
  • - Manage massive SQL bindings
    - Improve the user experience of SQL binding, making it convenient for users to create and manage a large number of execution plans to stabilize database performance. -
    • -
    -
  • - Enhance resource group control over complex SQL
    - Regularly assess the Request Unit (RU) consumption of complex SQL before completion of SQL execution to prevent excessively large impacts on the entire system during execution. -
    • -
    -
  • - Automatically switch resource groups for runaway queries
    - When a query is identified as a runaway query, users can choose to switch it to a specific resource group and set an upper limit on resource consumption. -
    • -
- 		-
    -
  • - Limit memory consumption of schema metadata
    - Enhance the stability of large-scale clusters. -
    • -
    -
  • - Distributed statistics collection
    - Statistics collection supports parallel execution across multiple TiDB nodes to improve collection efficiency. -
    • -
    -
  • - Multi-version statistics
    - After statistics are updated, users can view the historical versions and choose to restore them to an earlier version. -
    • -
    -
  • - Reliable data backup
    - Reduce potential issues like insufficient memory during data backup and ensure the availability of backup data. -
    • -
    -
  • - Common operators support spilling to disk
    - Common operators such as HashAgg, Sort, TopN, HashJoin, WindowFunction, IndexJoin, and IndexHashJoin support spilling to disk, reducing the risk of out-of-memory (OOM). -
    • -
- 		-
    -
  • - Adaptive resource group
    - Resource groups automatically adjust their Request Unit (RU) settings based on past execution patterns. -
    • -
    -
  • - Enhanced memory protection
    - TiDB actively monitors the memory usage of all components and prevents memory operations that might impact system stability. -
    • -
    -
  • - Instance-level execution plan cache
    - All sessions within the same TiDB instance can share the execution plan cache, improving memory utilization. -
    • -
-
- Database Operations and Observability -
Enhance DB manageability and its ecosystem - 		-
    -
  • - Reliable query termination
    - Running SQL statements can be immediately terminated, and the corresponding resources are released from TiDB and TiKV. -
    • -
    -
  • - Permission control for switching resource groups -
    Only users with specific permissions can switch their resource groups, thus preventing resource abuse. -
    • -
    -
  • - Mapping tables or SQL with hot Regions -
    • -
    -
  • - Logical data import mode with IMPORT INTO -
    • -
    -
- 		-
    -
  • - Fine-grained customization of statistics collection -
    Users can modify the statistics collection strategy for specific tables, such as healthiness and parallelism. -
    • -
    -
  • - Workload Repository -
    TiDB persists workload information in memory to permanent volume, including cumulative and real-time statistic data, which aids in troubleshooting and analysis. -
    • -
    -
  • - Automatic index advisor -
    TiDB automatically analyzes SQL statements that can be optimized and recommends creating or dropping indexes. -
    • -
    -
  • - Support modifying column types for partitioned tables -
    Users can modify the data type of columns in partitioned tables, regardless of whether a column is a partitioning key. -
    • -
    -
  • - Conflict strategy for IMPORT INTO -
    Users can set the conflict resolution strategy when importing data, such as exiting with an error, ignoring, or replacing in case of conflicts. -
    • -
    -
  • - End-to-End monitoring -
    Track the time consumption of individual SQL statements throughout their entire lifecycle, including consumption on TiDB, TiKV, PD, and TiFlash components. -
    • -
- 		-
    -
  • - Workload analysis -
    Analyze historical workload data from the Workload Repository and provide optimization recommendations, such as SQL tuning and statistics collection. -
    • -
    -
  • - Revisable primary key -
    • -
    -
  • - Export data as SQL statements -
    • -
    -
-
- Security -
Enhance data safety and privacy - 		-
    -
  • - Google Cloud KMS -
    Enhance the key management mechanism for static encryption based on Google Cloud KMS, making it generally available (GA). -
    • -
    -
  • - Improved dynamic privilege -
    Improve the dynamic privilege design and limit the implementation of Super privilege. -
    • -
    -
  • - Marker-based log desensitization -
    Support marking sensitive information in the cluster log. Then, you can determine whether to desensitize it according to the usage scenario. -
    • -
    -
  • - FIPS -
    Encryption scenarios comply with FIPS. -
    • -
- 		-
    -
  • - IAM authentication for AWS -
    TiDB as AWS third-party ARN for AWS IAM access. -
    • -
    -
  • - Kerberos -
    Support Kerberos-based authentication. -
    • -
  • - MFA -
    Support the multi-factor authentication mechanism. -
    • -
- 		-
    -
  • - Label-based access control -
    Support data access control by configuring labels. -
    • -
    -
  • - Enhanced client-side encryption -
    Support client-side encryption of key fields to enhance data security. -
    • -
    -
  • - Dynamic desensitization of business data -
    Support desensitizing data based on different data application scenarios to ensure data security in important fields. -
    • -
-
- -> **Note:** -> -> These are non-exhaustive plans and are subject to change. Features might differ per service subscriptions. \ No newline at end of file diff --git a/tidb-scheduling.md b/tidb-scheduling.md index 2cfe2a43fa0e3..90e03fdf5d778 100644 --- a/tidb-scheduling.md +++ b/tidb-scheduling.md @@ -67,7 +67,7 @@ Scheduling is based on information collection. In short, the PD scheduling compo - State information reported by each TiKV peer: - Each TiKV peer sends heartbeats to PD periodically. PD not only checks whether the store is alive, but also collects [`StoreState`](https://github.com/pingcap/kvproto/blob/master/proto/pdpb.proto#L473) in the heartbeat message. `StoreState` includes: + Each TiKV peer sends heartbeats to PD periodically. PD not only checks whether the store is alive, but also collects [`StoreState`](https://github.com/pingcap/kvproto/blob/release-8.1/proto/pdpb.proto#L473) in the heartbeat message. `StoreState` includes: * Total disk space * Available disk space @@ -83,13 +83,13 @@ Scheduling is based on information collection. In short, the PD scheduling compo + **Disconnect**: Heartbeat messages between the PD and the TiKV store are lost for more than 20 seconds. If the lost period exceeds the time specified by `max-store-down-time`, the status "Disconnect" changes to "Down". + **Down**: Heartbeat messages between the PD and the TiKV store are lost for a time longer than `max-store-down-time` (30 minutes by default). In this status, the TiKV store starts replenishing replicas of each Region on the surviving store. + **Offline**: A TiKV store is manually taken offline through PD Control. This is only an intermediate status for the store to go offline. The store in this status moves all its Regions to other "Up" stores that meet the relocation conditions. When `leader_count` and `region_count` (obtained through PD Control) both show `0`, the store status changes to "Tombstone" from "Offline". In the "Offline" status, **do not** disable the store service or the physical server where the store is located. During the process that the store goes offline, if the cluster does not have target stores to relocate the Regions (for example, inadequate stores to hold replicas in the cluster), the store is always in the "Offline" status. - + **Tombstone**: The TiKV store is completely offline. You can use the `remove-tombstone` interface to safely clean up TiKV in this status. + + **Tombstone**: The TiKV store is completely offline. You can use the `remove-tombstone` interface to safely clean up TiKV in this status. Starting from v6.5.0, if not manually handled, PD will automatically delete the Tombstone records stored internally one month after the node is converted to Tombstone. ![TiKV store status relationship](/media/tikv-store-status-relationship.png) - Information reported by Region leaders: - Each Region leader sends heartbeats to PD periodically to report [`RegionState`](https://github.com/pingcap/kvproto/blob/master/proto/pdpb.proto#L312), including: + Each Region leader sends heartbeats to PD periodically to report [`RegionState`](https://github.com/pingcap/kvproto/blob/release-8.1/proto/pdpb.proto#L312), including: * Position of the leader itself * Positions of other replicas diff --git a/tidb-storage.md b/tidb-storage.md index 8c801eb559988..624e0d941409e 100644 --- a/tidb-storage.md +++ b/tidb-storage.md @@ -106,4 +106,4 @@ Note that for multiple versions of the same Key, versions with larger numbers ar ## Distributed ACID transaction -Transaction of TiKV adopts the model used by Google in BigTable: [Percolator](https://research.google.com/pubs/pub36726.html). TiKV's implementation is inspired by this paper, with a lot of optimizations. See [transaction overview](/transaction-overview.md) for details. +Transaction of TiKV adopts the model used by Google in BigTable: [Percolator](https://research.google/pubs/large-scale-incremental-processing-using-distributed-transactions-and-notifications/). TiKV's implementation is inspired by this paper, with a lot of optimizations. See [transaction overview](/transaction-overview.md) for details. diff --git a/tidb-troubleshooting-map.md b/tidb-troubleshooting-map.md index 8d73f3151c494..885ea49a11295 100644 --- a/tidb-troubleshooting-map.md +++ b/tidb-troubleshooting-map.md @@ -1,7 +1,6 @@ --- title: TiDB Troubleshooting Map summary: Learn how to troubleshoot common errors in TiDB. -aliases: ['/docs/dev/tidb-troubleshooting-map/','/docs/dev/how-to/troubleshoot/diagnose-map/'] --- # TiDB Troubleshooting Map @@ -271,7 +270,7 @@ Check the specific cause for busy by viewing the monitor **Grafana** -> **TiKV** - `append log duration` is high, which causes slow processing of messages. You can refer to [4.5](#45-tikv-write-is-slow) to analyze why `append log duration` is high. - raftstore receives a large batch of messages in an instant (check in the TiKV Raft messages dashboard), and fails to process them. Usually the short-term `channel full` status does not affect the service. -- 4.3.4 TiKV coprocessor is in a queue. The number of piled up tasks exceeds `coprocessor threads * readpool.coprocessor.max-tasks-per-worker-[normal|low|high]`. Too many large queries leads to the tasks piling up in coprocessor. You need to check whether a execution plan change causes a large number of table scan operations. Refer to [3.3](#33-wrong-execution-plan). +- 4.3.4 TiKV coprocessor is in a queue. The number of piled up tasks exceeds `coprocessor threads * readpool.coprocessor.max-tasks-per-worker-[normal|low|high]`. Too many large queries leads to the tasks piling up in coprocessor. You need to check whether an execution plan change causes a large number of table scan operations. Refer to [3.3](#33-wrong-execution-plan). ### 4.4 Some TiKV nodes drop Leader frequently @@ -291,7 +290,7 @@ Check the specific cause for busy by viewing the monitor **Grafana** -> **TiKV** ### 4.5 TiKV write is slow -- 4.5.1 Check whether the TiKV write is low by viewing the `prewrite/commit/raw-put` duration of TiKV gRPC (only for RawKV clusters). Generally, you can locate the slow phase according to the [performance-map](https://github.com/pingcap/tidb-map/blob/master/maps/performance-map.png). Some common situations are listed as follows. +- 4.5.1 Check whether the TiKV write is slow by viewing the `prewrite/commit/raw-put` duration of TiKV gRPC (only for RawKV clusters). Generally, you can locate the slow phase according to the [performance-map](https://github.com/pingcap/tidb-map/blob/master/maps/performance-map.png). Some common situations are listed as follows. - 4.5.2 The scheduler CPU is busy (only for transaction kv). @@ -518,7 +517,7 @@ Check the specific cause for busy by viewing the monitor **Grafana** -> **TiKV** ### 6.3 TiDB Lightning -- 6.3.1 TiDB Lightning is a tool for fast full import of large amounts of data into a TiDB cluster. See [TiDB Lightning on GitHub](https://github.com/pingcap/tidb/tree/master/lightning). +- 6.3.1 TiDB Lightning is a tool for fast full import of large amounts of data into a TiDB cluster. See [TiDB Lightning on GitHub](https://github.com/pingcap/tidb/tree/release-8.1/lightning). - 6.3.2 Import speed is too slow. diff --git a/tidb-upgrade-migration-guide.md b/tidb-upgrade-migration-guide.md new file mode 100644 index 0000000000000..3f63dd6dad644 --- /dev/null +++ b/tidb-upgrade-migration-guide.md @@ -0,0 +1,327 @@ +--- +title: Migrate and Upgrade a TiDB Cluster +summary: Learn how to migrate and upgrade a TiDB cluster using BR for full backup and restore, along with TiCDC for incremental data replication. +--- + +# Migrate and Upgrade a TiDB Cluster + +This document describes how to migrate and upgrade a TiDB cluster (also known as a blue-green upgrade) using [BR](/br/backup-and-restore-overview.md) for full backup and restore, along with [TiCDC](/ticdc/ticdc-overview.md) for incremental data replication. This solution uses dual-cluster redundancy and incremental replication to enable smooth traffic switchover and fast rollback, providing a reliable and low-risk upgrade path for critical systems. It is recommended to regularly upgrade the database version to continuously benefit from performance improvements and new features, helping you maintain a secure and efficient database system. The key advantages of this solution include: + +- **Controllable risk**: supports rollback to the original cluster within minutes, ensuring business continuity. +- **Data integrity**: uses a multi-stage verification mechanism to prevent data loss. +- **Minimal business impact**: requires only a brief maintenance window for the final switchover. + +The core workflow for migration and upgrade is as follows: + +1. **Pre-check risks**: verify cluster status and solution feasibility. +2. **Prepare the new cluster**: create a new cluster from a full backup of the old cluster and upgrade it to the target version. +3. **Replicate incremental data**: establish a forward data replication channel using TiCDC. +4. **Switch and verify**: perform multi-dimensional verification, switch business traffic to the new cluster, and set up a TiCDC reverse replication channel. +5. **Observe status**: maintain the reverse replication channel. After the observation period, clean up the environment. + +**Rollback plan**: if the new cluster encounters issues during the migration and upgrade process, you can switch business traffic back to the original cluster at any time. + +The following sections describe the standardized process and general steps for migrating and upgrading a TiDB cluster. The example commands are based on a TiDB Self-Managed environment. + +## Step 1: Evaluate solution feasibility + +Before migrating and upgrading, evaluate the compatibility of relevant components and check cluster health status. + +- Check the TiDB cluster version: this solution applies to TiDB v6.5.0 or later versions. + +- Verify TiCDC compatibility: + + - **Table schema requirements**: ensure that tables to be replicated contain valid indexes. For more information, see [TiCDC valid index](/ticdc/ticdc-overview.md#best-practices). + - **Feature limitations**: TiCDC does not support Sequence or TiFlash DDL replication. For more information, see [TiCDC unsupported scenarios](/ticdc/ticdc-overview.md#unsupported-scenarios). + - **Best practices**: avoid executing DDL operations on the upstream cluster of TiCDC during switchover. + +- Verify BR compatibility: + + - Review the compatibility matrix of BR full backup. For more information, see [BR version compatibility matrix](/br/backup-and-restore-overview.md#version-compatibility). + - Check the known limitations of BR backup and restore. For more information, see [BR usage restrictions](/br/backup-and-restore-overview.md#restrictions). + +- Check the health status of the cluster, such as [Region](/glossary.md#regionpeerraft-group) health and node resource utilization. + +## Step 2: Prepare the new cluster + +### 1. Adjust the GC lifetime of the old cluster + +To ensure data replication stability, adjust the system variable [`tidb_gc_life_time`](/system-variables.md#tidb_gc_life_time-new-in-v50) to a value that covers the total duration of the following operations and intervals: BR backup, BR restore, cluster upgrade, and TiCDC Changefeed replication setup. Otherwise, the replication task might enter an unrecoverable `failed` state, requiring a restart of the entire migration and upgrade process from a new full backup. + +The following example sets `tidb_gc_life_time` to `60h`: + +```sql +-- Check the current GC lifetime setting. +SHOW VARIABLES LIKE '%tidb_gc_life_time%'; +-- Set GC lifetime. +SET GLOBAL tidb_gc_life_time=60h; +``` + +> **Note:** +> +> Increasing `tidb_gc_life_time` increases storage usage for [MVCC](/glossary.md#multi-version-concurrency-control-mvcc) data and might affect query performance. For more information, see [GC Overview](/garbage-collection-overview.md). Adjust the GC duration based on estimated operation time while considering storage and performance impacts. + +### 2. Migrate full data to the new cluster + +When migrating full data to the new cluster, note the following: + +- **Version compatibility**: the BR version used for backup and restore must match the major version of the old cluster. +- **Performance impact**: BR backup consumes system resources. To minimize business impact, perform backups during off-peak hours. +- **Time estimation**: under optimal hardware conditions (no disk I/O or network bandwidth bottlenecks), estimated times are: + + - Backup speed: backing up 1 TiB of data per TiKV node with 8 threads takes approximately 1 hour. + - Restore speed: restoring 1 TiB of data per TiKV node takes approximately 20 minutes. + +- **Configuration consistency**: ensure that the [`new_collations_enabled_on_first_bootstrap`](/tidb-configuration-file.md#new_collations_enabled_on_first_bootstrap) configuration is identical between the old and new clusters. Otherwise, BR restore will fail. +- **System table restore**: Use the `--with-sys-table` option during BR restore to recover system table data. + +To migrate full data to the new cluster, take the following steps: + +1. Perform a full backup on the old cluster: + + ```shell + tiup br:${cluster_version} backup full --pd ${pd_host}:${pd_port} -s ${backup_location} + ``` + +2. Record the TSO of the old cluster for later TiCDC Changefeed creation: + + ```shell + tiup br:${cluster_version} validate decode --field="end-version" \ + --storage "s3://xxx?access-key=${access-key}&secret-access-key=${secret-access-key}" | tail -n1 + ``` + +3. Deploy the new cluster: + + ```shell + tiup cluster deploy ${new_cluster_name} ${cluster_version} tidb-cluster.yaml + ``` + +4. Restore the full backup to the new cluster: + + ```shell + tiup br:${cluster_version} restore full --pd ${pd_host}:${pd_port} -s ${backup_location} --with-sys-table + ``` + +### 3. Upgrade the new cluster to the target version + +To save time, you can perform an offline upgrade using the following commands. For more upgrade methods, see [Upgrade TiDB Using TiUP](/upgrade-tidb-using-tiup.md). + +```shell +tiup cluster stop # Stop the cluster +tiup cluster upgrade --offline # Perform offline upgrade +tiup cluster start # Start the cluster +``` + +To maintain business continuity, you need to replicate essential configurations from the old cluster to the new cluster, such as configuration items and system variables. + +## Step 3: Replicate incremental data + +### 1. Establish a forward data replication channel + +At this stage, the old cluster remains at its original version, while the new cluster has been upgraded to the target version. In this step, you need to establish a forward data replication channel from the old cluster to the new cluster. + +> **Note:** +> +> The TiCDC component version must match the major version of the old cluster. + +- Create a Changefeed task and set the incremental replication starting point (`${tso}`) to the exact backup TSO recorded in [Step 2](#step-2-prepare-the-new-cluster) to prevent data loss: + + ```shell + tiup ctl:${cluster_version} cdc changefeed create --server http://${cdc_host}:${cdc_port} --sink-uri="mysql://${username}:${password}@${tidb_endpoint}:${port}" --config config.toml --start-ts ${tso} + ``` + +- Check the replication task status and confirm that `tso` or `checkpoint` is continuously advancing: + + ```shell + tiup ctl:${cluster_version} cdc changefeed list --server http://${cdc_host}:${cdc_port} + ``` + + The output is as follows: + + ```shell + [{ + "id": "cdcdb-cdc-task-standby", + "summary": { + "state": "normal", + "tso": 417886179132964865, + "checkpoint": "202x-xx-xx xx:xx:xx.xxx", + "error": null + } + }] + ``` + +During incremental data replication, continuously monitor the replication channel status and adjust settings if needed: + +- Latency metrics: ensure that `Changefeed checkpoint lag` remains within an acceptable range, such as within 5 minutes. +- Throughput health: ensure that `Sink flush rows/s` consistently exceeds the business write rate. +- Errors and alerts: regularly check TiCDC logs and alert information. +- (Optional) Test data replication: update test data and verify that Changefeed correctly replicates it to the new cluster. +- (Optional) Adjust the TiCDC configuration item [`gc-ttl`](/ticdc/ticdc-server-config.md) (defaults to 24 hours). + + If a replication task is unavailable or interrupted and cannot be resolved in time, `gc-ttl` ensures that data needed by TiCDC is retained in TiKV without being cleaned by garbage collection (GC). If this duration is exceeded, the replication task enters a `failed` state and cannot recover. In this case, PD's GC safe point continues advancing, requiring a new backup to restart the process. + + Increasing the value of `gc-ttl` accumulates more MVCC data, similar to increasing `tidb_gc_life_time`. It is recommended to set it to a reasonably long but appropriate value. + +### 2. Verify data consistency + +After data replication is complete, verify data consistency between the old and new clusters using the following methods: + +- Use the [sync-diff-inspector](/sync-diff-inspector/sync-diff-inspector-overview.md) tool: + + ```shell + ./sync_diff_inspector --config=./config.toml + ``` + +- Use the snapshot configuration of [sync-diff-inspector](/sync-diff-inspector/sync-diff-inspector-overview.md) with the [Syncpoint](/ticdc/ticdc-upstream-downstream-check.md) feature of TiCDC to verify data consistency without stopping Changefeed replication. For more information, see [Upstream and Downstream Clusters Data Validation and Snapshot Read](/ticdc/ticdc-upstream-downstream-check.md). + +- Perform manual validation of business data, such as comparing table row counts. + +### 3. Finalize the environment setup + +This migration procedure restores some system table data using the BR `--with-sys-table` option. For tables that are not included in the scope, you need to manually restore. Common items to check and supplement include: + +- User privileges: compare the `mysql.user` table. +- Configuration settings: ensure that configuration items and system variables are consistent. +- Auto-increment columns: clear auto-increment ID caches in the new cluster. +- Statistics: collect statistics manually or enable automatic collection in the new cluster. + +Additionally, you can scale out the new cluster to handle expected workloads and migrate operational tasks, such as alert subscriptions, scheduled statistics collection scripts, and data backup scripts. + +## Step 4: Switch business traffic and rollback + +### 1. Prepare for the switchover + +- Confirm replication status: + + - Monitor the latency of TiCDC Changefeed replication. + - Ensure that the incremental replication throughput is greater than or equal to the peak business write rate. + +- Perform multi-dimensional validation, such as: + + - Ensure that all data validation steps are complete and perform any necessary additional checks. + - Conduct sanity or integration tests on the application in the new cluster. + +### 2. Execute the switchover + +1. Stop application services to prevent the old cluster from handling business traffic. To further restrict access, you can use one of the following methods: + + - Lock user accounts in the old cluster: + + ```sql + ALTER USER ACCOUNT LOCK; + ``` + + - Set the old cluster to read-only mode. It is recommended to restart TiDB nodes in the old cluster to clear active business sessions and prevent connections that have not entered read-only mode: + + ```sql + SET GLOBAL tidb_super_read_only=ON; + ``` + +2. Ensure TiCDC catches up: + + - After setting the old cluster to read-only mode, retrieve the current `up-tso`: + + ```sql + SELECT tidb_current_ts(); + ``` + + - Monitor the Changefeed `checkpointTs` to confirm it has surpassed `up-tso`, indicating that TiCDC has completed data replication. + +3. Verify data consistency between the new and old clusters: + + - After TiCDC catches up, obtain the `down-tso` from the new cluster. + - Use the [sync-diff-inspector](/sync-diff-inspector/sync-diff-inspector-overview.md) tool to compare data consistency between the new and old clusters at `up-tso` and `down-tso`. + +4. Pause the forward Changefeed replication task: + + ```shell + tiup ctl:${cluster_version} cdc changefeed pause --server http://${cdc_host}:${cdc_port} -c + ``` + +5. Restart the TiDB nodes in the new cluster to clear the auto-increment ID cache. + +6. Check the operational status of the new cluster using the following methods: + + - Verify that the TiDB version matches the target version: + + ```shell + tiup cluster display + ``` + + - Log into the database and confirm component versions: + + ```sql + SELECT * FROM INFORMATION_SCHEMA.CLUSTER_INFO; + ``` + + - Use Grafana to monitor service status: navigate to [**Overview > Services Port Status**](/grafana-overview-dashboard.md) and confirm that all services are in the **Up** state. + +7. Set up reverse replication from the new cluster to the old cluster. + + 1. Unlock user accounts in the old cluster and restore read-write mode: + + ```sql + ALTER USER ACCOUNT UNLOCK; + SET GLOBAL tidb_super_read_only=OFF; + ``` + + 2. Record the current TSO of the new cluster: + + ```sql + SELECT tidb_current_ts(); + ``` + + 3. Configure the reverse replication link and ensure the Changefeed task is running properly: + + - Because business operations are stopped at this stage, you can use the current TSO. + - Ensure that `sink-uri` is set to the address of the old cluster to avoid loopback writing risks. + + ```shell + tiup ctl:${cluster_version} cdc changefeed create --server http://${cdc_host}:${cdc_port} --sink-uri="mysql://${username}:${password}@${tidb_endpoint}:${port}" --config config.toml --start-ts ${tso} + + tiup ctl:${cluster_version} cdc changefeed list --server http://${cdc_host}:${cdc_port} + ``` + +8. Redirect business traffic to the new cluster. + +9. Monitor the load and operational status of the new cluster using the following Grafana panels: + + - [**TiDB Dashboard > Query Summary**](/grafana-tidb-dashboard.md#query-summary): check the Duration, QPS, and Failed Query OPM metrics. + - [**TiDB Dashboard > Server**](/grafana-tidb-dashboard.md#server): monitor the **Connection Count** metric to ensure even distribution of connections across nodes. + +At this point, business traffic has successfully switched to the new cluster, and the TiCDC reverse replication channel is established. + +### 3. Execute emergency rollback + +The rollback plan is as follows: + +- Check data consistency between the new and old clusters regularly to ensure the reverse replication link is operating properly. +- Monitor the system for a specified period, such as one week. If issues occur, switch back to the old cluster. +- After the observation period, remove the reverse replication link and delete the old cluster. + +The following introduces the usage scenario and steps for an emergency rollback, which redirects traffic back to the old cluster: + +- Usage scenario: execute the rollback plan if critical issues cannot be resolved. +- Steps: + + 1. Stop business access to the new cluster. + 2. Reauthorize business accounts and restore read-write access to the old cluster. + 3. Check the reverse replication link, confirm TiCDC has caught up, and verify data consistency between the new and old clusters. + 4. Redirect business traffic back to the old cluster. + +## Step 5: Clean up + +After monitoring the new cluster for a period and confirming stable business operations, you can remove the TiCDC reverse replication and delete the old cluster. + +- Remove the TiCDC reverse replication: + + ```shell + tiup ctl:${cluster_version} cdc changefeed remove --server http://${cdc_host}:${cdc_port} -c + ``` + +- Delete the old cluster. If you choose to retain it, restore `tidb_gc_life_time` to its original value: + + ```sql + -- Restore to the original value before modification. + SET GLOBAL tidb_gc_life_time=10m; + ``` diff --git a/tiflash-deployment-topology.md b/tiflash-deployment-topology.md index 92d12f806af48..9a8c91c19d29f 100644 --- a/tiflash-deployment-topology.md +++ b/tiflash-deployment-topology.md @@ -1,7 +1,6 @@ --- title: TiFlash Deployment Topology summary: Learn the deployment topology of TiFlash based on the minimal TiDB topology. -aliases: ['/docs/dev/tiflash-deployment-topology/'] --- # TiFlash Deployment Topology @@ -20,6 +19,10 @@ TiFlash is a columnar storage engine, and gradually becomes the standard cluster | TiFlash | 1 | 32 VCore 64 GB 2TB (nvme ssd) * 1 | 10.0.1.11 | Default port
Global directory configuration | | Monitoring & Grafana | 1 | 4 VCore 8GB * 1 500GB (ssd) | 10.0.1.10 | Default port
Global directory configuration | +> **Note:** +> +> The IP addresses of the instances are given as examples only. In your actual deployment, replace the IP addresses with your actual IP addresses. + ### Topology templates - [The simple template for the TiFlash topology](https://github.com/pingcap/docs/blob/master/config-templates/simple-tiflash.yaml) diff --git a/tiflash-upgrade-guide.md b/tiflash-upgrade-guide.md index 0a4fcbb78c7e0..07fd5bf32bfc4 100644 --- a/tiflash-upgrade-guide.md +++ b/tiflash-upgrade-guide.md @@ -1,7 +1,6 @@ --- title: TiFlash Upgrade Guide summary: Learn the precautions when you upgrade TiFlash. -aliases: ['/tidb/dev/tiflash-620-upgrade-guide'] --- # TiFlash Upgrade Guide @@ -19,7 +18,7 @@ To learn the standard upgrade process, see the following documents: > > - It is not recommended that you upgrade TiDB that includes TiFlash across major versions, for example, from v4.x to v6.x. Instead, you need to upgrade from v4.x to v5.x first, and then to v6.x. > -> - v4.x is near the end of its life cycle. It is recommended that you upgrade to v5.x or later as soon as possible. For more information, see [TiDB Release Support Policy](https://en.pingcap.com/tidb-release-support-policy/). +> - v4.x is near the end of its life cycle. It is recommended that you upgrade to v5.x or later as soon as possible. For more information, see [TiDB Release Support Policy](https://www.pingcap.com/tidb-release-support-policy/). > > - PingCAP does not provide bug fixes for non-LTS versions, such as v6.0. It is recommended that you upgrade to v6.1 and later LTS versions whenever possible. > @@ -65,7 +64,7 @@ When you upgrade TiFlash from v5.x or v6.0 to v6.1, pay attention to the functio TiFlash Proxy is upgraded in v6.1.0 (aligned with TiKV v6.0.0). The new version has upgraded the RocksDB version. After you upgrade TiFlash to v6.1, the data format is converted to the new version automatically. -In regular upgrades, the data conversion does not involve any risks. However, if you need to downgrade TiFlash from v6.1 to any earlier version in special scenarios (for example, testing or verification scenarios), the earlier version might fail to parse the new RocksDB configuration. As as result, TiFlash will fail to restart. It is recommended that you fully test and verify the upgrade process and prepare an emergency plan. +In regular upgrades, the data conversion does not involve any risks. However, if you need to downgrade TiFlash from v6.1 to any earlier version in special scenarios (for example, testing or verification scenarios), the earlier version might fail to parse the new RocksDB configuration. As result, TiFlash will fail to restart. It is recommended that you fully test and verify the upgrade process and prepare an emergency plan. **Workaround for downgrading TiFlash in testing or other special scenarios** diff --git a/tiflash/create-tiflash-replicas.md b/tiflash/create-tiflash-replicas.md index aaff056997f88..fda69863cbaae 100644 --- a/tiflash/create-tiflash-replicas.md +++ b/tiflash/create-tiflash-replicas.md @@ -19,6 +19,10 @@ The parameter of the above command is described as follows: - `count` indicates the number of replicas. When the value is `0`, the replica is deleted. +> **Note:** +> +> For a [TiDB Cloud Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-cloud-serverless) cluster, the `count` of TiFlash replicas can only be `2`. If you set it to `1`, it will be automatically adjusted to `2` for execution. If you set it to a number larger than 2, you will get an error about the replica count. + If you execute multiple DDL statements on the same table, only the last statement is ensured to take effect. In the following example, two DDL statements are executed on the table `tpch50`, but only the second statement (to delete the replica) takes effect. Create two replicas for the table: @@ -49,6 +53,11 @@ ALTER TABLE `tpch50`.`lineitem` SET TIFLASH REPLICA 0; * In v5.1 and later versions, setting the replicas for the system tables is no longer supported. Before upgrading the cluster, you need to clear the replicas of the relevant system tables. Otherwise, you cannot modify the replica settings of the system tables after you upgrade the cluster to a later version. +* Currently, when you use TiCDC to replicate tables to a downstream TiDB cluster, creating TiFlash replicas for the tables is not supported, which means that TiCDC does not support replicating TiFlash-related DDL statements, such as: + + * `ALTER TABLE table_name SET TIFLASH REPLICA count;` + * `ALTER DATABASE db_name SET TIFLASH REPLICA count;` + ### Check replication progress You can check the status of the TiFlash replicas of a specific table using the following statement. The table is specified using the `WHERE` clause. If you remove the `WHERE` clause, you will check the replica status of all tables. @@ -99,6 +108,8 @@ Examples: > > - This statement skips system tables, views, temporary tables, and tables with character sets not supported by TiFlash. +> - You can control the number of tables allowed to remain unavailable during execution by setting the [`tidb_batch_pending_tiflash_count`](/system-variables.md#tidb_batch_pending_tiflash_count-new-in-v60) system variable. Lowering this value helps reduce the pressure on the cluster during replication. Note that this limit is not real-time, so it is still possible for the number of unavailable tables to exceed the limit after the setting is applied. + ### Check replication progress Similar to creating TiFlash replicas for tables, successful execution of the DDL statement does not mean the completion of replication. You can execute the following SQL statement to check the progress of replication on target tables: @@ -130,7 +141,7 @@ Before TiFlash replicas are added, each TiKV instance performs a full table scan ```sql -- The default value for both configurations are 100MiB, i.e. the maximum disk bandwidth used for writing snapshots is no more than 100MiB/s. SET CONFIG tikv `server.snap-io-max-bytes-per-sec` = '300MiB'; - SET CONFIG tiflash `raftstore-proxy.server.snap-max-write-bytes-per-sec` = '300MiB'; + SET CONFIG tiflash `raftstore-proxy.server.snap-io-max-bytes-per-sec` = '300MiB'; ``` After executing these SQL statements, the configuration changes take effect immediately without restarting the cluster. However, since the replication speed is still restricted by the PD limit globally, you cannot observe the acceleration for now. @@ -143,10 +154,10 @@ Before TiFlash replicas are added, each TiKV instance performs a full table scan tiup ctl:v pd -u http://:2379 store limit all engine tiflash 60 add-peer ``` - > In the preceding command, you need to replace `v` with the actual cluster version, such as `v8.0.0` and `:2379` with the address of any PD node. For example: + > In the preceding command, you need to replace `v` with the actual cluster version, such as `v8.1.2` and `:2379` with the address of any PD node. For example: > > ```shell - > tiup ctl:v8.0.0 pd -u http://192.168.1.4:2379 store limit all engine tiflash 60 add-peer + > tiup ctl:v8.1.2 pd -u http://192.168.1.4:2379 store limit all engine tiflash 60 add-peer > ``` Within a few minutes, you will observe a significant increase in CPU and disk IO resource usage of the TiFlash nodes, and TiFlash should create replicas faster. At the same time, the TiKV nodes' CPU and disk IO resource usage increases as well. @@ -169,7 +180,7 @@ Before TiFlash replicas are added, each TiKV instance performs a full table scan ```sql SET CONFIG tikv `server.snap-io-max-bytes-per-sec` = '100MiB'; - SET CONFIG tiflash `raftstore-proxy.server.snap-max-write-bytes-per-sec` = '100MiB'; + SET CONFIG tiflash `raftstore-proxy.server.snap-io-max-bytes-per-sec` = '100MiB'; ``` ## Set available zones diff --git a/tiflash/maintain-tiflash.md b/tiflash/maintain-tiflash.md index 3831ac2bc99c1..ab9d32dd0d612 100644 --- a/tiflash/maintain-tiflash.md +++ b/tiflash/maintain-tiflash.md @@ -1,7 +1,6 @@ --- title: Maintain a TiFlash Cluster summary: Learn common operations when you maintain a TiFlash cluster. -aliases: ['/docs/dev/tiflash/maintain-tiflash/','/docs/dev/reference/tiflash/maintain/'] --- # Maintain a TiFlash Cluster diff --git a/tiflash/monitor-tiflash.md b/tiflash/monitor-tiflash.md index f1cdb07cf1eb1..710fe2b66cb1f 100644 --- a/tiflash/monitor-tiflash.md +++ b/tiflash/monitor-tiflash.md @@ -1,7 +1,6 @@ --- title: Monitor the TiFlash Cluster summary: Learn the monitoring items of TiFlash. -aliases: ['/docs/dev/tiflash/monitor-tiflash/','/docs/dev/reference/tiflash/monitor/'] --- # Monitor the TiFlash Cluster diff --git a/tiflash/tiflash-alert-rules.md b/tiflash/tiflash-alert-rules.md index a28f9ea713ead..993a78c602c01 100644 --- a/tiflash/tiflash-alert-rules.md +++ b/tiflash/tiflash-alert-rules.md @@ -1,7 +1,6 @@ --- title: TiFlash Alert Rules summary: Learn the alert rules of the TiFlash cluster. -aliases: ['/docs/dev/tiflash/tiflash-alert-rules/','/docs/dev/reference/tiflash/alert-rules/'] --- # TiFlash Alert Rules diff --git a/tiflash/tiflash-command-line-flags.md b/tiflash/tiflash-command-line-flags.md index c76e0b4f2735d..8cfbb93e757b0 100644 --- a/tiflash/tiflash-command-line-flags.md +++ b/tiflash/tiflash-command-line-flags.md @@ -1,7 +1,6 @@ --- title: TiFlash Command-line Flags summary: Learn the command-line startup flags of TiFlash. -aliases: ['/docs/dev/tiflash/tiflash-command-line-flags/'] --- # TiFlash Command-Line Flags diff --git a/tiflash/tiflash-configuration.md b/tiflash/tiflash-configuration.md index 128d4afc01a33..1a06fdef1908e 100644 --- a/tiflash/tiflash-configuration.md +++ b/tiflash/tiflash-configuration.md @@ -1,7 +1,6 @@ --- title: Configure TiFlash summary: Learn how to configure TiFlash. -aliases: ['/docs/dev/tiflash/tiflash-configuration/','/docs/dev/reference/tiflash/configuration/'] --- # Configure TiFlash @@ -166,23 +165,26 @@ delta_index_cache_size = 0 ## The external access address of status-addr. If it is left empty, the value of "status-addr" is used by default. ## Should guarantee that other nodes can access through `advertise-status-addr` when you deploy the cluster on multiple nodes. advertise-status-addr = "" + ## The external access address of the TiFlash coprocessor service. + engine-addr = "10.0.1.20:3930" ## The data storage path of proxy. data-dir = "/tidb-data/tiflash-9000/flash" ## The configuration file path of proxy. config = "/tidb-deploy/tiflash-9000/conf/tiflash-learner.toml" ## The log path of proxy. log-file = "/tidb-deploy/tiflash-9000/log/tiflash_tikv.log" - ## The log level of proxy (available options: "trace", "debug", "info", "warn", "error"). The default value is "info" - # log-level = "info" [logger] + ## Note that the following parameters only take effect in tiflash.log and tiflash_error.log. If you need to configure log parameters of TiFlash Proxy, specify them in tiflash-learner.toml. ## log level (available options: "trace", "debug", "info", "warn", "error"). The default value is "info". level = "info" + ## The log of TiFlash. log = "/tidb-deploy/tiflash-9000/log/tiflash.log" + ## The error log of TiFlash. The "warn" and "error" level logs are also output to this log file. errorlog = "/tidb-deploy/tiflash-9000/log/tiflash_error.log" ## Size of a single log file. The default value is "100M". size = "100M" - ## Maximum number of log files to save. The default value is 10. + ## Maximum number of log files to save. The default value is 10. For TiFlash logs and TiFlash error logs, the maximum number of log files to save is `count` respectively. count = 10 [raft] @@ -205,6 +207,13 @@ delta_index_cache_size = 0 ## see known issue [#5576](https://github.com/pingcap/tiflash/issues/5576). # dt_enable_logical_split = false + ## `max_threads` indicates the internal thread concurrency when TiFlash executes an MPP task. + ## The default value is 0. When it is set to 0, + ## TiFlash uses the number of CPU cores as the execution concurrency. + ## This parameter only takes effect + ## when the system variable `tidb_max_tiflash_threads` is set to -1. + max_threads = 0 + ## The memory usage limit for the generated intermediate data in a single query. ## When the value is an integer, the unit is byte. For example, 34359738368 means 32 GiB of memory limit, and 0 means no limit. ## When the value is a floating-point number in the range of [0.0, 1.0), it means the ratio of the allowed memory usage to the total memory of the node. For example, 0.8 means 80% of the total memory, and 0.0 means no limit. @@ -221,6 +230,13 @@ delta_index_cache_size = 0 ## New in v5.0. This item specifies the maximum number of cop requests that TiFlash Coprocessor executes at the same time. If the number of requests exceeds the specified value, the exceeded requests will queue. If the configuration value is set to 0 or not set, the default value is used, which is twice the number of physical cores. cop_pool_size = 0 + + ## New in v5.0. This item specifies the maximum number of cop requests that TiFlash Coprocessor handles at the same time, including the requests being executed and the requests waiting in the queue. If the number of requests exceeds the specified value, the error "TiFlash Server is Busy" is returned. -1 indicates no limit; 0 indicates using the default value, which is 10 * cop_pool_size. + cop_pool_handle_limit = 0 + + ## New in v5.0. This item specifies the maximum time that a cop request can queue in TiFlash. If a cop request waits in the queue for a time longer than the value specified by this configuration, the error "TiFlash Server is Busy" is returned. A value less than or equal to 0 indicates no limit. + cop_pool_max_queued_seconds = 15 + ## New in v5.0. This item specifies the maximum number of batch requests that TiFlash Coprocessor executes at the same time. If the number of requests exceeds the specified value, the exceeded requests will queue. If the configuration value is set to 0 or not set, the default value is used, which is twice the number of physical cores. batch_cop_pool_size = 0 ## New in v6.1.0. This item specifies the number of requests that TiFlash can concurrently process when it receives ALTER TABLE ... COMPACT from TiDB. @@ -251,6 +267,15 @@ delta_index_cache_size = 0 ## New in v7.4.0. This item controls whether to enable the TiFlash resource control feature. When it is set to true, TiFlash uses the pipeline execution model. enable_resource_control = true + ## New in v6.0.0. This item is used for the MinTSO scheduler. It specifies the maximum number of threads that one resource group can use. The default value is 5000. For details about the MinTSO scheduler, see https://docs.pingcap.com/tidb/v8.1/tiflash-mintso-scheduler. + task_scheduler_thread_soft_limit = 5000 + + ## New in v6.0.0. This item is used for the MinTSO scheduler. It specifies the maximum number of threads in the global scope. The default value is 10000. For details about the MinTSO scheduler, see https://docs.pingcap.com/tidb/v8.1/tiflash-mintso-scheduler. + task_scheduler_thread_hard_limit = 10000 + + ## New in v6.4.0. This item is used for the MinTSO scheduler. It specifies the maximum number of queries that can run simultaneously in a TiFlash instance. The default value is 0, which means twice the number of vCPUs. For details about the MinTSO scheduler, see https://docs.pingcap.com/tidb/v8.1/tiflash-mintso-scheduler. + task_scheduler_active_set_soft_limit = 0 + ## Security settings take effect starting from v4.0.5. [security] ## New in v5.0. This configuration item enables or disables log redaction. If the configuration value @@ -269,9 +294,26 @@ delta_index_cache_size = 0 ### Configure the `tiflash-learner.toml` file +The parameters in `tiflash-learner.toml` are basically the same as those in TiKV. You can refer to [TiKV configuration](/tikv-configuration-file.md) for TiFlash Proxy configuration. The following are only commonly used parameters. Note that: + +- Compared with TiKV, TiFlash Proxy has an extra `raftstore.snap-handle-pool-size` parameter. +- The `label` whose key is `engine` is reserved and cannot be configured manually. + ```toml -[server] - engine-addr = The external access address of the TiFlash coprocessor service. +[log] + ## The log level of TiFlash Proxy (available options: "trace", "debug", "info", "warn", "error"). The default value is "info". Introduced in v5.4.0. + level = "info" + +[log.file] + ## The maximum number of log files to save. Introduced in v5.4.0. + ## If this parameter is not set or set to the default value `0`, TiFlash Proxy saves all log files. + ## If this parameter is set to a non-zero value, TiFlash Proxy retains at most the number of old log files specified by `max-backups`. For example, if you set it to `7`, TiFlash Proxy retains at most 7 old log files. + max-backups = 0 + ## The maximum number of days that the log files are retained. Introduced in v5.4.0. + ## If this parameter is not set or set to the default value `0`, TiFlash Proxy retains all log files. + ## If this parameter is set to a non-zero value, TiFlash Proxy cleans up outdated log files after the number of days specified by `max-days`. + max-days = 0 + [raftstore] ## The allowable number of threads in the pool that flushes Raft data to storage. apply-pool-size = 4 @@ -280,15 +322,10 @@ delta_index_cache_size = 0 store-pool-size = 4 ## The number of threads that handle snapshots. - ## The default number is 2. - ## If you set it to 0, the multi-thread optimization is disabled. + ## The default value is 2. If you set it to 0, the multi-thread optimization is disabled. + ## A specific parameter of TiFlash Proxy, introduced in v4.0.0. snap-handle-pool-size = 2 - ## The shortest interval at which Raft store persists WAL. - ## You can properly increase the latency to reduce IOPS usage. - ## The default value is "4ms". - ## If you set it to 0ms, the optimization is disabled. - store-batch-retry-recv-timeout = "4ms" [security] ## New in v5.0. This configuration item enables or disables log redaction. ## If the configuration value is set to true, @@ -310,8 +347,6 @@ delta_index_cache_size = 0 ## Specifies the old master key when rotating the new master key. The configuration format is the same as that of `master-key`. To learn how to configure a master key, see Configure encryption: https://docs.pingcap.com/tidb/dev/encryption-at-rest#configure-encryption . ``` -In addition to the items above, other parameters are the same as those of TiKV. Note that the `label` whose key is `engine` is reserved and cannot be configured manually. - ### Schedule replicas by topology labels See [Set available zones](/tiflash/create-tiflash-replicas.md#set-available-zones). diff --git a/tiflash/tiflash-late-materialization.md b/tiflash/tiflash-late-materialization.md index 6493753df0a22..40c4e41a0de38 100644 --- a/tiflash/tiflash-late-materialization.md +++ b/tiflash/tiflash-late-materialization.md @@ -14,7 +14,7 @@ TiFlash late materialization is an optimization method to accelerate queries in - When it is disabled, to process a `SELECT` statement with filter conditions (`WHERE` clause), TiFlash reads all the data from the columns required by the query, and then filters and aggregates the data based on the query conditions. - When it is enabled, TiFlash supports pushing down part of the filter conditions to the TableScan operator. That is, TiFlash first scans the column data related to the filter conditions that are pushed down to the TableScan operator, filters the rows that meet the condition, and then scans the other column data of these rows for further calculation, thereby reducing IO scans and computations of data processing. -To improve the performance of certain queries in OLAP scenarios, starting from v7.1.0, the TiFlash late materialization feature is enabled by default. The TiDB optimizer can determine which filter conditions to be pushed down based on statistics and filter conditions, and prioritize pushing down the filter conditions with high filtration rates. For detailed algorithms, see the [RFC document](https://github.com/pingcap/tidb/tree/master/docs/design/2022-12-06-support-late-materialization.md). +To improve the performance of certain queries in OLAP scenarios, starting from v7.1.0, the TiFlash late materialization feature is enabled by default. The TiDB optimizer can determine which filter conditions to be pushed down based on statistics and filter conditions, and prioritize pushing down the filter conditions with high filtration rates. For detailed algorithms, see the [RFC document](https://github.com/pingcap/tidb/tree/release-8.1/docs/design/2022-12-06-support-late-materialization.md). For example: diff --git a/tiflash/tiflash-mintso-scheduler.md b/tiflash/tiflash-mintso-scheduler.md new file mode 100644 index 0000000000000..0fc21995580cd --- /dev/null +++ b/tiflash/tiflash-mintso-scheduler.md @@ -0,0 +1,68 @@ +--- +title: TiFlash MinTSO Scheduler +summary: Learn the implementation principles of the TiFlash MinTSO Scheduler. +--- + +# TiFlash MinTSO Scheduler + +The TiFlash MinTSO scheduler is a distributed scheduler for [MPP](/glossary.md#mpp) tasks in TiFlash. This document describes the implementation principles of the TiFlash MinTSO scheduler. + +## Background + +When processing an MPP query, TiDB splits the query into one or more MPP tasks and sends these MPP tasks to the corresponding TiFlash nodes for compilation and execution. Before TiFlash uses the [pipeline execution model](/tiflash/tiflash-pipeline-model.md), TiFlash needs to use several threads to execute each MPP task, with the specific number of threads depending on the complexity of the MPP task and the concurrency parameters set in TiFlash. + +In high concurrency scenarios, TiFlash nodes receive multiple MPP tasks simultaneously. If the execution of MPP tasks is not controlled, the number of threads that TiFlash needs to request from the system will increase linearly along with the increasing number of MPP tasks. Too many threads can affect the execution efficiency of TiFlash, and because the operating system itself supports a limited number of threads, TiFlash will encounter errors when it requests more threads than the operating system can provide. + +To improve TiFlash's processing capability in high concurrency scenarios, an MPP task scheduler needs to be introduced into TiFlash. + +## Implementation principles + +As mentioned in the [background](#background), the initial purpose of introducing the TiFlash task scheduler is to control the number of threads used during MPP query execution. A simple scheduling strategy is to specify the maximum number of threads TiFlash can request. For each MPP task, the scheduler decides whether the MPP task can be scheduled based on the current number of threads used by the system and the expected number of threads the MPP task will use: + +![TiFlash MinTSO Scheduler v1](/media/tiflash/tiflash_mintso_v1.png) + +Although the preceding scheduling strategy can effectively control the number of system threads, an MPP task is not the smallest independent execution unit, and dependencies exist between different MPP tasks: + +```sql +EXPLAIN SELECT count(*) FROM t0 a JOIN t0 b ON a.id = b.id; +``` + +``` ++--------------------------------------------+----------+--------------+---------------+----------------------------------------------------------+ +| id | estRows | task | access object | operator info | ++--------------------------------------------+----------+--------------+---------------+----------------------------------------------------------+ +| HashAgg_44 | 1.00 | root | | funcs:count(Column#8)->Column#7 | +| └─TableReader_46 | 1.00 | root | | MppVersion: 2, data:ExchangeSender_45 | +| └─ExchangeSender_45 | 1.00 | mpp[tiflash] | | ExchangeType: PassThrough | +| └─HashAgg_13 | 1.00 | mpp[tiflash] | | funcs:count(1)->Column#8 | +| └─Projection_43 | 12487.50 | mpp[tiflash] | | test.t0.id | +| └─HashJoin_42 | 12487.50 | mpp[tiflash] | | inner join, equal:[eq(test.t0.id, test.t0.id)] | +| ├─ExchangeReceiver_22(Build) | 9990.00 | mpp[tiflash] | | | +| │ └─ExchangeSender_21 | 9990.00 | mpp[tiflash] | | ExchangeType: Broadcast, Compression: FAST | +| │ └─Selection_20 | 9990.00 | mpp[tiflash] | | not(isnull(test.t0.id)) | +| │ └─TableFullScan_19 | 10000.00 | mpp[tiflash] | table:a | pushed down filter:empty, keep order:false, stats:pseudo | +| └─Selection_24(Probe) | 9990.00 | mpp[tiflash] | | not(isnull(test.t0.id)) | +| └─TableFullScan_23 | 10000.00 | mpp[tiflash] | table:b | pushed down filter:empty, keep order:false, stats:pseudo | ++--------------------------------------------+----------+--------------+---------------+----------------------------------------------------------+ +``` + +For example, the preceding query generates two MPP tasks on each TiFlash node, where the MPP task containing the `ExchangeSender_45` executor depends on the MPP task containing the `ExchangeSender_21` executor. In high concurrency scenarios, if the scheduler schedules the MPP task containing `ExchangeSender_45` for each query, the system will enter a deadlock state. + +To avoid deadlock, TiFlash introduces the following two levels of thread limits: + +* thread_soft_limit: used to limit the number of threads used by the system. For specific MPP tasks, this limit can be broken to avoid deadlock. +* thread_hard_limit: used to protect the system. Once the number of threads used by the system exceeds the hard limit, TiFlash will report an error to avoid deadlock. + +The soft limit and hard limit work together to avoid deadlock as follows: the soft limit restricts the total number of threads used by all queries, enabling full use of resources while avoiding thread resource exhaustion; the hard limit ensures that in any situation, at least one query in the system can break the soft limit and continue to acquire thread resources and run, thus avoiding deadlock. As long as the number of threads does not exceed the hard limit, there will always be one query in the system where all its MPP tasks can be executed normally, thus preventing deadlock. + +The goal of the MinTSO scheduler is to control the number of system threads while ensuring that there is always one and only one special query in the system, where all its MPP tasks can be scheduled. The MinTSO scheduler is a fully distributed scheduler, with each TiFlash node scheduling MPP tasks based only on its own information. Therefore, all MinTSO schedulers on TiFlash nodes need to identify the same "special" query. In TiDB, each query carries a read timestamp (`start_ts`), and the MinTSO scheduler defines the "special" query as the query with the smallest `start_ts` on the current TiFlash node. Based on the principle that the global minimum is also the local minimum, the "special" query selected by all TiFlash nodes must be the same, called the MinTSO query. + +The scheduling process of the MinTSO Scheduler is as follows: + +![TiFlash MinTSO Scheduler v2](/media/tiflash/tiflash_mintso_v2.png) + +By introducing soft limit and hard limit, the MinTSO scheduler effectively avoids system deadlock while controlling the number of system threads. In high concurrency scenarios, however, most queries might only have part of their MPP tasks scheduled. Queries with only part of MPP tasks scheduled cannot execute normally, leading to low system execution efficiency. To avoid this situation, TiFlash introduces a query-level limit for the MinTSO scheduler, called active_set_soft_limit. This limit allows only MPP tasks of up to active_set_soft_limit queries to participate in scheduling; MPP tasks of other queries do not participate in scheduling, and only after the current queries finish can new queries participate in scheduling. This limit is only a soft limit because for the MinTSO query, all its MPP tasks can be scheduled directly as long as the number of system threads does not exceed the hard limit. + +## See also + +- [Configure TiFlash](/tiflash/tiflash-configuration.md): learn how to configure the MinTSO scheduler. diff --git a/tiflash/tiflash-overview.md b/tiflash/tiflash-overview.md index 2999f51cae4f4..93e6a18d08f78 100644 --- a/tiflash/tiflash-overview.md +++ b/tiflash/tiflash-overview.md @@ -1,12 +1,11 @@ --- title: TiFlash Overview summary: Learn the architecture and key features of TiFlash. -aliases: ['/docs/dev/tiflash/tiflash-overview/','/docs/dev/reference/tiflash/overview/','/docs/dev/tiflash/use-tiflash/','/docs/dev/reference/tiflash/use-tiflash/','/tidb/dev/use-tiflash'] --- # TiFlash Overview -[TiFlash](https://github.com/pingcap/tiflash) is the key component that makes TiDB essentially an Hybrid Transactional/Analytical Processing (HTAP) database. As a columnar storage extension of TiKV, TiFlash provides both good isolation level and strong consistency guarantee. +[TiFlash](https://github.com/pingcap/tiflash) is the key component that makes TiDB essentially a Hybrid Transactional/Analytical Processing (HTAP) database. As a columnar storage extension of TiKV, TiFlash provides both good isolation level and strong consistency guarantee. In TiFlash, the columnar replicas are asynchronously replicated according to the Raft Learner consensus algorithm. When these replicas are read, the Snapshot Isolation level of consistency is achieved by validating Raft index and multi-version concurrency control (MVCC). @@ -22,7 +21,7 @@ With TiDB Cloud, you can create an HTAP cluster easily by specifying one or more The above figure is the architecture of TiDB in its HTAP form, including TiFlash nodes. -TiFlash provides the columnar storage, with a layer of coprocessors efficiently implemented by ClickHouse. Similar to TiKV, TiFlash also has a Multi-Raft system, which supports replicating and distributing data in the unit of Region (see [Data Storage](https://en.pingcap.com/blog/tidb-internal-data-storage/) for details). +TiFlash provides the columnar storage, with a layer of coprocessors efficiently implemented by ClickHouse. Similar to TiKV, TiFlash also has a Multi-Raft system, which supports replicating and distributing data in the unit of Region (see [Data Storage](https://www.pingcap.com/blog/tidb-internal-data-storage/) for details). TiFlash conducts real-time replication of data in the TiKV nodes at a low cost that does not block writes in TiKV. Meanwhile, it provides the same read consistency as in TiKV and ensures that the latest data is read. The Region replica in TiFlash is logically identical to those in TiKV, and is split and merged along with the Leader replica in TiKV at the same time. @@ -38,9 +37,9 @@ It is recommended that you deploy TiFlash in different nodes from TiKV to ensure Currently, data cannot be written directly into TiFlash. You need to write data in TiKV and then replicate it to TiFlash, because it connects to the TiDB cluster as a Learner role. TiFlash supports data replication in the unit of table, but no data is replicated by default after deployment. To replicate data of a specified table, see [Create TiFlash replicas for tables](/tiflash/create-tiflash-replicas.md#create-tiflash-replicas-for-tables). -TiFlash has three components: the columnar storage module, `tiflash proxy`, and `pd buddy`. `tiflash proxy` is responsible for the communication using the Multi-Raft consensus algorithm. `pd buddy` works with PD to replicate data from TiKV to TiFlash in the unit of table. +TiFlash consists of two main components: the columnar storage component, and the TiFlash proxy component. The TiFlash proxy component is responsible for the communication using the Multi-Raft consensus algorithm. -When TiDB receives the DDL command to create replicas in TiFlash, the `pd buddy` component acquires the information of the table to be replicated via the status port of TiDB, and sends the information to PD. Then PD performs the corresponding data scheduling according to the information provided by `pd buddy`. +After receiving a DDL command to create replicas for a table in TiFlash, TiDB automatically creates the corresponding [placement rules](https://docs.pingcap.com/tidb/stable/configure-placement-rules) in PD, and then PD performs the corresponding data scheduling based on these rules. ## Key features diff --git a/tiflash/tiflash-results-materialization.md b/tiflash/tiflash-results-materialization.md index ed0eccdc1579b..afec777dde1b9 100644 --- a/tiflash/tiflash-results-materialization.md +++ b/tiflash/tiflash-results-materialization.md @@ -93,8 +93,8 @@ CREATE TABLE daily_data ( customer_id VARCHAR(20), -- Customer ID daily_fee DECIMAL(20,2)); -- Amount of fee for per day -ALTER TABLE detail_data SET TIFLASH REPLICA 1; -ALTER TABLE daily_data SET TIFLASH REPLICA 1; +ALTER TABLE detail_data SET TIFLASH REPLICA 2; +ALTER TABLE daily_data SET TIFLASH REPLICA 2; -- ... (detail_data table continues updating) INSERT INTO detail_data(ts,customer_id,detail_fee) VALUES diff --git a/tiflash/troubleshoot-tiflash.md b/tiflash/troubleshoot-tiflash.md index e5737061b1600..39f9ea11e398b 100644 --- a/tiflash/troubleshoot-tiflash.md +++ b/tiflash/troubleshoot-tiflash.md @@ -1,7 +1,6 @@ --- title: Troubleshoot a TiFlash Cluster summary: Learn common operations when you troubleshoot a TiFlash cluster. -aliases: ['/docs/dev/tiflash/troubleshoot-tiflash/'] --- # Troubleshoot a TiFlash Cluster @@ -53,22 +52,16 @@ This is because TiFlash is in an abnormal state caused by configuration errors o 3. Check whether the TiFlash proxy status is normal through `pd-ctl`. - {{< copyable "shell-regular" >}} - ```shell - echo "store" | /path/to/pd-ctl -u http://${pd-ip}:${pd-port} + tiup ctl:nightly pd -u http://${pd-ip}:${pd-port} store ``` The TiFlash proxy's `store.labels` includes information such as `{"key": "engine", "value": "tiflash"}`. You can check this information to confirm a TiFlash proxy. -4. Check whether `pd buddy` can correctly print the logs (the log path is the value of `log` in the [flash.flash_cluster] configuration item; the default log path is under the `tmp` directory configured in the TiFlash configuration file). - -5. Check whether the number of configured replicas is less than or equal to the number of TiKV nodes in the cluster. If not, PD cannot replicate data to TiFlash: - - {{< copyable "shell-regular" >}} +4. Check whether the number of configured replicas is less than or equal to the number of TiKV nodes in the cluster. If not, PD cannot replicate data to TiFlash. ```shell - echo 'config placement-rules show' | /path/to/pd-ctl -u http://${pd-ip}:${pd-port} + tiup ctl:nightly pd -u http://${pd-ip}:${pd-port} config placement-rules show | grep -C 10 default ``` Reconfirm the value of `default: count`. @@ -78,7 +71,7 @@ This is because TiFlash is in an abnormal state caused by configuration errors o > - When [Placement Rules](/configure-placement-rules.md) are enabled and multiple rules exist, the previously configured [`max-replicas`](/pd-configuration-file.md#max-replicas), [`location-labels`](/pd-configuration-file.md#location-labels), and [`isolation-level`](/pd-configuration-file.md#isolation-level) no longer take effect. To adjust the replica policy, use the interface related to Placement Rules. > - When [Placement Rules](/configure-placement-rules.md) are enabled and only one default rule exists, TiDB will automatically update this default rule when `max-replicas`, `location-labels`, or `isolation-level` configurations are changed. -6. Check whether the remaining disk space of the machine (where `store` of the TiFlash node is) is sufficient. By default, when the remaining disk space is less than 20% of the `store` capacity (which is controlled by the `low-space-ratio` parameter), PD cannot schedule data to this TiFlash node. +5. Check whether the remaining disk space of the machine (where `store` of the TiFlash node is) is sufficient. By default, when the remaining disk space is less than 20% of the `store` capacity (which is controlled by the [`low-space-ratio`](/pd-configuration-file.md#low-space-ratio) parameter), PD cannot schedule data to this TiFlash node. ## Some queries return the `Region Unavailable` error @@ -94,6 +87,65 @@ Take the following steps to handle the data file corruption: 2. Delete the related data of the TiFlash node. 3. Redeploy the TiFlash node in the cluster. +## Removing TiFlash nodes is slow + +Take the following steps to handle this issue: + +1. Check whether any table has more TiFlash replicas than the number of TiFlash nodes available after the cluster scale-in: + + ```sql + SELECT * FROM information_schema.tiflash_replica WHERE REPLICA_COUNT > 'tobe_left_nodes'; + ``` + + `tobe_left_nodes` is the number of TiFlash nodes after the scale-in. + + If the query result is not empty, you need to modify the number of TiFlash replicas for the corresponding tables. This is because, when the number of TiFlash replicas exceeds the number of TiFlash nodes after the scale-in, PD will not move Region peers away from the TiFlash nodes to be removed, causing the removal of these TiFlash nodes to fail. + +2. In scenarios where all TiFlash nodes need to be removed from a cluster, if the `INFORMATION_SCHEMA.TIFLASH_REPLICA` table shows that there are no TiFlash replicas in the cluster but removing TiFlash nodes still fails, check whether you have recently executed `DROP TABLE .` or `DROP DATABASE ` operations. + + For tables or databases with TiFlash replicas, after executing `DROP TABLE .` or `DROP DATABASE `, TiDB does not immediately delete the TiFlash replication rules for the corresponding tables in PD. Instead, it waits until the corresponding tables meet the garbage collection (GC) conditions before deleting these replication rules. After GC is complete, the corresponding TiFlash nodes can be successfully removed. + + To remove data replication rules of TiFlash manually before the GC conditions are met, you can do the following: + + > **Note:** + > + > After manually removing TiFlash replication rules for a table, if you perform `RECOVER TABLE`, `FLASHBACK TABLE`, or `FLASHBACK DATABASE` operations on this table, the TiFlash replicas of this table will not be restored. + + 1. View all data replication rules related to TiFlash in the current PD instance: + + ```shell + curl http://:/pd/api/v1/config/rules/group/tiflash + ``` + + ``` + [ + { + "group_id": "tiflash", + "id": "table-45-r", + "override": true, + "start_key": "7480000000000000FF2D5F720000000000FA", + "end_key": "7480000000000000FF2E00000000000000F8", + "role": "learner", + "count": 1, + "label_constraints": [ + { + "key": "engine", + "op": "in", + "values": [ + "tiflash" + ] + } + ] + } + ] + ``` + + 2. Remove all data replication rules related to TiFlash. Take the rule whose `id` is `table-45-r` as an example. Delete it by the following command: + + ```shell + curl -v -X DELETE http://:/pd/api/v1/config/rule/tiflash/table-45-r + ``` + ## TiFlash analysis is slow If a statement contains operators or functions not supported in the MPP mode, TiDB does not select the MPP mode. Therefore, the analysis of the statement is slow. In this case, you can execute the `EXPLAIN` statement to check for operators or functions not supported in the MPP mode. diff --git a/tiflash/tune-tiflash-performance.md b/tiflash/tune-tiflash-performance.md index c2ef222a5327f..30e62c924a755 100644 --- a/tiflash/tune-tiflash-performance.md +++ b/tiflash/tune-tiflash-performance.md @@ -1,7 +1,6 @@ --- title: Tune TiFlash Performance summary: Learn how to tune the performance of TiFlash by planning machine resources and tuning TiDB parameters. -aliases: ['/docs/dev/tiflash/tune-tiflash-performance/','/docs/dev/reference/tiflash/tune-performance/'] --- # Tune TiFlash Performance diff --git a/tiflash/use-fastscan.md b/tiflash/use-fastscan.md index 7683dc8026d1b..4108d81468bfc 100644 --- a/tiflash/use-fastscan.md +++ b/tiflash/use-fastscan.md @@ -1,7 +1,6 @@ --- title: Use FastScan summary: Introduces a way to speed up querying in OLAP scenarios by using FastScan. -aliases: ['/tidb/dev/sql-statement-set-tiflash-mode/','/tidb/dev/dev-guide-use-fastscan/'] --- # Use FastScan diff --git a/tikv-configuration-file.md b/tikv-configuration-file.md index bdb825bc70662..0a9d38c9c5c11 100644 --- a/tikv-configuration-file.md +++ b/tikv-configuration-file.md @@ -1,14 +1,13 @@ --- title: TiKV Configuration File summary: Learn the TiKV configuration file. -aliases: ['/docs/dev/tikv-configuration-file/','/docs/dev/reference/configuration/tikv-server/configuration-file/'] --- # TiKV Configuration File -The TiKV configuration file supports more options than command-line parameters. You can find the default configuration file in [etc/config-template.toml](https://github.com/tikv/tikv/blob/master/etc/config-template.toml) and rename it to `config.toml`. +The TiKV configuration file supports more options than command-line parameters. You can find the default configuration file in [etc/config-template.toml](https://github.com/tikv/tikv/blob/release-8.1/etc/config-template.toml) and rename it to `config.toml`. This document only describes the parameters that are not included in command-line parameters. For more details, see [command-line parameter](/command-line-flags-for-tikv-configuration.md). @@ -163,7 +162,7 @@ This document only describes the parameters that are not included in command-lin ### `grpc-raft-conn-num` -+ The maximum number of links among TiKV nodes for Raft communication ++ The maximum number of connections between TiKV nodes for Raft communication + Default value: `1` + Minimum value: `1` @@ -224,6 +223,17 @@ This document only describes the parameters that are not included in command-lin + Unit: KiB|MiB|GiB + Minimum value: `"1KiB"` +### `snap-min-ingest-size` New in v8.1.2 + ++ Specifies the minimum threshold for whether TiKV adopts the ingest method when processing snapshots. + + + When the snapshot size exceeds this threshold, TiKV adopts the ingest method, which imports SST files from the snapshot into RocksDB. This method is faster for large files. + + When the snapshot size does not exceed this threshold, TiKV adopts the direct write method, which writes each piece of data into RocksDB individually. This method is more efficient for small files. + ++ Default value: `"2MiB"` ++ Unit: KiB|MiB|GiB ++ Minimum value: `0` + ### `enable-request-batch` + Determines whether to process requests in batches @@ -422,7 +432,7 @@ Configuration items related to storage. > > This feature is experimental. It is not recommended that you use it in the production environment. This feature might be changed or removed without prior notice. If you find a bug, you can report an [issue](https://github.com/pingcap/tidb/issues) on GitHub. -+ Specifies the engine type. This configuration can only be specified when creating a new cluster and cannot be modifies once being specified. ++ Specifies the engine type. This configuration can only be specified when creating a new cluster and cannot be modified once being specified. + Default value: `"raft-kv"` + Value options: @@ -465,9 +475,9 @@ Configuration items related to storage. > **Warning:** > > - Set `enable-ttl` to `true` or `false` **ONLY WHEN** deploying a new TiKV cluster. **DO NOT** modify the value of this configuration item in an existing TiKV cluster. TiKV clusters with different `enable-ttl` values use different data formats. Therefore, if you modify the value of this item in an existing TiKV cluster, the cluster will store data in different formats, which causes the "can't enable TTL on a non-ttl" error when you restart the TiKV cluster. -> - Use `enable-ttl` **ONLY IN** a TiKV cluster. **DO NOT** use this configuration item in a cluster that has TiDB nodes (which means setting `enable-ttl` to `true` in such clusters). Otherwise, critical issues such as data corruption and the upgrade failure of TiDB clusters will occur. +> - Use `enable-ttl` **ONLY IN** a TiKV cluster. **DO NOT** use this configuration item in a cluster that has TiDB nodes (which means setting `enable-ttl` to `true` in such clusters) unless `storage.api-version = 2` is configured. Otherwise, critical issues such as data corruption and the upgrade failure of TiDB clusters will occur. -+ TTL is short for "Time to live". If this item is enabled, TiKV automatically deletes data that reaches its TTL. To set the value of TTL, you need to specify it in the requests when writing data via the client. If the TTL is not specified, it means that TiKV does not automatically delete the corresponding data. ++ [TTL](/time-to-live.md) is short for "Time to live". If this item is enabled, TiKV automatically deletes data that reaches its TTL. To set the value of TTL, you need to specify it in the requests when writing data via the client. If the TTL is not specified, it means that TiKV does not automatically delete the corresponding data. + Default value: `false` ### `ttl-check-poll-interval` @@ -489,9 +499,9 @@ Configuration items related to storage. + Value options: + `1`: Uses API V1, does not encode the data passed from the client, and stores data as it is. In versions earlier than v6.1.0, TiKV uses API V1 by default. + `2`: Uses API V2: - + The data is stored in the Multi-Version Concurrency Control (MVCC) format, where the timestamp is obtained from PD (which is TSO) by tikv-server. + + The data is stored in the [Multi-Version Concurrency Control (MVCC)](/glossary.md#multi-version-concurrency-control-mvcc) format, where the timestamp is obtained from PD (which is TSO) by tikv-server. + Data is scoped according to different usage and API V2 supports co-existence of TiDB, Transactional KV, and RawKV applications in a single cluster. - + When API V2 is used, you are expected to set `storage.enable-ttl = true` at the same time. Because API V2 supports the TTL feature, you must turn on `enable-ttl` explicitly. Otherwise, it will be in conflict because `storage.enable-ttl` defaults to `false`. + + When API V2 is used, you are expected to set `storage.enable-ttl = true` at the same time. Because API V2 supports the TTL feature, you must turn on [`enable-ttl`](#enable-ttl) explicitly. Otherwise, it will be in conflict because `storage.enable-ttl` defaults to `false`. + When API V2 is enabled, you need to deploy at least one tidb-server instance to reclaim obsolete data. This tidb-server instance can provide read and write services at the same time. To ensure high availability, you can deploy multiple tidb-server instances. + Client support is required for API V2. For details, see the corresponding instruction of the client for the API V2. + Since v6.2.0, Change Data Capture (CDC) for RawKV is supported. Refer to [RawKV CDC](https://tikv.org/docs/latest/concepts/explore-tikv-features/cdc/cdc). @@ -502,6 +512,11 @@ Configuration items related to storage. > - API V1 and API V2 are different from each other in the storage format. You can enable or disable API V2 directly **only** when TiKV contains only TiDB data. In other scenarios, you need to deploy a new cluster, and migrate data using [RawKV Backup & Restore](https://tikv.org/docs/latest/concepts/explore-tikv-features/backup-restore/). > - After API V2 is enabled, you **cannot** downgrade the TiKV cluster to a version earlier than v6.1.0. Otherwise, data corruption might occur. +## `txn-status-cache-capacity` New in v7.6.0 + ++ Sets the capacity of the transaction status cache in TiKV. Do not modify this parameter. ++ Default value: `5120000` + ## storage.block-cache Configuration items related to the sharing of block cache among multiple RocksDB Column Families (CF). @@ -516,6 +531,11 @@ Configuration items related to the sharing of block cache among multiple RocksDB + Unit: KiB|MiB|GiB +### `low-pri-pool-ratio` New in v8.0.0 + ++ Controls the proportion of the entire block cache that the Titan component can use. ++ Default value: `0.2` + ## storage.flow-control Configuration items related to the flow control mechanism in TiKV. This mechanism replaces the write stall mechanism in RocksDB and controls flow at the scheduler layer, which avoids secondary disasters caused by the stuck Raftstore or Apply threads. @@ -663,10 +683,6 @@ Configuration items related to Raftstore. ### `raft-max-size-per-msg` -> **Note:** -> -> This configuration item cannot be queried via SQL statements but can be configured in the configuration file. - + The soft limit on the size of a single message packet + Default value: `"1MiB"` + Minimum value: greater than `0` @@ -675,10 +691,6 @@ Configuration items related to Raftstore. ### `raft-max-inflight-msgs` -> **Note:** -> -> This configuration item cannot be queried via SQL statements but can be configured in the configuration file. - + The number of Raft logs to be confirmed. If this number is exceeded, the Raft state machine slows down log sending. + Default value: `256` + Minimum value: greater than `0` @@ -738,6 +750,20 @@ Configuration items related to Raftstore. + Default value: `"30s"` + Minimum value: `0` +### `max-apply-unpersisted-log-limit` New in v8.1.0 + +> **Warning:** +> +> The feature controlled by this variable is not fully functional in the current TiDB version. Do not change the default value. + ++ The maximum number of committed but not persisted Raft logs that can be applied. + + + Setting this configuration item to a value greater than `0` enables the TiKV node to apply committed but not persisted Raft logs in advance, effectively reducing long-tail latency caused by IO jitter on that node. However, it might also increase the memory usage of TiKV and the disk space occupied by Raft logs. + + Setting this configuration item to `0` disables this feature, meaning that TiKV must wait until Raft logs are both committed and persisted before applying them. This default behavior is consistent with the behavior before v8.1.0. + ++ Default value: `0` ++ Minimum value: `0` + ### `hibernate-regions` + Enables or disables Hibernate Region. When this option is enabled, a Region idle for a long time is automatically set as hibernated. This reduces the extra overhead caused by heartbeat messages between the Raft leader and the followers for idle Regions. You can use `peer-stale-state-check-interval` to modify the heartbeat interval between the leader and the followers of hibernated Regions. @@ -817,6 +843,17 @@ Configuration items related to Raftstore. + Default value: `"10s"` + Minimum value: `0` +### `pd-report-min-resolved-ts-interval` New in v7.6.0 + +> **Note:** +> +> This configuration item is renamed from [`report-min-resolved-ts-interval`](https://docs.pingcap.com/tidb/v7.5/tikv-configuration-file/#report-min-resolved-ts-interval-new-in-v600). Starting from v7.6.0, `report-min-resolved-ts-interval` is no longer effective. + ++ Specifies the minimum interval for TiKV to report Resolved TS to the PD leader. Setting it to `0` disables the reporting. ++ Default value: `"1s"`, which is the minimum positive value. Before v6.3.0, the default value is `"0s"`. ++ Minimum value: `0` ++ Unit: second + ### `snap-mgr-gc-tick-interval` + The time interval at which the recycle of expired snapshot files is triggered. `0` means that this feature is disabled. @@ -1008,7 +1045,7 @@ Configuration items related to Raftstore. + At a certain interval, TiKV inspects the latency of the Raftstore component. This parameter specifies the interval of the inspection. If the latency exceeds this value, this inspection is marked as timeout. + Judges whether the TiKV node is slow based on the ratio of timeout inspection. -+ Default value: `"500ms"` ++ Default value: `"100ms"` + Minimum value: `"1ms"` ### `raft-write-size-limit` New in v5.3.0 @@ -1017,13 +1054,6 @@ Configuration items related to Raftstore. + Default value: `1MiB` + Minimum value: `0` -### `report-min-resolved-ts-interval` New in v6.0.0 - -+ Determines the interval at which the minimum resolved timestamp is reported to the PD leader. If this value is set to `0`, it means that the reporting is disabled. -+ Default value: Before v6.3.0, the default value is `"0s"`. Starting from v6.3.0, the default value is `"1s"`, which is the smallest positive value. -+ Minimum value: `0` -+ Unit: second - ### `evict-cache-on-memory-ratio` New in v7.5.0 + When the memory usage of TiKV exceeds 90% of the system available memory, and the memory occupied by Raft entry cache exceeds the used memory * `evict-cache-on-memory-ratio`, TiKV evicts the Raft entry cache. @@ -1039,7 +1069,8 @@ Configuration items related to Raftstore. + Set the specific times that TiKV initiates periodic full compaction. You can specify multiple time schedules in an array. For example: + `periodic-full-compact-start-times = ["03:00", "23:00"]` indicates that TiKV performs full compaction daily at 03:00 AM and 11:00 PM, based on the local time zone of the TiKV node. - + `periodic-full-compact-start-times = ["03:00 +0000", "23:00 +0000"]` indicates that TiKV performs full compaction daily at 03:00 AM and 11:00 PM in UTC time. + + `periodic-full-compact-start-times = ["03:00 +0000", "23:00 +0000"]` indicates that TiKV performs full compaction daily at 03:00 AM and 11:00 PM in UTC timezone. + + `periodic-full-compact-start-times = ["03:00 +0800", "23:00 +0800"]` indicates that TiKV performs full compaction daily at 03:00 AM and 11:00 PM in UTC+08:00 timezone. + Default value: `[]`, which means periodic full compaction is disabled by default. ### `periodic-full-compact-start-max-cpu` New in v7.6.0 @@ -1047,6 +1078,42 @@ Configuration items related to Raftstore. + Limits the maximum CPU usage rate for TiKV periodic full compaction. + Default value: `0.1`, which means that the maximum CPU usage for periodic compaction processes is 10%. +### `follower-read-max-log-gap` New in v7.4.0 + ++ The maximum number of logs a follower is allowed to lag behind when processing read requests. If this limit is exceeded, the read request is rejected. ++ Default value: `100` + +### `inspect-cpu-util-thd` New in v7.6.0 + ++ The CPU utilization threshold for determining whether a TiKV node is busy during slow node detection. ++ Value range: `[0, 1]` ++ Default value: `0.4`, which means `40%`. + +### `inspect-kvdb-interval` New in v8.1.2 + ++ The interval and timeout for checking the KV disk during slow node detection in TiKV. If KVDB and RaftDB share the same mount path, this value is overridden by `0` (no detection). ++ Default value: `2s` + +### `min-pending-apply-region-count` New in v8.0.0 + ++ The maximum number of Regions in the busy state of applying Raft logs when TiKV starts. Raftstore accepts leader transfers only when the number of such Regions is below this value, reducing availability degradation during rolling restarts. ++ Default value: `10` + +### `request-voter-replicated-index-interval` New in v6.6.0 + ++ Controls the interval at which the Witness node periodically retrieves the replicated Raft log position from voter nodes. ++ Default value: `5m`, which means 5 minutes. + +### `slow-trend-unsensitive-cause` New in v6.6.0 + ++ When TiKV uses the SlowTrend detection algorithm, this configuration item controls the sensitivity of latency detection. A higher value indicates lower sensitivity. ++ Default value: `10` + +### `slow-trend-unsensitive-result` New in v6.6.0 + ++ When TiKV uses the SlowTrend detection algorithm, this configuration item controls the sensitivity of QPS detection. A higher value indicates lower sensitivity. ++ Default value: `0.5` + ## coprocessor Configuration items related to Coprocessor. @@ -1272,6 +1339,10 @@ Configuration items related to RocksDB ### `info-log-max-size` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.file.max-size`](#max-size-new-in-v540). + + The maximum size of Info log + Default value: `"1GiB"` + Minimum value: `0` @@ -1279,11 +1350,19 @@ Configuration items related to RocksDB ### `info-log-roll-time` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated. TiKV no longer supports automatic log splitting based on time. Instead, you can use the configuration item [`log.file.max-size`](#max-size-new-in-v540) to set the threshold for automatic log splitting based on file size. + + The time interval at which Info logs are truncated. If the value is `0s`, logs are not truncated. + Default value: `"0s"` ### `info-log-keep-log-file-num` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.file.max-backups`](#max-backups-new-in-v540). + + The maximum number of kept log files + Default value: `10` + Minimum value: `0` @@ -1295,6 +1374,10 @@ Configuration items related to RocksDB ### `info-log-level` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.level`](#level-new-in-v540). + + Log levels of RocksDB + Default value: `"info"` @@ -1325,7 +1408,7 @@ Configuration items related to RocksDB + Unit: KiB|MiB|GiB -### `track-and-verify-wals-in-manifest` New in v6.5.9, v7.1.5, and v8.0.0 +### `track-and-verify-wals-in-manifest` New in v6.5.9, v7.1.5, v7.5.2, and v8.0.0 + Controls whether to record information about Write Ahead Log (WAL) files in the RocksDB MANIFEST file and whether to verify the integrity of WAL files during startup. For more information, see RocksDB [Track WAL in MANIFEST](https://github.com/facebook/rocksdb/wiki/Track-WAL-in-MANIFEST). + Default value: `true` @@ -1333,6 +1416,11 @@ Configuration items related to RocksDB + `true`: records information about WAL files in the MANIFEST file and verifies the integrity of WAL files during startup. + `false`: does not record information about WAL files in the MANIFEST file and does not verify the integrity of WAL files during startup. +### `enable-multi-batch-write` New in v6.2.0 + ++ Controls whether to enable RocksDB write optimization, allowing the contents of WriteBatch to be written concurrently to the memtable, reducing write latency. ++ Default value: None. However, it is enabled by default unless explicitly set to `false` or if `rocksdb.enable-pipelined-write` or `rocksdb.enable-unordered-write` is enabled. + ## rocksdb.titan Configuration items related to Titan. @@ -1361,10 +1449,10 @@ Configuration items related to Titan. ### `max-background-gc` + The maximum number of GC threads in Titan. From the **TiKV Details** > **Thread CPU** > **RocksDB CPU** panel, if you observe that the Titan GC threads are at full capacity for a long time, consider increasing the size of the Titan GC thread pool. -+ Default value: `4` ++ Default value: `1`. Before v8.0.0, the default value is `4`. + Minimum value: `1` -## rocksdb.defaultcf | rocksdb.writecf | rocksdb.lockcf +## rocksdb.defaultcf | rocksdb.writecf | rocksdb.lockcf | rocksdb.raftcf Configuration items related to `rocksdb.defaultcf`, `rocksdb.writecf`, and `rocksdb.lockcf`. @@ -1623,6 +1711,11 @@ Configuration items related to `rocksdb.defaultcf`, `rocksdb.writecf`, and `rock + Default value: `"0s"`, meaning that periodic compaction is disabled by default. + Unit: s(second)|h(hour)|d(day) +### `max-compactions` New in v6.6.0 + ++ The maximum number of concurrent compaction tasks. The value `0` means no limit. ++ Default value: `0` + ## rocksdb.defaultcf.titan > **Note:** @@ -1659,7 +1752,7 @@ Configuration items related to `rocksdb.defaultcf.titan`. + The zstd dictionary compression size. The default value is `"0KiB"`, which means to disable the zstd dictionary compression. In this case, Titan compresses data based on single values, whereas RocksDB compresses data based on blocks (`32KiB` by default). When the average size of Titan values is less than `32KiB`, Titan's compression ratio is lower than that of RocksDB. Taking JSON as an example, the store size in Titan can be 30% to 50% larger than that of RocksDB. The actual compression ratio depends on whether the value content is suitable for compression and the similarity among different values. You can enable the zstd dictionary compression to increase the compression ratio by configuring `zstd-dict-size` (for example, set it to `16KiB`). The actual store size can be lower than that of RocksDB. But the zstd dictionary compression might lead to about 10% performance regression in specific workloads. + Default value: `"0KiB"` -+ Unit: KiB|MiB|GiB ++ Unit: KiB|MiB|GiB ### `blob-cache-size` @@ -1843,6 +1936,10 @@ Configuration items related to `raftdb` ### `info-log-max-size` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.file.max-size`](#max-size-new-in-v540). + + The maximum size of Info logs + Default value: `"1GiB"` + Minimum value: `0` @@ -1850,11 +1947,19 @@ Configuration items related to `raftdb` ### `info-log-roll-time` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated. TiKV no longer supports automatic log splitting based on time. Instead, you can use the configuration item [`log.file.max-size`](#max-size-new-in-v540) to set the threshold for automatic log splitting based on file size. + + The interval at which Info logs are truncated. If the value is `0s`, logs are not truncated. + Default value: `"0s"` (which means logs are not truncated) ### `info-log-keep-log-file-num` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.file.max-backups`](#max-backups-new-in-v540). + + The maximum number of Info log files kept in RaftDB + Default value: `10` + Minimum value: `0` @@ -1866,6 +1971,10 @@ Configuration items related to `raftdb` ### `info-log-level` +> **Warning:** +> +> Starting from v5.4.0, RocksDB logs are managed by the logging module of TiKV. Therefore, this configuration item is deprecated, and its function is replaced by the configuration item [`log.level`](#level-new-in-v540). + + Log levels of RaftDB + Default value: `"info"` @@ -1897,9 +2006,13 @@ Configuration items related to Raft Engine. ### `bytes-per-sync` +> **Warning:** +> +> Starting from v6.5.0, Raft Engine writes logs to disk directly without buffering. Therefore, this configuration item is deprecated and no longer functional. + + Specifies the maximum accumulative size of buffered writes. When this configuration value is exceeded, buffered writes are flushed to the disk. + If you set this configuration item to `0`, incremental sync is disabled. -+ Default value: `"4MiB"` ++ Before v6.5.0, the default value is `"4MiB"`. ### `target-file-size` @@ -1972,6 +2085,12 @@ Configuration items related to Raft Engine. + Determines whether to generate empty log files for log recycling in Raft Engine. When it is enabled, Raft Engine will automatically fill a batch of empty log files for log recycling during initialization, making log recycling effective immediately after initialization. + Default value: `false` +### `compression-level` New in v7.4.0 + ++ Sets the compression efficiency of the LZ4 algorithm used by Raft Engine when writing Raft log files. A lower value indicates faster compression speed but lower compression ratio. ++ Range: `[1, 16]` ++ Default value: `1` + ## security Configuration items related to security. @@ -2183,7 +2302,11 @@ Configuration items related to TiCDC. ### `min-ts-interval` + The interval at which Resolved TS is calculated and forwarded. -+ Default value: `"200ms"` ++ Default value: `"1s"`. + +> **Note:** +> +> In v6.5.0, the default value of `min-ts-interval` is changed from `"1s"` to `"200ms"` to reduce CDC latency. Starting from v6.5.1, this default value is changed back to `"1s"` to reduce network traffic. ### `old-value-cache-memory-quota` @@ -2211,6 +2334,12 @@ Configuration items related to TiCDC. + Default value: `6`, which means 6 tasks can be concurrent executed at most. + Note: The value of `incremental-scan-concurrency` must be greater than or equal to that of `incremental-scan-threads`; otherwise, TiKV will report an error at startup. +### `incremental-scan-concurrency-limit` New in v7.6.0 + ++ The maximum queue length for the tasks of incrementally scanning historical data waiting to be executed. When the number of tasks waiting to be executed exceeds this limit, new tasks will be rejected. ++ Default value: `10000`, which means that at most 10000 tasks can be queued for execution. ++ Note: `incremental-scan-concurrency-limit` must be greater than or equal to [`incremental-scan-concurrency`](#incremental-scan-concurrency); otherwise, TiKV uses `incremental-scan-concurrency` to override this configuration. + ## resolved-ts Configuration items related to maintaining the Resolved TS to serve Stale Read requests. @@ -2418,3 +2547,8 @@ Configuration items related to [Load Base Split](/configure-load-base-split.md). + Specifies the amount of data sampled by Heap Profiling each time, rounding up to the nearest power of 2. + Default value: `512KiB` + +### `enable-thread-exclusive-arena` New in v8.1.0 + ++ Controls whether to display the memory allocation status at the TiKV thread level to track the memory usage of each TiKV thread. ++ Default value: `true` diff --git a/tikv-control.md b/tikv-control.md index e9762c5dc68ca..9162184cb8a19 100644 --- a/tikv-control.md +++ b/tikv-control.md @@ -1,7 +1,6 @@ --- title: TiKV Control User Guide summary: Use TiKV Control to manage a TiKV cluster. -aliases: ['/docs/dev/tikv-control/','/docs/dev/reference/tools/tikv-control/'] --- # TiKV Control User Guide diff --git a/time-to-live.md b/time-to-live.md index 7316d00582afd..391c664c5e863 100644 --- a/time-to-live.md +++ b/time-to-live.md @@ -159,7 +159,7 @@ The preceding statement allows TTL jobs to be scheduled only between 1:00 and 5: > **Note:** > -> This section is only applicable to TiDB Self-Hosted. Currently, TiDB Cloud does not provide TTL metrics. +> This section is only applicable to TiDB Self-Managed. Currently, TiDB Cloud does not provide TTL metrics. @@ -258,7 +258,6 @@ Currently, the TTL feature has the following limitations: * A table with the TTL attribute does not support being referenced by other tables as the primary table in a foreign key constraint. * It is not guaranteed that all expired data is deleted immediately. The time when expired data is deleted depends on the scheduling interval and scheduling window of the background cleanup job. * For tables that use [clustered indexes](/clustered-indexes.md), if the primary key is neither an integer nor a binary string type, the TTL job cannot be split into multiple tasks. This will cause the TTL job to be executed sequentially on a single TiDB node. If the table contains a large amount of data, the execution of the TTL job might become slow. -* TTL is not available for [TiDB Serverless](https://docs.pingcap.com/tidbcloud/select-cluster-tier#tidb-serverless). ## FAQs diff --git a/tiproxy/tiproxy-api.md b/tiproxy/tiproxy-api.md new file mode 100644 index 0000000000000..ee05aea62ab25 --- /dev/null +++ b/tiproxy/tiproxy-api.md @@ -0,0 +1,128 @@ +--- +title: TiProxy API +summary: Learn how to use the TiProxy API to access its configuration, health status, and monitoring data. +--- + +# TiProxy API + +[TiProxy](/tiproxy/tiproxy-overview.md) provides API endpoints for accessing its configuration, health status, and monitoring data. + +> **Note:** +> +> TiProxy API is specifically designed for debugging purposes and might not be fully compatible with future capabilities introduced in TiProxy. It's not recommended to include this tool in application or utility development to get information. + +The address for accessing the TiProxy API is `http://${host}:${port}${path}`, where `${host}:${port}` is specified by the TiProxy configuration item [`api.addr`](/tiproxy/tiproxy-configuration.md#addr-1), and `${path}` is the specific API endpoint you want to access. For example: + +```bash +curl http://127.0.0.1:3080/api/admin/config/ +``` + +## Get TiProxy configuration + +### Request URI + +`GET /api/admin/config/` + +### Parameter descriptions + +The query parameter is as follows: + +- `format`: (optional) specifies the format of the returned configuration. Value options are `json` and `toml`. The default value is `toml`. + +### Example + +The following example gets the TiProxy configuration in JSON format: + +```bash +curl "http://127.0.0.1:3080/api/admin/config/?format=json" +``` + +## Set TiProxy configuration + +Currently, you can only use the TOML format to modify TiProxy configuration. Unspecified configuration items will remain unchanged, so you only need to specify the items that you want to modify. + +### Request URI + +`PUT /api/admin/config/` + +### Request body + +You need to provide the TiProxy configuration in TOML format. For example: + +```toml +[log] +level='warning' +``` + +### Example + +The following example sets `log.level` as `'warning'`, while leaving other configuration items unchanged. + +1. Get the current TiProxy configuration: + + ```bash + curl http://127.0.0.1:3080/api/admin/config/ + ``` + + The output is as follows: + + ```toml + [log] + encoder = 'tidb' + level = 'info' + ``` + +2. Specify the value of `log.level` in the `test.toml` file, and then send a `PUT /api/admin/config/` request to update the value of `log.level`: + + ```shell + $ cat test.toml + [log] + level='warning' + $ curl -X PUT --data-binary @test.toml http://127.0.0.1:3080/api/admin/config/ + ``` + +3. Get the modified TiProxy configuration: + + ```bash + curl http://127.0.0.1:3080/api/admin/config/ + ``` + + The output is as follows: + + ```toml + [log] + encoder = 'tidb' + level = 'warning' + ``` + +## Get TiProxy health status + +This endpoint is used to get the health status of TiProxy and the checksum of the configuration. When TiProxy is running normally, this endpoint returns the checksum of the configuration. When TiProxy is shutting down or offline, it returns an error. + +### Request URI + +`GET /api/debug/health` + +### Example + +```bash +curl http://127.0.0.1:3080/api/debug/health +``` + +The output is as follows: + +```bash +{"config_checksum":3006078629} +``` + +## Get TiProxy monitoring data + +### Request URI + +`GET /metrics/` + +### Example + +```bash +curl http://127.0.0.1:3080/metrics/ +``` diff --git a/tiproxy/tiproxy-command-line-flags.md b/tiproxy/tiproxy-command-line-flags.md index 072decc43b016..5468a5c30e17b 100644 --- a/tiproxy/tiproxy-command-line-flags.md +++ b/tiproxy/tiproxy-command-line-flags.md @@ -18,11 +18,29 @@ This section lists the flags of the server program `tiproxy`. + Default: `""` + You must specify the configuration file. For detailed configuration items, refer to [Configure TiProxy](/tiproxy/tiproxy-configuration.md). Note that TiProxy automatically reloads the configuration when the configuration file is modified. Therefore, do not directly modify the configuration file. It is recommended to modify the configuration by executing [`tiup cluster edit-config`](/tiup/tiup-component-cluster-edit-config.md) or [`kubectl edit tc`](https://docs.pingcap.com/tidb-in-kubernetes/stable/modify-tidb-configuration). -## TiProxy control +## TiProxy Control -This section lists the flags of the client program `tiproxyctl`. +This section introduces the syntax, options, and commands of the client program `tiproxyctl`. -### `--log_encoder` +> **Note:** +> +> TiProxy Control is specifically designed for debugging purposes and might not be fully compatible with future capabilities introduced in TiProxy. It's not recommended to include this tool in application or utility development to get information. + +### Syntax + +``` +tiproxyctl [flags] [command] +``` + +For example: + +``` +tiproxyctl --curls 127.0.0.1:3080 config get +``` + +### Options + +#### `--log_encoder` + Specifies the log format of `tiproxyctl`. + Type: `string` @@ -32,35 +50,67 @@ This section lists the flags of the client program `tiproxyctl`. - `console`: a more human-readable format - `json`: a structured log format -### `--log_level` +#### `--log_level` + Specifies the log level of tiproxyctl. + Type: `string` + Default: `"warn"` + You can specify `debug`, `info`, `warn`, `error`, `panic`. -### `--curls` +#### `--curls` + Specifies the server addresses. You can add multiple listening addresses. + Type: `comma separated lists of ip:port` + Default: `localhost:3080` + Server API gateway addresses. -### `-k`, `--insecure` +#### `-k, --insecure` + Specifies whether to skip TLS CA verification when dialing to the server. + Type: `boolean` + Default: `false` + Used for testing. -### `--ca` +#### `--ca` + Specifies the CA when dialing to the server. + Type: `string` + Default: `""` -### `--cert` +#### `--cert` + Specifies the certificate when dialing to the server. + Type: `string` + Default: `""` + +### Commands + +#### `config set` + +The `tiproxyctl config set` command reads a TOML-formatted configuration file from standard input and sets these configuration items to TiProxy. Unspecified configuration items will remain unchanged, so you only need to specify the items that you want to modify. + +The following example sets `log.level` as `'warning'`, while leaving other configuration items unchanged. + +```bash +$ cat test.toml +[log] +level='warning' +$ cat test.toml | tiproxyctl config set +"" +$ tiproxyctl config get | grep level +level = 'warning' +``` + +#### `config get` + +The `tiproxyctl config get` command is used to get the current TiProxy configuration in TOML format. + +#### `health` + +The `tiproxyctl health` command is used to get the health status of TiProxy and the checksum of the configuration. When TiProxy is running normally, it returns the checksum of the configuration. When TiProxy is shutting down or offline, it returns an error. + +Example output: + +```json +{"config_checksum":3006078629} +``` diff --git a/tiproxy/tiproxy-configuration.md b/tiproxy/tiproxy-configuration.md index 3544073e7c997..16bb8f5cb67bd 100644 --- a/tiproxy/tiproxy-configuration.md +++ b/tiproxy/tiproxy-configuration.md @@ -79,10 +79,10 @@ Configuration for SQL port. #### `proxy-protocol` -+ Default value: `` ++ Default value: `""` + Support hot-reload: yes, but only for new connections -+ Possible values: ``, `v2` -+ Enable the [PROXY protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt) on the port. By enabling the PROXY protocol, TiProxy can pass the real client IP address to TiDB. `v2` indicates using the PROXY protocol version 2, and `` indicates disabling the PROXY protocol. If the PROXY protocol is enabled on TiProxy, you need to also enable the [PROXY protocol](/tidb-configuration-file.md#proxy-protocol) on the TiDB server. ++ Possible values: `""`, `"v2"` ++ Enable the [PROXY protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt) on the port. By enabling the PROXY protocol, TiProxy can pass the real client IP address to TiDB. `"v2"` indicates using the PROXY protocol version 2, and `""` indicates disabling the PROXY protocol. If the PROXY protocol is enabled on TiProxy, you need to also enable the [PROXY protocol](/tidb-configuration-file.md#proxy-protocol) on the TiDB server. ### api @@ -90,16 +90,16 @@ Configurations for HTTP gateway. #### `addr` -+ Default value: `0.0.0.0:3090` ++ Default value: `0.0.0.0:3080` + Support hot-reload: no + API gateway address. You can specify `ip:port`. #### `proxy-protocol` -+ Default value: `` ++ Default value: `""` + Support hot-reload: no -+ Possible values: ``, `v2` -+ Enable the [PROXY protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt) on the port. `v2` indicates using the PROXY protocol version 2, and `` indicates disabling the PROXY protocol. ++ Possible values: `""`, `"v2"` ++ Enable the [PROXY protocol](https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt) on the port. `"v2"` indicates using the PROXY protocol version 2, and `""` indicates disabling the PROXY protocol. ### log @@ -123,7 +123,7 @@ Configurations for HTTP gateway. #### `filename` -+ Default value: `` ++ Default value: `""` + Support hot-reload: yes + Log file path. Non empty value will enable logging to file. When TiProxy is deployed with TiUP, the filename is set automatically. diff --git a/tiproxy/tiproxy-deployment-topology.md b/tiproxy/tiproxy-deployment-topology.md index c13a09648a6a4..d6b79a9d08730 100644 --- a/tiproxy/tiproxy-deployment-topology.md +++ b/tiproxy/tiproxy-deployment-topology.md @@ -16,9 +16,13 @@ TiProxy is a L7 proxy server for TiDB, which can balance connections and migrate | TiDB | 3 | 16 VCore 32GB * 3 | 10.0.1.4
10.0.1.5
10.0.1.6 | Default port
Global directory configuration | | PD | 3 | 4 VCore 8GB * 3 | 10.0.1.1
10.0.1.2
10.0.1.3 | Default port
Global directory configuration | | TiKV | 3 | 16 VCore 32GB 2TB (nvme ssd) * 3 | 10.0.1.7
10.0.1.8
10.0.1.9 | Default port
Global directory configuration | -| TiProxy | 3 | 4 VCore 8 GB * 1 | 10.0.1.11 | Default port
Global directory configuration | +| TiProxy | 1 | 4 VCore 8 GB * 1 | 10.0.1.11 | Default port
Global directory configuration | | Monitoring & Grafana | 1 | 4 VCore 8GB * 1 500GB (ssd) | 10.0.1.13 | Default port
Global directory configuration | +> **Note:** +> +> The IP addresses of the instances are given as examples only. In your actual deployment, replace the IP addresses with your actual IP addresses. + ### Topology templates For more information about the template for TiProxy, see [The simple template for the TiProxy topology](https://github.com/pingcap/docs/blob/master/config-templates/simple-tiproxy.yaml). diff --git a/tiproxy/tiproxy-overview.md b/tiproxy/tiproxy-overview.md index e0558f9a902df..5bbbfbfbe4bc8 100644 --- a/tiproxy/tiproxy-overview.md +++ b/tiproxy/tiproxy-overview.md @@ -11,7 +11,7 @@ TiProxy is an optional component. You can also use a third-party proxy component The following figure shows the architecture of TiProxy: -TiProxy architecture +TiProxy architecture ## Main features @@ -23,7 +23,7 @@ TiProxy can migrate connections from one TiDB server to another without breaking As shown in the following figure, the client originally connects to TiDB 1 through TiProxy. After the connection migration, the client actually connects to TiDB 2. When TiDB 1 is about to be offline or the ratio of connections on TiDB 1 to connections on TiDB 2 exceeds the set threshold, the connection migration is triggered. The client is unaware of the connection migration. -TiProxy connection migration +TiProxy connection migration Connection migration usually occurs in the following scenarios: @@ -206,4 +206,4 @@ Note that some connectors call the common library to connect to the database, an ## TiProxy resources - [TiProxy Release Notes](https://github.com/pingcap/tiproxy/releases) -- [TiProxy Issues](https://github.com/pingcap/tiup/issues): Lists TiProxy GitHub issues +- [TiProxy Issues](https://github.com/pingcap/tiproxy/issues): Lists TiProxy GitHub issues diff --git a/tispark-deployment-topology.md b/tispark-deployment-topology.md index 445ec4b744d9d..cc79d3f4087ff 100644 --- a/tispark-deployment-topology.md +++ b/tispark-deployment-topology.md @@ -25,6 +25,10 @@ For more information about the TiSpark architecture and how to use it, see [TiSp | TiSpark | 3 | 8 VCore 16GB * 1 | 10.0.1.21 (master)
10.0.1.22 (worker)
10.0.1.23 (worker) | Default port
Global directory configuration | | Monitoring & Grafana | 1 | 4 VCore 8GB * 1 500GB (ssd) | 10.0.1.11 | Default port
Global directory configuration | +> **Note:** +> +> The IP addresses of the instances are given as examples only. In your actual deployment, replace the IP addresses with your actual IP addresses. + ## Topology templates - [Simple TiSpark topology template](https://github.com/pingcap/docs/blob/master/config-templates/simple-tispark.yaml) diff --git a/tispark-overview.md b/tispark-overview.md index 747c124539bc2..5c76591110413 100644 --- a/tispark-overview.md +++ b/tispark-overview.md @@ -1,7 +1,6 @@ --- title: TiSpark User Guide summary: Use TiSpark to provide an HTAP solution to serve as a one-stop solution for both online transactions and analysis. -aliases: ['/docs/dev/tispark-overview/','/docs/dev/reference/tispark/','/docs/dev/get-started-with-tispark/','/docs/dev/how-to/get-started/tispark/','/docs/dev/how-to/deploy/tispark/','/tidb/dev/get-started-with-tispark/','/tidb/stable/get-started-with-tispark'] --- # TiSpark User Guide @@ -98,8 +97,14 @@ You can choose TiSpark version according to your TiDB and Spark version. | 2.5.x | 5.x, 4.x | 3.0.x, 3.1.x | 2.12 | | 3.0.x | 5.x, 4.x | 3.0.x, 3.1.x, 3.2.x|2.12| | 3.1.x | 6.x, 5.x, 4.x | 3.0.x, 3.1.x, 3.2.x, 3.3.x|2.12| +| 3.2.x | 6.x, 5.x, 4.x | 3.0.x, 3.1.x, 3.2.x, 3.3.x|2.12| -TiSpark 2.4.4, 2.5.2, 3.0.2 and 3.1.1 are the latest stable versions and are highly recommended. +TiSpark 2.4.4, 2.5.3, 3.0.3, 3.1.7, and 3.2.3 are the latest stable versions and are highly recommended. + +> **Note:** +> +> TiSpark does not guarantee compatibility with TiDB v7.0.0 and later versions. +> TiSpark does not guarantee compatibility with Spark v3.4.0 and later versions. ### Get TiSpark jar @@ -130,11 +135,11 @@ mvn clean install -Dmaven.test.skip=true -Pspark3.2.1 The Artifact ID of TiSpark varies with TiSpark versions. -| TiSpark version | Artifact ID | -|-------------------------------| -------------------------------------------------- | +| TiSpark version | Artifact ID | +|--------------------------------| -------------------------------------------------- | | 2.4.x-\${scala_version}, 2.5.0 | tispark-assembly | -| 2.5.1 | tispark-assembly-\${spark_version} | -| 3.0.x, 3.1.x | tispark-assembly-\${spark_version}-\${scala_version} | +| 2.5.1 | tispark-assembly-\${spark_version} | +| 3.0.x, 3.1.x, 3.2.x | tispark-assembly-\${spark_version}-\${scala_version} | ## Getting started @@ -201,7 +206,7 @@ customerDF.write See [Data Source API User Guide](https://github.com/pingcap/tispark/blob/master/docs/features/datasource_api_userguide.md) for more details. -You can also write with Spark SQL since TiSpark 3.1. See [insert SQL](https://github.com/pingcap/tispark/blob/master/docs/features/insert_sql_userguide.md) for more details. +Starting from TiSpark 3.1, you can write data to TiKV using Spark SQL. For more information, see [insert SQL](https://github.com/pingcap/tispark/blob/master/docs/features/insert_sql_userguide.md). ### Write data using JDBC DataSource diff --git a/tiup/tiup-bench.md b/tiup/tiup-bench.md index c69b483af73eb..051f3d835f1a6 100644 --- a/tiup/tiup-bench.md +++ b/tiup/tiup-bench.md @@ -1,7 +1,6 @@ --- title: Stress Test TiDB Using TiUP Bench Component summary: Learn how to stress test TiDB with TPC-C, TPC-H, CH, RawSQL, and YCSB workloads using TiUP. -aliases: ['/docs/dev/tiup/tiup-bench/','/docs/dev/reference/tools/tiup/bench/'] --- # Stress Test TiDB Using TiUP Bench Component diff --git a/tiup/tiup-cluster-no-sudo-mode.md b/tiup/tiup-cluster-no-sudo-mode.md new file mode 100644 index 0000000000000..7b2f80623397e --- /dev/null +++ b/tiup/tiup-cluster-no-sudo-mode.md @@ -0,0 +1,190 @@ +--- +title: Deploy and Maintain an Online TiDB Cluster Using TiUP No-sudo Mode +summary: Learn how to deploy and maintain an online TiDB cluster using the TiUP no-sudo mode. +--- + +# Deploy and Maintain an Online TiDB Cluster Using TiUP No-sudo Mode + +This document describes how to use the TiUP no-sudo mode to deploy a cluster. + +> **Note:** +> +> For CentOS, only CentOS 8 or later versions are supported. + +## Prepare the user and configure the SSH mutual trust + +1. Take the `tidb` user as an example. Log in to all the target machines and create a user named `tidb` using the `root` user with the following command. In no-sudo mode, configuring passwordless sudo for the `tidb` user is unnecessary, that is, you do not need to add the `tidb` user to the `sudoers` file. + + ```shell + adduser tidb + ``` + +2. Start the `systemd user` mode for the `tidb` user on each target machine. This step is required and do not skip it. + + 1. Use the `tidb` user to set the `XDG_RUNTIME_DIR` environment variable. + + ```shell + sudo -iu tidb # Switch to the tidb user + mkdir -p ~/.bashrc.d + echo "export XDG_RUNTIME_DIR=/run/user/$(id -u)" > ~/.bashrc.d/systemd + source ~/.bashrc.d/systemd + ``` + + 2. Use the `root` user to start the user service. + + ```shell + $ uid=$(id -u tidb) # Get the ID of the tidb user + $ systemctl start user@${uid}.service + $ systemctl status user@${uid}.service + user@1000.service - User Manager for UID 1000 + Loaded: loaded (/usr/lib/systemd/system/user@.service; static; vendor preset> + Active: active (running) since Mon 2024-01-29 03:30:51 EST; 1min 7s ago + Main PID: 3328 (systemd) + Status: "Startup finished in 420ms." + Tasks: 6 + Memory: 6.1M + CGroup: /user.slice/user-1000.slice/user@1000.service + ├─dbus.service + │ └─3442 /usr/bin/dbus-daemon --session --address=systemd: --nofork > + ├─init.scope + │ ├─3328 /usr/lib/systemd/systemd --user + │ └─3335 (sd-pam) + └─pulseaudio.service + └─3358 /usr/bin/pulseaudio --daemonize=no --log-target=journal + ``` + + 3. Execute `systemctl --user`. If no errors occur, it indicates that the `systemd user` mode has started successfully. + +3. Use the `root` user to execute the following command to enable lingering for the systemd user `tidb`. + + ```shell + loginctl enable-linger tidb + loginctl show-user -p Linger tidb # This should show: Linger=yes + ``` + + You can read the systemd documentation for reference, [Automatic start-up of systemd user instances](https://wiki.archlinux.org/title/Systemd/User#Automatic_start-up_of_systemd_user_instances). + +4. Generate a key using `ssh-keygen` on the control machine: + + ```shell + ssh-keygen + ``` + +5. Copy the public key to the other machines in the cluster to establish SSH trust. + + - If you have set a password for the `tidb` user, you can use `ssh-copy-id` command to copy the public key to the target machine. + + ```shell + ssh-copy-id tidb@host + ``` + + You need to replace `host` with the hostname of the target machine and run this command on each of the other machines in the cluster. + + - If you use a different method to copy the public key, make sure to check the permissions of the `/home/tidb/.ssh/authorized_keys` file after the copy. + + ```shell + chown -R tidb:tidb /home/tidb/.ssh/authorized_keys + chmod 600 /home/tidb/.ssh/authorized_keys + ``` + +## Prepare the topology file + +1. Execute the following command to generate the topology file. + + ```shell + tiup cluster template > topology.yaml + ``` + +2. Edit the topology file. + + Compared with the regular mode, when using TiUP in no-sudo mode, you need to add a line `systemd_mode: "user"` in the `global` module of the `topology.yaml` file. The `systemd_mode` parameter is used to set whether to use the `systemd user` mode. If this parameter is not set, the default value is `system`, meaning sudo permissions are required. + + Additionally, in no-sudo mode, because the non-root `tidb` user does not have permission to use the `/data` directory as `deploy_dir` or `data_dir`, you must select a path accessible to non-root users. The following example uses relative paths, and the actual paths used are `/home/tidb/data/tidb-deploy` and `/home/tidb/data/tidb-data`. The rest of the topology file remains the same as in the regular mode. Another option is to use the root user to create the directories and then use `chown` to change the ownership to `tidb:tidb`. + + ```yaml + global: + user: "tidb" + systemd_mode: "user" + ssh_port: 22 + deploy_dir: "data/tidb-deploy" + data_dir: "data/tidb-data" + arch: "amd64" + ... + ``` + +## Manually repair failed check items + +> **Note:** +> +> If you use a minimal install, make sure the `tar` package is installed. Otherwise, the `tiup cluster check` command will fail. + +Executing `tiup cluster check topology.yaml --user tidb` can generate some failed check items. The following is an example. + +```shell +Node Check Result Message +---- ----- ------ ------- +192.168.124.27 thp Fail THP is enabled, please disable it for best performance +192.168.124.27 command Pass numactl: policy: default +192.168.124.27 os-version Pass OS is CentOS Stream 8 +192.168.124.27 network Pass network speed of ens160 is 10000MB +192.168.124.27 disk Warn mount point / does not have 'noatime' option set +192.168.124.27 disk Fail multiple components tikv:/home/blackcat/data/tidb-deploy/tikv-20160/data/tidb-data,tikv:/home/blackcat/data/tidb-deploy/tikv-20161/data/tidb-data are using the same partition 192.168.124.27:/ as data dir +192.168.124.27 selinux Pass SELinux is disabled +192.168.124.27 cpu-cores Pass number of CPU cores / threads: 16 +192.168.124.27 cpu-governor Warn Unable to determine current CPU frequency governor policy +192.168.124.27 swap Warn swap is enabled, please disable it for best performance +192.168.124.27 memory Pass memory size is 9681MB +192.168.124.27 service Fail service firewalld is running but should be stopped +``` + +In no-sudo mode, the `tidb` user does not have sudo permissions. As a result, running `tiup cluster check topology.yaml --apply --user tidb` cannot automatically fix the failed check items. You need to manually fix it by using the `root` user on the target machines. + +For more information, see [TiDB Environment and System Configuration Check](/check-before-deployment.md). Note that you need to skip the step [Manually configure the SSH mutual trust and sudo without password](/check-before-deployment.md#manually-configure-the-ssh-mutual-trust-and-sudo-without-password) in the document. + +## Deploy and manage the cluster + +To use the `tidb` user created in preceding steps and avoid creating a new one, add `--user tidb` when running the following `deploy` command: + +```shell +tiup cluster deploy mycluster v8.5.0 topology.yaml --user tidb +``` + +> **Note:** +> +> You need to replace `v8.5.0` in the preceding command with the TiDB version that you want to deploy and `mycluster` with the name you want to give to your cluster. + +Start the cluster: + +```shell +tiup cluster start mycluster +``` + +Scale out the cluster: + +```shell +tiup cluster scale-out mycluster scale.yaml --user tidb +``` + +Scale in the cluster: + +```shell +tiup cluster scale-in mycluster -N 192.168.124.27:20160 +``` + +Upgrade the cluster: + +```shell +tiup cluster upgrade mycluster v8.2.0 +``` + +## FAQ + +### The `Trying to run as user instance, but $XDG_RUNTIME_DIR is not set.` error occurs when starting user@.service + +This issue might be caused by the absence of `pam_systemd.so` in your `/etc/pam.d/system-auth.ued` file. + +To resolve this issue, use the following command to check whether the `/etc/pam.d/system-auth.ued` file contains the `pam_systemd.so` module. If not, append `session optional pam_systemd.so` to the end of the file. + +```shell +grep 'pam_systemd.so' /etc/pam.d/system-auth.ued || echo 'session optional pam_systemd.so' >> /etc/pam.d/system-auth.ued +``` diff --git a/tiup/tiup-cluster-topology-reference.md b/tiup/tiup-cluster-topology-reference.md index 93db67ba6afd2..592a2d6ae390b 100644 --- a/tiup/tiup-cluster-topology-reference.md +++ b/tiup/tiup-cluster-topology-reference.md @@ -26,6 +26,7 @@ A topology configuration file for TiDB deployment using TiUP might contain the f - [tiproxy_servers](#tiproxy_servers): The configuration of the TiProxy instance. This configuration specifies the machines to which the TiProxy component is deployed. - [pump_servers](#pump_servers): The configuration of the Pump instance. This configuration specifies the machines to which the Pump component is deployed. - [drainer_servers](#drainer_servers): The configuration of the Drainer instance. This configuration specifies the machines to which the Drainer component is deployed. +- [kvcdc_servers](#kvcdc_servers): The configuration of the [TiKV-CDC](https://tikv.org/docs/7.1/concepts/explore-tikv-features/cdc/cdc/) instance. This configuration specifies the machines to which the TiKV-CDC component is deployed. - [cdc_servers](#cdc_servers): The configuration of the TiCDC instance. This configuration specifies the machines to which the TiCDC component is deployed. - [tispark_masters](#tispark_masters): The configuration of the TiSpark master instance. This configuration specifies the machines to which the TiSpark master component is deployed. Only one node of TiSpark master can be deployed. - [tispark_workers](#tispark_workers): The configuration of the TiSpark worker instance. This configuration specifies the machines to which the TiSpark worker component is deployed. @@ -41,6 +42,8 @@ The `global` section corresponds to the cluster's global configuration and has t - `group`: The user group to which a user belongs. It is specified when the user is created. The value defaults to that of the `` field. If the specified group does not exist, it is automatically created. +- `systemd_mode`: Specifies the `systemd` mode used on the target machine during cluster deployment. The default value is `system`. If set to `user`, sudo permissions are not required on the target machine, meaning [TiUP no-sudo mode](/tiup/tiup-cluster-no-sudo-mode.md) is used. + - `ssh_port`: Specifies the SSH port to connect to the target machine for operations. The default value is `22`. - `enable_tls`: Specifies whether to enable TLS for the cluster. After TLS is enabled, the generated TLS certificate must be used for connections between components or between the client and the component. The default value is `false`. @@ -369,7 +372,7 @@ tikv_servers: - `ssh_port`: Specifies the SSH port to connect to the target machine for operations. If it is not specified, the `ssh_port` of the `global` section is used. -- `tcp_port`: The port of the TiFlash TCP service. The default value is `9000`. +- `tcp_port`: The port of the TiFlash TCP service for internal testing purposes. The default value is `9000`. Starting from TiUP v1.12.5, this configuration item does not take effect on clusters that are v7.1.0 or later. - `flash_service_port`: The port via which TiFlash provides services. TiDB reads data from TiFlash via this port. The default value is `3930`. @@ -399,11 +402,10 @@ tikv_servers: - `resource_control`: Resource control for the service. If this field is configured, the field content is merged with the `resource_control` content in `global` (if the two fields overlap, the content of this field takes effect). Then, a systemd configuration file is generated and sent to the machine specified in `host`. The configuration rules of `resource_control` are the same as the `resource_control` content in `global`. -After the deployment, for the fields above, you can only add directories to `data_dir`; for the fields below, you cannot modified these fields: +After the deployment, for the fields above, you can only add directories to `data_dir`; for the fields below, you cannot modify these fields: - `host` - `tcp_port` -- `http_port` - `flash_service_port` - `flash_proxy_port` - `flash_proxy_status_port` @@ -566,6 +568,54 @@ drainer_servers: tbl-name: audit ``` +### `kvcdc_servers` + +`kvcdc_servers` specifies the machines to which the [TiKV-CDC](https://tikv.org/docs/7.1/concepts/explore-tikv-features/cdc/cdc/) services are deployed. It also specifies the service configuration on each machine. `kvcdc_servers` is an array. Each array element contains the following fields: + +- `host`: Specifies the machine to which the TiKV-CDC services are deployed. The field value is an IP address and is mandatory. + +- `ssh_port`: Specifies the SSH port to connect to the target machine for operations. If it is not specified, the `ssh_port` of the `global` section is used. + +- `port`: The listening port of the TiKV-CDC services. The default value is `8600`. + +- `deploy_dir`: Specifies the deployment directory. If it is not specified or specified as a relative directory, the directory is generated according to the `deploy_dir` directory configured in `global`. + +- `data-dir`: Specifies the directory that TiKV-CDC uses to store temporary files primarily for sorting (optional). The free disk space for this directory is recommended to be greater than or equal to 500 GiB. + +- `log_dir`: Specifies the log directory. If it is not specified or specified as a relative directory, the log is generated according to the `log_dir` directory configured in `global`. + +- `gc-ttl`: The TTL (Time to Live, in seconds) of the service-level GC safepoint in PD set by TiKV-CDC (optional). It is the duration that replication tasks can be suspended, defaulting to `86400`, which is 24 hours. Note that suspending replication tasks affects the progress of TiKV garbage collection safepoint. The longer the `gc-ttl`, the longer changefeeds can be suspended, but at the same time, more obsolete data will be kept and occupy more space. Vice versa. + +- `tz`: The time zone that the TiKV-CDC services use. TiKV-CDC uses this time zone when internally converting time data types such as timestamp and when replicating data to the downstream. The default value is the local time zone where the process runs. + +- `numa_node`: Allocates the NUMA policy to the instance. Before specifying this field, you need to make sure that the target machine has [numactl](https://linux.die.net/man/8/numactl) installed. If this field is specified, cpubind and membind policies are allocated using [numactl](https://linux.die.net/man/8/numactl). This field is the string type. The field value is the ID of the NUMA node, such as "0,1". + +- `config`: The address of the configuration file that TiKV-CDC uses (optional). + +- `os`: The operating system of the machine specified in `host`. If this field is not specified, the default value is the `os` value in `global`. + +- `arch`: The architecture of the machine specified in `host`. If this field is not specified, the default value is the `arch` value in `global`. + +- `resource_control`: Resource control for the service. If this field is configured, the field content is merged with the `resource_control` content in `global` (if the two fields overlap, the content of this field takes effect). Then, a systemd configuration file is generated and sent to the machine specified in `host`. The configuration rules of `resource_control` are the same as the `resource_control` content in `global`. + +For the above fields, you cannot modify these configured fields after the deployment: + +- `host` +- `port` +- `deploy_dir` +- `data_dir` +- `log_dir` +- `arch` +- `os` + +A `kvcdc_servers` configuration example is as follows: + +```yaml +kvcdc_servers: + - host: 10.0.1.21 + - host: 10.0.1.22 +``` + ### `cdc_servers` `cdc_servers` specifies the machines to which the TiCDC services are deployed. It also specifies the service configuration on each machine. `cdc_servers` is an array. Each array element contains the following fields: diff --git a/tiup/tiup-cluster.md b/tiup/tiup-cluster.md index 6e862a92dd132..e061d96d8d417 100644 --- a/tiup/tiup-cluster.md +++ b/tiup/tiup-cluster.md @@ -1,7 +1,6 @@ --- title: Deploy and Maintain an Online TiDB Cluster Using TiUP summary: Learns how to deploy and maintain an online TiDB cluster using TiUP. -aliases: ['/docs/dev/tiup/tiup-cluster/','/docs/dev/reference/tools/tiup/cluster/'] --- # Deploy and Maintain an Online TiDB Cluster Using TiUP @@ -62,7 +61,7 @@ To deploy the cluster, run the `tiup cluster deploy` command. The usage of the c tiup cluster deploy [flags] ``` -This command requires you to provide the cluster name, the TiDB cluster version (such as `v8.0.0`), and a topology file of the cluster. +This command requires you to provide the cluster name, the TiDB cluster version (such as `v8.1.2`), and a topology file of the cluster. To write a topology file, refer to [the example](https://github.com/pingcap/tiup/blob/master/embed/examples/cluster/topology.example.yaml). The following file is an example of the simplest topology: @@ -122,12 +121,12 @@ tidb_servers: ... ``` -Save the file as `/tmp/topology.yaml`. If you want to use TiDB v8.0.0 and your cluster name is `prod-cluster`, run the following command: +Save the file as `/tmp/topology.yaml`. If you want to use TiDB v8.1.2 and your cluster name is `prod-cluster`, run the following command: {{< copyable "shell-regular" >}} ```shell -tiup cluster deploy -p prod-cluster v8.0.0 /tmp/topology.yaml +tiup cluster deploy -p prod-cluster v8.1.2 /tmp/topology.yaml ``` During the execution, TiUP asks you to confirm your topology again and requires the root password of the target machine (the `-p` flag means inputting password): @@ -135,7 +134,7 @@ During the execution, TiUP asks you to confirm your topology again and requires ```bash Please confirm your topology: TiDB Cluster: prod-cluster -TiDB Version: v8.0.0 +TiDB Version: v8.1.2 Type Host Ports OS/Arch Directories ---- ---- ----- ------- ----------- pd 172.16.5.134 2379/2380 linux/x86_64 deploy/pd-2379,data/pd-2379 @@ -179,7 +178,7 @@ tiup cluster list Starting /root/.tiup/components/cluster/v1.12.3/cluster list Name User Version Path PrivateKey ---- ---- ------- ---- ---------- -prod-cluster tidb v8.0.0 /root/.tiup/storage/cluster/clusters/prod-cluster /root/.tiup/storage/cluster/clusters/prod-cluster/ssh/id_rsa +prod-cluster tidb v8.1.2 /root/.tiup/storage/cluster/clusters/prod-cluster /root/.tiup/storage/cluster/clusters/prod-cluster/ssh/id_rsa ``` ## Start the cluster @@ -209,7 +208,7 @@ tiup cluster display prod-cluster ``` Starting /root/.tiup/components/cluster/v1.12.3/cluster display prod-cluster TiDB Cluster: prod-cluster -TiDB Version: v8.0.0 +TiDB Version: v8.1.2 ID Role Host Ports OS/Arch Status Data Dir Deploy Dir -- ---- ---- ----- ------- ------ -------- ---------- 172.16.5.134:3000 grafana 172.16.5.134 3000 linux/x86_64 Up - deploy/grafana-3000 @@ -284,7 +283,7 @@ tiup cluster display prod-cluster ``` Starting /root/.tiup/components/cluster/v1.12.3/cluster display prod-cluster TiDB Cluster: prod-cluster -TiDB Version: v8.0.0 +TiDB Version: v8.1.2 ID Role Host Ports OS/Arch Status Data Dir Deploy Dir -- ---- ---- ----- ------- ------ -------- ---------- 172.16.5.134:3000 grafana 172.16.5.134 3000 linux/x86_64 Up - deploy/grafana-3000 @@ -396,12 +395,12 @@ Global Flags: -y, --yes Skip all confirmations and assumes 'yes' ``` -For example, the following command upgrades the cluster to v8.0.0: +For example, the following command upgrades the cluster to v8.1.2: {{< copyable "shell-regular" >}} ```bash -tiup cluster upgrade tidb-test v8.0.0 +tiup cluster upgrade tidb-test v8.1.2 ``` ## Update configuration @@ -586,11 +585,11 @@ tiup cluster audit Starting component `cluster`: /home/tidb/.tiup/components/cluster/v1.12.3/cluster audit ID Time Command -- ---- ------- -4BLhr0 2024-03-28T23:55:09+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v8.0.0 /tmp/topology.yaml -4BKWjF 2024-03-28T23:36:57+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v8.0.0 /tmp/topology.yaml -4BKVwH 2024-03-28T23:02:08+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v8.0.0 /tmp/topology.yaml -4BKKH1 2024-03-28T16:39:04+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster destroy test -4BKKDx 2024-03-28T16:36:57+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v8.0.0 /tmp/topology.yaml +4BLhr0 2024-12-26T23:55:09+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v8.1.2 /tmp/topology.yaml +4BKWjF 2024-12-26T23:36:57+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v8.1.2 /tmp/topology.yaml +4BKVwH 2024-12-26T23:02:08+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v8.1.2 /tmp/topology.yaml +4BKKH1 2024-12-26T16:39:04+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster destroy test +4BKKDx 2024-12-26T16:36:57+08:00 /home/tidb/.tiup/components/cluster/v1.12.3/cluster deploy test v8.1.2 /tmp/topology.yaml ``` The first column is `audit-id`. To view the execution log of a certain command, pass the `audit-id` of a command as the flag as follows: @@ -701,12 +700,12 @@ Environment checks are not necessary for deploying a cluster. For the production All operations above performed on the cluster machine use the SSH client embedded in TiUP to connect to the cluster and execute commands. However, in some scenarios, you might also need to use the SSH client native to the control machine system to perform such cluster operations. For example: -- To use a SSH plug-in for authentication +- To use an SSH plug-in for authentication - To use a customized SSH client Then you can use the `--ssh=system` command-line flag to enable the system-native command-line tool: -- Deploy a cluster: `tiup cluster deploy --ssh=system`. Fill in the name of your cluster for ``, the TiDB version to be deployed (such as `v8.0.0`) for ``, and the topology file for ``. +- Deploy a cluster: `tiup cluster deploy --ssh=system`. Fill in the name of your cluster for ``, the TiDB version to be deployed (such as `v8.1.2`) for ``, and the topology file for ``. - Start a cluster: `tiup cluster start --ssh=system` - Upgrade a cluster: `tiup cluster upgrade ... --ssh=system` diff --git a/tiup/tiup-command-mirror-rotate.md b/tiup/tiup-command-mirror-rotate.md index 602f6d6f38101..a2861f3410722 100644 --- a/tiup/tiup-command-mirror-rotate.md +++ b/tiup/tiup-command-mirror-rotate.md @@ -35,7 +35,7 @@ TiUP uses the command `tiup mirror rotate` to automate the above process. > **Note:** > -> + For TiUP versions earlier than v1.5.0, running this command does not returns a correct new `root.json` file. See [#983](https://github.com/pingcap/tiup/issues/983). +> + For TiUP versions earlier than v1.5.0, running this command does not return a correct new `root.json` file. See [#983](https://github.com/pingcap/tiup/issues/983). > + Before using this command, make sure that all TiUP clients are upgraded to v1.5.0 or a later version. ## Syntax diff --git a/tiup/tiup-command-mirror-sign.md b/tiup/tiup-command-mirror-sign.md index 012b03b53b722..b11a1c87feec3 100644 --- a/tiup/tiup-command-mirror-sign.md +++ b/tiup/tiup-command-mirror-sign.md @@ -5,7 +5,7 @@ summary: The `tiup mirror sign` command is used to sign metadata files in TiUP m # tiup mirror sign -The `tiup mirror sign` command is used to sign the metadata files (*.json）defined in TiUP [mirror](/tiup/tiup-mirror-reference.md). These metadata files might be stored on the local file system or remotely stored using the HTTP protocol to provide a signature entry. +The `tiup mirror sign` command is used to sign the metadata files (*.json) defined in TiUP [mirror](/tiup/tiup-mirror-reference.md). These metadata files might be stored on the local file system or remotely stored using the HTTP protocol to provide a signature entry. ## Syntax diff --git a/tiup/tiup-component-cluster-check.md b/tiup/tiup-component-cluster-check.md index 7921333d812cc..2e659b6fc61e2 100644 --- a/tiup/tiup-component-cluster-check.md +++ b/tiup/tiup-component-cluster-check.md @@ -11,7 +11,7 @@ For a formal production environment, before the environment goes live, you need ### Operating system version -Check the operating system distribution and version of the deployed machines. Currently, only CentOS 7 is supported for deployment. More system versions may be supported in later releases for compatibility improvement. +Check the operating system distribution and version of the deployed machines. For a list of supported versions, see [OS and platform requirements](/hardware-and-software-requirements.md#os-and-platform-requirements). ### CPU EPOLLEXCLUSIVE @@ -19,7 +19,7 @@ Check whether the CPU of the target machine supports EPOLLEXCLUSIVE. ### numactl -Check whether numactl is installed on the target machine. If tied cores are configured on the target machine, you must install numactl. +Check whether `numactl` is installed on the target machine. If tied cores are configured on the target machine, you must install `numactl`. ### System time @@ -66,7 +66,11 @@ Check the limit values in the `/etc/security/limits.conf` file: ### SELinux -Check whether SELinux is enabled. It is recommended to disable SELinux. +SELinux must be disabled or set to permissive mode. To check the current status, use the [getenforce(8)](https://linux.die.net/man/8/getenforce) utility. + +If SELinux is not disabled, open the `/etc/selinux/config` file, locate the line starting with `SELINUX=`, and change it to `SELINUX=disabled`. After making this change, you need to reboot the system because switching from `enforcing` or `permissive` to `disabled` does not take effect without a reboot. + +On some systems (such as Ubuntu), the `/etc/selinux/config` file might not exist, and the getenforce utility might not be installed. In that case, you can skip this step. ### Firewall diff --git a/tiup/tiup-component-cluster-deploy.md b/tiup/tiup-component-cluster-deploy.md index 2c44333b7cdb0..4d93d1a3b6946 100644 --- a/tiup/tiup-component-cluster-deploy.md +++ b/tiup/tiup-component-cluster-deploy.md @@ -14,7 +14,7 @@ tiup cluster deploy [flags] ``` - ``: the name of the new cluster, which cannot be the same as the existing cluster names. -- ``: the version number of the TiDB cluster to deploy, such as `v8.0.0`. +- ``: the version number of the TiDB cluster to deploy, such as `v8.1.2`. - ``: the prepared [topology file](/tiup/tiup-cluster-topology-reference.md). ## Options diff --git a/tiup/tiup-component-cluster-enable.md b/tiup/tiup-component-cluster-enable.md index 6a8e73cfff776..16b9627cd6bcd 100644 --- a/tiup/tiup-component-cluster-enable.md +++ b/tiup/tiup-component-cluster-enable.md @@ -9,7 +9,7 @@ The `tiup cluster enable` command is used to set the auto-enabling of the cluste > **Note:** > -> When all clusters are shut down and restarted, the order of service startup is determined by the node's operating system startup order. When the restart order is incorrect, in some cases, the restarted cluster still cannot provide services. For example, if TiKV is started first but PD is not started, systemd gives up if TiKV is restarted multiple times while PD is not found). +> When all clusters are shut down and restarted, the order of service startup is determined by the node's operating system startup order. When the restart order is incorrect, in some cases, the restarted cluster still cannot provide services. For example, if TiKV is started first but PD is not started, systemd gives up if TiKV is restarted multiple times while PD is not found. ## Syntax diff --git a/tiup/tiup-component-cluster-patch.md b/tiup/tiup-component-cluster-patch.md index 69ac1d04602b6..c5ecc8c8190dd 100644 --- a/tiup/tiup-component-cluster-patch.md +++ b/tiup/tiup-component-cluster-patch.md @@ -29,7 +29,7 @@ Before running the `tiup cluster patch` command, you need to pack the binary pac 1. Determine the following variables: - `${component}`: the name of the component to be replaced (such as `tidb`, `tikv`, or `pd`). - - `${version}`: the version of the component (such as `v8.0.0` or `v7.5.1`). + - `${version}`: the version of the component (such as `v8.1.2` or `v7.5.4`). - `${os}`: the operating system (`linux`). - `${arch}`: the platform on which the component runs (`amd64`, `arm64`). @@ -70,7 +70,7 @@ After you have completed the preceding steps, you can use `/tmp/${component}-hot ### --overwrite -- After you patch a certain component (such as TiDB or TiKV), when the tiup cluster scales out the component, TiUP uses the original component version by default. To use the version that you patch when the cluster scales out in the future, you need to specified the option `--overwrite` in the command. +- After you patch a certain component (such as TiDB or TiKV), when the tiup cluster scales out the component, TiUP uses the original component version by default. To use the version that you patch when the cluster scales out in the future, you need to specify the option `--overwrite` in the command. - Data type: `BOOLEAN` - This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. diff --git a/tiup/tiup-component-cluster-reload.md b/tiup/tiup-component-cluster-reload.md index da540b492a99f..ae904659e07ab 100644 --- a/tiup/tiup-component-cluster-reload.md +++ b/tiup/tiup-component-cluster-reload.md @@ -79,6 +79,26 @@ After you specify the `--skip-restart` option, it only refreshes the configurati - Data type: `BOOLEAN` - Default: false +### --pre-restart-script + +> **Warning:** +> +> This option is experimental and is not recommended for production deployments. + +- Runs a script before the reload. +- Data type: `STRINGS` +- This option specifies the path of a script to be run on the node that is to be reloaded. It does not take effect when `--skip-restart` is set to `true`. + +### --post-restart-script + +> **Warning:** +> +> This option is experimental and is not recommended for production deployments. + +- Runs a script after the reload. +- Data type: `STRINGS` +- This option specifies the path of a script to be run after the reload of a node. It does not take effect when `--skip-restart` is set to `true`. + ## Output The execution log of the tiup-cluster. diff --git a/tiup/tiup-component-cluster-upgrade.md b/tiup/tiup-component-cluster-upgrade.md index bd192ec8ddb42..7547b492ec35e 100644 --- a/tiup/tiup-component-cluster-upgrade.md +++ b/tiup/tiup-component-cluster-upgrade.md @@ -14,7 +14,7 @@ tiup cluster upgrade [flags] ``` - ``: the cluster name to operate on. If you forget the cluster name, you can check it with the [cluster list](/tiup/tiup-component-cluster-list.md) command. -- ``: the target version to upgrade to, such as `v8.0.0`. Currently, it is only allowed to upgrade to a version higher than the current cluster, that is, no downgrade is allowed. It is also not allowed to upgrade to the nightly version. +- ``: the target version to upgrade to, such as `v8.1.2`. Currently, it is only allowed to upgrade to a version higher than the current cluster, that is, no downgrade is allowed. It is also not allowed to upgrade to the nightly version. ## Options @@ -122,6 +122,26 @@ tiup cluster upgrade [flags] - Data type: `BOOLEAN` - This option is disabled by default with the `false` value. To enable this option, add this option to the command, and either pass the `true` value or do not pass any value. +### ---pre-upgrade-script + +> **Warning:** +> +> This option is experimental and is not recommended for production deployments. + +- Runs a script before the upgrade. +- Data type: `STRINGS` +- This option specifies the path of a script to be run on the node that is to be upgraded. + +### ---post-upgrade-script + +> **Warning:** +> +> This option is experimental and is not recommended for production deployments. + +- Runs a script after the upgrade. +- Data type: `STRINGS` +- This option specifies the path of a script to be run after the upgrade of a node. This script will be executed on the upgraded node itself. + ## Output The log of the upgrading progress. diff --git a/tiup/tiup-component-dm-patch.md b/tiup/tiup-component-dm-patch.md index 6d12d214187dc..2ac267aa32bc1 100644 --- a/tiup/tiup-component-dm-patch.md +++ b/tiup/tiup-component-dm-patch.md @@ -79,7 +79,7 @@ The following example shows how to apply `v5.3.0-hotfix` to the `v5.3.0` cluster > **Note:** > -> Hotfix is used only for emergency fixes. Its daily maintenance is complicated. It is recommend that you upgrade the DM cluster to an official version as soon as it is released. +> Hotfix is used only for emergency fixes. Its daily maintenance is complicated. It is recommended that you upgrade the DM cluster to an official version as soon as it is released. ### Preparations diff --git a/tiup/tiup-component-dm-upgrade.md b/tiup/tiup-component-dm-upgrade.md index 4e578aae8f37e..5d5172f232d01 100644 --- a/tiup/tiup-component-dm-upgrade.md +++ b/tiup/tiup-component-dm-upgrade.md @@ -14,7 +14,7 @@ tiup dm upgrade [flags] ``` - `` is the name of the cluster to be operated on. If you forget the cluster name, you can check it using the [`tiup dm list`](/tiup/tiup-component-dm-list.md) command. -- `` is the target version to be upgraded to, such as `v8.0.0`. Currently, only upgrading to a later version is allowed, and upgrading to an earlier version is not allowed, which means the downgrade is not allowed. Upgrading to a nightly version is not allowed either. +- `` is the target version to be upgraded to, such as `v8.1.2`. Currently, only upgrading to a later version is allowed, and upgrading to an earlier version is not allowed, which means the downgrade is not allowed. Upgrading to a nightly version is not allowed either. ## Options diff --git a/tiup/tiup-component-dm.md b/tiup/tiup-component-dm.md index f543de0c8fa7b..98c61e49eed5b 100644 --- a/tiup/tiup-component-dm.md +++ b/tiup/tiup-component-dm.md @@ -39,7 +39,7 @@ tiup dm [command] [flags] - Specifies the maximum waiting time (in seconds) for each step in the operation process. The operation process consists of many steps, such as specifying systemctl to start or stop services, and waiting for ports to be online or offline. Each step may take several seconds. If the execution time of a step exceeds the specified timeout, the step exits with an error. - Data type: `UINT` -- If this option is not specified in the command, the maximum waiting time for each steps is `120` seconds. +- If this option is not specified in the command, the maximum waiting time for each step is `120` seconds. ### -y, --yes diff --git a/tiup/tiup-component-management.md b/tiup/tiup-component-management.md index 84a0324bc0a0b..13ed66dd65141 100644 --- a/tiup/tiup-component-management.md +++ b/tiup/tiup-component-management.md @@ -1,7 +1,6 @@ --- title: Manage TiUP Components with TiUP Commands summary: Learn how to manage TiUP components using TiUP commands. -aliases: ['/tidb/dev/manage-tiup-component','/docs/dev/tiup/manage-tiup-component/','/docs/dev/reference/tools/tiup/manage-component/'] --- # Manage TiUP Components with TiUP Commands @@ -72,12 +71,12 @@ Example 2: Use TiUP to install the nightly version of TiDB. tiup install tidb:nightly ``` -Example 3: Use TiUP to install TiKV v8.0.0. +Example 3: Use TiUP to install TiKV v8.1.2. {{< copyable "shell-regular" >}} ```shell -tiup install tikv:v8.0.0 +tiup install tikv:v8.1.2 ``` ## Upgrade components @@ -130,12 +129,12 @@ Before the component is started, TiUP creates a directory for it, and then puts If you want to start the same component multiple times and reuse the previous working directory, you can use `--tag` to specify the same name when the component is started. After the tag is specified, the working directory will *not be automatically deleted* when the instance is terminated, which makes it convenient to reuse the working directory. -Example 1: Operate TiDB v8.0.0. +Example 1: Operate TiDB v8.1.2. {{< copyable "shell-regular" >}} ```shell -tiup tidb:v8.0.0 +tiup tidb:v8.1.2 ``` Example 2: Specify the tag with which TiKV operates. @@ -221,12 +220,12 @@ The following flags are supported in this command: - If the version is ignored, adding `--all` means to uninstall all versions of this component. - If the version and the component are both ignored, adding `--all` means to uninstall all components of all versions. -Example 1: Uninstall TiDB v8.0.0. +Example 1: Uninstall TiDB v8.1.2. {{< copyable "shell-regular" >}} ```shell -tiup uninstall tidb:v8.0.0 +tiup uninstall tidb:v8.1.2 ``` Example 2: Uninstall TiKV of all versions. diff --git a/tiup/tiup-documentation-guide.md b/tiup/tiup-documentation-guide.md index 5a1b2ae51c70d..2b36a33bc8333 100644 --- a/tiup/tiup-documentation-guide.md +++ b/tiup/tiup-documentation-guide.md @@ -1,7 +1,6 @@ --- title: TiUP Documentation Map summary: Guide you through TiUP documentation with links and introductions. -aliases: ['/docs/dev/tiup/tiup-documentation-guide/'] --- # TiUP Documentation Map diff --git a/tiup/tiup-faq.md b/tiup/tiup-faq.md index a62b49b05619c..ab55eaa6ed81b 100644 --- a/tiup/tiup-faq.md +++ b/tiup/tiup-faq.md @@ -1,7 +1,6 @@ --- title: TiUP FAQs summary: Provide answers to common questions asked by TiUP users. -aliases: ['/docs/dev/tiup/tiup-faq/'] --- # TiUP FAQs diff --git a/tiup/tiup-mirror.md b/tiup/tiup-mirror.md index 714fe1f30a9fd..f7ac3effa4b8d 100644 --- a/tiup/tiup-mirror.md +++ b/tiup/tiup-mirror.md @@ -1,7 +1,6 @@ --- title: Create a Private Mirror summary: Learn how to create a private mirror. -aliases: ['/tidb/dev/tiup-mirrors','/docs/dev/tiup/tiup-mirrors/','/docs/dev/reference/tools/tiup/mirrors/'] --- # Create a Private Mirror @@ -87,9 +86,9 @@ The `tiup mirror clone` command provides many optional flags (might provide more If you want to clone only one version (not all versions) of a component, use `--=` to specify this version. For example: - - Execute the `tiup mirror clone --tidb v8.0.0` command to clone the v8.0.0 version of the TiDB component. - - Run the `tiup mirror clone --tidb v8.0.0 --tikv all` command to clone the v8.0.0 version of the TiDB component and all versions of the TiKV component. - - Run the `tiup mirror clone v8.0.0` command to clone the v8.0.0 version of all components in a cluster. + - Execute the `tiup mirror clone --tidb v8.1.2` command to clone the v8.1.2 version of the TiDB component. + - Run the `tiup mirror clone --tidb v8.1.2 --tikv all` command to clone the v8.1.2 version of the TiDB component and all versions of the TiKV component. + - Run the `tiup mirror clone v8.1.2` command to clone the v8.1.2 version of all components in a cluster. After cloning, signing keys are set up automatically. diff --git a/tiup/tiup-overview.md b/tiup/tiup-overview.md index 24f93a81ed1d9..f26e8ad0c7ef0 100644 --- a/tiup/tiup-overview.md +++ b/tiup/tiup-overview.md @@ -1,7 +1,6 @@ --- title: TiUP Overview summary: Introduce the TiUP tool and its ecosystem. -aliases: ['/docs/dev/tiup/tiup-overview/','/docs/dev/reference/tools/tiup/overview/'] --- # TiUP Overview diff --git a/tiup/tiup-playground.md b/tiup/tiup-playground.md index bf3e0b325eb43..83f8132e1bbcc 100644 --- a/tiup/tiup-playground.md +++ b/tiup/tiup-playground.md @@ -1,7 +1,6 @@ --- title: Quickly Deploy a Local TiDB Cluster summary: Learn how to quickly deploy a local TiDB cluster using the playground component of TiUP. -aliases: ['/docs/dev/tiup/tiup-playground/','/docs/dev/reference/tools/tiup/playground/'] --- # Quickly Deploy a Local TiDB Cluster @@ -22,52 +21,14 @@ This command actually performs the following operations: - Because this command does not specify the version of the playground component, TiUP first checks the latest version of the installed playground component. Assume that the latest version is v1.12.3, then this command works the same as `tiup playground:v1.12.3`. - If you have not used TiUP playground to install the TiDB, TiKV, and PD components, the playground component installs the latest stable version of these components, and then start these instances. -- Because this command does not specify the version of the TiDB, PD, and TiKV component, TiUP playground uses the latest version of each component by default. Assume that the latest version is v8.0.0, then this command works the same as `tiup playground:v1.12.3 v8.0.0`. +- Because this command does not specify the version of the TiDB, PD, and TiKV component, TiUP playground uses the latest version of each component by default. Assume that the latest version is v8.1.2, then this command works the same as `tiup playground:v1.12.3 v8.1.2`. - Because this command does not specify the number of each component, TiUP playground, by default, starts a smallest cluster that consists of one TiDB instance, one TiKV instance, one PD instance, and one TiFlash instance. - After starting each TiDB component, TiUP playground reminds you that the cluster is successfully started and provides you some useful information, such as how to connect to the TiDB cluster through the MySQL client and how to access the [TiDB Dashboard](/dashboard/dashboard-intro.md). -The command-line flags of the playground component are described as follows: +You can use the following command to view the command-line flags of the playground component: -```bash -Flags: - --db int Specify the number of TiDB instances (default: 1) - --db.host host Specify the listening address of TiDB - --db.port int Specify the port of TiDB - --db.binpath string Specify the TiDB instance binary path (optional, for debugging) - --db.config string Specify the TiDB instance configuration file (optional, for debugging) - --db.timeout int Specify TiDB maximum wait time in seconds for starting. 0 means no limit - --drainer int Specify Drainer data of the cluster - --drainer.binpath string Specify the location of the Drainer binary files (optional, for debugging) - --drainer.config string Specify the Drainer configuration file - -h, --help help for tiup - --host string Specify the listening address of each component (default: `127.0.0.1`). Set it to `0.0.0.0` if provided for access of other machines - --kv int Specify the number of TiKV instances (default: 1) - --kv.binpath string Specify the TiKV instance binary path (optional, for debugging) - --kv.config string Specify the TiKV instance configuration file (optional, for debugging) - --mode string Specify the playground mode: 'tidb' (default) and 'tikv-slim' - --pd int Specify the number of PD instances (default: 1) - --pd.host host Specify the listening address of PD - --pd.binpath string Specify the PD instance binary path (optional, for debugging) - --pd.config string Specify the PD instance configuration file (optional, for debugging) - --pump int Specify the number of Pump instances. If the value is not `0`, TiDB Binlog is enabled. - --pump.binpath string Specify the location of the Pump binary files (optional, for debugging) - --pump.config string Specify the Pump configuration file (optional, for debugging) - -T, --tag string Specify a tag for playground - --ticdc int Specify the number of TiCDC instances (default: 0) - --ticdc.binpath string Specify the TiCDC instance binary path (optional, for debugging) - --ticdc.config string Specify the TiCDC instance configuration file (optional, for debugging) - --tiflash int Specify the number of TiFlash instances (default: 1) - --tiflash.binpath string Specify the TiFlash instance binary path (optional, for debugging) - --tiflash.config string Specify the TiFlash instance configuration file (optional, for debugging) - --tiflash.timeout int Specify TiFlash maximum wait time in seconds for starting. 0 means no limit - --tiproxy int TiProxy instance number - --tiproxy.binpath string TiProxy instance binary path - --tiproxy.config string TiProxy instance configuration file - --tiproxy.host host Playground TiProxy host. If not provided, TiProxy will still use host flag as its host - --tiproxy.port int Playground TiProxy port. If not provided, TiProxy will use 6000 as its port - --tiproxy.timeout int TiProxy max wait time in seconds for starting. 0 means no limit (default 60) - -v, --version Specify the version of playground - --without-monitor Disable the monitoring function of Prometheus and Grafana. If you do not add this flag, the monitoring function is enabled by default. +```shell +tiup playground --help ``` ## Examples @@ -96,7 +57,7 @@ In the command above, `nightly` indicates the latest development version of TiDB ### Override PD's default configuration -First, you need to copy the [PD configuration template](https://github.com/pingcap/pd/blob/master/conf/config.toml). Assume you place the copied file to `~/config/pd.toml` and make some changes according to your need, then you can execute the following command to override PD's default configuration: +First, you need to copy the [PD configuration template](https://github.com/pingcap/pd/blob/release-8.1/conf/config.toml). Assume you place the copied file to `~/config/pd.toml` and make some changes according to your need, then you can execute the following command to override PD's default configuration: ```shell tiup playground --pd.config ~/config/pd.toml @@ -118,12 +79,12 @@ By default, only one instance is started for each TiDB, TiKV, and PD component. tiup playground --db 3 --pd 3 --kv 3 ``` -### Specify a tag when starting the TiDB cluster +### Specify a tag when starting the TiDB cluster to store the data After you stop a TiDB cluster started using TiUP playground, all cluster data is cleaned up as well. To start a TiDB cluster using TiUP playground and ensure that the cluster data is not cleaned up automatically, you can specify a tag when starting the cluster. After specifying the tag, you can find the cluster data in the `~/.tiup/data` directory. Run the following command to specify a tag: ```shell -tiup playground --tag +tiup playground --tag ${tag_name} ``` For a cluster started in this way, the data files are retained after the cluster is stopped. You can use this tag to start the cluster next time so that you can use the data kept since the cluster was stopped. @@ -171,4 +132,39 @@ You can specify a `pid` in the `tiup playground scale-in` command to scale in th ```shell tiup playground scale-in --pid 86526 -``` \ No newline at end of file +``` + +## Deploy TiProxy + +[TiProxy](/tiproxy/tiproxy-overview.md) is the official proxy component from PingCAP, placed between the client and the TiDB server to provide load balancing, connection persistence, service discovery, and other features for TiDB. + +Starting from TiUP v1.15.0, you can deploy TiProxy for your cluster using TiUP Playground. + +1. Create a `tidb.toml` file and add the following configuration: + + ``` + graceful-wait-before-shutdown=15 + ``` + + This configuration item controls the duration (in seconds) that TiDB waits before shutting down the server, avoiding client disconnections during cluster scaling-in operations. + +2. Start the TiDB cluster: + + ```shell + tiup playground v8.1.2 --tiproxy 1 --db.config tidb.toml + ``` + + In the playground component, TiProxy-related command-line flags are as follows: + + ```bash + Flags: + --tiproxy int The number of TiProxy nodes in the cluster. If not specified, TiProxy is not deployed. + --tiproxy.binpath string TiProxy instance binary path. + --tiproxy.config string TiProxy instance configuration file. + --tiproxy.host host Playground TiProxy host. If not provided, TiProxy will still use host flag as its host. + --tiproxy.port int Playground TiProxy port. If not provided, TiProxy will use 6000 as its port. + --tiproxy.timeout int TiProxy maximum wait time in seconds for starting. 0 means no limit (default 60). + --tiproxy.version string The version of TiProxy. If not specified, the latest version of TiProxy is deployed. + ``` + +For more information about deploying and using TiProxy, see [TiProxy installation and usage](/tiproxy/tiproxy-overview.md#installation-and-usage). diff --git a/tiup/tiup-terminology-and-concepts.md b/tiup/tiup-terminology-and-concepts.md index a44de21943c36..7ff8d201ffaec 100644 --- a/tiup/tiup-terminology-and-concepts.md +++ b/tiup/tiup-terminology-and-concepts.md @@ -1,7 +1,6 @@ --- title: TiUP Terminology and Concepts summary: Explain the terms and concepts of TiUP. -aliases: ['/docs/dev/tiup/tiup-terminology-and-concepts/'] --- # TiUP Terminology and Concepts diff --git a/tiup/tiup-troubleshooting-guide.md b/tiup/tiup-troubleshooting-guide.md index be2ca01c79f95..f6d225ed952bf 100644 --- a/tiup/tiup-troubleshooting-guide.md +++ b/tiup/tiup-troubleshooting-guide.md @@ -1,7 +1,6 @@ --- title: TiUP Troubleshooting Guide summary: Introduce the troubleshooting methods and solutions if you encounter issues when using TiUP. -aliases: ['/docs/dev/tiup/tiup-troubleshooting-guide/'] --- # TiUP Troubleshooting Guide diff --git a/transaction-isolation-levels.md b/transaction-isolation-levels.md index 013458019638a..e24efb41e955f 100644 --- a/transaction-isolation-levels.md +++ b/transaction-isolation-levels.md @@ -1,7 +1,6 @@ --- title: TiDB Transaction Isolation Levels summary: Learn about the transaction isolation levels in TiDB. -aliases: ['/docs/dev/transaction-isolation-levels/','/docs/dev/reference/transactions/transaction-isolation/'] --- # TiDB Transaction Isolation Levels @@ -95,3 +94,25 @@ In transactions with many point-write statements but a few point-write conflicts ## Difference between TiDB and MySQL Read Committed The MySQL Read Committed isolation level is in line with the Consistent Read features in most cases. There are also exceptions, such as [semi-consistent read](https://dev.mysql.com/doc/refman/8.0/en/innodb-transaction-isolation-levels.html). This special behavior is not supported in TiDB. + +## View and modify transaction isolation levels + +You can view and modify the transaction isolation level as follows. + +View the transaction isolation level of the current session: + +```sql +SHOW VARIABLES LIKE 'transaction_isolation'; +``` + +Modify the transaction isolation level of the current session: + +```sql +SET SESSION transaction_isolation = 'READ-COMMITTED'; +``` + +For more information on configuring and using transaction isolation levels, see the following documents: + +- [The system variable `transaction_isolation`](/system-variables.md#transaction_isolation) +- [Isolation level](/pessimistic-transaction.md#isolation-level) +- [`SET TRANSACTION`](/sql-statements/sql-statement-set-transaction.md) diff --git a/transaction-overview.md b/transaction-overview.md index e12e891f22dcf..07da0672ddaaa 100644 --- a/transaction-overview.md +++ b/transaction-overview.md @@ -1,7 +1,6 @@ --- title: Transactions summary: Learn transactions in TiDB. -aliases: ['/docs/dev/transaction-overview/','/docs/dev/reference/transactions/overview/'] --- # Transactions diff --git a/troubleshoot-lock-conflicts.md b/troubleshoot-lock-conflicts.md index fc1399896168c..c98b6ddb1c0d9 100644 --- a/troubleshoot-lock-conflicts.md +++ b/troubleshoot-lock-conflicts.md @@ -326,7 +326,7 @@ Solutions: ### TTL manager has timed out -The transaction execution time cannot exceed the GC time limit. In addition, the TTL time of pessimistic transactions has an upper limit, whose default value is 1 hour. Therefore, a pessimistic transaction executed for more than 1 hour will fail to commit. This timeout threshold is controlled by the TiDB parameter [`performance.max-txn-ttl`](https://github.com/pingcap/tidb/blob/master/pkg/config/config.toml.example). +The transaction execution time cannot exceed the GC time limit. In addition, the TTL time of pessimistic transactions has an upper limit, whose default value is 1 hour. Therefore, a pessimistic transaction executed for more than 1 hour will fail to commit. This timeout threshold is controlled by the TiDB parameter [`performance.max-txn-ttl`](https://github.com/pingcap/tidb/blob/release-8.1/pkg/config/config.toml.example). When the execution time of a pessimistic transaction exceeds the TTL time, the following error message occurs in the TiDB log: diff --git a/troubleshoot-tidb-cluster.md b/troubleshoot-tidb-cluster.md index c43a975b77ab2..f7ea1742b1e01 100644 --- a/troubleshoot-tidb-cluster.md +++ b/troubleshoot-tidb-cluster.md @@ -1,7 +1,6 @@ --- title: TiDB Cluster Troubleshooting Guide summary: Learn how to diagnose and resolve issues when you use TiDB. -aliases: ['/docs/dev/troubleshoot-tidb-cluster/','/docs/dev/how-to/troubleshoot/cluster-setup/'] --- # TiDB Cluster Troubleshooting Guide diff --git a/troubleshoot-tidb-oom.md b/troubleshoot-tidb-oom.md index 5362ba9f74b9c..1c2f102cd0681 100644 --- a/troubleshoot-tidb-oom.md +++ b/troubleshoot-tidb-oom.md @@ -183,7 +183,7 @@ To locate the root cause of an OOM issue, you need to collect the following info - Run the following command to collect the TiDB Profile information when memory usage is high: ```shell - curl -G http://{TiDBIP}:10080/debug/zip?seconds=10" > profile.zip + curl -G "http://{TiDBIP}:10080/debug/zip?seconds=10" > profile.zip ``` - Run `grep "tidb-server has the risk of OOM" tidb.log` to check the path of the alert file collected by TiDB Server. The following is an example output: diff --git a/tso.md b/tso.md index 897006c0120e1..dccf611654fae 100644 --- a/tso.md +++ b/tso.md @@ -5,7 +5,7 @@ summary: Learn about TimeStamp Oracle (TSO) in TiDB. # TimeStamp Oracle (TSO) in TiDB -In TiDB, the Placement Driver (PD) plays a pivotal role in allocating timestamps to various components within a cluster. These timestamps are instrumental in the assignment of temporal markers to transactions and data, a mechanism crucial for enabling the [Percolator](https://research.google.com/pubs/pub36726.html) model within TiDB. The Percolator model is used to support Multi-Version Concurrency Control (MVCC) and [transaction management](/transaction-overview.md). +In TiDB, the Placement Driver (PD) plays a pivotal role in allocating timestamps to various components within a cluster. These timestamps are instrumental in the assignment of temporal markers to transactions and data, a mechanism crucial for enabling the [Percolator](https://research.google/pubs/large-scale-incremental-processing-using-distributed-transactions-and-notifications/) model within TiDB. The Percolator model is used to support [Multi-Version Concurrency Control (MVCC)](https://docs.pingcap.com/tidb/stable/glossary#multi-version-concurrency-control-mvcc) and [transaction management](/transaction-overview.md). The following example shows how to get the current TSO in TiDB: @@ -78,7 +78,7 @@ FROM_UNIXTIME((@ts >> 18)/1000): 2023-08-27 20:33:41.6870 1 row in set (0.00 sec) ``` -The `>> 18` operation signifies a bitwise [right shift](/functions-and-operators/bit-functions-and-operators.md) by 18 bits, which is used to extract the physical timestamp. Because the physical timestamp is expressed in milliseconds, deviating from the more common UNIX timestamp format measured in seconds, you need to divide it by 1000 to convert it into a format compatible with [`FROM_UNIXTIME()`](/functions-and-operators/date-and-time-functions.md). This process aligns with the functionality of `TIDB_PARSE_TSO()`. +The `>> 18` operation signifies a bitwise [right shift](/functions-and-operators/bit-functions-and-operators.md#-right-shift) by 18 bits, which is used to extract the physical timestamp. Because the physical timestamp is expressed in milliseconds, deviating from the more common UNIX timestamp format measured in seconds, you need to divide it by 1000 to convert it into a format compatible with [`FROM_UNIXTIME()`](/functions-and-operators/date-and-time-functions.md). This process aligns with the functionality of `TIDB_PARSE_TSO()`. You can also extract the logical timestamp `000000000000000100` in binary, which is equivalent to `4` in decimal. diff --git a/tune-operating-system.md b/tune-operating-system.md index ca118bacb0133..85f8999b39d29 100644 --- a/tune-operating-system.md +++ b/tune-operating-system.md @@ -1,7 +1,6 @@ --- title: Tune Operating System Performance summary: Learn how to tune the parameters of the operating system. -aliases: ['/docs/dev/tune-operating-system/'] --- # Tune Operating System Performance @@ -10,7 +9,7 @@ This document introduces how to tune each subsystem of CentOS 7. > **Note:** > -> + The default configuration of the CentOS 7 operating system is suitable for most services running under moderate workloads. Adjusting the performance of a particular subsystem might negatively affects other subsystems. Therefore, before tuning the system, back up all the user data and configuration information. +> + The default configuration of the CentOS 7 operating system is suitable for most services running under moderate workloads. Adjusting the performance of a particular subsystem might negatively affect other subsystems. Therefore, before tuning the system, back up all the user data and configuration information. > + Fully test all the changes in the test environment before applying them to the production environment. ## Performance analysis methods diff --git a/tune-region-performance.md b/tune-region-performance.md index 52b0ae71ebee8..869771f6ea47a 100644 --- a/tune-region-performance.md +++ b/tune-region-performance.md @@ -19,15 +19,13 @@ To reduce the performance overhead of many Regions, you can also enable [Hiberna > **Note:** > -> The recommended range for the Region size is [48MiB, 258MiB]. Commonly used sizes include 96 MiB, 128 MiB, and 256 MiB. It is NOT recommended to set the Region size beyond 1 GiB. Avoid setting the size to more than 10 GiB. An excessively large Region size might result in the following side effects: +> The recommended range for the Region size is [48 MiB, 256 MiB]. Commonly used sizes include 96 MiB, 128 MiB, and 256 MiB. It is NOT recommended to set the Region size beyond 1 GiB. Avoid setting the size to more than 10 GiB. An excessively large Region size might result in the following side effects: > > + Performance jitters > + Decreased query performance, especially for queries that deal with a large range of data > + Slower Region scheduling -To adjust the Region size, you can use the [`coprocessor.region-split-size`](/tikv-configuration-file.md#region-split-size) configuration item. When TiFlash is used, the Region size should not exceed 256 MiB. - -When the Dumpling tool is used, the Region size should not exceed 1 GiB. In this case, you need to reduce the concurrency after increasing the Region size; otherwise, TiDB might run out of memory. +To adjust the Region size, you can use the [`coprocessor.region-split-size`](/tikv-configuration-file.md#region-split-size) configuration item. When TiFlash or the Dumpling tool is used, the Region size should not exceed 1 GiB. After increasing the Region size, you need to reduce the concurrency if the Dumpling tool is used; otherwise, TiDB might run out of memory. ## Use bucket to increase concurrency diff --git a/tune-tikv-memory-performance.md b/tune-tikv-memory-performance.md index 7c1c45fa60c81..81b5765d9e2d3 100644 --- a/tune-tikv-memory-performance.md +++ b/tune-tikv-memory-performance.md @@ -1,12 +1,11 @@ --- title: Tune TiKV Memory Parameter Performance summary: Learn how to tune the TiKV parameters for optimal performance. -aliases: ['/docs/dev/tune-tikv-performance/','/docs/dev/reference/performance/tune-tikv/','/tidb/dev/tune-tikv-performance'] --- # Tune TiKV Memory Parameter Performance -This document describes how to tune the TiKV parameters for optimal performance. You can find the default configuration file in [etc/config-template.toml](https://github.com/tikv/tikv/blob/master/etc/config-template.toml). To modify the configuration, you can [use TiUP](/maintain-tidb-using-tiup.md#modify-the-configuration) or [modify TiKV dynamically](/dynamic-config.md#modify-tikv-configuration-dynamically) for a limited set of configuration items. For the complete configuration, see [TiKV configuration file](/tikv-configuration-file.md). +This document describes how to tune the TiKV parameters for optimal performance. You can find the default configuration file in [etc/config-template.toml](https://github.com/tikv/tikv/blob/release-8.1/etc/config-template.toml). To modify the configuration, you can [use TiUP](/maintain-tidb-using-tiup.md#modify-the-configuration) or [modify TiKV dynamically](/dynamic-config.md#modify-tikv-configuration-dynamically) for a limited set of configuration items. For the complete configuration, see [TiKV configuration file](/tikv-configuration-file.md). TiKV uses RocksDB for persistent storage at the bottom level of the TiKV architecture. Therefore, many of the performance parameters are related to RocksDB. TiKV uses two RocksDB instances: the default RocksDB instance stores KV data, the Raft RocksDB instance (RaftDB) stores Raft logs. @@ -30,7 +29,7 @@ Each CF also has a separate `write buffer`. You can configure the size by settin ## Parameter specification -``` +```toml # Log level: trace, debug, warn, error, info, off. log-level = "info" diff --git a/tune-tikv-thread-performance.md b/tune-tikv-thread-performance.md index 36dfc3dd5ce5a..eaee3aadc52ce 100644 --- a/tune-tikv-thread-performance.md +++ b/tune-tikv-thread-performance.md @@ -1,7 +1,6 @@ --- title: Tune TiKV Thread Pool Performance summary: Learn how to tune TiKV thread pools for optimal performance. -aliases: ['/docs/dev/tune-tikv-thread-performance/'] --- # Tune TiKV Thread Pool Performance diff --git a/two-data-centers-in-one-city-deployment.md b/two-data-centers-in-one-city-deployment.md index 6f8268b6e2beb..5f51d76ca6c43 100644 --- a/two-data-centers-in-one-city-deployment.md +++ b/two-data-centers-in-one-city-deployment.md @@ -1,7 +1,6 @@ --- title: Two Availability Zones in One Region Deployment summary: Learn the deployment solution of two availability zones in one region. -aliases: ['/tidb/dev/synchronous-replication'] --- # Two Availability Zones in One Region Deployment diff --git a/upgrade-monitoring-services.md b/upgrade-monitoring-services.md index b16c729ceb234..4e899ca668572 100644 --- a/upgrade-monitoring-services.md +++ b/upgrade-monitoring-services.md @@ -36,7 +36,7 @@ Download a new installation package from the [Prometheus download page](https:// > **Tip:** > - > `{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v8.0.0` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-toolkit-v8.0.0-linux-amd64.tar.gz`. + > `{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v8.1.2` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-toolkit-v8.1.2-linux-amd64.tar.gz`. 2. In the extracted files, locate `prometheus-v{version}-linux-amd64.tar.gz` and extract it. @@ -59,7 +59,7 @@ Download a new installation package from the [Prometheus download page](https:// Execute the following command to upgrade Prometheus: ```bash -tiup cluster patch prometheus-v{new-version}.tar.gz -R prometheus +tiup cluster patch prometheus-v{new-version}.tar.gz -R prometheus --overwrite ``` After the upgrade, you can go to the home page of the Prometheus server (usually at `http://:9090`), click **Status** in the top navigation menu, and then open the **Runtime & Build Information** page to check the Prometheus version and confirm whether the upgrade is successful. @@ -85,7 +85,7 @@ In the following upgrade steps, you need to download the Grafana installation pa > **Tip:** > - > `{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v8.0.0` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-toolkit-v8.0.0-linux-amd64.tar.gz`. + > `{version}` in the link indicates the version number of TiDB and `{arch}` indicates the architecture of the system, which can be `amd64` or `arm64`. For example, the download link for `v8.1.2` in the `amd64` architecture is `https://download.pingcap.org/tidb-community-toolkit-v8.1.2-linux-amd64.tar.gz`. 2. In the extracted files, locate `grafana-v{version}-linux-amd64.tar.gz` and extract it. @@ -108,7 +108,7 @@ In the following upgrade steps, you need to download the Grafana installation pa Execute the following command to upgrade Grafana: ```bash -tiup cluster patch grafana-v{new-version}.tar.gz -R grafana +tiup cluster patch grafana-v{new-version}.tar.gz -R grafana --overwrite ``` @@ -127,7 +127,7 @@ Download the `alertmanager` installation package from the [Prometheus download p Execute the following command to upgrade Alertmanager: ```bash -tiup cluster patch alertmanager-v{new-version}-linux-amd64.tar.gz -R alertmanager +tiup cluster patch alertmanager-v{new-version}-linux-amd64.tar.gz -R alertmanager --overwrite ``` After the upgrade, you can go to the home page of the Alertmanager server (usually at `http://:9093`), click **Status** in the top navigation menu, and then check the Alertmanager version to confirm whether the upgrade is successful. \ No newline at end of file diff --git a/upgrade-tidb-using-tiup.md b/upgrade-tidb-using-tiup.md index 37729d251ea38..256881bd45103 100644 --- a/upgrade-tidb-using-tiup.md +++ b/upgrade-tidb-using-tiup.md @@ -1,29 +1,29 @@ --- title: Upgrade TiDB Using TiUP summary: Learn how to upgrade TiDB using TiUP. -aliases: ['/docs/dev/upgrade-tidb-using-tiup/','/docs/dev/how-to/upgrade/using-tiup/','/tidb/dev/upgrade-tidb-using-tiup-offline','/docs/dev/upgrade-tidb-using-tiup-offline/'] --- # Upgrade TiDB Using TiUP This document is targeted for the following upgrade paths: -- Upgrade from TiDB 4.0 versions to TiDB 8.0. -- Upgrade from TiDB 5.0-5.4 versions to TiDB 8.0. -- Upgrade from TiDB 6.0-6.6 to TiDB 8.0. -- Upgrade from TiDB 7.0-7.6 to TiDB 8.0. +- Upgrade from TiDB 4.0 versions to TiDB 8.1. +- Upgrade from TiDB 5.0-5.4 versions to TiDB 8.1. +- Upgrade from TiDB 6.0-6.6 to TiDB 8.1. +- Upgrade from TiDB 7.0-7.6 to TiDB 8.1. +- Upgrade from TiDB 8.0 to TiDB 8.1. > **Warning:** > > 1. You cannot upgrade TiFlash online from versions earlier than 5.3 to 5.3 or later. Instead, you must first stop all the TiFlash instances of the early version, and then upgrade the cluster offline. If other components (such as TiDB and TiKV) do not support an online upgrade, follow the instructions in warnings in [Online upgrade](#online-upgrade). > 2. **DO NOT** run DDL statements during the upgrade process. Otherwise, the issue of undefined behavior might occur. > 3. **DO NOT** upgrade a TiDB cluster when a DDL statement is being executed in the cluster (usually for the time-consuming DDL statements such as `ADD INDEX` and the column type changes). Before the upgrade, it is recommended to use the [`ADMIN SHOW DDL`](/sql-statements/sql-statement-admin-show-ddl.md) command to check whether the TiDB cluster has an ongoing DDL job. If the cluster has a DDL job, to upgrade the cluster, wait until the DDL execution is finished or use the [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) command to cancel the DDL job before you upgrade the cluster. -> -> If the TiDB version before upgrade is v7.1.0 or later, you can ignore the preceding warnings 2 and 3. For more information, see [TiDB Smooth Upgrade](/smooth-upgrade-tidb.md). +> 4. If the TiDB version before upgrade is 7.1.0 or later, you can ignore the preceding warnings 2 and 3. For more information, see [limitations on using TiDB smooth upgrade](/smooth-upgrade-tidb.md#limitations). +> 5. Be sure to read [limitations on user operations](/smooth-upgrade-tidb.md#limitations-on-user-operations) before upgrading your TiDB cluster using TiUP. > **Note:** > -> - If your cluster to be upgraded is v3.1 or an earlier version (v3.0 or v2.1), the direct upgrade to v8.0.0 is not supported. You need to upgrade your cluster first to v4.0 and then to v8.0.0. +> - If your cluster to be upgraded is v3.1 or an earlier version (v3.0 or v2.1), the direct upgrade to v8.1.0 or a later v8.1.x version is not supported. You need to upgrade your cluster first to v4.0 and then to the target TiDB version. > - If your cluster to be upgraded is earlier than v6.2, the upgrade might get stuck when you upgrade the cluster to v6.2 or later versions in some scenarios. You can refer to [How to fix the issue](#how-to-fix-the-issue-that-the-upgrade-gets-stuck-when-upgrading-to-v620-or-later-versions). > - TiDB nodes use the value of the [`server-version`](/tidb-configuration-file.md#server-version) configuration item to verify the current TiDB version. Therefore, to avoid unexpected behaviors, before upgrading the TiDB cluster, you need to set the value of `server-version` to empty or the real version of the current TiDB cluster. > - Setting the [`performance.force-init-stats`](/tidb-configuration-file.md#force-init-stats-new-in-v657-and-v710) configuration item to `ON` prolongs the TiDB startup time, which might cause startup timeouts and upgrade failures. To avoid this issue, it is recommended to set a longer waiting timeout for TiUP. @@ -55,16 +55,16 @@ This document is targeted for the following upgrade paths: ## Upgrade caveat - TiDB currently does not support version downgrade or rolling back to an earlier version after the upgrade. -- For the v4.0 cluster managed using TiDB Ansible, you need to import the cluster to TiUP (`tiup cluster`) for new management according to [Upgrade TiDB Using TiUP (v4.0)](https://docs.pingcap.com/tidb/v4.0/upgrade-tidb-using-tiup#import-tidb-ansible-and-the-inventoryini-configuration-to-tiup). Then you can upgrade the cluster to v8.0.0 according to this document. -- To update versions earlier than v3.0 to v8.0.0: +- For the v4.0 cluster managed using TiDB Ansible, you need to import the cluster to TiUP (`tiup cluster`) for new management according to [Upgrade TiDB Using TiUP (v4.0)](https://docs.pingcap.com/tidb/v4.0/upgrade-tidb-using-tiup#import-tidb-ansible-and-the-inventoryini-configuration-to-tiup). Then you can upgrade the cluster to v8.1.2 according to this document. +- To update versions earlier than v3.0 to v8.1.2: 1. Update this version to 3.0 using [TiDB Ansible](https://docs.pingcap.com/tidb/v3.0/upgrade-tidb-using-ansible). 2. Use TiUP (`tiup cluster`) to import the TiDB Ansible configuration. 3. Update the 3.0 version to 4.0 according to [Upgrade TiDB Using TiUP (v4.0)](https://docs.pingcap.com/tidb/v4.0/upgrade-tidb-using-tiup#import-tidb-ansible-and-the-inventoryini-configuration-to-tiup). - 4. Upgrade the cluster to v8.0.0 according to this document. + 4. Upgrade the cluster to v8.1.2 according to this document. - Support upgrading the versions of TiDB Binlog, TiCDC, TiFlash, and other components. - When upgrading TiFlash from versions earlier than v6.3.0 to v6.3.0 and later versions, note that the CPU must support the AVX2 instruction set under the Linux AMD64 architecture and the ARMv8 instruction set architecture under the Linux ARM64 architecture. For details, see the description in [v6.3.0 Release Notes](/releases/release-6.3.0.md#others). - For detailed compatibility changes of different versions, see the [Release Notes](/releases/release-notes.md) of each version. Modify your cluster configuration according to the "Compatibility Changes" section of the corresponding release notes. -- For clusters that upgrade from versions earlier than v5.3 to v5.3 or later versions, the default deployed Prometheus will upgrade from v2.8.1 to v2.27.1. Prometheus v2.27.1 provides more features and fixes a security issue. Compared with v2.8.1, alert time representation in v2.27.1 is changed. For more details, see [Prometheus commit](https://github.com/prometheus/prometheus/commit/7646cbca328278585be15fa615e22f2a50b47d06) for more details. +- When updating clusters from versions earlier than v5.3 to v5.3 or later versions, note that there is a time format change in the alerts generated by the default deployed Prometheus. This format change is introduced starting from Prometheus v2.27.1. For more information, see [Prometheus commit](https://github.com/prometheus/prometheus/commit/7646cbca328278585be15fa615e22f2a50b47d06). ## Preparations @@ -72,7 +72,13 @@ This section introduces the preparation works needed before upgrading your TiDB ### Step 1: Review compatibility changes -Review [the compatibility changes](/releases/release-8.0.0.md#compatibility-changes) in TiDB v8.0.0 release notes. If any changes affect your upgrade, take actions accordingly. +Review compatibility changes in TiDB release notes. If any changes affect your upgrade, take actions accordingly. + +The following provides compatibility changes you need to know when you upgrade from v8.0.0 to the current version (v8.1.2). If you are upgrading from v7.6.0 or earlier versions to the current version, you might also need to check the compatibility changes introduced in intermediate versions in the corresponding [release notes](/releases/release-notes.md). + +- TiDB v8.1.0 [compatibility changes](/releases/release-8.1.0.md#compatibility-changes) +- TiDB v8.1.1 [compatibility changes](/releases/release-8.1.1.md#compatibility-changes) +- TiDB v8.1.2 [release notes](/releases/release-8.1.2.md) ### Step 2: Upgrade TiUP or TiUP offline mirror @@ -147,7 +153,7 @@ Now, the offline mirror has been upgraded successfully. If an error occurs durin > Skip this step if one of the following situations applies: > > + You have not modified the configuration parameters of the original cluster. Or you have modified the configuration parameters using `tiup cluster` but no more modification is needed. -> + After the upgrade, you want to use v8.0.0's default parameter values for the unmodified configuration items. +> + After the upgrade, you want to use v8.1.2's default parameter values for the unmodified configuration items. 1. Enter the `vi` editing mode to edit the topology file: @@ -163,13 +169,17 @@ Now, the offline mirror has been upgraded successfully. If an error occurs durin > **Note:** > -> Before you upgrade the cluster to v6.6.0, make sure that the parameters you have modified in v4.0 are compatible in v8.0.0. For details, see [TiKV Configuration File](/tikv-configuration-file.md). +> Before you upgrade the cluster to v6.6.0, make sure that the parameters you have modified in v4.0 are compatible in v8.1.2. For details, see [TiKV Configuration File](/tikv-configuration-file.md). ### Step 4: Check the DDL and backup status of the cluster To avoid undefined behaviors or other unexpected problems during the upgrade, it is recommended to check the following items before the upgrade. -- Cluster DDLs: It is recommended to execute the [`ADMIN SHOW DDL`](/sql-statements/sql-statement-admin-show-ddl.md) statement to check whether there is an ongoing DDL job. If yes, wait for its execution or cancel it by executing the [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) statement before performing an upgrade. +- Cluster DDLs: + + - If you use [smooth upgrade](/smooth-upgrade-tidb.md) to upgrade TiDB to v8.1.0 or later, and the [Distributed eXecution Framework (DXF)](/tidb-distributed-execution-framework.md) is enabled, it is recommended to disable the DXF before upgrading. Otherwise, the indexes added during the upgrade process might be inconsistent with the data, leading to upgrade failures. + - If you do not use [smooth upgrade](/smooth-upgrade-tidb.md), it is recommended to use the [`ADMIN SHOW DDL`](/sql-statements/sql-statement-admin-show-ddl.md) statement to check whether ongoing DDL jobs exist. If an ongoing DDL job exists, wait for the completion of its execution or cancel it using the [`ADMIN CANCEL DDL`](/sql-statements/sql-statement-admin-cancel-ddl.md) statement before performing an upgrade. + - Cluster backup: It is recommended to execute the [`SHOW [BACKUPS|RESTORES]`](/sql-statements/sql-statement-show-backups.md) statement to check whether there is an ongoing backup or restore task in the cluster. If yes, wait for its completion before performing an upgrade. ### Step 5: Check the health status of the current cluster @@ -207,12 +217,12 @@ If your application has a maintenance window for the database to be stopped for tiup cluster upgrade ``` -For example, if you want to upgrade the cluster to v8.0.0: +For example, if you want to upgrade the cluster to v8.1.2: {{< copyable "shell-regular" >}} ```shell -tiup cluster upgrade v8.0.0 +tiup cluster upgrade v8.1.2 ``` > **Note:** @@ -260,7 +270,7 @@ tiup cluster upgrade -h | grep "version" tiup cluster stop ``` -2. Use the `upgrade` command with the `--offline` option to perform the offline upgrade. Fill in the name of your cluster for `` and the version to upgrade to for ``, such as `v8.0.0`. +2. Use the `upgrade` command with the `--offline` option to perform the offline upgrade. Fill in the name of your cluster for `` and the version to upgrade to for ``, such as `v8.1.2`. {{< copyable "shell-regular" >}} @@ -289,7 +299,7 @@ tiup cluster display ``` Cluster type: tidb Cluster name: -Cluster version: v8.0.0 +Cluster version: v8.1.2 ``` ## FAQ @@ -344,7 +354,7 @@ Starting from v6.2.0, TiDB enables the [concurrent DDL framework](/ddl-introduct ### The evict leader has waited too long during the upgrade. How to skip this step for a quick upgrade? -You can specify `--force`. Then the processes of transferring PD leader and evicting TiKV leader are skipped during the upgrade. The cluster is directly restarted to update the version, which has a great impact on the cluster that runs online. In the following command, `` is the version to upgrade to, such as `v8.0.0`. +You can specify `--force`. Then the processes of transferring PD leader and evicting TiKV leader are skipped during the upgrade. The cluster is directly restarted to update the version, which has a great impact on the cluster that runs online. In the following command, `` is the version to upgrade to, such as `v8.1.2`. {{< copyable "shell-regular" >}} @@ -359,5 +369,5 @@ You can upgrade the tool version by using TiUP to install the `ctl` component of {{< copyable "shell-regular" >}} ```shell -tiup install ctl:v8.0.0 +tiup install ctl:v8.1.2 ``` diff --git a/user-account-management.md b/user-account-management.md index feffdfc245d26..7fa15b9abf5ae 100644 --- a/user-account-management.md +++ b/user-account-management.md @@ -1,7 +1,6 @@ --- title: TiDB User Account Management summary: Learn how to manage a TiDB user account. -aliases: ['/docs/dev/user-account-management/','/docs/dev/reference/security/user-account-management/'] --- # TiDB User Account Management @@ -10,7 +9,7 @@ This document describes how to manage a TiDB user account. ## User names and passwords -TiDB stores the user accounts in the table of the `mysql.user` system database. Each account is identified by a user name and the client host. Each account may have a password. +TiDB stores the user accounts in the table of the [`mysql.user`](/mysql-schema/mysql-schema-user.md) system table. Each account is identified by a user name and the client host. Each account may have a password. You can connect to the TiDB server using the MySQL client, and use the specified account and password to login. For each user name, make sure that it contains no more than 32 characters. @@ -39,9 +38,7 @@ You can also create accounts by using third party GUI tools. CREATE USER [IF NOT EXISTS] user [IDENTIFIED BY 'auth_string']; ``` -After you assign the password, TiDB encrypts and stores the `auth_string` in the `mysql.user` table. - -{{< copyable "sql" >}} +After you assign the password, TiDB hashes and stores the `auth_string` in the [`mysql.user`](/mysql-schema/mysql-schema-user.md) table. ```sql CREATE USER 'test'@'127.0.0.1' IDENTIFIED BY 'xxx'; @@ -151,7 +148,7 @@ To remove a user account, use the `DROP USER` statement: DROP USER 'test'@'localhost'; ``` -This operation clears the user's records in the `mysql.user` table and the related records in the privilege table. +This operation clears the user's records in the [`mysql.user`](/mysql-schema/mysql-schema-user.md) table and the related records in the privilege table. ## Reserved user accounts @@ -163,7 +160,7 @@ TiDB can limit the resources consumed by users using resource groups. For more i ## Assign account passwords -TiDB stores passwords in the `mysql.user` system database. Operations that assign or update passwords are permitted only to users with the `CREATE USER` privilege, or, alternatively, privileges for the `mysql` database (`INSERT` privilege to create new accounts, `UPDATE` privilege to update existing accounts). +TiDB stores passwords in the [`mysql.user`](/mysql-schema/mysql-schema-user.md) system table. Operations that assign or update passwords are permitted only to users with the `CREATE USER` privilege, or, alternatively, privileges for the `mysql` database (`INSERT` privilege to create new accounts, `UPDATE` privilege to update existing accounts). - To assign a password when you create a new account, use `CREATE USER` and include an `IDENTIFIED BY` clause: diff --git a/user-defined-variables.md b/user-defined-variables.md index 3f2ebe74c8ea8..ef81a7f0e6f99 100644 --- a/user-defined-variables.md +++ b/user-defined-variables.md @@ -1,7 +1,6 @@ --- title: User-Defined Variables summary: Learn how to use user-defined variables. -aliases: ['/docs/dev/user-defined-variables/','/docs/dev/reference/sql/language-structure/user-defined-variables/'] --- # User-Defined Variables diff --git a/views.md b/views.md index 2229ab7e2f275..edc5b783f51f3 100644 --- a/views.md +++ b/views.md @@ -1,7 +1,6 @@ --- title: Views summary: Learn how to use views in TiDB. -aliases: ['/docs/dev/views/','/docs/dev/reference/sql/views/'] --- # Views diff --git a/wrong-index-solution.md b/wrong-index-solution.md index 7d01e42dc8e4a..46a9cc4706dd5 100644 --- a/wrong-index-solution.md +++ b/wrong-index-solution.md @@ -7,20 +7,61 @@ summary: Learn how to solve the wrong index issue. If you find that the execution speed of some query does not reach the expectation, the optimizer might choose the wrong index to run the query. +There are multiple reasons why the optimizer might choose an unexpected index: + +- **Outdated statistics**: the optimizer relies on statistics to estimate query costs. If the statistics are outdated, the optimizer might make suboptimal choices. +- **Statistics mismatch**: even if statistics are current, they might not accurately reflect the data distribution, leading to incorrect cost estimations. +- **Incorrect cost calculation**: the optimizer might miscalculate the cost of using an index due to complex query structures or data distribution. +- **Inappropriate engine selection**: in some cases, the optimizer might select a storage engine that is not optimal for the query. +- **Function pushdown limitations**: certain functions or operations might not be pushed down to storage engines, potentially affecting query performance. + +## Statistics health + You can first view the [health state of tables](/statistics.md#health-state-of-tables) in the statistics, and then solve this issue according to the different health states. -## Low health state +### Low health state The low health state means TiDB has not performed the`ANALYZE` statement for a long time. You can update the statistics by running the `ANALYZE` command. After the update, if the optimizer still uses the wrong index, refer to the next section. -## Near 100% health state +### Near 100% health state The near 100% health state suggests that the `ANALYZE` statement is just completed or was completed a short time ago. In this case, the wrong index issue might be related to TiDB's estimation logic for the number of rows. -For equivalence queries, the cause might be [Count-Min Sketch](/statistics.md#count-min-sketch). You can check whether Count-Min Sketch is the cause and take corresponding solutions. +For equivalence queries, the cause might be [Count-Min Sketch](/statistics.md#count-min-sketch). You can check whether Count-Min Sketch is the cause and take corresponding solutions. -If the cause above does not apply to your problem, you can force-select indexes by using the `USE_INDEX` or `use index` optimzer hint (see [USE_INDEX](/optimizer-hints.md#use_indext1_name-idx1_name--idx2_name-) for details). Also, you can change the query behavior by using [SQL Plan Management](/sql-plan-management.md) in a non-intrusive way. +If the cause above does not apply to your problem, you can force-select indexes by using the `USE_INDEX` or `use index` optimizer hint (see [USE_INDEX](/optimizer-hints.md#use_indext1_name-idx1_name--idx2_name-) for details). Also, you can change the query behavior by using [SQL Plan Management](/sql-plan-management.md) in a non-intrusive way. -## Other situations +### Other situations Apart from the aforementioned situations, the wrong index issue might also be caused by data updates which renders all the indexes no longer applicable. In such cases, you need to perform analysis on the conditions and data distribution to see whether new indexes can speed up the query. If so, you can add new indexes by running the [`ADD INDEX`](/sql-statements/sql-statement-add-index.md) command. + +## Statistics mismatch + +When data distribution is highly skewed, the statistics might not accurately reflect the actual data. In such cases, try configuring the options of the [`ANALYZE TABLE`](/sql-statements/sql-statement-analyze-table.md) statement. It might help improve the accuracy of statistics and better match the index. + +For example, suppose you have an `orders` table with an index on the `customer_id` column, and more than 50% of the orders share the same `customer_id`. In this case, the statistics might not represent the data distribution well, affecting query performance. + +## Cost information + +To view detailed information on execution costs, you can execute the [`EXPLAIN`](/sql-statements/sql-statement-explain.md) and [`EXPLAIN ANALYZE`](/sql-statements/sql-statement-explain-analyze.md) statements with the `FORMAT=verbose` option. According to the information, you can see cost differences between different execution paths. + +## Engine selection + +By default, TiDB chooses TiKV or TiFlash for table access based on cost estimation. You can experiment with different engines for the same query by applying engine isolation. + +For more information, see [Engine isolation](/tiflash/use-tidb-to-read-tiflash.md#engine-isolation). + +## Function pushdown + +To enhance query performance, TiDB can push down certain functions to the TiKV or TiFlash storage engine for execution. However, some functions do not support pushdown, which might limit available execution plans and potentially affect query performance. + +For expressions that support pushdown, see [TiKV supported pushdown calculations](/functions-and-operators/expressions-pushed-down.md) and [TiFlash supported pushdown calculations](/tiflash/tiflash-supported-pushdown-calculations.md). + +Note that you can also disable the pushdown of specific expressions. For more information, see [Blocklist of optimization rules and expression pushdown](/blocklist-control-plan.md). + +## See also + +- [Statistics](/statistics.md) +- [Index selection](/choose-index.md) +- [Optimizer hints](/optimizer-hints.md) +- [SQL Plan Management](/sql-plan-management.md) \ No newline at end of file