diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index b615e478f2..90deb95b3c 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -35,7 +35,4 @@ _Use this template for reporting bugs in the docs._ ## Contributing to documentation -We welcome documentation contributions! - -* For information about how to propose a change, see the [contributing guide](https://github.com/timescale/docs/blob/latest/CONTRIBUTING.md) in our GitHub repository. -* For information on style and word usage, see the [style guide](https://docs.timescale.com/about/latest/contribute-to-docs) +We welcome documentation contributions! For guidelines, see the [contributing guide](https://github.com/timescale/docs/blob/latest/CONTRIBUTING.md). diff --git a/.github/ISSUE_TEMPLATE/rfc.md b/.github/ISSUE_TEMPLATE/rfc.md index bcfc053546..1ca2efd1fb 100644 --- a/.github/ISSUE_TEMPLATE/rfc.md +++ b/.github/ISSUE_TEMPLATE/rfc.md @@ -25,7 +25,4 @@ assignees: '' ## Contributing to documentation -We welcome documentation contributions! - -* For information about how to suggest a change, see the [contributing guide](https://github.com/timescale/docs/blob/latest/CONTRIBUTING.md) in our GitHub repository. -* For information on style and word usage, see the [style guide](https://docs.timescale.com/about/latest/contribute-to-docs) +We welcome documentation contributions! For guidelines, see the [contributing guide](https://github.com/timescale/docs/blob/latest/CONTRIBUTING.md). diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 567d11e083..aa46627a82 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -8,7 +8,7 @@ Fixes #[insert issue link, if any] # Writing help -For information about style and word usage, see the [style guide](https://docs.timescale.com/about/latest/contribute-to-docs/) +For information about style and word usage, see the [Contribution guide](https://github.com/timescale/docs/blob/latest/CONTRIBUTING.md) # Review checklists diff --git a/.github/styles/Google/WordList.yml b/.github/styles/Google/WordList.yml index fba544db4a..a3026b9ca0 100644 --- a/.github/styles/Google/WordList.yml +++ b/.github/styles/Google/WordList.yml @@ -55,7 +55,6 @@ swap: static IP address: static external IP address time weighted: time-weighted url: URL - user defined actions: user-defined actions vanilla: regular|standard vs\.: versus World Wide Web: web diff --git a/.github/styles/Vocab/Timescale/accept.txt b/.github/styles/Vocab/Timescale/accept.txt index 0f12e77ca5..cd7dfb7f28 100644 --- a/.github/styles/Vocab/Timescale/accept.txt +++ b/.github/styles/Vocab/Timescale/accept.txt @@ -2,6 +2,7 @@ Aiven Alertmanager API +api Anthropic async [Aa]utoscal(?:e|ing) @@ -16,6 +17,7 @@ https? BSD [Cc]allouts? COPY +[Cc]opy [Cc]lickstreams? Cloudformation Cloudwatch diff --git a/.github/styles/config/vocabularies/Timescale/accept.txt b/.github/styles/config/vocabularies/Timescale/accept.txt new file mode 100644 index 0000000000..32ad4ac714 --- /dev/null +++ b/.github/styles/config/vocabularies/Timescale/accept.txt @@ -0,0 +1,160 @@ +[Aa]ccessor +Aiven +Alertmanager +API +api +Anthropic +async +[Aa]utoscal(?:e|ing) +[Bb]ackfill(?:ing)? +cron +csv +https? +(?i)bigint +[Bb]itemporal +[Bb]lockchains? +[Bb]oolean +BSD +[Cc]allouts? +COPY +[Cc]opy +[Cc]lickstreams? +Cloudformation +Cloudwatch +config +[Cc]onstif(?:y|ied) +[Cc]rypto +[Cc]ryptocurrenc(?:y|ies) +Datadog +[Dd]efragment(:?s|ed)? +Django +distincts +Docker +[Dd]ownsampl(?:e|ing) +erroring +Ethereum +[Ff]ailover +[Ff]inalizers? +[Ff]orex +[Gg]apfill(?:ing)? +[Gg]eospatial +GitHub +GNU +Grafana +GUC +gsed +gzip(?:ped)? +Hasura +HipChat +[Hh]ostname +href +[Hh]ypercore +[Hh]yperloglog +[Hh]yperfunctions? +[Hh]ypertables? +[Hh]ypershift +img +Indri +[Ii]nserter +[Ii]ntraday +[Iin]validation +ivfflat +jpg +JDBC +JDK +JSON +Kafka +Kaggle +Kinesis +[Ll]ambda +LangChain +LlamaIndex +LLMs +[Ll]ookups? +loopback +[Mm]atchers? +[Mm]aterializer +(?i)MST +matplotlib +[Mm]utators? +Nagios +[Nn]amespaces? +[Nn]ullable +Outflux +[Pp]ageviews? +[Pp]aralleliza(?:ble|tion) +[Pp]athname +Patroni +Paulo +[Pp]erformant +pg_dump +pg_restore +[Pp]gvector +[Pp]laintext +Plotly +pre +POSIX +PostgreSQL +[Pp]ooler? +Prometheus +PromLens +PromQL +Promscale +Protobuf +psql +[Qq]uantiles? +qStudio +RDS +[Rr]edistribut(?:e|able) +[Rr]eindex(?:ed)? +reltuples +[Rr]eusability +[Rr]unbooks? +[Ss]crollable +Sequelize +[Ss]ignups? +[Ss]iloed +(?i)smallint +sed +src +[Ss]ubquer(?:y|ies) +[Ss]ubsets? +[Ss]upersets? +[Tt]ablespaces? +Telegraf +Thanos +Threema +[Tt]iering +[Tt]imevectors? +Timescale(?:DB)? +tobs +[Tt]ransactionally +tsdbadmin +Uber +[Uu]nary +[Uu]ncomment +[Uu]nencrypted +Unix +[Uu]nmaterialized +[Uu]nregister +[Uu]nthrottled? +[Uu]ntier +[Uu]pserts? +[Rr]ebalanc(?:e|ing) +[Rr]epos? +[Rr]ollups? +[Ss]erverless +[Ss]hard(?:s|ing)? +SkipScan +(?i)timestamptz +URLs? +URIs? +UUID +versionContent +[Vv]irtualenv +WAL +[Ww]ebsockets? +Worldmap +www +Zabbix +Zipkin diff --git a/.github/styles/templates/integration.md b/.github/styles/templates/integration.md index 120033bc5c..a6eace669c 100644 --- a/.github/styles/templates/integration.md +++ b/.github/styles/templates/integration.md @@ -12,7 +12,7 @@ import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.md // Provide context for the integration steps, for example, if an additional connector is used. -// See https://docs.timescale.com/use-timescale/latest/integrations/grafana/ for an example. +// See https://docs.timescale.com/integrations/latest/grafana/ for an example. ## Prerequisites @@ -56,7 +56,7 @@ Take the following steps to with $CLOUD_LONG. -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ diff --git a/.github/workflows/delete-preview.yml b/.github/workflows/delete-preview.yml new file mode 100644 index 0000000000..8a3e230ccc --- /dev/null +++ b/.github/workflows/delete-preview.yml @@ -0,0 +1,22 @@ +name: PR preview cleanup + +on: + pull_request: + types: [closed] + +jobs: + remove-rpeview: + runs-on: ubuntu-latest + name: Remove preview + steps: + - name: Set variables + id: timescale + run: | + echo "DEV_FOLDER=$(echo ${GITHUB_HEAD_REF})" >> $GITHUB_OUTPUT + - name: Repository Dispatch + uses: peter-evans/repository-dispatch@26b39ed245ab8f31526069329e112ab2fb224588 + with: + token: ${{ secrets.ORG_AUTOMATION_TOKEN }} + repository: timescale/web-documentation + event-type: remove-docs-content + client-payload: '{"branch": "${{ steps.timescale.outputs.DEV_FOLDER }}", "pr_number": "${{ env.PR_NUMBER }}"}' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5d2be4e9bc..5ebd236f9d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,202 +1,164 @@ -# Introduction +# Contribute to Timescale documentation + +Timescale documentation is open for contribution from all community members. The current source is in this repository. + +This page explains the structure and language guidelines for contributing to Timescale documentation. See the [README][readme] for how to contribute. + +## Language + +Write in a clear, concise, and actionable manner. Timescale documentation uses the [Google Developer Documentation Style Guide][google-style] with the following exceptions: + +- Do not capitalize the first word after a colon. +- Use code font (back ticks) for UI elements instead of semi-bold. + +## Edit individual pages + +Each major doc section has a dedicated directory with `.md` files inside, representing its child pages. This includes an `index.md` file that serves as a landing page for that doc section by default, unless specifically changed in the navigation tree. To edit a page, modify the corresponding `.md` file following these recommendations: + +- **Regular pages** should include: + + - A short intro describing the main subject of the page. + - A visual illustrating the main concept, if relevant. + - Paragraphs with descriptive headers, organizing the content into logical sections. + - Procedures to describe the sequence of steps to reach a certain goal. For example, create a Timescale service. + - Other visual aids, if necessary. + - Links to other relevant resources. + +- **API pages** should include: + + - The function name, with empty parentheses if it takes arguments. + - A brief, specific description of the function, including any possible warnings. + - One or two samples of the function being used to demonstrate argument syntax. + - An argument table with `Name`, `Type`, `Default`, `Required`, `Description` columns. + - A return table with `Column`, `Type`, and `Description` columns. -There are multiple ways to help make Timescale better, including contributing -to the documentation. All of our documentation is available to use and review -with GitHub. +- **Troubleshooting pages** are not written as whole Markdown files, but are programmatically assembled from individual files in the`_troubleshooting` folder. Each entry describes a single troubleshooting case and its solution, and contains the following front matter: + + |Key| Type |Required| Description | + |-|-------|-|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| + |`title`| string |✅| The title of the troubleshooting entry, displayed as a heading above it | + |`section`| The literal string `troubleshooting` |✅| Must be `troubleshooting`, used to identify troubleshooting entries during site build | + |`products` or `topics`| array of strings |✅ (can have either or both, but must have at least one)| The products or topics related to the entry. The entry shows up on the troubleshooting pages for the listed products and topics. | + |`errors`| object of form `{language: string, message: string}` |❌| The error, if any, related to the troubleshooting entry. Displayed as a code block right underneath the title. `language` is the programming language to use for syntax highlighting. | + |`keywords`| array of strings |❌| These are displayed at the bottom of every troubleshooting page. Each keyword links to a collection of all pages associated with that keyword. | + |`tags`| array of strings |❌| Concepts, actions, or things associated with the troubleshooting entry. These are not displayed in the UI, but they affect the calculation of related pages. | + + Beneath the front matter, describe the error and its solution in regular Markdown. You can also use any other components allowed within the docs site. + + The entry shows up on the troubleshooting pages for its associated products and topics. If the page doesn't already exist, add an entry for it in the page + index, setting `type` to `placeholder`. See [Navigation tree](#navigation-tree). -## First contribution +## Edit the navigation hierarchy -You can make contributions to the documentation by creating a fork of the -repository. However, if you have write access to the repository, use a branch -instead. Some of our automation does not work correctly on forks. +The navigation hierarchy of a doc section is governed by `page-index/page-index.js` within the corresponding directory. For example: - +```js + { + title: "Timescale Cloud services", + href: "services", + excerpt: "About Timescale Cloud services", + children: [ + { + title: "Services overview", + href: "service-overview", + excerpt: "Timescale services overview", + }, + { + title: "Service explorer", + href: "service-explorer", + excerpt: "Timescale services explorer", + }, + { + title: "Troubleshooting Timescale services", + href: "troubleshooting", + type: "placeholder", + }, + ], + }, +``` -### Contributing using a fork +See [Use Timescale section navigation][use-navigation] for reference. -1. Make sure you have a [GitHub](https://github.com) account, and that - you're signed in. -1. Navigate to the - [Timescale documentation repo](https://github.com/timescale/docs), - click the `Fork` button in the top-right corner, and select the account you - want to use. -1. Wait for GitHub to create your fork and redirect you. -1. Clone the repository to your local machine. To find this URL, click the green - `Code` button and copy the HTTPS URL: +To change the structure, add or delete pages in a section, modify the corresponding `page-index.js`. An entry in a `page-index.js` includes the following fields: - ```bash - git clone https://github.com//docs.git - ``` +| Key | Type | Required | Description | +|--------------------|-----------------------------------------------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `href` | string | ✅ | The URL segment to use for the page. If there is a corresponding Markdown file, `href` must match the name of the Markdown file, minus the file extension. | +| `title` | string | ✅ | The title of the page, used as the page name within the TOC on the left. Must be the same as the first header in the corresponding Markdown file. | +| `excerpt` | string | ✅ | The short description of the page, used for the page card if `pageComponents` is set to `featured-cards`. Should be up to 100 characters. See `pageComponents` for details. | +| `type` | One of `[directory, placeholder, redirect-to-child-page]` | ❌ | If no type is specified, the page is built as a regular webpage. The structure of its children, if present, is defined by `children` entries and the corresponding structure of subfolders. If the type is `directory`, the corresponding file becomes a directory. The difference of the directory page is that its child pages sit at the same level as the `directory` page. They only become children during the site build. If the type is `placeholder`, the corresponding page is produced programmatically upon site build. If not produced, the link in the navigation tree returns a 404. In particular, this is used for troubleshooting pages. If the type is `redirect-to-child-page`, no page is built and the link in the navigation tree goes directly to the first child. | +| `children` | Array of page entries | ❌ | Child pages of the current page. For regular pages, the children should be located in a directory with the same name as the parent. The parent is the `index.md` file in that directory. For`directory` pages, the children should be located in the same directory as the parent. | +| `pageComponents` | One of `[['featured-cards'], ['content-list']]` | ❌ | Any page that has child pages can list its children in either card or list style at the bottom of the page. Specify the desired style with this key. | +| `featuredChildren` | Array of URLs | ❌ | Similar to `pageComponents`, this displays the children of the current page, but only the selected ones. | +| `index` | string | ❌ | If a section landing page needs to be different from the `index.md` file in that directory, this field specifies the corresponding Markdown file name. | -1. List the current remote branches: +## Reuse text in multiple pages - ```bash - git remote -v - ``` +Partials allow you to reuse snippets of content in multiple places. All partials +live in the `_partials` top-level directory. To make a new partial, create a new +`.md` file in this directory. The filename must start with an underscore. Then import it into the target page as an `.mdx` file and reference in the relevant place. See [Formatting examples][formatting]. - This command should list two remotes, both marked `origin`, like this: +## Formatting - ```bash - origin https://github.com//docs.git (fetch) - origin https://github.com//docs.git (push) - ``` +In addition to all the [regular Markdown formatting][markdown-syntax], the following elements are available for Timescale docs: - The `origin` remotes are your own fork, and you can do whatever you want - here without changing the upstream repository. -1. Add the docs repo as an upstream: +- Procedure blocks +- Highlight blocks +- Tabs +- Code blocks without line numbers and the copy button +- Multi-tab code blocks +- Tags - ```bash - git remote add upstream https://github.com/timescale/docs.git - ``` +See [Formatting examples][formatting] for how to use them. -1. Check: +## Variables - ```bash - git remote -v - ``` +Timescale documentation uses variables for its product names, features, and UI elements in Timescale Console with the following syntax: `$VARIABLE_NAME`. Variables do not work inside the following: - This command should now have the same two `origin` remotes as before, plus - two more labelled `upstream`, like this: +- Front matter on each page +- HTML tables and tabs + +See the [full list of available variables][variables]. - ```bash - origin https://github.com//docs.git (fetch) - origin https://github.com//docs.git (push) - upstream https://github.com/timescale/docs.git (fetch) - upstream https://github.com/timescale/docs.git (push) - ``` +## Links -1. Fetch the branches in the upstream repository: +- Internal page links: internal links do not need to include the domain name `https://docs.timescale.com`. Use the `:currentVersion:` variable instead of `latest` in the URL. +- External links: input external links as is. - ```bash - git fetch upstream - ``` +See [Formatting examples][formatting] for details. + +## Visuals -1. Merge the changes from the upstream `latest` branch, into your fork's - `latest` branch: +When adding screenshots to the docs, aim for a full-screen view to provide better context. Reduce the size of your browser so there is as little wasted space as possible. - ```bash - git merge upstream/latest - ``` +Attach the image to your issue or PR, and the doc team uploads and inserts it for you. -1. Create a new branch for the work you want to do. Make sure you give it an - appropriate name, and include your username: +## SEO optimization - ```bash - git checkout -b update-readme-username - ``` +To make a documentation page more visible and clear for Google: - +- Include the `title` and `excerpt` meta tags at the top of the page. These represent meta title and description required for SEO optimization. - + - `title`: up to 60 characters, a short description of the page contents. In most cases a variation of the page title. + - `excerpt`: under 200 characters, a longer description of the page contents. In most cases a variation of the page intro. -### Committing changes and creating a pull request +- Summarize the contents of each paragraph in the first sentence of that paragraph. +- Include main page keywords into the meta tags, page title, first header, and intro. These are usually the names of features described in the page. For example, for a page dedicated to creating hypertables, you can use the keyword **hypertable** in the following way: -1. Make your changes. -1. Add the updated files to your commit: + - Title: Create a hypertable in Timescale Cloud + - Description: Turn a regular PostgreSQL table into a hypertable in a few steps, using Timescale Console. + - First header: Create a hypertable - ```bash - git add . - ``` +## Docs for deprecated products -1. Commit your changes: +The previous documentation source is in the deprecated repository called [docs.timescale.com-content][legacy-source]. - ```bash - git commit -m "Commit message here" - ``` - -1. Push your changes: - - ```bash - git push - ``` - - If git prompts you to set an upstream in order to push, use this command: - - ```bash - git push --set-upstream origin - ``` - -1. Create a pull request (PR) by navigating to - and clicking - `Compare and Create Pull Request`. Write an informative commit message - detailing your changes, choose reviewers, and save your PR. If you haven't - yet finished the work you want to do, make sure you create a draft PR by - selecting it from the drop down box in the GitHub web UI. This lets your - reviewers know that you haven't finished work yet, while still being - transparent about what you are working on, and making sure we all understand - current progress. - - - - -Choose your reviewers carefully! If you have made changes to the technical -detail of the documentation, choose an appropriate subject matter expert (SME) -to review those changes. Additionally, every change requires at least one -documentation team member to approve. Ask the documentation team for a review by -adding the `timescale/documentation` group as a reviewer. - - -## Second contribution - -When you have checked out the repo, if you want to keep working on things, you -need to make sure that your local copy of the repo stays up to date. If you -don't do this, you *will* end up with merge conflicts. - - - -### Second contribution - -1. Check out your fork's `latest` branch: - - ```bash - git checkout latest - ``` - - You get a message like this: - - ```bash - Switched to branch 'latest' - Your branch is up to date with 'origin/latest'. - ``` - - BEWARE! This is usually a lie! -1. Fetch the branches in the upstream repository: - - ```bash - git fetch upstream - ``` - -1. Merge the changes from the upstream `latest` branch, into your fork's - `latest` branch: - - ```bash - git merge upstream/latest - ``` - -1. If you are continuing work you began earlier, check out the branch that - contains your work. For new work, create a new branch. Doing this regularly - as you are working means you keep your local copies up to date and avoid - conflicts. You should do it at least every day before you begin work, and - again whenever you switch branches. - - - - -Never leave branches lying around on your local system. Create your PR as soon -as possible, and make good use of the Draft feature. Commit to your feature -branch early and often! Update your local copy from latest whenever you switch -branches. - - -## Reviewing standards - -When you create a pull request, a member of the documentation team will review -it for accuracy and adherance to our standards. You can see a list of the things -that reviewers check for in the pull request template. - -## Writing standards - -Timescale has comprehensive writing and style standards, that are constantly -being updated and improved. For the current guidelines, see -[contributing to documentation](https://docs.timescale.com/about/latest/contribute-to-docs/). +[legacy-source]: https://github.com/timescale/docs.timescale.com-content +[google-style]: https://developers.google.com/style +[markdown-syntax]: https://www.markdownguide.org/extended-syntax/ +[github-docs]: https://github.com/timescale/docs +[use-navigation]: use-timescale/page-index/page-index.js +[formatting]: _partials/_formatting_examples.md +[variables]: https://docs.timescale.com/variables-for-contributors/ +[readme]: README.md \ No newline at end of file diff --git a/README.md b/README.md index 6148df9aca..0197c58c5e 100644 --- a/README.md +++ b/README.md @@ -1,331 +1,55 @@ -# README +
+ + + + Timescale logo + +
-This is the source for content for docs.timescale.com, starting with release 2.0. -All documentation for previous versions is in the deprecated repository called -[docs.timescale.com-content](https://github.com/timescale/docs.timescale.com-content). +
-The documentation site is statically generated with -[Gatsby](https://www.gatsbyjs.com/). The source code is in a separate -repository, which pulls in content from this repository on each build. +

Timescale Cloud is the modern PostgreSQL data platform for all your applications. It enhances PostgreSQL to handle time series, events, real-time analytics, and vector search—all in a single database alongside transactional workloads. +

-**All files are written in standard markdown.** +[![Docs](https://img.shields.io/badge/Read_the_Timescale_docs-black?style=for-the-badge&logo=readthedocs&logoColor=white)](https://docs.timescale.com/) +[![SLACK](https://img.shields.io/badge/Ask_the_Timescale_community-black?style=for-the-badge&logo=slack&logoColor=white)](https://timescaledb.slack.com/archives/C4GT3N90X) +[![Try Timescale Cloud for free](https://img.shields.io/badge/Try_Timescale_for_free-black?style=for-the-badge&logo=timescale&logoColor=white)](https://console.cloud.timescale.com/signup) -## Contributing +
-We welcome and appreciate any help the community can provide to make -Timescale's documentation better. +This repository contains the current source for Timescale documentation available at https://docs.timescale.com/. -You can help either by opening an -[issue](https://github.com/timescale/docs/issues) with -any suggestions or bug reports, or by forking this repository, making your own -contribution, and submitting a pull request. +We welcome contributions! You can contribute to Timescale documentation in the following ways: -Before we accept any contributions, Timescale contributors need to -sign the [Contributor License Agreement](https://cla-assistant.io/timescale/docs)(CLA). -By signing a CLA, we can ensure that the community is free and confident in its -ability to use your contributions. +- [Create an issue][docs-issues] in this repository and describe the proposed change. Our doc team takes care of it. +- Update the docs yourself and have your change reviewed and published by our doc team. -## Docs review +## Contribute to the Timescale docs -Once a PR has been started for any branch, a GitHub action will attach a unique -URL to preview your changes and allow others to more effectively review your -updates. The URL will be added as a comment to your PR. +To make the contribution yourself: -Any new commits to that PR will rebuild and be visible at the same URL. +1. Get the documentation source: -## Main sections and tree navigation + - No write access? [Fork this repository][github-fork]. + - Already have a write access? [Clone this repository][github-clone]. -Each major section that is incorporated into docs.timescale.com has a navigation -hierarchy governed by the appropriate `page-index.js` file and underlying -directory structure. Depending on the content you are adding or editing, you may -have to modify the `page-index.js` file that exists for that specific sub-project. +2. Create a branch from `latest`, make your changes, and raise a pull request back to `latest`. -For instance, if you are adding a new function to the 'API Reference' section, -then you need to modify the `page-index.js` file inside of `api-reference` folder, -specifically at `api-reference/page-index/page-index.js`. +3. Sign a Contributor License Agreement (CLA). -Most content is then held within the parent folder as defined by the hierarchy. For -instance, the **Overview** section is a parent to a number of other pages and -folders. Therefore, there is a folder in the repo for `overview` with more Markdown -files and folder inside as the hierarchy dictates. + You have to sign the CLA only the first time you raise a PR. This helps to ensure that the community is free to use your contributions. -As a general rule, every folder must have an `index.md` file inside which will -have the content and layout for the "landing page" of that parent folder. +4. Review your changes. -### `page-index.js` layout + The documentation site is generated in a separate private repository using [Gatsby][gatsby]. Once you raise a PR for any branch, GitHub **automatically** generates a preview for your changes and attach the link in the comments. Any new commits are visible at the same URL. If you don't see the latest changes, try an incognito browser window. Automated builds are not available for PRs from forked repositories. -The format of this file is generally self explanatory, but the following -rules should be understood to ensure you get the results you expect. +See the [Contributing guide](CONTRIBUTING.md) for style and language guidance. -|Key|Type|Required|Description| -|-|-|-|-| -|`href`|string|Yes|The URL segment to use for the page. If there is a corresponding Markdown file, `href` must also match the name of the Markdown file, minus the file extension.| -|`title`|string|Yes|The title of the page, used as the page name within the navigation tree| -|`type`|One of `[directory, placeholder, redirect-to-child-page]`|No|If no type is specified, the page is built as a default page, turning the corresponding Markdown file into a webpage. If the type is `directory`, the corresponding file is turned into a webpage, _and_ the page becomes a directory. `directory` pages are the exception to the rule that the page index matches the file directory structure. Child pages of `directory` pages sit on the same level as the `directory` page inside the repository. They only become children during the site build. If the type is `placeholder`, an entry is made in the navigation tree, but a Markdown file is not converted into a webpage. The Markdown file doesn't even need to exist. Rather, the corresponding page is produced programmatically upon site build. If not produced, the link in the navigation tree returns a 404. If the type is `redirect-to-child-page`, an entry is made in the navigation tree, no page is built, and the link in the navigation tree goes directly to the first child.| -|`children`|Array of page entries|No|Child pages of the current page. If the parent page's type is not `directory`, the children should be located in a directory with the same name as the parent. The parent is the `index.md` file in that directory. If the parent page's type is `directory`, the children should be located in the same directory as the parent.| -|`pageComponents`|One of `[['featured-cards'], ['content-list']]`|No|Any page that has child pages can list its children in either card or list style at the bottom of the page. Specify the desired style with this key.| +## Learn about Timescale -**Example** +Timescale is PostgreSQL made powerful. To learn more about the company and its products, visit [timescale.com](https://www.timescale.com). -```js - href: "overview", - pageComponents: ["featured-cards"], - children: [ - { - title: "What is time-series data?", - href: "what-is-time-series-data", - }, - { - title: "Core concepts", - href: "core-concepts", - children : [ - { - title: "Hypertables & Chunks", - href: "hypertables-and-chunks", - }, - { - title: "Scaling", - href: "scaling", - }, - ... - ], - }, - ] -``` - -## Formatting and content rules - -### Internal page links - -None of the internal page links within these files work on GitHub inside of -the raw Markdown files that are being reviewed. Instead, the review link discussed -above should be utilized for verifying all internal links. - -Internal links do not need to include the domain name, . - -### External links: - -Input as-is. - -### Anchor tags - -By default, H1, H2, H3, and H4 headings have automatically generated anchor -tags. H2 headings also show up in the Table of Contents (on the right side of -the screen on wide windows). - -### Code blocks - -#### Command formatting - -When showing commands being entered from a command line, do not include a -character for the prompt. Do this: - -```bash -some_command -``` - -instead of this: - -```bash -some_command -``` - -or this: - -```bash -> some_command -``` - -Otherwise the code highlighter may be disrupted. - -#### Syntax highlighting - -When using a code block, add the appropriate language identifier after the -initial three backticks to provide the appropriate highlighting on the -rendered documentation site. - -Programming language samples aside, most code blocks are one of: -`bash`, `sql`, `json`. - -#### Line numbers and copy button - -Code blocks have line numbers and copy buttons by default. To remove line -numbers and copy buttons on a case-by-case basis, use the `CodeBlock` component -directly rather than a native Markdown code block. - -To remove the copy button, set `canCopy` to `false`. - -To remove line numbers, set `showLineNumbers` to `false`. - -```markdown - -``` - -### Partials - -Partials allow you to reuse snippets of content in multiple places. All partials -live in the `_partials` top-level directory. To make a new partial, create a new -`.mdx` file. The filename must start with an underscore. - -To insert the partial in another document, put an import statement in the -document. The import statement goes before the content and after any -frontmatter. For example: - -`import PartialName from 'versionContent/_partials/_partial-name.mdx';` - -`PartialName` can be any CamelCased name, but it is recommended to use the -CamelCased filename to prevent name clashes, because partial names are global -across all MDX files. - -Where you need the partial to display, insert it as a self-closing tag: - -`` - -### General formatting conventions - -To maintain consistency, please follow these general rules. - -* Maintain text editor width for paragraphs at 80 characters. We ask you to do -this to assist in reviewing documentation changes. When text is very wide, it -is difficult to visually see where text has changed within a paragraph and keeping -a narrow width on text assists in making PRs easier to review. **Most editors such -as Visual Studio Code have settings to do this visually.** -* Most links should be reference-style links where the link address is at the -bottom of the page. The two exceptions are: - * Links within highlight blocks (Note, Important, or Warning). These must be inline links for now - * Links to anchors on the same page as the link itself. -* All functions, commands and standalone function arguments (ex. `SELECT`, -`time_bucket`) should be set as inline code within backticks ("\`command\`"). -* Functions should not be written with parentheses unless the function is -being written with arguments within the parentheses. -* "PostgreSQL" is the way to write the elephant database name, rather than -"Postgres." "TimescaleDB" refers to the database, "Timescale" refers to the -company. -* Use backticks when referring to the object of a user interface action. -For example: Click `Get started` to proceed with the tutorial. - -### Callout and highlight blocks - -To create a callout around a paragraph of text, wrap it with the following custom -React component tag. **Reminder**, any links within the callout text MUST have -inline link styles. - -The `type` can currently support a value of `"note"`, `"warning"`, -`"important"`, `"deprecation"` or `"cloud"`". `cloud` produces a callout -for content specifically referring to hosted Timescale. - -```html - -Callout text goes here... - -Example link style would [look like this](http://example.com/) - -``` - -### Tags - -You can use tags to indicate links to downloadable files, or to indicate -metadata about functions. Available tags: - -* Download tags: `Markdown link to download` -* Experimental tags: `` or - `` -* Toolkit tag: `` -* Community tag: `` - -By default, tags have a solid background and gray text. There is also a hollow -variant: - -```markdown -Text to display in a tag -``` - -### Procedures - -Procedures are used to indicate a sequence of steps. For syntax, see [the -procedure example](./_procedure-block.md). - -### Optional label - -Used to indicate an optional step within a procedure. Syntax: `` - -### Multi-code blocks - -Multi-code blocks are code blocks with a language or OS selector. For syntax, -see [the multi-code-block example](./_multi-code-block.md). - -### Tabs - -Tabs can be used to display content that differs based on a user selection. The -syntax is: - -```md - - - - -Content goes here - - - - - -Content goes here - - - - -``` - -Note that spacing is important. - -### Editing troubleshooting sections - -Troubleshooting pages are not written as whole Markdown files, but are -programmatically assembled from troubleshooting entries in the -`_troubleshooting` folder. Each entry describes a single troubleshooting case -and its solution, and contains the following frontmatter: - -|Key|Type|Required|Description| -|-|-|-|-| -|`title`|string|Yes|The title of the troubleshooting entry, displayed as a heading above it| -|`section`|the literal string `troubleshooting`|Yes|Must be `troubleshooting`, used to identify troubleshooting entries during site build| -|`products` or `topics`|array of strings|Yes (can have either or both, but must have at least one)|The products or topics related to the entry. The entry will show up on the troubleshooting pages for the listed products and topics.| -|`errors`|object of form `{language: string, message: string}`|No|The error, if any, related to the troubleshooting entry. Displayed as a code block right underneath the title. `language` is the programming language to use for syntax highlighting.| -|`keywords`|array of strings|No|These are displayed at the bottom of every troubleshooting page. Each keyword links to a collection of all pages associated with that keyword.| -|`tags`|array of strings|No|Concepts, actions, or things associated with the troubleshooting entry. These are not displayed in the UI, but they affect the calculation of related pages.| - -Beneath the frontmatter, describe the error and its solution in normal Markdown. -You can also use any other components allowed within the docs site. - -The entry shows up on the troubleshooting pages for its associated products and -topics. If the page doesn't already exist, add an entry for it in the page -index, setting `type` to `placeholder`. For more information, see the section on -[page index layout](#page-indexjs-layout). - -### Editing the API section - -There is a specific format for the API section which consists of: - -* **Function name** with empty parentheses (if function takes arguments). Ex. `add_dimension()` -* A brief, specific description of the function -* Any warnings necessary -* **Required Arguments** - * A table with columns for "Name," "Type," and "Description" -* **Optional Arguments** - * A table with columns for "Name," "Type," and "Description" -* Any specific instructions about the arguments, including valid types -* **Sample Usage** - * One or two literal examples of the function being used to demonstrate argument syntax. - -See the API file to get an idea. +[docs-issues]: https://github.com/timescale/docs/issues +[github-fork]: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo +[github-clone]: https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository +[gatsby]: https://www.gatsbyjs.com/ \ No newline at end of file diff --git a/_code-samples/toolkit/two-step_aggregation.md b/_code-samples/toolkit/two-step_aggregation.md index 279efd2bc8..e6aa015457 100644 --- a/_code-samples/toolkit/two-step_aggregation.md +++ b/_code-samples/toolkit/two-step_aggregation.md @@ -151,6 +151,6 @@ GROUP BY id, time_bucket('1 day'::interval, bucket) ##### NB: There are some two-step aggregates like `tdigest` __ADD LINK? and expose and other bits...__ when we document that function where two-step aggregation can lead to more error or different results, because the algorithm is not deterministic in its re-aggregation, but we will note that clearly in the documentation when that happens, it is unusual. ### Retrospective analysis over downsampled data -[Continuous aggregates][continuous-aggregates] (or separate aggregation tables powered by a cron job or [user-defined action]( __LINK__ ) ) aren't just used for speeding up queries, they're also used for [data retention]( __LINK__ ). But this can mean that they are very difficult to modify as your data ages. Unfortunately this is also when you are learning more things about the analysis you want to do on your data. By keeping them in their raw aggregate form, the user has the flexibility to apply different accessors to do retrospective analysis. With a one-step aggregate the user needs to determine, say, which percentiles are important when we create the continous aggregate, with a two-step aggregate the user can simply determine they're going to want an approximate percentile, and then determine when doing the analysis whether they want the median, the 90th, 95th or 1st percentile. No need to modify the aggregate or try to re-calculate from data that may no longer exist in the system. +[Continuous aggregates][continuous-aggregates] (or separate aggregation tables powered by a cron job or a [custom job]( __LINK__ ) ) aren't just used for speeding up queries, they're also used for [data retention]( __LINK__ ). But this can mean that they are very difficult to modify as your data ages. Unfortunately this is also when you are learning more things about the analysis you want to do on your data. By keeping them in their raw aggregate form, the user has the flexibility to apply different accessors to do retrospective analysis. With a one-step aggregate the user needs to determine, say, which percentiles are important when we create the continous aggregate, with a two-step aggregate the user can simply determine they're going to want an approximate percentile, and then determine when doing the analysis whether they want the median, the 90th, 95th or 1st percentile. No need to modify the aggregate or try to re-calculate from data that may no longer exist in the system. [continuous-aggregates]: /use-timescale/:currentVersion:/continuous-aggregates/about-continuous-aggregates/ \ No newline at end of file diff --git a/_multi-code-block.md b/_multi-code-block.md deleted file mode 100644 index fe2c36d061..0000000000 --- a/_multi-code-block.md +++ /dev/null @@ -1,37 +0,0 @@ - - - - - -```ruby -class AddTimescale < ActiveRecord::Migration[5.2] - def change - enable_extension("timescaledb") unless extensions.include? "timescaledb" - end -end -``` - - - - - -```python -def start: - if this: - return - -``` - - - - - -```python -def different_python: - return - -``` - - - - diff --git a/_partials/_add-data-blockchain.md b/_partials/_add-data-blockchain.md index 924f31d044..5e359ee738 100644 --- a/_partials/_add-data-blockchain.md +++ b/_partials/_add-data-blockchain.md @@ -1,16 +1,14 @@ ## Load financial data -This tutorial uses Bitcoin transactions from the past five days. - -## Ingest the dataset +The dataset contains around 1.5 million Bitcoin transactions, the trades for five days. It includes +information about each transaction, along with the value in [satoshi][satoshi-def]. It also states if a +trade is a [coinbase][coinbase-def] transaction, and the reward a coin miner receives for mining the coin. To ingest data into the tables that you created, you need to download the dataset and copy the data to your database. -### Ingesting the dataset - 1. Download the `bitcoin_sample.zip` file. The file contains a `.csv` file that contains Bitcoin transactions for the past five days. Download: @@ -37,3 +35,6 @@ dataset and copy the data to your database. resources. + +[satoshi-def]: https://www.pcmag.com/encyclopedia/term/satoshi +[coinbase-def]: https://www.pcmag.com/encyclopedia/term/coinbase-transaction diff --git a/_partials/_add-data-energy.md b/_partials/_add-data-energy.md index a10dc6b3e6..f9d5e32c4c 100644 --- a/_partials/_add-data-energy.md +++ b/_partials/_add-data-energy.md @@ -5,8 +5,6 @@ into the `metrics` hypertable. -### Loading energy consumption data - This is a large dataset, so it might take a long time, depending on your network connection. @@ -48,5 +46,3 @@ connection. ``` - -[parallel-copy]: https://github.com/timescale/timescaledb-parallel-copy diff --git a/_partials/_add-data-nyctaxis.md b/_partials/_add-data-nyctaxis.md index 60b9f5231a..e8aafd1537 100644 --- a/_partials/_add-data-nyctaxis.md +++ b/_partials/_add-data-nyctaxis.md @@ -5,8 +5,6 @@ When you have your database set up, you can load the taxi trip data into the -### Loading trip data - This is a large dataset, so it might take a long time, depending on your network connection. @@ -60,5 +58,3 @@ tolls_amount | 0 improvement_surcharge | 0.3 total_amount | 10.3 ``` - -[parallel-copy]: https://github.com/timescale/timescaledb-parallel-copy diff --git a/_partials/_add-data-twelvedata-crypto.md b/_partials/_add-data-twelvedata-crypto.md index 5e35ecb48a..3021b352d4 100644 --- a/_partials/_add-data-twelvedata-crypto.md +++ b/_partials/_add-data-twelvedata-crypto.md @@ -1,29 +1,29 @@ ## Load financial data This tutorial uses real-time cryptocurrency data, also known as tick data, from -[Twelve Data][twelve-data]. A direct download link is provided below. +[Twelve Data][twelve-data]. To ingest data into the tables that you created, you need to +download the dataset, then upload the data to your $SERVICE_LONG. -### Ingest the dataset + -To ingest data into the tables that you created, you need to download the -dataset and copy the data to your database. - +1. Unzip [crypto_sample.zip](https://assets.timescale.com/docs/downloads/candlestick/crypto_sample.zip) to a ``. -#### Ingesting the dataset + This test dataset contains second-by-second trade data for the most-traded crypto-assets + and a regular table of asset symbols and company names. -1. Download the `crypto_sample.zip` file. The file contains two `.csv` - files; one with company information, and one with real-time stock trades for - the past month. Download: - [crypto_sample.zip](https://assets.timescale.com/docs/downloads/candlestick/crypto_sample.zip) - + To import up to 100GB of data directly from your current PostgreSQL based database, + [migrate with downtime][migrate-with-downtime] using native PostgreSQL tooling. To seamlessly import 100GB-10TB+ + of data, use the [live migration][migrate-live] tooling supplied by $COMPANY. To add data from non-PostgreSQL + data sources, see [Import and ingest data][data-ingest]. -1. In a new terminal window, run this command to unzip the `.csv` files: - ```bash - unzip crypto_sample.zip - ``` + +1. In Terminal, navigate to `` and connect to your $SERVICE_SHORT. + ```bash + psql -d "postgres://:@:/" + ``` + The connection information for a $SERVICE_SHORT is available in the file you downloaded when you created it. 1. At the `psql` prompt, use the `COPY` command to transfer data into your Timescale instance. If the `.csv` files aren't in your current directory, @@ -44,3 +44,6 @@ dataset and copy the data to your database. [twelve-data]: https://twelvedata.com/ +[migrate-with-downtime]: /migrate/:currentVersion:/pg-dump-and-restore/ +[migrate-live]: /migrate/:currentVersion:/live-migration/ +[data-ingest]: /use-timescale/:currentVersion:/ingest-data/ diff --git a/_partials/_add-data-twelvedata-stocks.md b/_partials/_add-data-twelvedata-stocks.md index 7be487e717..fb3a13b13d 100644 --- a/_partials/_add-data-twelvedata-stocks.md +++ b/_partials/_add-data-twelvedata-stocks.md @@ -3,15 +3,11 @@ This tutorial uses real-time stock trade data, also known as tick data, from [Twelve Data][twelve-data]. A direct download link is provided below. -## Ingest the dataset - To ingest data into the tables that you created, you need to download the dataset and copy the data to your database. -#### Ingesting the dataset - 1. Download the `real_time_stock_data.zip` file. The file contains two `.csv` files; one with company information, and one with real-time stock trades for the past month. Download: diff --git a/_partials/_caggs-intro.md b/_partials/_caggs-intro.md index aa8f0d6bcc..b56fb0f0ef 100644 --- a/_partials/_caggs-intro.md +++ b/_partials/_caggs-intro.md @@ -1,12 +1,12 @@ -Time-series data usually grows very quickly. And that means that aggregating the -data into useful summaries can become very slow. Continuous aggregates makes -aggregating data lightning fast. +In modern applications, data usually grows very quickly. This means that aggregating +it into useful summaries can become very slow. $CLOUD_LONG continuous aggregates make +aggregating data lightning fast, accurate, and easy. If you are collecting data very frequently, you might want to aggregate your -data into minutes or hours instead. For example, if you have a table of -temperature readings taken every second, you can find the average temperature +data into minutes or hours instead. For example, if an IoT device takes +temperature readings every second, you might want to find the average temperature for each hour. Every time you run this query, the database needs to scan the -entire table and recalculate the average every time. +entire table and recalculate the average. Continuous aggregates are a kind of hypertable that is refreshed automatically in the background as new data is added, or old data is modified. Changes to your @@ -22,14 +22,15 @@ database. Because continuous aggregates are based on hypertables, you can query them in exactly the same way as your other tables, and enable [compression][compression] -or [tiered storage][data-tiering] on your continuous aggregates. You can even +or [tiered storage][data-tiering] on them. You can even create -[continuous aggregates on top of your continuous aggregates][hierarchical-caggs]. +[continuous aggregates on top of your continuous aggregates][hierarchical-caggs] - for an even more fine-tuned aggregation. By default, querying continuous aggregates provides you with real-time data. Pre-aggregated data from the materialized view is combined with recent data that hasn't been aggregated yet. This gives you up-to-date results on every query. +[compression]: /use-timescale/:currentVersion:/compression/about-compression [data-tiering]: /use-timescale/:currentVersion:/data-tiering/ -[compression]: /use-timescale/:currentVersion:/compression/ +[hypercore]: /use-timescale/:currentVersion:/hypercore/ [hierarchical-caggs]: /use-timescale/:currentVersion:/continuous-aggregates/hierarchical-continuous-aggregates/ diff --git a/_partials/_cloud-connect-service.md b/_partials/_cloud-connect-service.md new file mode 100644 index 0000000000..cb02abbf8b --- /dev/null +++ b/_partials/_cloud-connect-service.md @@ -0,0 +1,110 @@ + + + +1. **Check your $SERVICE_SHORT is running correctly** + + In [$CONSOLE][services-portal], check that your $SERVICE_SHORT is marked as `Running`. + + ![Check $SERVICE_SHORT is running](https://assets.timescale.com/docs/images/console-services-view.png) + +1. **Connect to your $SERVICE_SHORT** + + Connect using data mode or SQL editor in $CONSOLE, or psql in the command line: + + + + + + + + 1. In $CONSOLE, toggle `Data`. + + 1. Select your $SERVICE_SHORT in the connection drop-down in the top right. + + ![Select a connection](https://assets.timescale.com/docs/images/data-mode-connection-dropdown.png) + + 1. Run a test query: + + ```sql + SELECT CURRENT_DATE; + ``` + + This query gives you the current date, you have successfully connected to your $SERVICE_SHORT. + + + + And that is it, you are up and running. Enjoy developing with $COMPANY. + + + + + + + + 1. In $CONSOLE, select your $SERVICE_SHORT. + + 1. Click `SQL editor`. + + ![SQL editor](https://assets.timescale.com/docs/images/ops-view-sql-editor.png) + + 1. Run a test query: + + ```sql + SELECT CURRENT_DATE; + ``` + + This query gives you the current date, you have successfully connected to your $SERVICE_SHORT. + + + + And that is it, you are up and running. Enjoy developing with $COMPANY. + + + + + + + + 1. Install [psql][psql]. + + 1. Run the following command in the terminal using the service URL from the config file you have saved during service creation: + + ``` + psql "" + ``` + + 1. Run a test query: + + ```sql + SELECT CURRENT_DATE; + ``` + + This query returns the current date. You have successfully connected to your $SERVICE_SHORT. + + + + And that is it, you are up and running. Enjoy developing with $COMPANY. + + + + + + + +Quick recap. You: +- Manage your $SERVICE_SHORTs in the [$OPS_MODE][portal-ops-mode] in $CONSOLE: add $READ_REPLICAs and enable + high availability, configure compression, change parameters, and so on. +- Analyze your data in the [$DATA_MODE][portal-data-mode] in $CONSOLE: write queries with + autocomplete, save them in folders, share them, create charts/dashboards, and much more. +- Store configuration and security information in your config file. + +[portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services +[portal-data-mode]: https://console.cloud.timescale.com/dashboard/services?popsql +[account-portal]: https://console.cloud.timescale.com/dashboard/account +[services-portal]: https://console.cloud.timescale.com/dashboard/services +[install-psql]: /integrations/:currentVersion:/psql/ +[popsql]: /getting-started/:currentVersion:/run-queries-from-console/#data-mode +[run-sqleditor]: /getting-started/:currentVersion:/run-queries-from-console/#sql-editor +[install-psql]: /integrations/:currentVersion:/psql/ +[hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/#hypertable-partitioning +[psql]: /integrations/:currentVersion:/psql/ diff --git a/_partials/_cloud-connect.md b/_partials/_cloud-connect.md deleted file mode 100644 index 4f057c6123..0000000000 --- a/_partials/_cloud-connect.md +++ /dev/null @@ -1,57 +0,0 @@ - - - -To connect to a service: - -1. **Check your service is running correctly** - - In [Timescale Console][services-portal], check that your service is marked as `Running`. - - ![Check service is running](https://assets.timescale.com/docs/images/console-services-view.png) - -1. **Connect to your service** - - Use either: - - [Data mode][popsql] in Timescale Console: - 1. In the [data mode][portal-data-mode] in Timescale Console, select a service and enter your password, then click **Connect**. - - You find your password in the config file you just downloaded. - 2. Select a query to edit: - ![Select a query to edit](https://assets.timescale.com/docs/images/data-mode-query-window.png) - - - [SQL editor][run-sqleditor] in Timescale Console: - - In the [ops mode][portal-ops-mode] in Timescale Console, select a service, then click **SQL editor**. - - ![Check service is running](https://assets.timescale.com/docs/images/ops-view-sql-editor.png) - - - [psql][install-psql] on the command line: - - Connect to your service with the value of `Service URL` from the config file you - just saved: - - @:/tsdb?sslmode=require" - `} /> - - You can now run queries for this service. - -Quick recap. You: -- Manage your services in the [ops mode][portal-ops-mode] in Timescale Console: add read replicas and enable - high-availability, configure compression, change parameters, and so on. -- Analyze your data in the [data mode][portal-data-mode] in Timescale Console: write queries with - autocomplete, save them in folders, share them, create charts/dashboards, and much more. -- Store configuration and security information in your config file. - - - - -[portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services -[portal-data-mode]: https://console.cloud.timescale.com/dashboard/services?popsql -[account-portal]: https://console.cloud.timescale.com/dashboard/account -[services-portal]: https://console.cloud.timescale.com/dashboard/services -[install-psql]: /use-timescale/:currentVersion:/integrations/psql/ -[popsql]: /getting-started/:currentVersion:/run-queries-from-console/#data-mode -[run-sqleditor]: /getting-started/:currentVersion:/run-queries-from-console/#sql-editor -[install-psql]: /use-timescale/:currentVersion:/integrations/psql/ -[hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/#hypertable-partitioning diff --git a/_partials/_cloud-create-connect-tutorials.md b/_partials/_cloud-create-connect-tutorials.md index 55bae061e2..df05b50753 100644 --- a/_partials/_cloud-create-connect-tutorials.md +++ b/_partials/_cloud-create-connect-tutorials.md @@ -6,9 +6,7 @@ command-line utility. If you've used PostgreSQL before, you might already have -### Create a Timescale service and connect to the service - -1. In the [Timescale portal][timescale-portal], click `Create service`. +1. In the [$CONSOLE][timescale-portal], click `Create service`. 1. Click `Download the cheatsheet` to download an SQL file that contains the login details for your new service. You can also copy the details directly from this page. When you have copied your password, @@ -39,4 +37,4 @@ command-line utility. If you've used PostgreSQL before, you might already have [timescale-portal]: https://console.cloud.timescale.com/ -[install-psql]: /use-timescale/:currentVersion:/integrations/psql/ +[install-psql]: /integrations/:currentVersion:/psql/ diff --git a/_partials/_cloud-installation.md b/_partials/_cloud-installation.md index 01b7f71ca0..a8d8130c40 100644 --- a/_partials/_cloud-installation.md +++ b/_partials/_cloud-installation.md @@ -1,21 +1,72 @@ -## Create a Timescale Cloud account +## Create a $CLOUD_LONG account -You create a $CLOUD_LONG account to manage your services and data in a centralized and efficient manner. From $CONSOLE you can create and delete services, run queries, manage access and billing, integrate other services, contact support, and more. +You create a $CLOUD_LONG account to manage your $SERVICE_SHORTs and data in a centralized and efficient manner in $CONSOLE. From there, you can create and delete $SERVICE_SHORTs, run queries, manage access and billing, integrate other services, contact support, and more. + + + + + +You create a standalone account to manage $CLOUD_LONG as a separate unit in your infrastructure, which includes separate billing and invoicing. + + + +To set up $CLOUD_LONG: + +1. **Sign up for a 30-day free trial** + + Open [Sign up for Timescale][timescale-signup] and add your details, then click `Start your free trial`. You receive a confirmation email in your inbox. + +1. **In the confirmation email, click the link supplied and sign in to [$CONSOLE][tsc-portal]** + +1. **Answer the requirements questions** + + Your answers help us optimize $SERVICE_LONGs for your use cases. + + + + + + + +You create an account through AWS Marketplace to have $CLOUD_LONG as a part of your AWS infrastructure. In this case, $CLOUD_LONG is a line item in your AWS invoice. -To set up Timescale: +To set up $CLOUD_LONG via AWS: + +1. **Open [AWS Marketplace][aws-marketplace] and search for `Timescale Cloud`** + + You see two pricing options, [pay-as-you-go][aws-paygo] and [annual commit][aws-annual-commit]. + +1. **Select the pricing option that suits you and click `View purchase options`** -1. Sign up for a [30 day free trial][sign-up]. +1. **Review and configure the purchase details, then click `Subscribe`** - You receive a confirmation email in your inbox. -1. In the confirmation email, click the link supplied and sign in to [Timescale Console][tsc-portal]. +1. **Click `Set up your account` at the top of the page** - Answer the requirements questions, they help us optimize the Timescale service for your use case. You can now create a Timescale service. + You are redirected to $CONSOLE. +1. **Sign up for a 30-day free trial** + Add your details, then click `Start your free trial`. If you want to link an existing $CLOUD_LONG account to AWS, log in with your existing credentials. + +1. **In `Confirm AWS Marketplace connection`, click `Connect`** + + Your $CLOUD_LONG and AWS accounts are now connected. + +1. **Answer the requirements questions** + + Your answers help us optimize $SERVICE_LONGs for your use cases. -[sign-up]: https://console.cloud.timescale.com/signup + + + + [tsc-portal]: https://console.cloud.timescale.com/ +[timescale-website]: https://www.timescale.com/ +[aws-marketplace]: https://aws.amazon.com/marketplace +[aws-paygo]: https://aws.amazon.com/marketplace/pp/prodview-iestawpo5ihca?applicationId=AWSMPContessa&ref_=beagle&sr=0-1 +[aws-annual-commit]: https://aws.amazon.com/marketplace/pp/prodview-ezxwlmjyr6x4u?applicationId=AWSMPContessa&ref_=beagle&sr=0-2 +[timescale-signup]: https://console.cloud.timescale.com/signup diff --git a/_partials/_cloud-intro.md b/_partials/_cloud-intro.md index ee0298abe4..f8341c0779 100644 --- a/_partials/_cloud-intro.md +++ b/_partials/_cloud-intro.md @@ -1,24 +1,26 @@ -$CLOUD_LONG is a cloud-based PostgreSQL platform for resource-intensive workloads. We help you build faster, -scale further, and stay under budget. A $SERVICE_LONG is a single optimized 100% PostgreSQL database instance that you -use as is, or extend with capabilities specific to your business needs. The available capabilities are: +$CLOUD_LONG is the modern $PG data platform for all your applications. It enhances $PG to handle time series, events, +real-time analytics, and vector search—all in a single database alongside transactional workloads. -- **Time-series and analytics**: PostgreSQL with TimescaleDB. The PostgreSQL you know and love, +You get one system that handles live data ingestion, late and out-of-order updates, and low latency queries, with the performance, reliability, and scalability your app needs. Ideal for IoT, crypto, finance, SaaS, and a myriad other domains, $CLOUD_LONG allows you to build data-heavy, mission-critical apps while retaining the familiarity and reliability of $PG. + +A $SERVICE_LONG is a single optimized 100% $PG database instance that you +use as is, or extend with the capabilities specific to your business needs: + +- **Time-series and analytics**: $PG with $TIMESCALE_DB. The $PG you know and love, supercharged with functionality for storing and querying [time-series data][what-is-time-series] at scale for - analytics and other use cases. - Get faster time-based queries with hypertables, continuous aggregates, and columnar storage. Save on storage with - native compression, data retention policies, and bottomless data tiering to Amazon S3. -- **AI and vector**: PostgreSQL with vector extensions. Use PostgreSQL as a vector database with + real-time analytics and other use cases. Get faster time-based queries with $HYPERTABLEs, $CAGGs, and columnar storage. Save on storage with native compression, data retention policies, and bottomless data tiering to Amazon S3. +- **AI and vector**: $PG with vector extensions. Use $PG as a vector database with purpose built extensions for building AI applications from start to scale. Get fast and accurate similarity search with the pgvector and pgvectorscale extensions. Create vector embeddings and perform LLM reasoning on your data with the pgai extension. - **PostgreSQL**: the trusted industry-standard RDBMS. Ideal for applications requiring strong data consistency, complex relationships, and advanced querying capabilities. Get ACID compliance, extensive SQL support, JSON handling, and extensibility through custom functions, data types, and extensions. $CLOUD_LONG continuously - monitors your services and prevents common PostgreSQL out-of-memory crashes. + monitors your $SERVICE_SHORTs and prevents common $PG out-of-memory crashes. All $SERVICE_LONGs include the tooling you expect for production and developer environments: [live migration][live-migration], -[automatic backups and PITR][automatic-backups], [high availability][high-availability], [read replicas][readreplica], [data forking][operations-forking], [connection pooling][connection-pooling], [tiered storage][data-tiering], -[usage-based storage][how-plans-work], secure in-Console [SQL editing][in-console-editors], service [metrics][metrics] +[automatic backups and PITR][automatic-backups], [high availability][high-availability], [$READ_REPLICAs][readreplica], [data forking][operations-forking], [connection pooling][connection-pooling], [tiered storage][data-tiering], +[usage-based storage][how-plans-work], secure in-$CONSOLE [SQL editing][in-console-editors], $SERVICE_SHORT [metrics][metrics] and [insights][insights], [streamlined maintenance][maintain-upgrade], and much more. [what-is-time-series]: https://www.timescale.com/blog/what-is-a-time-series-database/#what-is-a-time-series-database diff --git a/_partials/_cloud_self_configuration.md b/_partials/_cloud_self_configuration.md new file mode 100644 index 0000000000..b478a63f1e --- /dev/null +++ b/_partials/_cloud_self_configuration.md @@ -0,0 +1,81 @@ +import EarlyAccess from "versionContent/_partials/_early_access_2_18_0.mdx"; + +## Policies + +### `timescaledb.max_background_workers (int)` + +Max background worker processes allocated to TimescaleDB. Set to at least 1 + +the number of databases loaded with the TimescaleDB extension in a PostgreSQL +instance. Default value is 16. + +## Hypercore features + +### `timescaledb.default_hypercore_use_access_method (bool)` + +The default value for `hypercore_use_access_method` for functions that have this parameter. This function is in `user` context, meaning that any user can set it for the session. The default value is `false`. + + + +## $SERVICE_LONG tuning + +### `timescaledb.disable_load (bool)` + +Disable the loading of the actual extension + +### `timescaledb.enable_cagg_reorder_groupby (bool)` +Enable group by reordering + +### `timescaledb.enable_chunk_append (bool)` +Enable chunk append node + +### `timescaledb.enable_constraint_aware_append (bool)` +Enable constraint-aware append scans + +### `timescaledb.enable_constraint_exclusion (bool)` +Enable constraint exclusion + +### `timescaledb.enable_job_execution_logging (bool)` +Enable job execution logging + +### `timescaledb.enable_optimizations (bool)` +Enable TimescaleDB query optimizations + +### `timescaledb.enable_ordered_append (bool)` +Enable ordered append scans + +### `timescaledb.enable_parallel_chunk_append (bool)` +Enable parallel chunk append node + +### `timescaledb.enable_runtime_exclusion (bool)` +Enable runtime chunk exclusion + +### `timescaledb.enable_tiered_reads (bool)` + +Enable [tiered reads][enabling-data-tiering] to query your data normally when it's distributed across different storage tiers. +Your hypertable is spread across the tiers, so queries and `JOIN`s work and fetch the same data as usual. + +By default, tiered data is not accessed by queries. Querying tiered data may slow down query performance +as the data is not stored locally on Timescale's high-performance storage tier. + + +### `timescaledb.enable_transparent_decompression (bool)` +Enable transparent decompression + + +### `timescaledb.restoring (bool)` +Stop any background workers which could have been performing tasks. This is especially useful if you +migrate data to your [$SERVICE_LONG][pg-dump-and-restore] or [self-hosted database][migrate-entire]. + +### `timescaledb.max_cached_chunks_per_hypertable (int)` +Maximum cached chunks + +### `timescaledb.max_open_chunks_per_insert (int)` +Maximum open chunks per insert + +### `timescaledb.max_tuples_decompressed_per_dml_transaction (int)` + +The max number of tuples that can be decompressed during an INSERT, UPDATE, or DELETE. + +[enabling-data-tiering]: /use-timescale/:currentVersion:/data-tiering/enabling-data-tiering/ +[pg-dump-and-restore]: /migrate/:currentVersion:/pg-dump-and-restore/ +[migrate-entire]: /self-hosted/:currentVersion:/migration/entire-database/ diff --git a/_partials/_create-hypertable-blockchain.md b/_partials/_create-hypertable-blockchain.md index 84761f1e02..9d277e6130 100644 --- a/_partials/_create-hypertable-blockchain.md +++ b/_partials/_create-hypertable-blockchain.md @@ -1,5 +1,5 @@ -## Create a hypertable +## Optimize time-series data in hypertables Hypertables are the core of Timescale. Hypertables enable Timescale to work efficiently with time-series data. Because Timescale is PostgreSQL, all the @@ -9,8 +9,6 @@ with Timescale tables similar to standard PostgreSQL. -### Creating a hypertable - 1. Create a standard PostgreSQL table to store the Bitcoin blockchain data using `CREATE TABLE`: diff --git a/_partials/_create-hypertable-energy.md b/_partials/_create-hypertable-energy.md index 235a90d788..ff75bf552e 100644 --- a/_partials/_create-hypertable-energy.md +++ b/_partials/_create-hypertable-energy.md @@ -1,4 +1,4 @@ -## Create a hypertable +## Optimize time-series data in hypertables Hypertables are the core of Timescale. Hypertables enable Timescale to work efficiently with time-series data. Because Timescale is PostgreSQL, all the @@ -8,8 +8,6 @@ with Timescale tables similar to standard PostgreSQL. -### Creating a hypertable - 1. Create a standard PostgreSQL table to store the energy consumption data using `CREATE TABLE`: diff --git a/_partials/_create-hypertable-nyctaxis.md b/_partials/_create-hypertable-nyctaxis.md index b41a579de4..3400e849b6 100644 --- a/_partials/_create-hypertable-nyctaxis.md +++ b/_partials/_create-hypertable-nyctaxis.md @@ -1,17 +1,16 @@ -## Create a hypertable +## Optimize time-series data in hypertables -Hypertables are the core of Timescale. Hypertables enable Timescale to work -efficiently with time-series data. Because Timescale is PostgreSQL, all the -standard PostgreSQL tables, indexes, stored procedures and other objects can be -created alongside your Timescale hypertables. This makes creating and working -with Timescale tables similar to standard PostgreSQL. +Time-series data represents how a system, process, or behavior changes over time. [Hypertables][hypertables-section] +are PostgreSQL tables that help you improve insert and query performance by automatically partitioning your data by +time. Each hypertable is made up of child tables called chunks. Each chunk is assigned a range of time, and only +contains data from that range. - +Hypertables exist alongside regular PostgreSQL tables. You use regular PostgreSQL tables for relational data, and +interact with hypertables and regular PostgreSQL tables in the same way. -### Creating a hypertable + -1. Create a standard PostgreSQL table to store the taxi trip data - using `CREATE TABLE`: +1. **Create a standard PostgreSQL table to store the taxi trip data** ```sql CREATE TABLE "rides"( @@ -36,7 +35,8 @@ with Timescale tables similar to standard PostgreSQL. ); ``` -1. Convert the standard table into a hypertable partitioned on the `time` +1. **Convert the standard table into a hypertable** + Partitioned on the `time` column using the `create_hypertable()` function provided by Timescale. You must provide the name of the table and the column in that table that holds the timestamp data to use for partitioning: @@ -69,8 +69,6 @@ there are two other tables of data, called `payment_types` and `rates`. -### Creating standard PostgreSQL tables - 1. Add a table to store the payment types data: ```sql @@ -117,3 +115,6 @@ the `psql` command line. You should see this: public | rides | table | tsdbadmin (3 rows) ``` + + +[hypertables-section]: /use-timescale/:currentVersion:/hypertables/ diff --git a/_partials/_create-hypertable-twelvedata-crypto.md b/_partials/_create-hypertable-twelvedata-crypto.md index 4412295ea7..86955d9859 100644 --- a/_partials/_create-hypertable-twelvedata-crypto.md +++ b/_partials/_create-hypertable-twelvedata-crypto.md @@ -1,18 +1,22 @@ -## Create a hypertable +## Optimize time-series data in a hypertable -Hypertables are the core of Timescale. Hypertables enable Timescale to work -efficiently with time-series data. Because Timescale is PostgreSQL, all the -standard PostgreSQL tables, indexes, stored procedures, and other objects can be -created alongside your Timescale hypertables. This makes creating and working -with Timescale tables similar to standard PostgreSQL. +Hypertables are the core of $TIMESCALE_DB, they enable $CLOUD_LONG to work +efficiently with time-series data. Hypertables are PostgreSQL tables that automatically +partition your time-series data by time. When you run a query, $CLOUD_LONG identifies the +correct partition and runs the query on it, instead of going through the entire table. + +Because $TIMESCALE_DB is 100% PostgreSQL, you can create standard PostgreSQL tables, indexes, stored +procedures, and other objects alongside your Timescale hypertables. This makes creating and working +with hypertables similar to standard PostgreSQL. -### Creating a hypertable +1. Connect to your $SERVICE_LONG. + + In [$CONSOLE][services-portal] open an [SQL editor][in-console-editors]. You can also connect to your service using [psql][connect-using-psql]. -1. Create a standard PostgreSQL table to store the real-time cryptocurrency data - using `CREATE TABLE`: +1. Create a standard PostgreSQL table to store the real-time cryptocurrency data: ```sql CREATE TABLE crypto_ticks ( @@ -32,23 +36,16 @@ with Timescale tables similar to standard PostgreSQL. SELECT create_hypertable('crypto_ticks', by_range('time')); ``` - - The `by_range` dimension builder is an addition to TimescaleDB 2.13. - - -## Create standard PostgreSQL tables for relational data +## Create a standard PostgreSQL table for relational data -When you have other relational data that enhances your time-series data, you can -create standard PostgreSQL tables just as you would normally. For this dataset, -there is one other table of data called `crypto_assets`. +When you have relational data that enhances your time-series data, store that data in +standard PostgreSQL relational tables. -### Creating standard PostgreSQL tables - -1. Add a table to store the company name and symbol for the stock trade data: +1. Add a table to store the asset symbol and name in a relational table: ```sql CREATE TABLE crypto_assets ( @@ -57,7 +54,11 @@ there is one other table of data called `crypto_assets`. ); ``` -1. You now have two tables within your Timescale database. One hypertable - named `crypto_ticks`, and one normal PostgreSQL table named `crypto_assets`. - + +You now have two tables within your $SERVICE_LONG. A hypertable named `crypto_ticks`, and a normal +PostgreSQL table named `crypto_assets`. + +[in-console-editors]: /getting-started/:currentVersion:/run-queries-from-console/ +[services-portal]: https://console.cloud.timescale.com/dashboard/services +[connect-using-psql]: /integrations/:currentVersion:/psql diff --git a/_partials/_create-hypertable-twelvedata-stocks.md b/_partials/_create-hypertable-twelvedata-stocks.md index 85727a9fdc..6e06273e9b 100644 --- a/_partials/_create-hypertable-twelvedata-stocks.md +++ b/_partials/_create-hypertable-twelvedata-stocks.md @@ -1,5 +1,5 @@ -## Create a hypertable +## Optimize time-series data in hypertables Hypertables are the core of Timescale. Hypertables enable Timescale to work efficiently with time-series data. Because Timescale is PostgreSQL, all the @@ -9,8 +9,6 @@ with Timescale tables similar to standard PostgreSQL. -### Creating a hypertable - 1. Create a standard PostgreSQL table to store the real-time stock trade data using `CREATE TABLE`: diff --git a/_partials/_create-hypertable.md b/_partials/_create-hypertable.md index 99bb23bb4a..3cebc22003 100644 --- a/_partials/_create-hypertable.md +++ b/_partials/_create-hypertable.md @@ -40,8 +40,7 @@ To create a hypertable: [services-portal]: https://console.cloud.timescale.com/dashboard/services -[install-psql]: /use-timescale/:currentVersion:/integrations/psql/ +[install-psql]: /integrations/:currentVersion:/psql/ [popsql]: /getting-started/:currentVersion:/run-queries-from-console/#data-mode [run-sqleditor]: /getting-started/:currentVersion:/run-queries-from-console/#sql-editor -[install-psql]: /use-timescale/:currentVersion:/integrations/psql/ [hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/#hypertable-partitioning diff --git a/_partials/_deprecated.md b/_partials/_deprecated.md index c07cc2db29..53763784a1 100644 --- a/_partials/_deprecated.md +++ b/_partials/_deprecated.md @@ -1,5 +1,7 @@ + This section describes a feature that is deprecated on Timescale. We strongly recommend that you do not use this feature in a production environment. If you need more information, [contact us](https://www.timescale.com/contact/). + diff --git a/_partials/_deprecated_2_18_0.md b/_partials/_deprecated_2_18_0.md new file mode 100644 index 0000000000..77bf9772d6 --- /dev/null +++ b/_partials/_deprecated_2_18_0.md @@ -0,0 +1 @@ +Old API since [TimescaleDB v2.18.0](https://github.com/timescale/timescaledb/releases/tag/2.18.0) diff --git a/_partials/_early_access.md b/_partials/_early_access.md index b27534d7f6..66072e60a1 100644 --- a/_partials/_early_access.md +++ b/_partials/_early_access.md @@ -1,5 +1 @@ - -This feature is early access. Early access features might be subject to billing -changes in the future. If you have feedback, reach out to your customer success -manager, or [contact us](https://www.timescale.com/contact/). - +Early access diff --git a/_partials/_early_access_2_18_0.md b/_partials/_early_access_2_18_0.md new file mode 100644 index 0000000000..89ad944fba --- /dev/null +++ b/_partials/_early_access_2_18_0.md @@ -0,0 +1 @@ +Early access: TimescaleDB v2.18.0 diff --git a/_partials/_financial-industry-data-analysis.md b/_partials/_financial-industry-data-analysis.md new file mode 100644 index 0000000000..bfd6293928 --- /dev/null +++ b/_partials/_financial-industry-data-analysis.md @@ -0,0 +1 @@ +The financial industry is extremely data-heavy and relies on real-time and historical data for decision-making, risk assessment, fraud detection, and market analysis. Timescale simplifies management of these large volumes of data, while also providing you with meaningful analytical insights and optimizing storage costs. \ No newline at end of file diff --git a/_partials/_foreign-data-wrappers.md b/_partials/_foreign-data-wrappers.md index b720b66a8f..3b7da794d4 100644 --- a/_partials/_foreign-data-wrappers.md +++ b/_partials/_foreign-data-wrappers.md @@ -191,6 +191,6 @@ You create PostgreSQL FDWs with the `postgres_fdw` extension. See [documenation] [vpc-peering]: /use-timescale/:currentVersion:/security/vpc/ [sql-editor]: /getting-started/:currentVersion:/run-queries-from-console/#ops-mode-sql-editor/ [connect]: /getting-started/:currentVersion:/run-queries-from-console/ -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ [enable-fdw-docs]: https://www.postgresql.org/docs/current/postgres-fdw.html -[psql]: /use-timescale/:currentVersion:/integrations/psql/ +[psql]: /integrations/:currentVersion:/psql/ diff --git a/_partials/_formatting_examples.md b/_partials/_formatting_examples.md new file mode 100644 index 0000000000..885e4d4330 --- /dev/null +++ b/_partials/_formatting_examples.md @@ -0,0 +1,287 @@ +# Formatting examples + +This page illustrates and provides examples of the formatting available for Timescale documentation. Note that for most elements, spacing is important. + +## Procedure + +Use for a logical sequence of steps to achieve a goal. For example, create a hypertable. + +![Procedure example](https://assets.timescale.com/docs/images/procedure-syntax.png) + +Syntax: + + + + 1. **Step 1 summary in bold** + + Step 1 explanation in details. + + ``` + step 1 code + ``` + + 1. **Step 2 summary in bold** + + Step 2 explanation in details. + + 1. Sub-step 1. + + ``` + Sub-step 1 code + ``` + + 1. Sub-step 2. + + 1. Sub-step 3. + + + +See a [use example][data-tiering] in the docs. + +## Highlight blocks + +Use sparingly and only if it's essential to attract the reader's attention. + +- Note + + ![Note highlight](https://assets.timescale.com/docs/images/highlight-note.png) + + Syntax: + + + + Additional relevant information worth highlighting. + + + + See a [use example][disable-chunk-skipping] in the docs. + +- Important + + ![Important highlight](https://assets.timescale.com/docs/images/highlight-important.png) + + Syntax: + + + + Important things to keep in mind. + + + + See a [use example][decompress-chunks] in the docs. + +- Warning + + ![Caution highlight](https://assets.timescale.com/docs/images/highlight-warning.png) + + Syntax: + + + + Caution! + + + + See a [use example][alerting] in the docs. + +- Deprecation + + ![Deprecated highlight](https://assets.timescale.com/docs/images/highlight-deprecation.png) + + Syntax: + + + + A deprecation notice. + + + + See a [use example][deprecation] in the docs. + +- Cloud + + ![Cloud highlight](https://assets.timescale.com/docs/images/highlight-cloud.png) + + Syntax: + + + + A note dealing specifically with Timescale Cloud. + + + +## Tabs + +![Tabs](https://assets.timescale.com/docs/images/tabs-example.png) + +Syntax: + + + + + + First tab content + + + + + + Second tab content + + + + + +See a [use example][live-migration] in the docs. + +## Code blocks + +As a default, use [fenced Markdown code blocks][fenced-code-blocks]: + +![Regular code block](https://assets.timescale.com/docs/images/markdown-code-block.png) + +To remove line numbers and the copy button, use the `CodeBlock` component with `canCopy` and `showLineNumbers` set to `false`: + +![Custom code block](https://assets.timescale.com/docs/images/custom-code-block.png) + +Syntax: + + + +See a [use example][connection-pooling] in the docs. + +## Multi-tab code blocks + +![Multi-tab code block](https://assets.timescale.com/docs/images/multi-tab-code.png) + +Syntax: + + + + + + ```ruby + ruby code + ``` + + + + + + ```python + pyhon code + ``` + + + + + + ```go + go code + ``` + + + + + +## Tags + +- Download + + ![Download tag](https://assets.timescale.com/docs/images/tag-download.png) + + Syntax: + + Markdown link to download + + See a [use example][installation-windows] in the docs. + +- Experimental + + ![Experimental tag](https://assets.timescale.com/docs/images/tag-experimental.png) + + Syntax: + + Experimental + + See a [use example][time-bucket] in the docs. + +- Toolkit + + ![Tooklit tag](https://assets.timescale.com/docs/images/tag-toolkit.png) + + Syntax: + + Toolkit + + See a [use example][time-weighted-average] in the docs. + +- Community + + ![Community tag](https://assets.timescale.com/docs/images/tag-community.png) + + Syntax: + + Community + + See a [use example][remove-reorder-policy] in the docs. + +- Hollow + + ![Hollow tag](https://assets.timescale.com/docs/images/hollow-tag.png) + + Syntax: + + ```text + Text to display in a tag + ``` + +## Partials + +Import a partial from the `_partials` directory and then reference it in the relevant part of the page. + +Syntax: + + import PartialName from "versionContent/_partials/_partial.mdx"; + + + +See a [use example][live-migration] in the docs. + +## Links + +Links should be [reference-style Markdown links][reference-links]. + +Syntax: + + [Anchor][link-label] + + [link-label]: absolute or relative URL + +For example: + +[A link to the data tiering section in docs][data-tiering] + +[data-tiering]: ../use-timescale/data-tiering/enabling-data-tiering.md +[disable-chunk-skipping]: ../api/hypertable/disable_chunk_skipping.md +[decompress-chunks]: ../use-timescale/compression/decompress-chunks.md +[alerting]: ../use-timescale/alerting.md +[deprecation]: ../_partials/_deprecated.md +[live-migration]: ../migrate/live-migration.md +[fenced-code-blocks]: https://www.markdownguide.org/extended-syntax/#fenced-code-blocks +[connection-pooling]: ../use-timescale/services/connection-pooling.md +[installation-windows]: ../self-hosted/install/installation-windows.md +[time-bucket]: ../api/time_bucket_ng.md +[time-weighted-average]: ../api/time-weighted-averages.md +[remove-reorder-policy]: ../api/hypertable/remove_reorder_policy.md +[reference-links]: https://www.markdownguide.org/basic-syntax/#reference-style-links + + diff --git a/_partials/_grafana-connect.md b/_partials/_grafana-connect.md index fdc3619001..42078641cc 100644 --- a/_partials/_grafana-connect.md +++ b/_partials/_grafana-connect.md @@ -1,14 +1,6 @@ -## Prerequisites +## Connect Grafana to $CLOUD_LONG -import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; - - - -* Install [self-managed Grafana][grafana-self-managed] or sign up for [Grafana Cloud][grafana-cloud]. - -## Add your $SERVICE_SHORT as a data source - -To connect the data in your $SERVICE_SHORT to Grafana: +To visualize the results of your queries, enable Grafana to read the data in your $SERVICE_SHORT: @@ -21,15 +13,16 @@ To connect the data in your $SERVICE_SHORT to Grafana: 1. Open `Connections` > `Data sources`, then click `Add new data source`. 1. Select `PostgreSQL` from the list. 1. Configure the connection: - - `Host URL`, `Username`, `Password`, and `Database`: configure using your [connection details][connection-info]. - - `Database name`: provide the name for your dataset. - - `TLS/SSL Mode`: select `require`. - - `PostgreSQL options`: enable `TimescaleDB`. - - Leave the default setting for all other fields. - - 1. **Click `Save & test`** + - `Host URL`, `Database name`, `Username`, and `Password` + + Configure using your [connection details][connection-info]. `Host URL` is in the format `:`. + - `TLS/SSL Mode`: select `require`. + - `PostgreSQL options`: enable `TimescaleDB`. + - Leave the default setting for all other fields. + + 1. Click `Save & test`. - Grafana checks that your details are set correctly. + Grafana checks that your details are set correctly. @@ -37,4 +30,4 @@ To connect the data in your $SERVICE_SHORT to Grafana: [grafana-cloud]: https://grafana.com/get/ [cloud-login]: https://console.cloud.timescale.com/ [create-service]: /getting-started/:currentVersion:/services/ -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ diff --git a/_partials/_graphing-ohlcv-data.md b/_partials/_graphing-ohlcv-data.md index 2e7a51d53b..272ac3d156 100644 --- a/_partials/_graphing-ohlcv-data.md +++ b/_partials/_graphing-ohlcv-data.md @@ -9,12 +9,12 @@ up to connect to your TimescaleDB database. ### Graphing OHLCV data 1. Ensure you have Grafana installed, and you are using the TimescaleDB - database that contains the Twelve Data stocks dataset set up as a + database that contains the Twelve Data dataset set up as a data source. 1. In Grafana, from the `Dashboards` menu, click `New Dashboard`. In the `New Dashboard` page, click `Add a new panel`. 1. In the `Visualizations` menu in the top right corner, select `Candlestick` - from the list. Ensure you have set the Twelve Data stocks dataset as + from the list. Ensure you have set the Twelve Data dataset as your data source. 1. Click `Edit SQL` and paste in the query you used to get the OHLCV values. 1. In the `Format as` section, select `Table`. diff --git a/_partials/_high-availability-setup.md b/_partials/_high-availability-setup.md new file mode 100644 index 0000000000..5eb9eb15d1 --- /dev/null +++ b/_partials/_high-availability-setup.md @@ -0,0 +1,13 @@ + + +1. In [Timescale Console][cloud-login], select the service to enable replication for. +1. Click `Operations`, then select `High availability`. +1. Choose your replication strategy, then click `Change configuration`. + + ![Creating a service replica in Timescale Cloud](https://assets.timescale.com/docs/images/tsc-replication-add.png) + +1. In `Change high availability configuration`, click `Change config`. + + + +[cloud-login]: https://console.cloud.timescale.com diff --git a/_partials/_hypercore-conversion-overview.md b/_partials/_hypercore-conversion-overview.md new file mode 100644 index 0000000000..7cc2bf5ebc --- /dev/null +++ b/_partials/_hypercore-conversion-overview.md @@ -0,0 +1,21 @@ +When you convert chunks from the rowstore to the columnstore, multiple records are grouped into a single row. +The columns of this row hold an array-like structure that stores all the data. For example, data in the following +rowstore chunk: + +| Timestamp | Device ID | Device Type | CPU |Disk IO| +|---|---|---|---|---| +|12:00:01|A|SSD|70.11|13.4| +|12:00:01|B|HDD|69.70|20.5| +|12:00:02|A|SSD|70.12|13.2| +|12:00:02|B|HDD|69.69|23.4| +|12:00:03|A|SSD|70.14|13.0| +|12:00:03|B|HDD|69.70|25.2| + +Is converted and compressed into arrays in a row in the columnstore: + +|Timestamp|Device ID|Device Type|CPU|Disk IO| +|-|-|-|-|-| +|[12:00:01, 12:00:01, 12:00:02, 12:00:02, 12:00:03, 12:00:03]|[A, B, A, B, A, B]|[SSD, HDD, SSD, HDD, SSD, HDD]|[70.11, 69.70, 70.12, 69.69, 70.14, 69.70]|[13.4, 20.5, 13.2, 23.4, 13.0, 25.2]| + +Because a single row takes up less disk space, you can reduce your chunk size by more than 90%, and can also +speed up your queries. This saves on storage costs, and keeps your queries operating at lightning speed. diff --git a/_partials/_hypercore_manual_workflow.md b/_partials/_hypercore_manual_workflow.md new file mode 100644 index 0000000000..a5d57b81c0 --- /dev/null +++ b/_partials/_hypercore_manual_workflow.md @@ -0,0 +1,44 @@ +import EarlyAccess from "versionContent/_partials/_early_access_2_18_0.mdx"; + +1. **Stop the jobs that are automatically adding chunks to the columnstore** + + Retrieve the list of jobs from the [timescaledb_information.jobs][informational-views] view + to find the job you need to [alter_job][alter_job]. + + ``` sql + SELECT alter_job(JOB_ID, scheduled => false); + ``` + +1. **Convert a chunk to update back to the rowstore** + + ``` sql + CALL convert_to_rowstore('_timescaledb_internal._hyper_2_2_chunk'); + ``` + +1. **Update the data in the chunk you added to the rowstore** + + Best practice is to structure your [INSERT][insert] statement to include appropriate + partition key values, such as the timestamp. TimescaleDB adds the data to the correct chunk: + + ``` sql + INSERT INTO metrics (time, value) + VALUES ('2025-01-01T00:00:00', 42); + ``` + +1. **Convert the updated chunks back to the columnstore** + + ``` sql + CALL convert_to_columnstore('_timescaledb_internal._hyper_1_2_chunk'); + ``` + +1. **Restart the jobs that are automatically converting chunks to the columnstore** + + ``` sql + SELECT alter_job(JOB_ID, scheduled => true); + ``` + +[alter_job]: /api/:currentVersion:/actions/alter_job/ +[informational-views]: /api/:currentVersion:/informational-views/jobs/ +[insert]: /use-timescale/:currentVersion:/write-data/insert/ +[setup-hypercore]: /use-timescale/:currentVersion:/hypercore/real-time-analytics-in-hypercore/ +[compression_alter-table]: /api/:currentVersion:/hypercore/alter_table/ diff --git a/_partials/_hypercore_policy_workflow.md b/_partials/_hypercore_policy_workflow.md new file mode 100644 index 0000000000..871b325990 --- /dev/null +++ b/_partials/_hypercore_policy_workflow.md @@ -0,0 +1,118 @@ + + + +1. **Connect to your $SERVICE_LONG** + + In [$CONSOLE][services-portal] open an [SQL editor][in-console-editors]. You can also connect to your service using [psql][connect-using-psql]. + +1. **Enable columnstore on a hypertable** + + Create a [job][job] that automatically moves chunks in a hypertable to the columnstore at a specific time interval. + By default, your table is `orderedby` the time column. For efficient queries on columnstore data, remember to + `segmentby` the column you will use most often to filter your data: + + * [Use `ALTER TABLE` for a hypertable][alter_table_hypercore] + ```sql + ALTER TABLE crypto_ticks SET ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'symbol'); + ``` + * [Use ALTER MATERIALIZED VIEW for a continuous aggregate][compression_continuous-aggregate] + ```sql + ALTER MATERIALIZED VIEW assets_candlestick_daily set ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'symbol' ); + ``` + Before you say `huh`, a continuous aggregate is a specialized hypertable. + +1. **Add a policy to convert chunks to the columnstore at a specific time interval** + + For example, move yesterday's crypto trading data to the columnstore: + ``` sql + CALL add_columnstore_policy('crypto_ticks', after => INTERVAL '1d'); + ``` + See [add_columnstore_policy][add_columnstore_policy]. + +1. **Check the columstore policy** + + 1. View your data space saving: + + When you convert data to the columnstore, as well as being optimized for analytics, it is compressed by more than + 90%. This saves on storage costs and keeps your queries operating at lightning speed. To see the amount of space + saved: + + ``` sql + SELECT + pg_size_pretty(before_compression_total_bytes) as before, + pg_size_pretty(after_compression_total_bytes) as after + FROM hypertable_compression_stats('crypto_ticks'); + ``` + You see something like: + + | before | after | + |---------|--------| + | 194 MB | 24 MB | + + 1. View the policies that you set or the policies that already exist: + + ``` sql + SELECT * FROM timescaledb_information.jobs + WHERE proc_name='policy_compression'; + ``` + See [timescaledb_information.jobs][informational-views]. + +1. **Pause a columnstore policy** + + If you need to modify or add a lot of data to a chunk in the columnstore, best practice is to stop any jobs moving + chunks to the columnstore, [convert the chunk back to the rowstore][convert_to_rowstore], then modify the data. + After the update, [convert the chunk to the columnstore][convert_to_columnstore] and restart the jobs. + + ``` sql + SELECT * FROM timescaledb_information.jobs where + proc_name = 'policy_compression' AND relname = 'crypto_ticks' + + -- Select the JOB_ID from the results + + SELECT alter_job(JOB_ID, scheduled => false); + ``` + See [alter_job][alter_job]. + +1. **Restart a columnstore policy** + + ``` sql + SELECT alter_job(JOB_ID, scheduled => true); + ``` + See [alter_job][alter_job]. + +1. **Remove a columnstore policy** + + ``` sql + CALL remove_columnstore_policy('crypto_ticks'); + ``` + See [remove_columnstore_policy][remove_columnstore_policy]. + +1. **Disable columnstore** + + If your table has chunks in the columnstore, you have to + [convert the chunks back to the rowstore][convert_to_rowstore] before you disable the columnstore. + ``` sql + ALTER TABLE crypto_ticks SET (timescaledb.enable_columnstore = false); + ``` + See [alter_table_hypercore][alter_table_hypercore]. + + + +[job]: /api/:currentVersion:/actions/add_job/ +[alter_table_hypercore]: /api/:currentVersion:/hypercore/alter_table/ +[compression_continuous-aggregate]: /api/:currentVersion:/hypercore/alter_materialized_view/ +[convert_to_rowstore]: /api/:currentVersion:/hypercore/convert_to_rowstore/ +[convert_to_columnstore]: /api/:currentVersion:/hypercore/convert_to_columnstore/ +[informational-views]: /api/:currentVersion:/informational-views/jobs/ +[add_columnstore_policy]: /api/:currentVersion:/hypercore/add_columnstore_policy/ +[hypercore_workflow]: /api/:currentVersion:/hypercore/#hypercore-workflow +[alter_job]: /api/:currentVersion:/actions/alter_job/ +[remove_columnstore_policy]: /api/:currentVersion:/hypercore/remove_columnstore_policy/ +[in-console-editors]: /getting-started/:currentVersion:/run-queries-from-console/ +[services-portal]: https://console.cloud.timescale.com/dashboard/services +[connect-using-psql]: /integrations/:currentVersion:/psql/#connect-to-your-service +[insert]: /use-timescale/:currentVersion:/write-data/insert/ diff --git a/api/hypertable_detailed_size.md b/_partials/_hypertable-detailed-size-api.md similarity index 85% rename from api/hypertable_detailed_size.md rename to _partials/_hypertable-detailed-size-api.md index 6643eface4..275e29eb88 100644 --- a/api/hypertable_detailed_size.md +++ b/_partials/_hypertable-detailed-size-api.md @@ -1,14 +1,3 @@ ---- -api_name: hypertable_detailed_size() -excerpt: Get detailed information about disk space used by a hypertable -topics: [hypertables] -keywords: [hypertables, information] -tags: [statistics, size, disk space] -api: - license: apache - type: function ---- - # hypertable_detailed_size() Get detailed information about disk space used by a hypertable or @@ -27,13 +16,13 @@ instead. For more information about using hypertables, including chunk size partitioning, see the [hypertable section][hypertable-docs]. -### Required arguments +## Required arguments |Name|Type|Description| |---|---|---| | `hypertable` | REGCLASS | Hypertable or continuous aggregate to show detailed size of. | -### Returns +## Returns |Column|Type|Description| |-|-|-| @@ -48,7 +37,7 @@ If executed on a relation that is not a hypertable, the function returns `NULL`. -### Sample usage +## Sample usage Get the size information for a hypertable. @@ -68,4 +57,4 @@ The access node is listed without a user-given node name. Normally, the access node holds no data, but still maintains, for example, index information that occupies a small amount of disk space. -[hypertable-docs]: /use-timescale/:currentVersion:/hypertables/ +[hypertable-docs]: /use-timescale/:currentVersion:/hypertables/ \ No newline at end of file diff --git a/api/hypertable_size.md b/_partials/_hypertable-size-api.md similarity index 82% rename from api/hypertable_size.md rename to _partials/_hypertable-size-api.md index 83959b59c2..b5379b8675 100644 --- a/api/hypertable_size.md +++ b/_partials/_hypertable-size-api.md @@ -1,14 +1,3 @@ ---- -api_name: hypertable_size() -excerpt: Get the total disk space used by a hypertable -topics: [hypertables] -keywords: [hypertables, information] -tags: [disk space, size] -api: - license: apache - type: function ---- - # hypertable_size() Get the total disk space used by a hypertable or continuous aggregate, @@ -26,13 +15,13 @@ instead. For more information about using hypertables, including chunk size partitioning, see the [hypertable section][hypertable-docs]. -### Required arguments +## Required arguments |Name|Type|Description| |-|-|-| |`hypertable`|REGCLASS|Hypertable or continuous aggregate to show size of.| -### Returns +## Returns |Name|Type|Description| |-|-|-| @@ -42,7 +31,7 @@ see the [hypertable section][hypertable-docs]. `NULL` is returned if the function is executed on a non-hypertable relation. -### Sample usage +## Sample usage Get the size information for a hypertable. @@ -71,4 +60,4 @@ SELECT hypertable_size('device_stats_15m'); 73728 ``` -[hypertable-docs]: /use-timescale/:currentVersion:/hypertables/ +[hypertable-docs]: /use-timescale/:currentVersion:/hypertables/ \ No newline at end of file diff --git a/_partials/_hypertables-intro.md b/_partials/_hypertables-intro.md index 5b3d450076..fb2d811fc3 100644 --- a/_partials/_hypertables-intro.md +++ b/_partials/_hypertables-intro.md @@ -1,15 +1,12 @@ -Hypertables are PostgreSQL tables that automatically partition your data by -time. You interact with hypertables in the same way as regular PostgreSQL -tables, but with extra features that makes managing your time-series data much -easier. - -In Timescale, hypertables exist alongside regular PostgreSQL tables. Use -hypertables to store time-series data. This gives you improved insert and query -performance, and access to useful time-series features. Use regular PostgreSQL -tables for other relational data. - -With hypertables, Timescale makes it easy to improve insert and query -performance by partitioning time-series data on its time parameter. Behind the -scenes, the database performs the work of setting up and maintaining the -hypertable's partitions. Meanwhile, you insert and query your data as if it all -lives in a single, regular PostgreSQL table. +$CLOUD_LONG supercharges your real-time analytics by letting you run complex queries continuously, with near-zero latency. Under the hood, this is achieved by using hypertables - PostgreSQL tables that automatically partition your time-series data by time and optionally by space. When you run a query, $CLOUD_LONG identifies the correct partition and runs the query on it, instead of going through the entire table. + +Hypertables offer a range of other features, such as [skipping partitions][chunk-skipping] or running [hyperfunctions][hyperfunctions], that boost the performance of your analytical queries even more. + +To top it all, there is no added complexity - you interact with hypertables in the same way as you would with regular PostgreSQL tables. All the optimization magic happens behind the scenes. + +[chunk-skipping]: /use-timescale/:currentVersion:/hypertables/improve-query-performance/ +[hyperfunctions]: /use-timescale/:currentVersion:/hyperfunctions/ + + + + diff --git a/_partials/_import-data-iot.md b/_partials/_import-data-iot.md new file mode 100644 index 0000000000..834a0c09cb --- /dev/null +++ b/_partials/_import-data-iot.md @@ -0,0 +1,86 @@ +Time-series data represents how a system, process, or behavior changes over time. [Hypertables][hypertables-section] +are PostgreSQL tables that help you improve insert and query performance by automatically partitioning your data by +time, speeding up queries for real-time analytics and other challenging workloads. Each hypertable is made up of child +tables called chunks. Each chunk is assigned a range of time, and only contains data from that range. + + + +1. **Import time-series data into a hypertable** + + 1. Unzip [metrics.csv.gz](https://assets.timescale.com/docs/downloads/metrics.csv.gz) to a ``. + + This test dataset contains energy consumption data. + + To import up to 100GB of data directly from your current PostgreSQL based database, + [migrate with downtime][migrate-with-downtime] using native PostgreSQL tooling. To seamlessly import 100GB-10TB+ + of data, use the [live migration][migrate-live] tooling supplied by $COMPANY. To add data from non-PostgreSQL + data sources, see [Import and ingest data][data-ingest]. + + 1. In Terminal, navigate to `` and update the following string with [your connection details][connection-info] + to connect to your $SERVICE_SHORT. + + ```bash + psql -d "postgres://:@:/?sslmode=require" + ``` + + 1. Create tables to import time-series data: + + 1. In your sql client, create a normal PostgreSQL table: + + ```sql + CREATE TABLE "metrics"( + created timestamp with time zone default now() not null, + type_id integer not null, + value double precision not null + ); + ``` + + 1. Convert `metrics` to a hypertable and partitioned on time: + ```sql + SELECT create_hypertable('metrics', by_range('created')); + ``` + To more fully understand how hypertables work, and how to optimize them for performance by + tuning chunk intervals and enabling chunk skipping, see [the hypertables documentation][hypertables-section]. + + 1. Upload the dataset to your $SERVICE_SHORT + ```sql + \COPY metrics FROM metrics.csv CSV; + ``` + To more fully understand how hypertables work, and how to optimize them for performance by + tuning chunk intervals and enabling chunk skipping, see [the hypertables documentation][hypertables-section]. + +1. **Have a quick look at your data** + + You query hypertables in exactly the same way as you would a relational PostgreSQL table. + Use one of the following SQL editors to run a query and see the data you uploaded: + - **Data mode**: write queries, visualize data, and share your results in [$CONSOLE][portal-data-mode] for all your $SERVICE_LONGs. + - **SQL editor**: write, fix, and organize SQL faster and more accurately in [$CONSOLE][portal-ops-mode] for a $SERVICE_LONG. + - **psql**: easily run queries on your $SERVICE_LONGs or self-hosted TimescaleDB deployment from Terminal. + + ```sql + SELECT time_bucket('1 day', created, 'Europe/Berlin') AS "time", + round((last(value, created) - first(value, created)) * 100.) / 100. AS value + FROM metrics + WHERE type_id = 5 + GROUP BY 1; + ``` + + On this amount of data, this query on data in the rowstore takes about 3.6 seconds. You see something like: + + | Time | value | + |------------------------------|-------| + | 2023-05-29 22:00:00+00 | 23.1 | + | 2023-05-28 22:00:00+00 | 19.5 | + | 2023-05-30 22:00:00+00 | 25 | + | 2023-05-31 22:00:00+00 | 8.1 | + + + +[hypertables-section]: /use-timescale/:currentVersion:/hypertables/ +[portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services +[portal-data-mode]: https://console.cloud.timescale.com/dashboard/services?popsql +[connection-info]: /integrations/:currentVersion:/find-connection-details/ +[migrate-with-downtime]: /migrate/:currentVersion:/pg-dump-and-restore/ +[migrate-live]: /migrate/:currentVersion:/live-migration/ +[data-ingest]: /use-timescale/:currentVersion:/ingest-data/ + diff --git a/_partials/_import-data-nyc-taxis.md b/_partials/_import-data-nyc-taxis.md new file mode 100644 index 0000000000..ef24b447d3 --- /dev/null +++ b/_partials/_import-data-nyc-taxis.md @@ -0,0 +1,168 @@ +Time-series data represents how a system, process, or behavior changes over time. [Hypertables][hypertables-section] +are PostgreSQL tables that help you improve insert and query performance by automatically partitioning your data by +time, speeding up queries for real-time analytics and other challenging workloads. Each hypertable is made up of child +tables called chunks. Each chunk is assigned a range of time, and only contains data from that range. + + + +1. **Import time-series data into a hypertable** + + 1. Unzip [nyc_data.tar.gz](https://assets.timescale.com/docs/downloads/nyc_data.tar.gz) to a ``. + + This test dataset contains historical data from New York's yellow taxi network. + + To import up to 100GB of data directly from your current PostgreSQL based database, + [migrate with downtime][migrate-with-downtime] using native PostgreSQL tooling. To seamlessly import 100GB-10TB+ + of data, use the [live migration][migrate-live] tooling supplied by $COMPANY. To add data from non-PostgreSQL + data sources, see [Import and ingest data][data-ingest]. + + 1. In Terminal, navigate to `` and update the following string with [your connection details][connection-info] + to connect to your $SERVICE_SHORT. + + ```bash + psql -d "postgres://:@:/?sslmode=require" + ``` + + 1. Create tables to import time-series data: + + 1. In your sql client, create a normal PostgreSQL table: + + ```sql + CREATE TABLE "rides"( + vendor_id TEXT, + pickup_datetime TIMESTAMP WITHOUT TIME ZONE NOT NULL, + dropoff_datetime TIMESTAMP WITHOUT TIME ZONE NOT NULL, + passenger_count NUMERIC, + trip_distance NUMERIC, + pickup_longitude NUMERIC, + pickup_latitude NUMERIC, + rate_code INTEGER, + dropoff_longitude NUMERIC, + dropoff_latitude NUMERIC, + payment_type INTEGER, + fare_amount NUMERIC, + extra NUMERIC, + mta_tax NUMERIC, + tip_amount NUMERIC, + tolls_amount NUMERIC, + improvement_surcharge NUMERIC, + total_amount NUMERIC + ); + ``` + + 1. Convert `rides` to a hypertable and partitioned on time: + ```sql + SELECT create_hypertable('rides', by_range('pickup_datetime'), create_default_indexes=>FALSE); + SELECT add_dimension('rides', by_hash('payment_type', 2)); + ``` + To more fully understand how hypertables work, and how to optimize them for performance by + tuning chunk intervals and enabling chunk skipping, see [the hypertables documentation][hypertables-section]. + + 1. Create an index to support efficient queries by vendor, rate code, and passenger count: + ```sql + CREATE INDEX ON rides (vendor_id, pickup_datetime DESC); + CREATE INDEX ON rides (rate_code, pickup_datetime DESC); + CREATE INDEX ON rides (passenger_count, pickup_datetime DESC); + ``` + + 1. Create tables for relational data: + + 1. Add a table to store the payment types data: + + ```sql + CREATE TABLE IF NOT EXISTS "payment_types"( + payment_type INTEGER, + description TEXT + ); + INSERT INTO payment_types(payment_type, description) VALUES + (1, 'credit card'), + (2, 'cash'), + (3, 'no charge'), + (4, 'dispute'), + (5, 'unknown'), + (6, 'voided trip'); + ``` + + 1. Add a table to store the rates data: + + ```sql + CREATE TABLE IF NOT EXISTS "rates"( + rate_code INTEGER, + description TEXT + ); + INSERT INTO rates(rate_code, description) VALUES + (1, 'standard rate'), + (2, 'JFK'), + (3, 'Newark'), + (4, 'Nassau or Westchester'), + (5, 'negotiated fare'), + (6, 'group ride'); + ``` + + 1. Upload the dataset to your $SERVICE_SHORT + ```sql + \COPY rides FROM nyc_data_rides.csv CSV; + ``` + To more fully understand how hypertables work, and how to optimize them for performance by + tuning chunk intervals and enabling chunk skipping, see [the hypertables documentation][hypertables-section]. + +1. **Have a quick look at your data** + + You query hypertables in exactly the same way as you would a relational PostgreSQL table. + Use one of the following SQL editors to run a query and see the data you uploaded: + - **Data mode**: write queries, visualize data, and share your results in [$CONSOLE][portal-data-mode] for all your $SERVICE_LONGs. + - **SQL editor**: write, fix, and organize SQL faster and more accurately in [$CONSOLE][portal-ops-mode] for a $SERVICE_LONG. + - **psql**: easily run queries on your $SERVICE_LONGs or self-hosted TimescaleDB deployment from Terminal. + + For example: + - Display the number of rides for each fare type: + ```sql + SELECT rate_code, COUNT(vendor_id) AS num_trips + FROM rides + WHERE pickup_datetime < '2016-01-08' + GROUP BY rate_code + ORDER BY rate_code; + ``` + This simple query runs in 3 seconds. You see something like: + + | rate_code | num_trips | + |-----------------|-----------| + |1 | 2266401| + |2 | 54832| + |3 | 4126| + |4 | 967| + |5 | 7193| + |6 | 17| + |99 | 42| + + - To select all rides taken in the first week of January 2016, and return the total number of trips taken for each rate code: + ```sql + SELECT rates.description, COUNT(vendor_id) AS num_trips + FROM rides + JOIN rates ON rides.rate_code = rates.rate_code + WHERE pickup_datetime < '2016-01-08' + GROUP BY rates.description + ORDER BY LOWER(rates.description); + ``` + On this large amount of data, this analytical query on data in the rowstore takes about 59 seconds. You see something like: + + | description | num_trips | + |-----------------|-----------| + | group ride | 17 | + | JFK | 54832 | + | Nassau or Westchester | 967 | + | negotiated fare | 7193 | + | Newark | 4126 | + | standard rate | 2266401 | + + + + +[hypertables-section]: /use-timescale/:currentVersion:/hypertables/ +[portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services +[portal-data-mode]: https://console.cloud.timescale.com/dashboard/services?popsql +[connection-info]: /integrations/:currentVersion:/find-connection-details/ +[migrate-with-downtime]: /migrate/:currentVersion:/pg-dump-and-restore/ +[migrate-live]: /migrate/:currentVersion:/live-migration/ +[data-ingest]: /use-timescale/:currentVersion:/ingest-data/ + diff --git a/_partials/_install-self-hosted-redhat-based.md b/_partials/_install-self-hosted-redhat-based.md index 2f06e54f3c..cdc3f53773 100644 --- a/_partials/_install-self-hosted-redhat-based.md +++ b/_partials/_install-self-hosted-redhat-based.md @@ -72,6 +72,8 @@ 1. **Install TimescaleDB** + To avoid errors, **do not** install $TDB_APACHE and $TDB_COMMUNITY at the same time. + ```bash sudo yum install timescaledb-2-postgresql-17 postgresql17 ``` @@ -87,7 +89,7 @@ - 1. Initialize the PostgreSQL instance: + 1. **Initialize the PostgreSQL instance** ```bash sudo /usr/pgsql-17/bin/postgresql-17-setup initdb diff --git a/_partials/_integration-prereqs-cloud-only.md b/_partials/_integration-prereqs-cloud-only.md index b46e842697..e2be24287e 100644 --- a/_partials/_integration-prereqs-cloud-only.md +++ b/_partials/_integration-prereqs-cloud-only.md @@ -1,5 +1,10 @@ -To follow the procedure on this page, you need to: -* Create a target [$SERVICE_LONG][create-service]. +To follow the steps on this page: + +* Create a target [$SERVICE_LONG][create-service] with time-series and analytics enabled. + + You need your [connection details][connection-info]. + [create-service]: /getting-started/:currentVersion:/services/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ diff --git a/_partials/_integration-prereqs.md b/_partials/_integration-prereqs.md index 26a70caf25..5bcd9da0c2 100644 --- a/_partials/_integration-prereqs.md +++ b/_partials/_integration-prereqs.md @@ -1,10 +1,11 @@ -To follow the procedure on this page, you need to: +To follow the steps on this page: -* Create a target [$SERVICE_LONG][create-service] +* Create a target [$SERVICE_LONG][create-service] with time-series and analytics enabled. - You need your [connection details][connection-info] to follow the steps in this page. This procedure also + You need [your connection details][connection-info]. This procedure also works for [self-hosted $TIMESCALE_DB][enable-timescaledb]. + [create-service]: /getting-started/:currentVersion:/services/ [enable-timescaledb]: /self-hosted/:currentVersion:/install/ -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ diff --git a/_partials/_livesync-configure-source-database-awsrds.md b/_partials/_livesync-configure-source-database-awsrds.md new file mode 100644 index 0000000000..0cb2d35809 --- /dev/null +++ b/_partials/_livesync-configure-source-database-awsrds.md @@ -0,0 +1,86 @@ +import EnableReplication from "versionContent/_partials/_migrate_live_setup_enable_replication.mdx"; + +Updating parameters on a PostgreSQL instance will cause an outage. Choose a time that will cause the least issues to tune this database. + +1. **Tune the Write Ahead Log (WAL) on the RDS/Aurora PostgreSQL source database** + + 1. In [https://console.aws.amazon.com/rds/home#databases:][databases], + select the RDS instance to migrate. + + 1. Click `Configuration`, scroll down and note the `DB instance parameter group`, then click `Parameter Groups` + + Create security rule to enable RDS EC2 connection + + 1. Click `Create parameter group`, fill in the form with the following values, then click `Create`. + - **Parameter group name** - whatever suits your fancy. + - **Description** - knock yourself out with this one. + - **Engine type** - `PostgreSQL` + - **Parameter group family** - the same as `DB instance parameter group` in your `Configuration`. + 1. In `Parameter groups`, select the parameter group you created, then click `Edit`. + 1. Update the following parameters, then click `Save changes`. + - `rds.logical_replication` set to `1`: record the information needed for logical decoding. + - `wal_sender_timeout` set to `0`: disable the timeout for the sender process. + + 1. In RDS, navigate back to your [databases][databases], select the RDS instance to migrate and click `Modify`. + + 1. Scroll down to `Database options` select your new parameter group and click `Continue`. + 1. Either `Apply immediately` or choose a maintence window, then click `Modify DB instance`. + + Changing parameters will cause an outage. Wait for the database instance to reboot before continuing. + 1. Verify that the settings are live in your database. + +1. **Create a user for livesync and assign permissions** + + 1. Create ``: + + ```sql + psql $SOURCE -c "CREATE USER PASSWORD ''" + ``` + + You can use an existing user. However, you must ensure that the user has the following permissions. + + 1. Grant permissions to create a replication slot: + + ```sql + psql $SOURCE -c "GRANT rds_replication TO " + ``` + + 1. Grant permissions to create a publication: + + ```sql + psql $SOURCE -c "GRANT CREATE ON DATABASE TO " + ``` + + 1. Assign the user permissions on the source database: + + ```sql + psql $SOURCE <; + GRANT SELECT ON ALL TABLES IN SCHEMA "public" TO ; + ALTER DEFAULT PRIVILEGES IN SCHEMA "public" GRANT SELECT ON TABLES TO ; + EOF + ``` + + If the tables you are syncing are not in the `public` schema, grant the user permissions for each schema you are syncing.: + ```sql + psql $SOURCE < TO ; + GRANT SELECT ON ALL TABLES IN SCHEMA TO ; + ALTER DEFAULT PRIVILEGES IN SCHEMA GRANT SELECT ON TABLES TO ; + EOF + ``` + + 1. On each table you want to sync, make `` the owner: + + ```sql + psql $SOURCE -c 'ALTER TABLE OWNER TO ;' + ``` + You can skip this step if the replicating user is already the owner of the tables. + +1. **Enable replication `DELETE` and`UPDATE` operations** + + + +[databases]: https://console.aws.amazon.com/rds/home#databases: diff --git a/_partials/_livesync-configure-source-database.md b/_partials/_livesync-configure-source-database.md new file mode 100644 index 0000000000..3ce73d7a4e --- /dev/null +++ b/_partials/_livesync-configure-source-database.md @@ -0,0 +1,69 @@ +import EnableReplication from "versionContent/_partials/_migrate_live_setup_enable_replication.mdx"; + +1. **Tune the Write Ahead Log (WAL) on the PostgreSQL source database** + + ```sql + psql $SOURCE <`: + + ```sql + psql $SOURCE -c "CREATE USER PASSWORD ''" + ``` + + You can use an existing user. However, you must ensure that the user has the following permissions. + + 1. Grant permissions to create a replication slot: + + ```sql + psql $SOURCE -c "ALTER ROLE REPLICATION" + ``` + + 1. Grant permissions to create a publication: + + ```sql + psql $SOURCE -c "GRANT CREATE ON DATABASE TO " + ``` + + 1. Assign the user permissions on the source database: + + ```sql + psql $SOURCE <; + GRANT SELECT ON ALL TABLES IN SCHEMA "public" TO ; + ALTER DEFAULT PRIVILEGES IN SCHEMA "public" GRANT SELECT ON TABLES TO ; + EOF + ``` + + If the tables you are syncing are not in the `public` schema, grant the user permissions for each schema you are syncing.: + ```sql + psql $SOURCE < TO ; + GRANT SELECT ON ALL TABLES IN SCHEMA TO ; + ALTER DEFAULT PRIVILEGES IN SCHEMA GRANT SELECT ON TABLES TO ; + EOF + ``` + + 1. On each table you want to sync, make `` the owner: + + ```sql + psql $SOURCE -c 'ALTER TABLE
OWNER TO ;' + ``` + You can skip this step if the replicating user is already the owner of the tables. + + +1. **Enable replication `DELETE` and`UPDATE` operations** + + diff --git a/_partials/_livesync-console.md b/_partials/_livesync-console.md new file mode 100644 index 0000000000..d9ca0192ef --- /dev/null +++ b/_partials/_livesync-console.md @@ -0,0 +1,116 @@ +import PrereqCloud from "versionContent/_partials/_prereqs-cloud-only.mdx"; +import LivesyncLimitations from "versionContent/_partials/_livesync-limitations.mdx"; +import LivesyncConfigureSourceDatabase from "versionContent/_partials/_livesync-configure-source-database.mdx"; +import TuneSourceDatabaseAWSRDS from "versionContent/_partials/_livesync-configure-source-database-awsrds.mdx"; + +## Prerequisites + + + +- Install the [PostgreSQL client tools][install-psql] on your sync machine. + +- Ensure that the source $PG instance and the target $SERVICE_LONG have the same extensions installed. + + LiveSync does not create extensions on the target. If the table uses column types from an extension, + first create the extension on the target $SERVICE_LONG before syncing the table. + +## Limitations + +* Indexes(including Primary Key and Unique constraints) are not migrated by $SERVICE_LONG. + + We recommend that you create only necessary indexes on the target $SERVICE_LONG depending on your query patterns. + +* Tables with user defined types are not migrated by $SERVICE_LONG. + + You need to create the user defined types on the target $SERVICE_LONG before syncing the table. + + + +## Set your connection string + +This variable holds the connection information for the source database. In Terminal on your migration machine, +set the following: + +```bash +export SOURCE="postgres://:@:/" +``` + + +Avoid using connection strings that route through connection poolers like PgBouncer or similar tools. This tool +requires a direct connection to the database to function properly. + + + +## Tune your source database + + + + + + + + + + + + + + + + + + + + + + + +## Synchronize data to your $SERVICE_LONG + +To sync data from your PostgreSQL database to your $SERVICE_LONG using $CONSOLE: + + + +1. **Connect to your $SERVICE_LONG** + + In [$CONSOLE][portal-ops-mode], select the service to sync live data to. +1. **Start livesync** + 1. Click `Actions` > `livesync for PostgreSQL`. + +1. **Connect the source database and target $SERVICE_SHORT** + + ![Livesync wizard](https://assets.timescale.com/docs/images/livesync-wizard.png) + + In `livesync for PostgreSQL`: + 1. Set the `Livesync Name`. + 1. Set the` PostgreSQL Connection String` to point to the source database you want to sync to Timescale. + + This is the connection string for [``][livesync-tune-source-db]. + 1. Press `Continue`. + $CONSOLE connects to the source database and retrieves the schema information. + +1. **Optimize the data to synchronize in hypertables** + + ![livesync start](https://assets.timescale.com/docs/images/livesync-start.png) + 1. Select the table to sync, and press `+`. + $CONSOLE checks the table schema and, if possible suggests the column to use as the time dimension in a hypertable. + 1. Repeat this step for each table you want to sync. + 1. Press `Start Livesync`. + + $CONSOLE starts livesync between the source database and the target $SERVICE_SHORT and displays the progress. + +1. **Monitor syncronization** + 1. To view the progress of the livesync, click the name of the livesync process: + ![livesync view status](https://assets.timescale.com/docs/images/livesync-view-status.png) + 1. To pause and restart livesync, click the buttons on the right of the livesync process and select an action: + ![livesync start stop](https://assets.timescale.com/docs/images/livesync-start-stop.png) + + + +And that is it, you are using Livesync to synchronize all the data, or specific tables, from a PostgreSQL database +instance to your $SERVICE_LONG in real-time. + +[install-psql]: /integrations/:currentVersion:/psql/ +[portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services +[livesync-tune-source-db]: /migrate/:currentVersion:/livesync-for-postgresql/#tune-your-source-database + diff --git a/_partials/_livesync-limitations.md b/_partials/_livesync-limitations.md new file mode 100644 index 0000000000..69949ec0c0 --- /dev/null +++ b/_partials/_livesync-limitations.md @@ -0,0 +1,10 @@ +* Schema changes must be co-ordinated. + + Make compatible changes to the schema in your $SERVICE_LONG first, then make + the same changes to the source PostgreSQL instance. +* Ensure that the source $PG instance and the target $SERVICE_LONG have the same extensions installed. + + LiveSync does not create extensions on the target. If the table uses column types from an extension, + first create the extension on the target $SERVICE_LONG before syncing the table. +* There is WAL volume growth on the source PostgreSQL instance during large table copy. +* This works for PostgreSQL databases only as source. TimescaleDB is not yet supported. diff --git a/migrate/livesync.md b/_partials/_livesync-terminal.md similarity index 50% rename from migrate/livesync.md rename to _partials/_livesync-terminal.md index d89d0180ce..d133cd86f1 100644 --- a/migrate/livesync.md +++ b/_partials/_livesync-terminal.md @@ -1,112 +1,63 @@ ---- -title: Livesync from Postgres to Timescale Cloud -excerpt: Synchronize updates to your primary PostgreSQL database with the corresponding Timescale Cloud service in real time -products: [cloud] -keywords: [migration, low-downtime, backup] -tags: [recovery, logical backup, replication] ---- - import MigrationPrerequisites from "versionContent/_partials/_migrate_prerequisites.mdx"; import SetupConnectionStrings from "versionContent/_partials/_migrate_live_setup_connection_strings.mdx"; +import LivesyncLimitations from "versionContent/_partials/_livesync-limitations.mdx"; +import LivesyncConfigureSourceDatabase from "versionContent/_partials/_livesync-configure-source-database.mdx"; +import TuneSourceDatabaseAWSRDS from "versionContent/_partials/_migrate_live_tune_source_database_awsrds.mdx"; +## Prerequisites -# Livesync from PostgreSQL to Timescale Cloud - -You use the Livesync Docker image to synchronize all data, or specific tables, from a PostgreSQL database -instance to a $SERVICE_LONG in real-time. You run Livesync continuously, turning PostgreSQL into a primary database -with a $SERVICE_LONG as a logical replica. This enables you to leverage $CLOUD_LONG’s real-time analytics capabilities -on your replica data. - - -You use Livesync for for data synchronization, rather than migration. It is in alpha and is not recommended for -production use. - - -Livesync leverages the PostgreSQL logical replication protocol, a well-established and widely -understood feature in the PostgreSQL ecosystem. By relying on this protocol, Livesync ensures -compatibility, familiarity, and a broader knowledge base, making it easier for you to adopt and -integrate. - -You use Livesync to: -* Copy existing data from a PostgreSQL instance to a $SERVICE_LONG: - - Copy data at up to 150 GB/hr. You need at least a 4 CPU/16GB source database, and a 4 CPU/16GB target $SERVICE_SHORT. - - Copy the publication tables in parallel. However, large tables are still copied using a single connection. - Parallel copying is in the backlog. - - Forget foreign key relationships. Livesync disables foreign key validation during the sync. For example, if a - `metrics` table refers to the `id` column on the `tags` table, you can still sync only the `metrics` table - without worrying about their foreign key relationships. - - Track progress. PostgreSQL expose `COPY` progress under in `pg_stat_progress_copy`. -* Synchronize real-time changes from a PostgreSQL instance to a $SERVICE_LONG. -* Add and remove tables on demand using the [PostgreSQL PUBLICATION interface](https://www.postgresql.org/docs/current/sql-createpublication.html). -* Enable features such as [hypertables][about-hypertables], [columnstore][compression], and - [continuous aggregates][caggs] on your logical replica. - -If you have an questions or feedback, talk to us in [#livesync in Timescale Community][join-livesync-on-slack]. + -# Prerequisites +- Ensure that the source $PG instance and the target $SERVICE_LONG have the same extensions installed. - + LiveSync does not create extensions on the target. If the table uses column types from an extension, + first create the extension on the target $SERVICE_LONG before syncing the table. - [Install Docker][install-docker] on your sync machine. - You need a minimum of a 4 CPU/16GB EC2 instance to run Livesync -- Install the PostgreSQL client tools on your sync machine. + You need a minimum of a 4 CPU/16GB EC2 instance to run Livesync. + +- Install the [PostgreSQL client tools][install-psql] on your sync machine. This includes `psql`, `pg_dump`, and `pg_dumpall`. + ## Limitations -* The Schema is not migrated by Livesync, you use pg_dump/restore to migrate schema -* Schema changes must be co-ordinated. Make compatible changes to the schema in your $SERVICE_LONG first, then make - the same changes to the source PostgreSQL instance. -* There is WAL volume growth on the source PostgreSQL instance during large table copy. -* This works for PostgreSQL databases only as source. Timescaledb is not yet supported. + + +- The Schema is not migrated by Livesync, you use pg_dump/restore to migrate schema ## Set your connection strings -The `` in the `SOURCE` connection must have the replication role granted in order to create a replication slot. +The `` in the `SOURCE` connection must have the replication role granted in order to create a replication slot. -## Configure the source database -You need to tune the Write Ahead Log (WAL) on the PostgreSQL source database: -* PostgreSQL[ GUC “wal_level” as “logical”](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) -* PostgreSQL[GUC “max_wal_senders” as 10](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-MAX-WAL-SENDERS) +## Tune your source database -To do this: + -```sql -psql $SOURCE -c "SET wal_level=’logical’;" -psql $SOURCE -c "SET max_wal_sender=10;" -``` + -## Enable update and delete replication on the source database + -Replica identity assists data replication by identifying the rows being modified. -By default each table and hypertable in the source database defaults to the primary key of the table being replicated. -However, you can also have: + -- **A viable unique index**: each table has a unique, non-partial, non-deferrable index that includes only columns - marked as `NOT NULL`. If a `UNIQUE` index does not exists, create one to assist the migration. You can delete it after - live sync. For each table, set `REPLICA IDENTITY` to the viable unique index: + - ```sql - psql -X -d $SOURCE -c 'ALTER TABLE
REPLICA IDENTITY USING INDEX <_index_name>' - + + + + -- **No primary key or viable unique index**: use brute force. For each table, set `REPLICA IDENTITY` to `FULL`: + - ```sql - psql -X -d $SOURCE -c 'ALTER TABLE
REPLICA IDENTITY FULL' - ``` - For each `UPDATE` or `DELETE` statement, PostgreSQL reads the whole table to find all matching rows. - This results in significantly slower replication. If you are expecting a large number of `UPDATE` or `DELETE` - operations on the table, best practice is to not use `FULL` + -To capture only `INSERT` and ignore `UPDATE`s and `DELETE`s, use a -[publish config](https://www.postgresql.org/docs/current/sql-createpublication.html#SQL-CREATEPUBLICATION-PARAMS-WITH-PUBLISH) -while [creating the publication][lives-sync-specify-tables]. + + ## Migrate the table schema to the $SERVICE_LONG @@ -172,11 +123,11 @@ events data, and tables that are already partitioned using PostgreSQL declarativ COMMIT; EOF ``` - + -## Syncronize data from your source database to the $SERVICE_LONG +## Synchronize data to your $SERVICE_LONG You use the Livesync docker image to synchronize changes in real-time from a PostgreSQL database instance to a $SERVICE_LONG: @@ -188,7 +139,7 @@ instance to a $SERVICE_LONG: As you run Livesync continuously, best practice is to run it as a background process. ```shell - docker run -d --rm --name livesync timescale/live-sync:v0.0.0-alpha.7 run --publication analytics --subscription livesync --source $SOURCE --target $TARGET + docker run -d --rm --name livesync timescale/live-sync:v0.1.11 run --publication analytics --subscription livesync --source $SOURCE --target $TARGET ``` 1. **Trace progress** @@ -209,13 +160,13 @@ instance to a $SERVICE_LONG: ``` Possible values for `state` are: - - d: initial table data sync - - - f: initial table data sync completed - - - s: catching up with the latest change - - - r: table is ready, synching live changes + - d: initial table data sync + + - f: initial table data sync completed + + - s: catching up with the latest change + + - r: table is ready, synching live changes 1. **Stop Livesync** @@ -265,9 +216,9 @@ For example: 1. **Publish PostgreSQL declarative partitioned table** - To publish declaratively partitioned table changes to your $SERVICE_LONG, set the `publish_via_partition_root` + To publish declaratively partitioned table changes to your $SERVICE_LONG, set the `publish_via_partition_root` special `PUBLICATION` config to `true`: - + ```sql ALTER PUBLICATION analytics SET(publish_via_partition_root=true); ``` @@ -285,7 +236,8 @@ For example: [alter-publication]: https://www.postgresql.org/docs/current/sql-alterpublication.html [install-docker]: https://docs.docker.com/engine/install/ [about-hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/ -[lives-sync-specify-tables]: /migrate/:currentVersion:/livesync/#specify-the-tables-to-synchronize +[lives-sync-specify-tables]: /migrate/:currentVersion:/livesync-for-postgresql/#specify-the-tables-to-synchronize [compression]: /use-timescale/:currentVersion:/compression/about-compression [caggs]: /use-timescale/:currentVersion:/continuous-aggregates/about-continuous-aggregates/ [join-livesync-on-slack]: https://app.slack.com/client/T4GT3N2JK/C086NU9EZ88 +[install-psql]: /integrations/:currentVersion:/psql/ diff --git a/_partials/_manage-data-exporter.md b/_partials/_manage-a-data-exporter.md similarity index 97% rename from _partials/_manage-data-exporter.md rename to _partials/_manage-a-data-exporter.md index cfc23cabef..64cc7c2bc7 100644 --- a/_partials/_manage-data-exporter.md +++ b/_partials/_manage-a-data-exporter.md @@ -1,4 +1,4 @@ -## Attach a data exporter to a $SERVICE_LONG +### Attach a data exporter to a $SERVICE_LONG To send telemetry data to an external monitoring tool, you attach a data exporter to your $SERVICE_LONG. You can attach only one exporter to a $SERVICE_SHORT. @@ -14,7 +14,7 @@ To attach an exporter: -## Monitor $SERVICE_LONG metrics +### Monitor $SERVICE_LONG metrics You can now monitor your $SERVICE_SHORT metrics. Use the following metrics to check the service is running correctly: @@ -36,7 +36,7 @@ Additionally, use the following tags to filter your results. |`role`|`replica` or `primary`| For $SERVICE_SHORT with replicas | |`node-id`|| For multi-node services | -## Edit a data exporter +### Edit a data exporter To update a data exporter: @@ -50,7 +50,7 @@ You cannot change fields such as the provider or the AWS region. -## Delete a data exporter +### Delete a data exporter To remove a data exporter that you no longer need: @@ -73,7 +73,7 @@ To remove a data exporter that you no longer need: -## Reference +### Reference When you create the IAM OIDC provider, the URL must match the region you create the exporter in. It must be one of the following: diff --git a/_partials/_migrate_awsrds_connect_intermediary.md b/_partials/_migrate_awsrds_connect_intermediary.md index dd056c0c14..86c1abaa2a 100644 --- a/_partials/_migrate_awsrds_connect_intermediary.md +++ b/_partials/_migrate_awsrds_connect_intermediary.md @@ -10,7 +10,7 @@ - **Key pair**: use an existing pair or create a new one that you will use to access the intermediary machine. - **VPC**: by default, this is the same as the database instance. - **Configure Storage**: adjust the volume to at least the size of RDS/Aurora PostgreSQL instance you are migrating from. - You can reduce the space used by your data on Timescale Cloud using [data compression][data-compression]. + You can reduce the space used by your data on Timescale Cloud using [Hypercore][hypercore]. 1. Click `Lauch instance`. AWS creates your EC2 instance, then click `Connect to instance` > `SSH client`. Follow the instructions to create the connection to your intermediary EC2 instance. @@ -89,5 +89,5 @@ [about-hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/ -[data-compression]: /use-timescale/:currentVersion:/compression/about-compression/ +[hypercore]: /use-timescale/:currentVersion:/hypercore/ [databases]: https://console.aws.amazon.com/rds/home#databases: diff --git a/_partials/_migrate_dump_roles_schema_data_mst.md b/_partials/_migrate_dump_roles_schema_data_mst.md index dd293cdc30..b928040c45 100644 --- a/_partials/_migrate_dump_roles_schema_data_mst.md +++ b/_partials/_migrate_dump_roles_schema_data_mst.md @@ -23,15 +23,29 @@ ```bash sed -i -E \ - -e '/CREATE ROLE "postgres";/d' \ - -e '/ALTER ROLE "postgres"/d' \ - -e '/CREATE ROLE "tsdbadmin";/d' \ - -e '/ALTER ROLE "tsdbadmin"/d' \ - -e '/GRANT "pg_read_all_stats" TO "tsdbadmin"/d' \ - -e 's/(NO)*SUPERUSER//g' \ - -e 's/(NO)*REPLICATION//g' \ - -e 's/(NO)*BYPASSRLS//g' \ - -e 's/GRANTED BY "[^"]*"//g' \ + -e '/DROP ROLE IF EXISTS "postgres";/d' \ + -e '/DROP ROLE IF EXISTS "tsdbadmin";/d' \ + -e '/CREATE ROLE "postgres";/d' \ + -e '/ALTER ROLE "postgres"/d' \ + -e '/CREATE ROLE "rds/d' \ + -e '/ALTER ROLE "rds/d' \ + -e '/TO "rds/d' \ + -e '/GRANT "rds/d' \ + -e '/GRANT "pg_read_all_stats" TO "tsdbadmin"/d' \ + -e 's/(NO)*SUPERUSER//g' \ + -e 's/(NO)*REPLICATION//g' \ + -e 's/(NO)*BYPASSRLS//g' \ + -e 's/GRANTED BY "[^"]*"//g' \ + -e '/CREATE ROLE "tsdbadmin";/d' \ + -e '/ALTER ROLE "tsdbadmin"/d' \ + -e 's/WITH ADMIN OPTION,/WITH /g' \ + -e 's/WITH ADMIN OPTION//g' \ + -e 's/GRANTED BY ".*"//g' \ + -e '/GRANT "pg_.*" TO/d' \ + -e '/CREATE ROLE "_aiven";/d' \ + -e '/ALTER ROLE "_aiven"/d' \ + -e '/GRANT SET ON PARAMETER "pgaudit.log_max_string_length" TO "_tsdbadmin_auditing"/d' \ + -e '/GRANT SET ON PARAMETER "pgaudit.log_nested_statements" TO "_tsdbadmin_auditing"/d' \ roles.sql ``` diff --git a/_partials/_migrate_install_psql_ec2_instance.md b/_partials/_migrate_install_psql_ec2_instance.md index 00f0d5326c..a90af1989e 100644 --- a/_partials/_migrate_install_psql_ec2_instance.md +++ b/_partials/_migrate_install_psql_ec2_instance.md @@ -71,4 +71,3 @@ [about-hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/ -[data-compression]: /use-timescale/:currentVersion:/compression/about-compression/ diff --git a/_partials/_migrate_live_tune_source_database_awsrds.md b/_partials/_migrate_live_tune_source_database_awsrds.md index 6bfe993060..118885fe04 100644 --- a/_partials/_migrate_live_tune_source_database_awsrds.md +++ b/_partials/_migrate_live_tune_source_database_awsrds.md @@ -31,7 +31,7 @@ Updating parameters on a PostgreSQL instance will cause an outage. Choose a time Changing parameters will cause an outage. Wait for the database instance to reboot before continuing. 1. Verify that the settings are live in your database. -1. **Enable live-migration to replicate `DELETE` and`UPDATE` operations** +1. **Enable replication `DELETE` and`UPDATE` operations** diff --git a/_partials/_migrate_post_schema_caggs_etal.md b/_partials/_migrate_post_schema_caggs_etal.md index b03f4566dd..db130636a7 100644 --- a/_partials/_migrate_post_schema_caggs_etal.md +++ b/_partials/_migrate_post_schema_caggs_etal.md @@ -131,8 +131,7 @@ separately. Recreate them on your Timescale database. 1. Recreate each policy. For more information about recreating policies, see the sections on [continuous-aggregate refresh policies][cagg-policy], - [retention policies][retention-policy], [compression - policies][compression-policy], and [reorder policies][reorder-policy]. + [retention policies][retention-policy], [Hypercore policies][setup-hypercore], and [reorder policies][reorder-policy]. @@ -160,7 +159,7 @@ accessed. Skipping them does not affect statistics on your data. [analyze]: https://www.postgresql.org/docs/10/sql-analyze.html [cagg-policy]: /use-timescale/:currentVersion:/continuous-aggregates/refresh-policies/ -[compression-policy]: /use-timescale/:currentVersion:/compression/ +[setup-hypercore]: /use-timescale/:currentVersion:/hypercore/real-time-analytics-in-hypercore/ [retention-policy]: /use-timescale/:currentVersion:/data-retention/create-a-retention-policy/ [reorder-policy]: /api/:currentVersion:/hypertable/add_reorder_policy/ [timescaledb-parallel-copy]: https://github.com/timescale/timescaledb-parallel-copy diff --git a/_partials/_migrate_self_postgres_plan_migration_path.md b/_partials/_migrate_self_postgres_plan_migration_path.md index 0ecb649776..c00f5ffd42 100644 --- a/_partials/_migrate_self_postgres_plan_migration_path.md +++ b/_partials/_migrate_self_postgres_plan_migration_path.md @@ -7,10 +7,10 @@ Cloud and always get latest update without any hassle. Check the following support matrix against the versions of TimescaleDB and PostgreSQL that you are running currently and the versions you want to update to, then choose your upgrade path. -For example, to upgrade from TimescaleDB 2.13 on PostgreSQL 13 to TimescaleDB 2.17.2 you need to: -1. Upgrade TimescaleDB to 2.16 -1. Upgrade PostgreSQL to 14 or higher -1. Upgrade TimescaleDB to 2.17.2. +For example, to upgrade from TimescaleDB 2.13 on PostgreSQL 13 to TimescaleDB 2.18.2 you need to: +1. Upgrade TimescaleDB to 2.15 +1. Upgrade PostgreSQL to 14, 15 or 16. +1. Upgrade TimescaleDB to 2.18.2. You may need to [upgrade to the latest PostgreSQL version][upgrade-pg] before you upgrade TimescaleDB. Also, if you use [Timescale Toolkit][toolkit-install], ensure the `timescaledb_toolkit` extension is >= diff --git a/_partials/_migrate_self_postgres_timescaledb_compatibility.md b/_partials/_migrate_self_postgres_timescaledb_compatibility.md index 4e4b9473f2..65d5ef78cc 100644 --- a/_partials/_migrate_self_postgres_timescaledb_compatibility.md +++ b/_partials/_migrate_self_postgres_timescaledb_compatibility.md @@ -1,19 +1,20 @@ -| Version number |PostgreSQL 17|PostgreSQL 16|PostgreSQL 15|PostgreSQL 14|PostgreSQL 13|PostgreSQL 12|PostgreSQL 11|PostgreSQL 10| -|---------------------------|-|-|-|-|-|-|-|-| -| TimescaleDB
2.18.x |✅|✅|✅|✅|❌|❌|❌|❌|❌| -| TimescaleDB
2.17.x |✅|✅|✅|✅|❌|❌|❌|❌|❌| -| TimescaleDB
2.16.x |❌|✅|✅|✅|❌|❌|❌|❌|❌|❌| -| TimescaleDB
2.15.x |❌|✅|✅|✅|✅|❌|❌|❌|❌|❌| -| TimescaleDB
2.14.x |❌|✅|✅|✅|✅|❌|❌|❌|❌|❌| -| TimescaleDB
2.13.x |❌|✅|✅|✅|✅|❌|❌|❌|❌| -| TimescaleDB
2.12.x |❌|❌|✅|✅|✅|❌|❌|❌|❌| -| TimescaleDB
2.10.x |❌|❌|✅|✅|✅|✅|❌|❌|❌| -| TimescaleDB
2.5 - 2.9 |❌|❌|❌|✅|✅|✅|❌|❌|❌| -| TimescaleDB
2.4 |❌|❌|❌|❌|✅|✅|❌|❌|❌| -| TimescaleDB
2.1 - 2.3 |❌|❌|❌|❌|✅|✅|✅|❌|❌| -| TimescaleDB
2.0 |❌|❌|❌|❌|❌|✅|✅|❌|❌ -| TimescaleDB
1.7 |❌|❌|❌|❌|❌|✅|✅|✅|✅| +| Version number |PostgreSQL 17|PostgreSQL 16|PostgreSQL 15|PostgreSQL 14|PostgreSQL 13|PostgreSQL 12|PostgreSQL 11|PostgreSQL 10| +|----------------------------------|-|-|-|-|-|-|-|-| +| TimescaleDB
2.19 and higher |✅|✅|✅|✅|❌|❌|❌|❌|❌| +| TimescaleDB
2.18.x |✅|✅|✅|✅|❌|❌|❌|❌|❌| +| TimescaleDB
2.17.x |✅|✅|✅|✅|❌|❌|❌|❌|❌| +| TimescaleDB
2.16.x |❌|✅|✅|✅|❌|❌|❌|❌|❌|❌| +| TimescaleDB
2.15.x |❌|✅|✅|✅|✅|❌|❌|❌|❌|❌| +| TimescaleDB
2.14.x |❌|✅|✅|✅|✅|❌|❌|❌|❌|❌| +| TimescaleDB
2.13.x |❌|✅|✅|✅|✅|❌|❌|❌|❌| +| TimescaleDB
2.12.x |❌|❌|✅|✅|✅|❌|❌|❌|❌| +| TimescaleDB
2.10.x |❌|❌|✅|✅|✅|✅|❌|❌|❌| +| TimescaleDB
2.5 - 2.9 |❌|❌|❌|✅|✅|✅|❌|❌|❌| +| TimescaleDB
2.4 |❌|❌|❌|❌|✅|✅|❌|❌|❌| +| TimescaleDB
2.1 - 2.3 |❌|❌|❌|❌|✅|✅|✅|❌|❌| +| TimescaleDB
2.0 |❌|❌|❌|❌|❌|✅|✅|❌|❌ +| TimescaleDB
1.7 |❌|❌|❌|❌|❌|✅|✅|✅|✅| We recommend not using TimescaleDB with PostgreSQL 17.1, 16.5, 15.9, 14.14, 13.17, 12.21. These minor versions [introduced a breaking binary interface change][postgres-breaking-change] that, diff --git a/_partials/_migrate_using_parallel_copy.md b/_partials/_migrate_using_parallel_copy.md index d8b6db3fc9..c01b6046e5 100644 --- a/_partials/_migrate_using_parallel_copy.md +++ b/_partials/_migrate_using_parallel_copy.md @@ -17,8 +17,9 @@ --connection "host= \ user=tsdbadmin password= \ port= \ - sslmode=require" \ - --db-name tsdb \ + dbname=tsdb \ + sslmode=require + " \ --table \ --file .csv \ --workers \ diff --git a/_partials/_multi-node-deprecation.md b/_partials/_multi-node-deprecation.md index a5e1eda837..31f68e1bd0 100644 --- a/_partials/_multi-node-deprecation.md +++ b/_partials/_multi-node-deprecation.md @@ -1,9 +1,10 @@ -[Multi-node support is deprecated][multi-node-deprecation]. +[Multi-node support is sunsetted][multi-node-deprecation]. TimescaleDB v2.13 is the last release that includes multi-node support for PostgreSQL versions 13, 14, and 15. + [multi-node-deprecation]: https://github.com/timescale/timescaledb/blob/main/docs/MultiNodeDeprecation.md diff --git a/_partials/_prereqs-cloud-and-self.md b/_partials/_prereqs-cloud-and-self.md new file mode 100644 index 0000000000..e55dacb480 --- /dev/null +++ b/_partials/_prereqs-cloud-and-self.md @@ -0,0 +1,8 @@ +To follow the procedure on this page you need to: + +* Create a [target $SERVICE_LONG][create-service]. + + This procedure also works for [self-hosted $TIMESCALE_DB][enable-timescaledb]. + +[create-service]: /getting-started/:currentVersion:/services/ +[enable-timescaledb]: /self-hosted/:currentVersion:/install/ diff --git a/_partials/_prereqs-cloud-only.md b/_partials/_prereqs-cloud-only.md new file mode 100644 index 0000000000..a176783bf9 --- /dev/null +++ b/_partials/_prereqs-cloud-only.md @@ -0,0 +1,9 @@ +To follow the steps on this page: + +* Create a target [$SERVICE_LONG][create-service] with time-series and analytics enabled. + + You need your [connection details][connection-info]. + + +[create-service]: /getting-started/:currentVersion:/services/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ diff --git a/_partials/_prometheus-integrate.md b/_partials/_prometheus-integrate.md index 9828c0423d..d05a6b4c5e 100644 --- a/_partials/_prometheus-integrate.md +++ b/_partials/_prometheus-integrate.md @@ -99,11 +99,11 @@ You can further [visualize your data][grafana-prometheus] with Grafana. Use the [install-exporter]: https://grafana.com/oss/prometheus/exporters/postgres-exporter/?tab=installation [postgresql-exporter-dashboard]: https://grafana.com/oss/prometheus/exporters/postgres-exporter/?tab=dashboards [install-prometheus]: https://prometheus.io/docs/prometheus/latest/installation/ -[grafana]: /use-timescale/:currentVersion:/integrations/grafana/ +[grafana]: /integrations/:currentVersion:/grafana/ [grafana-prometheus]: https://grafana.com/docs/grafana-cloud/send-data/metrics/metrics-prometheus/ [prometheus]: https://prometheus.io/docs/introduction/overview/ [run-queries]: /getting-started/:currentVersion:/run-queries-from-console/ -[psql]: /use-timescale/:currentVersion:/integrations/psql/ -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[psql]: /integrations/:currentVersion:/psql/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ [postgresql-exporter]: https://grafana.com/oss/prometheus/exporters/postgres-exporter/ [open-console]: https://console.cloud.timescale.com/dashboard/services diff --git a/_partials/_quickstart-intro.md b/_partials/_quickstart-intro.md index a788977d18..d9c564c75f 100644 --- a/_partials/_quickstart-intro.md +++ b/_partials/_quickstart-intro.md @@ -1,3 +1,2 @@ -This code quick start enables you to easily integrate your app with $CLOUD_LONG. -Use your favorite programming language to connect to your $SERVICE_LONG, create and manage -hypertables, then ingest and query data. +Easily integrate your app with $CLOUD_LONG. Use your favorite programming language to connect to your +$SERVICE_LONG, create and manage hypertables, then ingest and query data. diff --git a/_partials/_service-overview.md b/_partials/_service-overview.md index 9588148b44..25d576dd21 100644 --- a/_partials/_service-overview.md +++ b/_partials/_service-overview.md @@ -1,46 +1,9 @@ You manage your $SERVICE_LONGs and interact with your data in $CONSOLE using the following modes: -
- - - - - - - - +| **$OPS_MODE_CAP** | **$DATA_MODE_CAP** | +|-------------------|---------------------| +| ![$CONSOLE $OPS_MODE][ops-mode] | ![$CONSOLE $DATA_MODE][data-mode] | +| **You use the $OPS_MODE to:**
  • Ensure data security with high availability and $READ_REPLICAs
  • Save money using compressed and tiered storage
  • Enable $PG extensions to add extra functionality
  • Increase security using $VPCs
  • Perform day-to-day administration
| **Powered by $POPSQL, you use the $DATA_MODE to:**
  • Write queries with autocomplete
  • Visualize data with charts and dashboards
  • Schedule queries and dashboards for alerts or recurring reports
  • Share queries and dashboards
| - - - - - - -
Ops modeData mode
- Timescale Console ops mode - - Timescale Console data mode -
-You use the ops mode to: -
    -
  • Ensure data security with high availability and read replicas
    • -
  • Save money using compressed and tiered storage
    • -
  • Enable PostgreSQL extensions to add extra functionality
    • -
  • Increase security using VPCs
    • -
  • Perform day-to-day administration
    • -
-		 -Powered by PopSQL, you use the data mode to: -
    -
  • Write queries with autocomplete
    • -
  • Visualize data with charts and dashboards
    • -
  • Schedule queries and dashboards for alerts or recurring reports
    • -
  • Share queries and dashboards
    • -
-
- -[how-plans-work]: /about/:currentVersion:/pricing-and-account-management/#how-plans-work +[ops-mode]: https://assets.timescale.com/docs/images/ops-mode-overview.png +[data-mode]: https://assets.timescale.com/docs/images/data-mode-overview.png \ No newline at end of file diff --git a/_partials/_since_2_18_0.md b/_partials/_since_2_18_0.md new file mode 100644 index 0000000000..19a03870be --- /dev/null +++ b/_partials/_since_2_18_0.md @@ -0,0 +1 @@ +Since [TimescaleDB v2.18.0](https://github.com/timescale/timescaledb/releases/tag/2.18.0) diff --git a/quick-start/golang.md b/_partials/_start-coding-golang.md similarity index 93% rename from quick-start/golang.md rename to _partials/_start-coding-golang.md index e1c6eac4cd..59b6579582 100644 --- a/quick-start/golang.md +++ b/_partials/_start-coding-golang.md @@ -1,41 +1,21 @@ ---- -title: "Quick Start: Go and TimescaleDB" -excerpt: Get started with Timescale Cloud or TimescaleDB using Go -keywords: [Golang, Go] ---- -import Intro from "versionContent/_partials/_quickstart-intro.mdx"; - -# Go quick start - - - -This quick start guide walks you through: - -* [Connecting to TimescaleDB][connect] -* [Creating a relational table][create-table] -* [Creating a hypertable][create-a-hypertable] -* [Inserting data][insert] -* [Executing a query][query] +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; ## Prerequisites -Before you start, make sure you have: + -* Installed [TimescaleDB][install]. -* Installed [Go][golang-install]. -* Installed the [PGX driver for Go][pgx-driver-github]. +- Install [Go][golang-install]. +- Install the [PGX driver for Go][pgx-driver-github]. -## Connect to TimescaleDB +## Connect to your $SERVICE_SHORT -In this section, you create a connection to TimescaleDB using the PGX driver. +In this section, you create a connection to $CLOUD_LONG using the PGX driver. PGX is a toolkit designed to help Go developers work directly with PostgreSQL. You can use it to help your Go application interact directly with TimescaleDB. - - 1. Locate your TimescaleDB credentials and use them to compose a connection string for PGX. @@ -107,9 +87,7 @@ You can use it to help your Go application interact directly with TimescaleDB. ```go os.Getenv("DATABASE_CONNECTION_STRING") ``` - - - + Alternatively, you can connect to TimescaleDB using a connection pool. @@ -118,8 +96,6 @@ result in faster database queries: - - 1. To create a connection pool that can be used for concurrent connections to your database, use the `pgxpool.New()` function instead of `pgx.Connect()`. Also note that this script imports @@ -159,8 +135,6 @@ result in faster database queries: } ``` - - ## Create a relational table @@ -173,8 +147,6 @@ percentage of the sensors. - - 1. Compose a string that contains the SQL statement to create a relational table. This example creates a table called `sensors`, with columns for ID, type, and location: @@ -223,8 +195,6 @@ percentage of the sensors. } ``` - - ## Generate a hypertable @@ -235,8 +205,6 @@ and most other tasks are executed on the hypertable. - - 1. Create a variable for the `CREATE TABLE SQL` statement for your hypertable. Notice how the hypertable has the compulsory time column: @@ -313,8 +281,6 @@ and most other tasks are executed on the hypertable. } ``` - - ## Insert rows of data @@ -329,8 +295,6 @@ the process. - - 1. Open a connection pool to the database, then use the prepared statements to formulate an `INSERT` SQL statement, and execute it: @@ -381,8 +345,6 @@ the process. } ``` - - Instead of inserting a single row of data at a time, you can use this procedure @@ -390,8 +352,6 @@ to insert multiple rows of data, instead: - - 1. This example uses PostgreSQL to generate some sample time-series to insert into the `sensor_data` hypertable. Define the SQL statement to generate the data, called `queryDataGeneration`. Then use the `.Query()` function to @@ -603,8 +563,6 @@ to insert multiple rows of data, instead: } ``` - - Inserting multiple rows of data using this method executes as many `insert` @@ -616,8 +574,6 @@ the previous procedure. It uses the pgx `Batch` object: - - 1. This example batch inserts data into the database: ```go @@ -742,8 +698,6 @@ the previous procedure. It uses the pgx `Batch` object: } ``` - - ## Execute a query @@ -752,8 +706,6 @@ This section covers how to execute queries against your database. - - 1. Define the SQL query you'd like to run on the database. This example uses a SQL query that combines time-series and relational data. It returns the average CPU values for every 5 minute interval, for sensors located on @@ -904,9 +856,7 @@ This section covers how to execute queries against your database. } } ``` - - - + ## Next steps @@ -927,13 +877,13 @@ your Go application, be sure to check out these advanced TimescaleDB tutorials: [parallel-copy-tool]: https://github.com/timescale/timescaledb-parallel-copy [pgx-docs]: https://pkg.go.dev/github.com/jackc/pgx [pgx-driver-github]: https://github.com/jackc/pgx -[install]: /getting-started/latest/ -[connect]: #connect-to-timescaledb -[create-table]: #create-a-relational-table -[create-a-hypertable]: #create-a-hypertable -[insert]: #insert-rows-of-data -[query]: #execute-a-query +[install]: /getting-started/:currentVersion:/ +[connect]: /getting-started/:currentVersion:/start-coding-with-timescale/#connect-to-timescaledb +[create-table]: /getting-started/:currentVersion:/start-coding-with-timescale/#create-a-relational-table +[create-a-hypertable]: /getting-started/:currentVersion:/start-coding-with-timescale/#generate-a-hypertable +[insert]: /getting-started/:currentVersion:/start-coding-with-timescale/#insert-rows-of-data +[query]: /getting-started/:currentVersion:/start-coding-with-timescale/#execute-a-query [create-hypertable-docs]: /use-timescale/:currentVersion:/hypertables/create/ -[insert]: /quick-start/:currentVersion:/golang/#insert-a-row-into-your-timescale-database -[query]: /quick-start/:currentVersion:/golang/#execute-a-query-on-your-timescale-database -[create-hypertable]: /quick-start/:currentVersion:/python/#create-hypertable +[insert]: /getting-started/:currentVersion:/start-coding-with-timescale/#insert-a-row-into-your-timescale-database +[query]: /getting-started/:currentVersion:/start-coding-with-timescale/#execute-a-query-on-your-timescale-database +[create-hypertable]: /getting-started/:currentVersion:/start-coding-with-timescale/#generate-a-hypertable diff --git a/quick-start/java.md b/_partials/_start-coding-java.md similarity index 91% rename from quick-start/java.md rename to _partials/_start-coding-java.md index f3b9389477..2066fef8b7 100644 --- a/quick-start/java.md +++ b/_partials/_start-coding-java.md @@ -1,45 +1,23 @@ ---- -title: "Quick Start: Java and TimescaleDB" -excerpt: Get started with Timescale Cloud or TimescaleDB using Java -keywords: [Java] ---- - -import Intro from "versionContent/_partials/_quickstart-intro.mdx"; - -# Java quick start - - - -This quick start guide walks you through: - -* [Connecting to TimescaleDB][connect] -* [Creating a relational table][create-table] -* [Creating a hypertable][create-a-hypertable] -* [Inserting data][insert] -* [Executing a query][query] +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; ## Prerequisites -Before you start, make sure you have: + -* Installed TimescaleDB. For more information, see the - [installation documentation][install]. -* Installed the [Java Development Kit (JDK)][jdk]. -* Installed the [PostgreSQL JDBC driver][pg-jdbc-driver]. +* Install the [Java Development Kit (JDK)][jdk]. +* Install the [PostgreSQL JDBC driver][pg-jdbc-driver]. -All code in this quick start is for Java 16 and later. If you are working +All code in this quick start is for Java 16 and later. If you are working with older JDK versions, use legacy coding techniques. -## Connect to TimescaleDB +## Connect to your $SERVICE_SHORT -In this section, you create a connection to TimescaleDB using an application in +In this section, you create a connection to your $SERVICE_SHORT using an application in a single file. You can use any of your favorite build tools, including `gradle` or `maven`. - - 1. Create a directory containing a text file called `Main.java`, with this content: ```java @@ -91,9 +69,9 @@ or `maven`. java -cp *.jar Main.java ``` -If the command is successful, a string similar to -`org.postgresql.Driver@7f77e91b` is printed to your console. This means that you -are ready to connect to TimescaleDB from Java. + If the command is successful, a string similar to + `org.postgresql.Driver@7f77e91b` is printed to your console. This means that you + are ready to connect to TimescaleDB from Java. 1. Locate your TimescaleDB credentials and use them to compose a connection string for JDBC. @@ -145,8 +123,6 @@ are ready to connect to TimescaleDB from Java. If the command is successful, a string similar to `{ApplicationName=PostgreSQL JDBC Driver}` is printed to your console. - - ## Create a relational table @@ -159,8 +135,6 @@ percentage of the sensors. - - 1. Compose a string which contains the SQL statement to create a relational table. This example creates a table called `sensors`, with columns `id`, `type` and `location`: @@ -210,9 +184,7 @@ percentage of the sensors. } } ``` - - - + ## Create a hypertable @@ -223,8 +195,6 @@ and most other tasks are executed on the hypertable. - - 1. Create a `CREATE TABLE` SQL statement for your hypertable. Notice how the hypertable has the compulsory time column: @@ -297,8 +267,6 @@ and most other tasks are executed on the hypertable. } ``` - - ## Insert data @@ -308,8 +276,6 @@ section, you can insert single rows, or insert by batches of rows. - - 1. Open a connection to the database, use prepared statements to formulate the `INSERT` SQL statement, then execute the statement: @@ -329,8 +295,6 @@ section, you can insert single rows, or insert by batches of rows. } ``` - - If you want to insert a batch of rows by using a batching mechanism. In this @@ -339,11 +303,6 @@ example, you generate some sample time-series data to insert into the - - 1. Insert batches of rows: ```java @@ -367,8 +326,6 @@ defaultExpanded={false}> } ``` - - ## Execute a query @@ -377,8 +334,6 @@ This section covers how to execute queries against your database. - - ## Execute queries on TimescaleDB 1. Define the SQL query you'd like to run on the database. This example @@ -428,8 +383,6 @@ This section covers how to execute queries against your database. ... ``` - - ## Next steps @@ -446,7 +399,7 @@ advanced TimescaleDB tutorials: This section contains complete code samples. - +### Complete code sample ```java package com.timescale.java; @@ -534,9 +487,7 @@ public class Main { } ``` - - - +### Execute more complex queries ```java package com.timescale.java; @@ -645,8 +596,6 @@ public class Main { } ``` - - [jdk]: https://openjdk.java.net [pg-jdbc-driver-artifact]: https://jdbc.postgresql.org/download/ [pg-jdbc-driver-conn-docs]: https://jdbc.postgresql.org/documentation/datasource/ diff --git a/quick-start/node.md b/_partials/_start-coding-node.md similarity index 88% rename from quick-start/node.md rename to _partials/_start-coding-node.md index a0160a19ec..0520c53c25 100644 --- a/quick-start/node.md +++ b/_partials/_start-coding-node.md @@ -1,31 +1,11 @@ ---- -title: "Quick Start: Node.js and TimescaleDB" -excerpt: Get started with Timescale Cloud or TimescaleDB using Node.js -keywords: [Node, node.js] ---- - -import Intro from "versionContent/_partials/_quickstart-intro.mdx"; - -# Node.js quick start - - - -This quick start guide walks you through: - -* [Connecting to TimescaleDB][connect] -* [Creating a relational table][create-table] -* [Creating a hypertable][create-a-hypertable] -* [Inserting data][insert] -* [Executing a query][query] +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; ## Prerequisites -Before you start, make sure you have: + -* Installed TimescaleDB. For more information, see the - [installation documentation][install]. -* Installed [Node.js][node-install]. -* Installed the Node.js package manager [npm][npm-install]. +* Install [Node.js][node-install]. +* Install the Node.js package manager [npm][npm-install]. ## Connect to TimescaleDB @@ -34,8 +14,6 @@ ORM (object relational mapper) called [Sequelize][sequelize-info]. - - 1. At the command prompt, initialize a new Node.js app: ```bash @@ -157,8 +135,6 @@ ORM (object relational mapper) called [Sequelize][sequelize-info]. Connection has been established successfully. ``` - - ## Create a relational table @@ -167,8 +143,6 @@ In this section, you create a relational table called `page_loads`. - - 1. Use the Sequelize command line tool to create a table and model called `page_loads`: ```bash @@ -237,8 +211,6 @@ In this section, you create a relational table called `page_loads`. 1. Instantiate a `PageLoads` object and save it to the database. - - ## Create a hypertable @@ -249,8 +221,6 @@ and most other tasks are executed on the hypertable. - - 1. Create a migration to modify the `page_loads` relational table, and change it to a hypertable by first running the following command: @@ -304,8 +274,6 @@ and most other tasks are executed on the hypertable. == 20200601202912-add_hypertable: migrated (0.426s) ``` - - ## Insert rows of data @@ -314,10 +282,6 @@ This section covers how to insert data into your hypertables. - - -## Insert rows into TimescaleDB - 1. In the `index.js` file, modify the `/` route to get the `user-agent` from the request object (`req`) and the current timestamp. Then, call the `create` method on `PageLoads` model, supplying the user agent and timestamp @@ -342,9 +306,7 @@ This section covers how to insert data into your hypertables. } }) ``` - - - + ## Execute a query @@ -355,8 +317,6 @@ is displayed. - - 1. Modify the `/` route in the `index.js` file to call the Sequelize `findAll` function and retrieve all data from the `page_loads` table using the `PageLoads` model: @@ -385,8 +345,6 @@ is displayed. Now, when you reload the page, you should see all of the rows currently in the `page_loads` table. - - [node-install]: https://nodejs.org diff --git a/quick-start/python.md b/_partials/_start-coding-python.md similarity index 90% rename from quick-start/python.md rename to _partials/_start-coding-python.md index 13b39c3590..77d3d8bf2d 100644 --- a/quick-start/python.md +++ b/_partials/_start-coding-python.md @@ -1,32 +1,13 @@ ---- -title: "Quick Start: Python and TimescaleDB" -excerpt: Get started with Timescale Cloud or TimescaleDB using Python -keywords: [Python] ---- - -import Intro from "versionContent/_partials/_quickstart-intro.mdx"; - -# Python quick start - - - -This quick start guide walks you through: - -* [Connecting to TimescaleDB][connect] -* [Creating a relational table][create-table] -* [Creating a hypertable][create-a-hypertable] -* [Inserting data][insert] -* [Executing a query][query] +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; ## Prerequisites -Before you start, make sure you have: + + +* Install the `psycopg2` library. -* Installed TimescaleDB. For more information, see the - [installation documentation][install]. -* Installed the `psycopg2` library. For more information, see the - [psycopg2 documentation][psycopg2-docs]. -* [](#)A [Python virtual environment][virtual-env]. + For more information, see the [psycopg2 documentation][psycopg2-docs]. +* Create a [Python virtual environment][virtual-env]. [](#) ## Connect to TimescaleDB @@ -37,8 +18,6 @@ prevents common attacks such as SQL injection. - - 1. Import the psycogpg2 library: ```python @@ -108,8 +87,6 @@ prevents common attacks such as SQL injection. print(cursor.fetchone()) ``` - - ## Create a relational table @@ -122,8 +99,6 @@ percentage of the sensors. - - 1. Compose a string which contains the SQL statement to create a relational table. This example creates a table called `sensors`, with columns `id`, `type` and `location`: @@ -149,8 +124,6 @@ percentage of the sensors. cursor.close() ``` - - ## Create a hypertable @@ -161,8 +134,6 @@ and most other tasks are executed on the hypertable. - - 1. Create a string variable that contains the `CREATE TABLE` SQL statement for your hypertable. Notice how the hypertable has the compulsory time column: @@ -202,9 +173,7 @@ and most other tasks are executed on the hypertable. conn.commit() cursor.close() ``` - - - + ## Insert rows of data @@ -215,8 +184,6 @@ section, you can use `psycopg2` with prepared statements, or you can use - - 1. This example inserts a list of tuples, or relational data, called `sensors`, into the relational table named `sensors`. Open a cursor with a connection to the database, use prepared statements to formulate the `INSERT` SQL @@ -251,8 +218,6 @@ section, you can use `psycopg2` with prepared statements, or you can use conn.commit() ``` - - If you choose to use `pgcopy` instead, install the `pgcopy` package @@ -265,8 +230,6 @@ from pgcopy import CopyManager - - 1. Generate some random sensor data using the `generate_series` function provided by PostgreSQL. This example inserts a total of 480 rows of data (4 readings, every 5 minutes, for 24 hours). In your application, this would be @@ -350,8 +313,6 @@ from pgcopy import CopyManager print(cursor.fetchall()) ``` - - ## Execute a query @@ -367,9 +328,9 @@ For more information about properly using placeholders in `psycopg2`, see the For more information about how to execute more complex queries in `psycopg2`, see the [psycopg2 documentation][psycopg2-docs-basics]. - +### Execute a query - + 1. Define the SQL query you'd like to run on the database. This example is a simple `SELECT` statement querying each row from the previously created @@ -412,16 +373,14 @@ see the [psycopg2 documentation][psycopg2-docs-basics]. Using this cursor, `cursor.fetchall()` returns a list of dictionary-like objects. - - For more complex queries, you can use prepared statements to ensure queries are executed safely against the database. - +### Execute queries using prepared statements - + 1. Write the query using prepared statements: @@ -442,9 +401,7 @@ executed safely against the database. cursor.execute(query, data) results = cursor.fetchall() ``` - - - + [install]: /getting-started/latest/ diff --git a/quick-start/ruby.md b/_partials/_start-coding-ruby.md similarity index 98% rename from quick-start/ruby.md rename to _partials/_start-coding-ruby.md index 1d839a3e81..145952a2a8 100644 --- a/quick-start/ruby.md +++ b/_partials/_start-coding-ruby.md @@ -7,10 +7,6 @@ keywords: [Ruby] import QuickstartIntro from "versionContent/_partials/_quickstart-intro.mdx"; import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; -# Ruby quick start - - - ## Prerequisites @@ -423,9 +419,9 @@ Now that you have integrated the ruby gem into your app: [add-policies]: #manage-chunks-and-compression [manage-chunks]: #manage-chunks-and-compression [install]: /getting-started/latest/ -[psql-install]: /use-timescale/:currentVersion:/integrations/psql/ +[psql-install]: /integrations/:currentVersion:/psql/ [rails-guide]: https://guides.rubyonrails.org/install_ruby_on_rails.html#installing-rails -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ [about-hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/ [rails-compostite-primary-keys]: https://guides.rubyonrails.org/active_record_composite_primary_keys.html [ruby-blog-post]: https://www.timescale.com/blog/building-a-better-ruby-orm-for-time-series-and-analytics diff --git a/_partials/_tiered-storage-billing.md b/_partials/_tiered-storage-billing.md new file mode 100644 index 0000000000..81674e4786 --- /dev/null +++ b/_partials/_tiered-storage-billing.md @@ -0,0 +1 @@ +$COMPANY charges only for the storage that your data occupies in S3, regardless of whether it was compressed in $CLOUD_LONG before tiering. There are no additional expenses, such as data transfer or compute. \ No newline at end of file diff --git a/_partials/_timescale-intro.md b/_partials/_timescale-intro.md index c78b8c69e7..b69b5205e7 100644 --- a/_partials/_timescale-intro.md +++ b/_partials/_timescale-intro.md @@ -1,2 +1,2 @@ -Timescale extends PostgreSQL for all of your resource-intensive production workloads, so you +$COMPANY extends $PG for all of your resource-intensive production workloads, so you can build faster, scale further, and stay under budget. diff --git a/_partials/_transit-gateway.md b/_partials/_transit-gateway.md new file mode 100644 index 0000000000..2c4cb51f1e --- /dev/null +++ b/_partials/_transit-gateway.md @@ -0,0 +1,43 @@ +1. **Create a Peering $VPC in [$CONSOLE][console-login]** + + 1. In `Security` > `VPC`, click `Create a VPC`: + + ![$CLOUD_LONG new $VPC](https://assets.timescale.com/docs/images/add-peering-vpc.png) + + 1. Choose your region and IP range, name your VPC, then click `Create VPC`: + + ![Create a new VPC in $CLOUD_LONG](https://assets.timescale.com/docs/images/configure-peering-vpc.png) + + Your $SERVICE_SHORT and Peering $VPC must be in the same AWS region. The number of Peering $VPCs you can create in your project depends on your [price tier][pricing-plans]. If you need another Peering $VPC, either contact [support@timescale.com](mailto:support@timescale.com) or change your tier in [$CONSOLE][console-login]. + + 1. Add a peering connection: + + 1. In the `VPC Peering` column, click `Add`. + 1. Provide your AWS account ID, Transit Gateway ID, CIDR ranges, and AWS region. + 1. Click `Add connection`. + + ![Add peering](https://assets.timescale.com/docs/images/add-peering.png) + +1. **Accept and configure peering connection in your AWS account** + + Once your peering connection appears as `Processing`, you can accept and configure it in AWS: + + 1. Accept the peering request coming from $CLOUD_LONG. The request can take up to 5 min to arrive. Within 5 more minutes after accepting, the peering should appear as `Connected` in $CONSOLE. + + 1. Configure at least the following in yur AWS account networking: + + - Your subnet route table to route traffic to your Transit Gateway for the Peering VPC CIDRs. + - Your Transit Gateway route table to route traffic to the newly created Transit Gateway peering attachment for the Peering VPC CIDRs. + - Security groups to allow outbound TCP 5432. + +1. **Attach a $CLOUD_LONG service to the Peering VPC In [$CONSOLE][console-services]** + + 1. Select the $SERVICE_SHORT you want to connect to the Peering VPC. + 1. Click `Operations` > `Security` > `VPC`. + 1. Select the VPC, then click `Attach VPC`. + + You cannot attach a $SERVICE_LONG to multiple $CLOUD_LONG $VPCs at the same time. + +[console-login]: https://console.cloud.timescale.com/ +[console-services]: https://console.cloud.timescale.com/dashboard/services +[pricing-plans]: /about/:currentVersion:/pricing-and-account-management/ diff --git a/_partials/_usage-based-storage-intro.md b/_partials/_usage-based-storage-intro.md index 6823fdbb60..5bee4f7a55 100644 --- a/_partials/_usage-based-storage-intro.md +++ b/_partials/_usage-based-storage-intro.md @@ -1,9 +1,9 @@ $CLOUD_LONG charges are based on the amount of storage you use. You don't pay for fixed storage size, and you don't need to worry about scaling disk size as your -data grows; We handle it all for you. To reduce your data costs further, -use [compression][compression], a [data retention policy][data-retention], and +data grows - we handle it all for you. To reduce your data costs further, +combine [Hypercore][hypercore], a [data retention policy][data-retention], and [tiered storage][data-tiering]. -[compression]: /use-timescale/:currentVersion:/compression/about-compression +[hypercore]: /api/:currentVersion:/hypercore/ [data-retention]: /use-timescale/:currentVersion:/data-retention/ [data-tiering]: /use-timescale/:currentVersion:/data-tiering/ diff --git a/_partials/_use-case-iot-create-cagg.md b/_partials/_use-case-iot-create-cagg.md new file mode 100644 index 0000000000..7a8f13940f --- /dev/null +++ b/_partials/_use-case-iot-create-cagg.md @@ -0,0 +1,87 @@ +1. **Monitor energy consumption on a day-to-day basis** + + 1. Create a continuous aggregate `kwh_day_by_day` for energy consumption: + + ```sql + CREATE MATERIALIZED VIEW kwh_day_by_day(time, value) + with (timescaledb.continuous) as + SELECT time_bucket('1 day', created, 'Europe/Berlin') AS "time", + round((last(value, created) - first(value, created)) * 100.) / 100. AS value + FROM metrics + WHERE type_id = 5 + GROUP BY 1; + ``` + + 1. Add a refresh policy to keep `kwh_day_by_day` up-to-date: + + ```sql + SELECT add_continuous_aggregate_policy('kwh_day_by_day', + start_offset => NULL, + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); + ``` + +1. **Monitor energy consumption on an hourly basis** + + 1. Create a continuous aggregate `kwh_hour_by_hour` for energy consumption: + + ```sql + CREATE MATERIALIZED VIEW kwh_hour_by_hour(time, value) + with (timescaledb.continuous) as + SELECT time_bucket('01:00:00', metrics.created, 'Europe/Berlin') AS "time", + round((last(value, created) - first(value, created)) * 100.) / 100. AS value + FROM metrics + WHERE type_id = 5 + GROUP BY 1; + ``` + + 1. Add a refresh policy to keep the continuous aggregate up-to-date: + + ```sql + SELECT add_continuous_aggregate_policy('kwh_hour_by_hour', + start_offset => NULL, + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); + ``` + +1. **Analyze your data** + + Now you have made continuous aggregates, it could be a good idea to use them to perform analytics on your data. + For example, to see how average energy consumption changes during weekdays over the last year, run the following query: + ```sql + WITH per_day AS ( + SELECT + time, + value + FROM kwh_day_by_day + WHERE "time" at time zone 'Europe/Berlin' > date_trunc('month', time) - interval '1 year' + ORDER BY 1 + ), daily AS ( + SELECT + to_char(time, 'Dy') as day, + value + FROM per_day + ), percentile AS ( + SELECT + day, + approx_percentile(0.50, percentile_agg(value)) as value + FROM daily + GROUP BY 1 + ORDER BY 1 + ) + SELECT + d.day, + d.ordinal, + pd.value + FROM unnest(array['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat']) WITH ORDINALITY AS d(day, ordinal) + LEFT JOIN percentile pd ON lower(pd.day) = lower(d.day); + ``` + + You see something like: + + | day | ordinal | value | + | --- | ------- | ----- | + | Mon | 2 | 23.08078714975423 | + | Sun | 1 | 19.511430831944395 | + | Tue | 3 | 25.003118897837307 | + | Wed | 4 | 8.09300571759772 | diff --git a/_partials/_use-case-setup-blockchain-dataset.md b/_partials/_use-case-setup-blockchain-dataset.md new file mode 100644 index 0000000000..c1ddd2669f --- /dev/null +++ b/_partials/_use-case-setup-blockchain-dataset.md @@ -0,0 +1,18 @@ + + +import CreateHypertableBlockchain from "versionContent/_partials/_create-hypertable-blockchain.mdx"; +import AddDataBlockchain from "versionContent/_partials/_add-data-blockchain.mdx"; +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; + +# Ingest data into a $SERVICE_LONG + +This tutorial uses a dataset that contains Bitcoin blockchain data for +the past five days, in a hypertable named `transactions`. + +## Prerequisites + + + + + + diff --git a/_partials/_use-case-transport-geolocation.md b/_partials/_use-case-transport-geolocation.md new file mode 100644 index 0000000000..fbdefd6cad --- /dev/null +++ b/_partials/_use-case-transport-geolocation.md @@ -0,0 +1,88 @@ + +### Set up your data for geospatial queries + +To add geospatial analysis to your ride count visualization, you need geospatial data to work out which trips +originated where. As $TIMESCALE_DB is compatible with all PostgreSQL extensions, use [PostGIS][postgis] to slice +data by time and location. + + + +1. Connect to your [$SERVICE_LONG][in-console-editors] and add the PostGIS extension: + + ```sql + CREATE EXTENSION postgis; + ``` + +1. Add geometry columns for pick up and drop off locations: + + ```sql + ALTER TABLE rides ADD COLUMN pickup_geom geometry(POINT,2163); + ALTER TABLE rides ADD COLUMN dropoff_geom geometry(POINT,2163); + ``` + +1. Convert the latitude and longitude points into geometry coordinates that work with PostGIS: + + ```sql + UPDATE rides SET pickup_geom = ST_Transform(ST_SetSRID(ST_MakePoint(pickup_longitude,pickup_latitude),4326),2163), + dropoff_geom = ST_Transform(ST_SetSRID(ST_MakePoint(dropoff_longitude,dropoff_latitude),4326),2163); + ``` + This updates 10,906,860 rows of data on both columns, it takes a while. Coffee is your friend. + + + +### Visualize the area where you can make the most money + +In this section you visualize a query that returns rides longer than 5 miles for +trips taken within 2 km of Times Square. The data includes the distance travelled and +is `GROUP BY` `trip_distance` and location so that Grafana can plot the data properly. + +This enables you to see where a taxi driver is most likely to pick up a passenger who wants a longer ride, +and make more money. + + + +1. **Create a geolocalization dashboard** + + 1. In Grafana, create a new dashboard that is connected to your $SERVICE_LONG data source with a Geomap + visualization. + + 1. In the `Queries` section, select `Code`, then select the Time series `Format`. + + ![Real-time analytics geolocation](https://assets.timescale.com/docs/images/use-case-rta-grafana-timescale-configure-dashboard.png) + + 1. To find rides longer than 5 miles in Manhattan, paste the following query: + + ```sql + SELECT time_bucket('5m', rides.pickup_datetime) AS time, + rides.trip_distance AS value, + rides.pickup_latitude AS latitude, + rides.pickup_longitude AS longitude + FROM rides + WHERE rides.pickup_datetime BETWEEN '2016-01-01T01:41:55.986Z' AND '2016-01-01T07:41:55.986Z' AND + ST_Distance(pickup_geom, + ST_Transform(ST_SetSRID(ST_MakePoint(-73.9851,40.7589),4326),2163) + ) < 2000 + GROUP BY time, + rides.trip_distance, + rides.pickup_latitude, + rides.pickup_longitude + ORDER BY time + LIMIT 500; + ``` + You see a world map with a dot on New York. + 1. Zoom into your map to see the visualization clearly. + +1. **Customize the visualization** + + 1. In the Geomap options, under `Map Layers`, click `+ Add layer` and select `Heatmap`. + You now see the areas where a taxi driver is most likely to pick up a passenger who wants a + longer ride, and make more money. + + ![Real-time analytics geolocation](https://assets.timescale.com/docs/images/use-case-rta-grafana-heatmap.png) + + + + + +[in-console-editors]: /getting-started/:currentVersion:/run-queries-from-console/ +[postgis]: http://postgis.net/ diff --git a/_partials/_vpc-limitations.md b/_partials/_vpc-limitations.md new file mode 100644 index 0000000000..29a314a828 --- /dev/null +++ b/_partials/_vpc-limitations.md @@ -0,0 +1,12 @@ +* You **can attach**: + * Up to 50 Customer $VPCs to a Peering $VPC. + * A $SERVICE_LONG to a single Peering $VPC at a time. + The $SERVICE_SHORT and the Peering $VPC must be in the same AWS region. However, you can peer a Customer $VPC and a Peering $VPC that are in different regions. + * Multiple $SERVICE_LONGs to the same Peering $VPC. +* You **cannot attach** a $SERVICE_LONG to multiple Peering $VPCs at the same time. + + The number of Peering $VPCs you can create in your project depends on your [$PRICING_PLAN][pricing-plans]. + If you need another Peering $VPC, either contact [support@timescale.com](mailto:support@timescale.com) or change your $PRICING_PLAN in [$CONSOLE][console-login]. + +[console-login]: https://console.cloud.timescale.com/ +[pricing-plans]: /about/:currentVersion:/pricing-and-account-management/ \ No newline at end of file diff --git a/_partials/_where-to-next.md b/_partials/_where-to-next.md index 5d1124a297..6c4aa2ac8e 100644 --- a/_partials/_where-to-next.md +++ b/_partials/_where-to-next.md @@ -1,7 +1,11 @@ -Now you have TimescaleDB running, have a look at the: +What next? [Try the main features offered by Timescale][try-timescale-features], see the [use case tutorials][tutorials], +interact with the data in your $SERVICE_LONG using [your favorite programming language][connect-with-code], integrate +your $SERVICE_LONG with a range of [third-party tools][integrations], plain old [Use Timescale][use-timescale], or dive +into [the API][use-the-api]. -* [Tutorials][tutorials]: walk through a variety of business scenarios using example datasets. -* [Use Timescale][tsdb-docs]: browse the features available with TimescaleDB. - -[tsdb-docs]: /use-timescale/:currentVersion:/ [tutorials]: /tutorials/:currentVersion:/ +[integrations]: /integrations/:currentVersion:/ +[connect-with-code]: /getting-started/:currentVersion:/start-coding-with-timescale/ +[use-the-api]: /api/:currentVersion:/ +[use-timescale]: /use-timescale/:currentVersion:/ +[try-timescale-features]: /getting-started/:currentVersion:/try-key-features-timescale-products/ diff --git a/_procedure-block.md b/_procedure-block.md deleted file mode 100644 index 38921bc352..0000000000 --- a/_procedure-block.md +++ /dev/null @@ -1,28 +0,0 @@ - - -### Procedure Title - -1. Do this - - ```bash - code code code - ``` - -2. Then do this - - ```ruby - def function - print('hello') - ``` - -3. TimescaleDB is a time-series database, built on top of PostgreSQL. More than that, -however, it's a relational database for time-series. Developers who use TimescaleDB -get the benefit of a purpose-built time-series database, plus a classic relational -database (PostgreSQL), all in one, with full SQL support. - - ```python - def start: - print('start') - ``` - - diff --git a/_queries/getting-started-crypto-cagg.md b/_queries/getting-started-crypto-cagg.md new file mode 100644 index 0000000000..2834d8ea02 --- /dev/null +++ b/_queries/getting-started-crypto-cagg.md @@ -0,0 +1,19 @@ +SELECT * FROM assets_candlestick_daily +ORDER BY day DESC, symbol +LIMIT 10; + +-- Output + +day | symbol | high | open | close | low +-----------------------+--------+----------+--------+----------+---------- +2025-01-30 00:00:00+00 | ADA/USD | 0.9708 | 0.9396 | 0.9607 | 0.9365 +2025-01-30 00:00:00+00 | ATOM/USD | 6.114 | 5.825 | 6.063 | 5.776 +2025-01-30 00:00:00+00 | AVAX/USD | 34.1 | 32.8 | 33.95 | 32.44 +2025-01-30 00:00:00+00 | BNB/USD | 679.3 | 668.12 | 677.81 | 666.08 +2025-01-30 00:00:00+00 | BTC/USD | 105595.65 | 103735.84 | 105157.21 | 103298.84 +2025-01-30 00:00:00+00 | CRO/USD | 0.13233 | 0.12869 | 0.13138 | 0.12805 +2025-01-30 00:00:00+00 | DAI/USD | 1 | 1 | 0.9999 | 0.99989998 +2025-01-30 00:00:00+00 | DOGE/USD | 0.33359 | 0.32392 | 0.33172 | 0.32231 +2025-01-30 00:00:00+00 | DOT/USD | 6.01 | 5.779 | 6.004 | 5.732 +2025-01-30 00:00:00+00 | ETH/USD | 3228.9 | 3113.36 | 3219.25 | 3092.92 +(10 rows) diff --git a/_queries/getting-started-crypto-srt-orderby.md b/_queries/getting-started-crypto-srt-orderby.md new file mode 100644 index 0000000000..4aaad26f1c --- /dev/null +++ b/_queries/getting-started-crypto-srt-orderby.md @@ -0,0 +1,20 @@ +SELECT * FROM crypto_ticks srt +WHERE symbol='ETH/USD' +ORDER BY time DESC +LIMIT 10; + +-- Output + +time | symbol | price | day_volume +-----------------------+--------+----------+------------ +2025-01-30 12:05:09+00 | ETH/USD | 3219.25 | 39425 +2025-01-30 12:05:00+00 | ETH/USD | 3219.26 | 39425 +2025-01-30 12:04:42+00 | ETH/USD | 3219.26 | 39459 +2025-01-30 12:04:33+00 | ETH/USD | 3219.91 | 39458 +2025-01-30 12:04:15+00 | ETH/USD | 3219.6 | 39458 +2025-01-30 12:04:06+00 | ETH/USD | 3220.68 | 39458 +2025-01-30 12:03:57+00 | ETH/USD | 3220.68 | 39483 +2025-01-30 12:03:48+00 | ETH/USD | 3220.12 | 39483 +2025-01-30 12:03:20+00 | ETH/USD | 3219.79 | 39482 +2025-01-30 12:03:11+00 | ETH/USD | 3220.06 | 39472 +(10 rows) diff --git a/_queries/getting-started-srt-orderby.md b/_queries/getting-started-srt-orderby.md index a0afde5a41..a1831ca158 100644 --- a/_queries/getting-started-srt-orderby.md +++ b/_queries/getting-started-srt-orderby.md @@ -1,20 +1,20 @@ SELECT * FROM stocks_real_time srt -WHERE symbol='TSLA' and day_volume is not null -ORDER BY time DESC, day_volume desc +WHERE symbol='TSLA' +ORDER BY time DESC LIMIT 10; -- Output -time | symbol | price | day_volume +time | symbol | price | day_volume -----------------------+--------+----------+------------ -2023-07-31 16:52:54+00 | TSLA | 267.88 | 55220135 -2023-07-31 16:52:53+00 | TSLA | 267.9141 | 55218795 -2023-07-31 16:52:52+00 | TSLA | 267.8987 | 55218669 -2023-07-31 16:52:51+00 | TSLA | 267.89 | 55218368 -2023-07-31 16:52:49+00 | TSLA | 267.87 | 55217109 -2023-07-31 16:52:49+00 | TSLA | 267.862 | 55216861 -2023-07-31 16:52:48+00 | TSLA | 267.8649 | 55216275 -2023-07-31 16:52:46+00 | TSLA | 267.88 | 55215737 -2023-07-31 16:52:44+00 | TSLA | 267.88 | 55215060 -2023-07-31 16:52:44+00 | TSLA | 267.88 | 55214966 +2025-01-30 00:51:00+00 | TSLA | 405.32 | NULL +2025-01-30 00:41:00+00 | TSLA | 406.05 | NULL +2025-01-30 00:39:00+00 | TSLA | 406.25 | NULL +2025-01-30 00:32:00+00 | TSLA | 406.02 | NULL +2025-01-30 00:32:00+00 | TSLA | 406.10 | NULL +2025-01-30 00:25:00+00 | TSLA | 405.95 | NULL +2025-01-30 00:24:00+00 | TSLA | 406.04 | NULL +2025-01-30 00:24:00+00 | TSLA | 406.04 | NULL +2025-01-30 00:22:00+00 | TSLA | 406.38 | NULL +2025-01-30 00:21:00+00 | TSLA | 405.77 | NULL (10 rows) diff --git a/_troubleshooting/chunk-temp-file-limit.md b/_troubleshooting/chunk-temp-file-limit.md index e1582582ad..a196fae1d5 100644 --- a/_troubleshooting/chunk-temp-file-limit.md +++ b/_troubleshooting/chunk-temp-file-limit.md @@ -2,7 +2,7 @@ title: Temporary file size limit exceeded when compressing chunks section: troubleshooting products: [cloud, mst, self_hosted] -topics: [hypertables, distributed hypertables, compression] +topics: [hypertables, distributed hypertables, compression, hypercore] errors: - language: text message: |- diff --git a/_troubleshooting/compression-alter-hypertable.md b/_troubleshooting/compression-alter-hypertable.md index e7ae4b99df..452e6bb564 100644 --- a/_troubleshooting/compression-alter-hypertable.md +++ b/_troubleshooting/compression-alter-hypertable.md @@ -2,7 +2,7 @@ title: Cannot add column to a compressed hypertable section: troubleshooting products: [cloud, mst, self_hosted] -topics: [hypertables, compression] +topics: [hypertables, compression, hypercore] errors: - language: text message: |- diff --git a/_troubleshooting/compression-dml-tuple-limit.md b/_troubleshooting/compression-dml-tuple-limit.md index 466267c962..a4516ed41e 100644 --- a/_troubleshooting/compression-dml-tuple-limit.md +++ b/_troubleshooting/compression-dml-tuple-limit.md @@ -2,7 +2,7 @@ title: Tuple decompression limit exceeded by operation section: troubleshooting products: [cloud, mst, self_hosted] -topics: [hypertables, compression] +topics: [hypertables, compression, hypercore] errors: - language: text message: |- diff --git a/_troubleshooting/compression-inefficient-chunk-interval.md b/_troubleshooting/compression-inefficient-chunk-interval.md new file mode 100644 index 0000000000..b3e2ce4d27 --- /dev/null +++ b/_troubleshooting/compression-inefficient-chunk-interval.md @@ -0,0 +1,18 @@ +--- +title: Inefficient `compress_chunk_time_interval` configuration +section: troubleshooting +products: [cloud, mst, self_hosted] +topics: [compression, hypercore] +errors: + - language: text + message: |- + compress_chunk_time_interval configured and primary dimension not first column in compress_orderby. + consider setting "" as first compress_orderby column +keywords: [compression, alter_table] +tags: [compression, alter_table] +--- + +When you configure `compress_chunk_time_interval` but do not set the primary dimension as the first column in `compress_orderby`, $TIMESCALE_DB decompresses chunks before merging. This makes merging less efficient. Set the primary dimension of the chunk as the first column in `compress_orderby` to improve efficiency. + + + diff --git a/_troubleshooting/compression-userperms.md b/_troubleshooting/compression-userperms.md index 71098006e7..54c9fdbf23 100644 --- a/_troubleshooting/compression-userperms.md +++ b/_troubleshooting/compression-userperms.md @@ -2,7 +2,7 @@ title: User permissions do not allow chunks to be compressed or decompressed section: troubleshooting products: [cloud, mst, self_hosted] -topics: [hypertables, compression, configuration] +topics: [hypertables, compression, configuration, hypercore] errors: - language: text message: |- diff --git a/_troubleshooting/invalid-attribute-reindex-hypertable.md b/_troubleshooting/invalid-attribute-reindex-hypertable.md index bad92c6bb9..129ad8d64e 100644 --- a/_troubleshooting/invalid-attribute-reindex-hypertable.md +++ b/_troubleshooting/invalid-attribute-reindex-hypertable.md @@ -2,7 +2,7 @@ title: Reindex hypertables to fix large indexes section: troubleshooting products: [cloud, mst, self_hosted] -topics: [hypertables, distributed hypertables, schema management, data retention, compression] +topics: [hypertables, distributed hypertables, schema management, data retention, compression, hypercore] errors: - language: bash message: |- diff --git a/_troubleshooting/mst/low-disk-memory-cpu.md b/_troubleshooting/mst/low-disk-memory-cpu.md index 39f03b0eb0..d752c9f8ef 100644 --- a/_troubleshooting/mst/low-disk-memory-cpu.md +++ b/_troubleshooting/mst/low-disk-memory-cpu.md @@ -34,11 +34,11 @@ for. In the `Overview` tab, locate the `Service Plan` section, and click `Upgrade` to enable the additional resources. If you run out of resources regularly, you might need to consider using your -resources more efficiently. Consider enabling [compression][howto-compression], +resources more efficiently. Consider enabling [Hypercore][setup-hypercore], using [continuous aggregates][howto-caggs], or [configuring data retention][howto-dataretention] to reduce the amount of resources your database uses. -[howto-compression]: /use-timescale/:currentVersion:/compression +[setup-hypercore]: /use-timescale/:currentVersion:/hypercore/real-time-analytics-in-hypercore/ [howto-caggs]: /use-timescale/:currentVersion:/continuous-aggregates [howto-dataretention]: /use-timescale/:currentVersion:/data-retention diff --git a/_troubleshooting/scheduled-jobs-stop-running.md b/_troubleshooting/scheduled-jobs-stop-running.md index 76b7fc128a..9e70965450 100644 --- a/_troubleshooting/scheduled-jobs-stop-running.md +++ b/_troubleshooting/scheduled-jobs-stop-running.md @@ -2,7 +2,7 @@ title: Scheduled jobs stop running section: troubleshooting products: [cloud, mst, self_hosted] -topics: [jobs, continuous aggregates, data retention, compression] +topics: [jobs, continuous aggregates, data retention, compression, hypercore] apis: - [continuous aggregates, add_continuous_aggregate_policy()] - [continuous aggregates, add_policies()] diff --git a/_troubleshooting/self-hosted/background-worker-failed-start.md b/_troubleshooting/self-hosted/background-worker-failed-start.md index 5191c790d8..c96b81a9fd 100644 --- a/_troubleshooting/self-hosted/background-worker-failed-start.md +++ b/_troubleshooting/self-hosted/background-worker-failed-start.md @@ -16,7 +16,7 @@ apis: - [hypertables, add_compression_policy()] - [hypertables, add_continuous_aggregate_policy()] keywords: [jobs, policies, actions] -tags: [jobs, scheduled jobs, background jobs, background workers, automation framework, policies, user-defined actions] +tags: [jobs, scheduled jobs, background jobs, background workers, automation framework, policies] --- - -```txt -1. Use numbers -1. If the order of the steps -1. Is important - -* Otherwise -* Use -* Bullets -``` - - - -## Processes and procedures - -We use processes and procedures to provide readers with numbered steps to -achieve a specific goal. - -Processes contain procedures. If the task you are describing is very lengthy, or -has a series of distinct components, break it up into a series of procedures -within a process. - -Procedures contain these elements: - - - -1. Level two section title in simple verb form -1. Short introduction -1. Open `` tag -1. Level three procedure title in gerund verb form -1. Numbered steps -1. Screenshot -1. Close `` tag - - - -For example: - -```txt -## Install Timescale - -This section describes how to install Timescale on premise. - - - -### Installing Timescale - -1. Start each step with a verb, or a location. -1. For example, "Run the `example` command", or "At the command prompt, open the - `example` file." - - - - - -``` - -In general, do not use result statements. However, if you feel it is absolutely -necessary, include it immediately before the closing procedure tag, and do not -put it as a step. - -## Code blocks - -Offset code blocks from the surrounding text by using three backticks and -declaring the language in use in the block. Usually, this is either `sql`, for -queries, or `bash`, for commands given at the command prompt. For a full list of -available languages see [the prismjs site][prismjs]. - -Make sure that the code block is indented at the same level as the surrounding text. - -When writing commands in a code block, do not use `#`, `$`, or any other prompt. -Additionally, for Linux commands, do not use `sudo`, instead, use the stem -sentence to indicate that the command must be run as the root user. - -For example, a bash command: - -```txt - - At the command prompt, as root, use this command: - - ```bash - apt install postgresql - ``` - -``` - -For example, an SQL query: - -```txt - - At the `psql` prompt, use this query: - - ```sql - SELECT * FROM table LIMIT 5; - ``` - -``` - -## Verbs - -* `Click` a button in a graphical user interface using a mouse. Do not `Click on`. -* `Press` a key or key combination on a keyboard. -* `Type` words or numbers using a keyboard. -* `Check` or `uncheck` a checkbox. -* `Select` or `deselect` an item in a menu. -* `Navigate` to a page or location in a graphical user interface. - -## Word usage - -### A - -*Above* -: Avoid all directional words. You cannot guarantee that things will stay in the -same position, or be in the position you expect on an individual reader's -device. - -*Adverbs* -: Do not use. - -❌ Simply install Timescale. - -👍 Install Timescale. - -*AWS* -: Amazon Web Services. No need to expand this acronym. - -*And/Or* -: Do not use. You can usually pick one. If you're not sure, pick "and." - -❌ I like apples and/or oranges. - -👍 I like apples and oranges. - -*Appears* -: Do not use. - -### B - -*Bare metal* -: Noun. - -❌ "Perform a bare metal installation." - -👍 "Install Timescale on bare metal." - -*Bare-metal* -: Adjective. - -❌ "Install Timescale on bare-metal." - -👍 "Perform a bare-metal installation." - -*Backend* -: One word. - -*Below* -: Avoid all directional words. You cannot guarantee that things will stay in the -same position, or be in the position you expect on an individual reader's -device. - -### C - -*Contractions* -: Absolutely fine to use, but try not to overdo it. - -*Cybersecurity* -: One word. - -### D - -*Dataset* -: One word. - -❌ "Ingest the example data set." - -👍 "Ingest the example dataset." - -### E - -### F - -*Fail over* -: Use as a verb - -*Failover* -: Use as a noun or adjective. - -*File system* -: Two words. - -*Frontend* -: One word. - -### G - -*GCP* -: Google Cloud Platform. No need to expand this acronym. - -### H - -*Hypershift* -: One word, always capitalized. Check the spelling of this word -carefully before publishing. - -### I - -### J - -### K - -### L - -*Latin abbreviations* -: Do not use. - -❌ eg - -👍 For example - -❌ I.E: - -👍 That is, - -*Log in* -: Verb. - -❌ "Enter your log in information." - -👍 "Log in to Timescale." - -*Login* -: Adjective or noun. - -❌ "Login to Timescale." - -👍 "Enter your login information." - -### M - -*Managed Service for TimescaleDB* -: The name of the product. Always capitalize the initial letters. Do not -abbreviate. - -❌ "I am using Managed Service for Timescale to manage my time-series data." - -❌ "I am using Managed Service to manage my time-series data." - -👍 "I am using Managed Service for TimescaleDB to manage my time-series -data." - -*Master/Slave* -: Do not use. Use "Primary" and "Secondary" instead. - -*MST* -: Do not use. Use "Managed Service for TimescaleDB" instead. - -### N - -*Next* -: Avoid all directional words. You cannot guarantee that things will stay in the -same position, or be in the position you expect on an individual reader's -device. - -### O - -*Once* -: Do not use. Use "when" instead. - -❌ "Once you have finished the installation, you can..." - -👍 "When you have finished the installation, you can." - -### P - -*Postgres* -: Do not use. Use "PostgreSQL" instead. - -❌ "Run the Postgres query" - -👍 "Run the PostgreSQL query." - -*PostgreSQL* -: Always refer to PostgreSQL by its full name and with correct capitalization, -as shown. - -*Previous* -: Avoid all directional words. You cannot guarantee that things will stay in the -same position, or be in the position you expect on an individual reader's -device. - -### Q - -### R - -### S - -*Simply* -: Don't use. - -❌ Simply install Timescale. - -👍 Install Timescale. - -*Superuser* -: One word. - -### T - -*Thus* -: Do not use. - -*Timescale* -: The name of the company, and the name of the product in a general -sense. Do not use camel case. - -❌ TimeScale - -👍 Timescale is hosting a virtual event. - -👍 I have installed Timescale to manage my time-series data. - -*Timescale* -: The name of the product. Always capitalize the initial letters. - -❌ "I am using timescale to manage my time-series data." - -❌ "I am using Timescale Cloud to manage my time-series data." - -👍 "I am using Timescale to manage my time-series data." - -*TimescaleDB* -: The name of the PostgreSQL extension. Always capitalize the initial letter and -the DB at the end if using in prose. If referring directly to the name of the -extension, use all lowercase. - -❌ "I installed timescaleDB to manage my time-series data." - -👍 "I installed the `timescaledb` extension to manage my time-series data." - -👍 "I installed TimescaleDB to manage my time-series data." - -*tobs* -: The observability suite. Do not use capitalization, even when it begins -a sentence. If possible, rewrite the sentence to avoid this. - -❌ "Tobs can be used to install." - -👍 "Install using tobs" - -*TSDB* -: Do not use. Use "Timescale database" instead. - -### U - -*Update* -: An update is a small or minor improvement, often delivered in a patch. Updates -are done frequently, and require little or no downtime. - -👍 Install the security update to patch this version. - -*Upgrade* -: An upgrade is a large or major improvement, and usually requires a new -version. Upgrades are done less frequently, and could require planning, -prepatory backups, and planned downtime. - -👍 Upgrade from TimescaleDB 1 to TimescaleDB 2. - -👍 Upgrade from TimescaleDB 2.3 to TimescaleDB 2.4. - -*Utilize* -: Do not use. Use "use" instead. - -### V - -*Vanilla PostgreSQL* -: Do not use. If you want to differentiate between regular PostgreSQL tables, -and tables that contain time-series, use "standard PostgreSQL". - -❌ "You can also create services that run vanilla PostgreSQL." - -👍 "You can also create services that run standard PostgreSQL." - -*Via* -: Do not use. There is usually a more accurate English word, like -"through," "with," or "using." - -### W - -*Will* -: Do not use. It usually indicates that you are writing in future tense. -Always write in simple present tense. - -❌ After installation, you will see a message. - -👍 After installation, you see a message. - -### X - -### Y - -### Z - -[contributors]: https://github.com/timescale/docs/blob/latest/CONTRIBUTING.md -[github-docs]: https://github.com/timescale/docs -[google-style]: https://developers.google.com/style -[readme]: https://github.com/timescale/docs#readme -[prismjs]: https://prismjs.com/#supported-languages diff --git a/about/contribute-to-timescale.md b/about/contribute-to-timescale.md index c53468a1df..0e16623c63 100644 --- a/about/contribute-to-timescale.md +++ b/about/contribute-to-timescale.md @@ -6,57 +6,36 @@ keywords: [contribute] tags: [github] --- -# Contributing to Timescale +# Contribute to $COMPANY -There are multiple ways to help make TimescaleDB better. All of the documentation -and source for the PostgreSQL extension are available to use and review on -GitHub. +$TIMESCALE_DB, $PGAI_SHORT, $PGVECTORSCALE, $TOOLKIT_LONG, and the $COMPANY documentation are all open source. They are available in GitHub for you to use, review, and update. This page shows you where you can add to $COMPANY products. -## Contributing to Timescale documentation +## Contribute to the code for $COMPANY products -Timescale documentation is hosted in a [GitHub repository][github-docs] -and is open for contribution from all community members. If you -find errors or would like to add content to the docs, this tutorial -walks you through the process. +$COMPANY appreciates any help the community can provide to make its products better! You can: -### Making minor changes +* Open an issue with a bug report, build issue, feature request or suggestion. +* Fork a corresponding repository and submit a pull request. -If you want to make only minor changes to docs, you can make corrections -and submit pull requests on the GitHub website. Go to the file you want to -correct and click the 'pencil' icon to edit. Once done, GitHub gives you -an option to submit a pull request at the bottom of the page. +Head over to the $COMPANY source repositories to learn, review, and help improve our products! -### Making larger contributions to docs +* [$TIMESCALE_DB][timescaledb]: a $PG extension for high-performance real-time analytics on time-series and event data. +* [$PGAI_SHORT][pgai]: a suite of tools to develop RAG, semantic search, and other AI applications more easily with $PG. +* [$PGVECTORSCALE][pgvectorscale]: a complement to pgvector for higher performance embedding search and cost-efficient storage for AI applications. +* [$TOOLKIT_LONG][toolkit]: all things analytics when using $TIMESCALE_DB, with a particular focus on developer ergonomics and performance. -In order to modify documentation, you should have a working knowledge -of [git][install-git] and [Markdown][markdown-tutorial]. You -also need to create a GitHub account. +## Contribute to $COMPANY documentation -Be sure to read the [Timescale docs contribution styleguide][timescale-docs-style]. -You'll see information about how we refer to aspects of Timescale, -how we format our docs, and special Markdown tags available to -you as you author your contribution. +$COMPANY documentation is hosted in the [docs GitHub repository][github-docs] +and open for contribution from all community members. -Before we accept any contributions, Timescale contributors need to -sign the Contributor License Agreement (CLA). By signing a CLA, we -can ensure that the community is free and confident in its -ability to use your contributions. You are prompted to sign the -CLA during the pull request process. - -## Contributing to TimescaleDB code - -Timescale appreciates any help the community can provide to make TimescaleDB better! - -There are multiple ways you can help: - -* Open an issue with a bug report, build issue, feature request, suggestion, etc. -* Fork this repository and submit a pull request - -[Head over to our GitHub repository][github-timescaledb] for TimescaleDB to learn -more about how you can help and to review our coding style guide! +See the [README][readme] and [contribution guide][contribution-guide] for details. +[contribution-guide]: https://github.com/timescale/docs/blob/latest/CONTRIBUTING.md [github-docs]: https://github.com/timescale/docs -[github-timescaledb]: https://github.com/timescale/timescaledb/blob/master/CONTRIBUTING.md -[install-git]: https://git-scm.com/book/en/v2/Getting-Started-Installing-Git -[markdown-tutorial]: https://www.markdownguide.org/basic-syntax/ -[timescale-docs-style]: https://github.com/timescale/docs/blob/master/README.md +[timescaledb]: https://github.com/timescale/timescaledb/blob/main/CONTRIBUTING.md +[pgai]: https://github.com/timescale/pgai/blob/main/CONTRIBUTING.md +[pgvectorscale]: https://github.com/timescale/pgvectorscale/blob/main/CONTRIBUTING.md +[toolkit]: https://github.com/timescale/timescaledb-toolkit +[readme]: https://github.com/timescale/docs/blob/latest/README.md + diff --git a/about/index.md b/about/index.md index c8fddbabbe..01afd5180e 100644 --- a/about/index.md +++ b/about/index.md @@ -1,5 +1,5 @@ --- -title: About Timescale +title: About Timescale products excerpt: Learn about Timescale Cloud and TimescaleDB, including pricing, release notes, feature overview, and contribution guidelines --- diff --git a/about/page-index/page-index.js b/about/page-index/page-index.js index 56d53008db..6b827dbc43 100644 --- a/about/page-index/page-index.js +++ b/about/page-index/page-index.js @@ -1,27 +1,29 @@ module.exports = [ { - title: "About Timescale", + title: "About Timescale products", href: "about", + defaultOpen: true, filePath: "index.md", pageComponents: ["featured-cards"], excerpt: "Additional information about Timescale, including how to contribute, and release notes", children: [ { - title: "Changelog", - href: "changelog", - excerpt: "A summary of the latest changes to all Timescale products.", - }, - { - title: "Release notes", - href: "release-notes", - excerpt: "Release information for TimescaleDB v2.0.0 - v2.15.1", + title: "Timescale architecture for real-time analytics", + href: "whitepaper", + excerpt: + "A whitepaper detailing the architectural choices and optimizations for real-time analytics that power Timescale", }, { title: "Pricing plans and account management", href: "pricing-and-account-management", excerpt: "Pricing plans for Timescale Cloud services", }, + { + title: "Changelog", + href: "changelog", + excerpt: "A summary of the latest changes to all Timescale products.", + }, { title: "TimescaleDB editions", href: "timescaledb-editions", @@ -33,9 +35,9 @@ module.exports = [ excerpt: "Contribute to the Timescale codebase", }, { - title: "Contribute to Timescale documentation", - href: "contribute-to-docs", - excerpt: "Contribute to Timescale documentation", + title: "Release notes", + href: "release-notes", + excerpt: "Release information for TimescaleDB v2.0.0 - v2.15.1", }, ], }, diff --git a/about/pricing-and-account-management.md b/about/pricing-and-account-management.md index 9f9ea8978e..78bd4c427f 100644 --- a/about/pricing-and-account-management.md +++ b/about/pricing-and-account-management.md @@ -9,7 +9,7 @@ cloud_ui: - [billing] --- -import UsageBasedStorage from "versionContent/_partials/_usage-based-storage-intro.mdx"; +import TieredStorageBilling from "versionContent/_partials/_tiered-storage-billing.mdx"; # Pricing plans and account management @@ -34,7 +34,9 @@ Pricing plans give you: * **Cost efficiency**: by aligning our pricing with the value delivered, we ensure that you get the most out of every dollar spent. Our goal is to help you achieve more with less. -Using self-hosted TimescaleDB and our open-source products is still free. +Using self-hosted TimescaleDB and our open-source products is still free. + +If you create a $COMPANY account from AWS Marketplace, the pricing options are pay-as-you-go and annual commit. See [AWS pricing][aws-pricing] for details. This page explains pricing plans for Timescale Cloud, and how to easily manage your Timescale account. @@ -109,6 +111,8 @@ storage tier and a low-cost bottomless storage tier. You can keep up to 16 TB co (typically 80-100 TBs uncompressed) in the high-performance storage tier, and configure less-frequently accessed data to be moved to our low-cost storage tier built on S3. + + Timescale Cloud is very efficient and generally needs less compute than other databases to deliver the same performance. The best way to size your needs is to signup for a free trial and to test with a realistic workload. @@ -231,10 +235,16 @@ alt="Adding a payment method in Timescale"/> - **Plans**: choose the plan supplying the [features][plan-features] that suit your business and engineering needs. -- **Add-ons**: add `Production support` and improved database performance for mission critical workloads. +- **Add-ons**: add `Production support` and improved database performance for mission-critical workloads. + +## AWS Marketplace pricing + +When you get $CLOUD_LONG at AWS Marketplace, the following pricing options are available: + +- **Pay-as-you-go**: your consumption is calculated at the end of the month and included in your AWS invoice. No upfront costs, standard $CLOUD_LONG rates apply. +- **Annual commit**: your consumption is calculated at the end of the month ensuring predictable pricing and seamless billing through your AWS account. We confirm the contract terms with you before finalizing the commitment. [cloud-login]: https://console.cloud.timescale.com/ -[compression]: /use-timescale/:currentVersion:/compression/ [data-tiering]: /use-timescale/:currentVersion:/data-tiering/ [cloud-billing]: https://console.cloud.timescale.com/dashboard/billing/details [commercial-sla]: https://www.timescale.com/legal/timescale-cloud-terms-of-service @@ -243,3 +253,4 @@ alt="Adding a payment method in Timescale"/> [production-support]: https://www.timescale.com/support [get-in-touch]: https://www.timescale.com/contact [hipaa-compliance]: https://www.hhs.gov/hipaa/for-professionals/index.html +[aws-pricing]: /about/:currentVersion:/pricing-and-account-management/#aws-marketplace-pricing diff --git a/about/release-notes.md b/about/release-notes.md index 13425d94fc..43bab8884e 100644 --- a/about/release-notes.md +++ b/about/release-notes.md @@ -16,7 +16,7 @@ notes about our downloadable products, see: * [pgspot](https://github.com/timescale/pgspot/releases) - spot vulnerabilities in PostgreSQL extension scripts. * [live-migration](https://hub.docker.com/r/timescale/live-migration/tags) - a Docker image to migrate data to a Timescale Cloud service. - +This documentation is based on TimescaleDB v2.18.0 and compatible products. diff --git a/about/timescaledb-editions.md b/about/timescaledb-editions.md index 70f9a98c5f..f8e23764bc 100644 --- a/about/timescaledb-editions.md +++ b/about/timescaledb-editions.md @@ -6,6 +6,9 @@ keywords: [Apache, community, license] tags: [learn, contribute] --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; +import Since2180 from "versionContent/_partials/_since_2_18_0.mdx"; + # TimescaleDB Apache 2 and TimescaleDB Community Edition There are two versions of TimescaleDB available: @@ -162,40 +165,55 @@ You can access a hosted version of TimescaleDB Community Edition through - Compression + Hypercore - ALTER TABLE (Compression) + ALTER MATERIALIZED VIEW (Hypercore) ❌ ✅ - add_compression_policy + ALTER TABLE (Hypercore) ❌ ✅ - remove_compression_policy + add_columnstore_policy ❌ ✅ - compress_chunk + remove_columnstore_policy ❌ ✅ - decompress_chunk + convert_to_columnstore ❌ ✅ - hypertable_compression_stats + convert_to_rowstore ❌ ✅ - chunk_compression_stats + hypertable_columnstore_settings + ❌ + ✅ + + + hypertable_columnstore_stats + ❌ + ✅ + + + chunk_columnstore_settings + ❌ + ✅ + + + chunk_columnstore_stats ❌ ✅ @@ -246,7 +264,7 @@ You can access a hosted version of TimescaleDB Community Edition through ✅ - Actions and automation + Jobs and automation add_job @@ -452,6 +470,44 @@ You can access a hosted version of TimescaleDB Community Edition through ✅ ✅ + + Compression replaced by Hypercore + + + ALTER TABLE (Compression) + ❌ + ✅ + + + add_compression_policy + ❌ + ✅ + + + remove_compression_policy + ❌ + ✅ + + + compress_chunk + ❌ + ✅ + + + decompress_chunk + ❌ + ✅ + + + hypertable_compression_stats + ❌ + ✅ + + + chunk_compression_stats + ❌ + ✅ + diff --git a/about/whitepaper.md b/about/whitepaper.md new file mode 100644 index 0000000000..7fa97e0956 --- /dev/null +++ b/about/whitepaper.md @@ -0,0 +1,468 @@ +--- +title: Timescale architecture for real-time analytics +excerpt: A whitepaper detailing the architectural choices and optimizations for real-time analytics that power Timescale +products: [cloud, mst, self_hosted] +keywords: [real-time analytics, timescale cloud, timescaledb, time-series, whitepaper] +--- +# Timescale architecture for real-time analytics + +Timescale provides a powerful application database for real-time analytics on time-series data. It integrates seamlessly with the PostgreSQL ecosystem and enhances it with automatic time-based partitioning, hybrid row-columnar storage, and vectorized execution—enabling high-ingest performance, sub-second queries, and full SQL support at scale. + +By making use of incrementally updated materialized views and advanced analytical functions, Timescale reduces compute overhead and improves query efficiency. Developers can continue using familiar SQL workflows and tools, while benefiting from a database purpose-built for fast, scalable analytics. + +Timescale combines TimescaleDB, an open-source PostgreSQL extension, and Timescale Cloud, a cloud-native database service. + +This document outlines the architectural choices and optimizations that power Timescale’s performance and scalability while preserving PostgreSQL’s reliability and transactional guarantees. + +Want to read this whitepaper from the comfort of your own computer? + +
+ [Timescale architecture for real-time analytics (PDF)](https://assets.timescale.com/docs/downloads/Timescale_Architecture_for_Real-time_Analytics.pdf) +
+ + +## Introduction + +### What is real-time analytics? + +Real-time analytics enables applications to process and query data as it is generated and as it accumulates, delivering immediate and ongoing insights for decision-making. Unlike traditional analytics, which relies on batch processing and delayed reporting, real-time analytics supports *both* instant queries on fresh data and fast exploration of historical trends—powering applications with sub-second query performance across vast, continuously growing datasets. + +Many modern applications depend on real-time analytics to drive critical functionality: + + + +* **IoT monitoring systems** track sensor data over time, identifying long-term performance patterns while still surfacing anomalies as they arise. This allows businesses to optimize maintenance schedules, reduce costs, and improve reliability. +* **Financial and business intelligence platforms** analyze both current and historical data to detect trends, assess risk, and uncover opportunities—from tracking stock performance over a day, week, or year to identifying spending patterns across millions of transactions. +* **Interactive customer dashboards** empower users to explore live and historical data in a seamless experience—whether it's a SaaS product providing real-time analytics on business operations, a media platform analyzing content engagement, or an e-commerce site surfacing personalized recommendations based on recent and past behavior. + +Real-time analytics isn't just about reacting to the latest data, although that is critically important. It's also about delivering fast, interactive, and scalable insights across all your data, enabling better decision-making and richer user experiences. Unlike traditional ad-hoc analytics used by analysts, real-time analytics powers applications—driving dynamic dashboards, automated decisions, and user-facing insights at scale. + +To achieve this, real-time analytics systems must meet several key requirements: + +* **Low-latency queries** ensure sub-second response times even under high load, enabling fast insights for dashboards, monitoring, and alerting. +* **Low-latency ingest** minimizes the lag between when data is created and when it becomes available for analysis, ensuring fresh and accurate insights. +* **Data mutability** allows for efficient updates, corrections, and backfills, ensuring analytics reflect the most accurate state of the data. +* **Concurrency and scalability** enable systems to handle high query volumes and growing workloads without degradation in performance. +* **Seamless access to both recent and historical data** ensures fast queries across time, whether analyzing live, streaming data, or running deep historical queries on days or months of information. +* **Query flexibility** provides full SQL support, allowing for complex queries with joins, filters, aggregations, and analytical functions. + + +### Timescale: real-time analytics from PostgreSQL + +Timescale is a high-performance database that brings real-time analytics to applications. It combines fast queries, high ingest performance, and full SQL support—all while ensuring scalability and reliability. Timescale extends PostgreSQL with the TimescaleDB extension. It enables sub-second queries on vast amounts of incoming data while providing optimizations designed for continuously updating datasets. + +Timescale achieves this through the following optimizations: + + + +* **Efficient data partitioning:** automatically and transparently partitioning data into chunks, ensuring fast queries, minimal indexing overhead, and seamless scalability +* **Row-columnar storage:** providing the flexibility of a row store for transactions and the performance of a column store for analytics +* **Optimized query execution: **using techniques like chunk and batch exclusion, columnar storage, and vectorized execution to minimize latency +* **Continuous aggregates:** precomputing analytical results for fast insights without expensive reprocessing +* **Cloud-native operation: **compute/compute separation, elastic usage-based storage, horizontal scale out, data tiering to object storage +* **Operational simplicity: **offering high availability, connection pooling, and automated backups for reliable and scalable real-time applications + +With Timescale, developers can build low-latency, high-concurrency applications that seamlessly handle streaming data, historical queries, and real-time analytics while leveraging the familiarity and power of PostgreSQL. + + +## Data model + +Today's applications demand a database that can handle real-time analytics and transactional queries without sacrificing speed, flexibility, or SQL compatibility (including joins between tables). Timescale achieves this with **hypertables**, which provide an automatic partitioning engine, and **hypercore**, a hybrid row-columnar storage engine designed to deliver high-performance queries and efficient compression (up to 95 %) within PostgreSQL. + + +### Efficient data partitioning + +Timescale provides hypertables, a table abstraction that automatically partitions data into chunks in real time (using time stamps or incrementing IDs) to ensure fast queries and predictable performance as datasets grow. Unlike traditional relational databases that require manual partitioning, hypertables automate all aspects of partition management, keeping locking minimal even under high ingest load. + +At ingest time, hypertables ensure that PostgreSQL can deal with a constant stream of data without suffering from table bloat and index degradation by automatically partitioning data across time. Because each chunk is ordered by time and has its own indexes and storage, writes are usually isolated to small, recent chunks—keeping index sizes small, improving cache locality, and reducing the overhead of vacuum and background maintenance operations. This localized write pattern minimizes write amplification and ensures consistently high ingest performance, even as total data volume grows. + +At query time, hypertables efficiently exclude irrelevant chunks from the execution plan when the partitioning column is used in a `WHERE` clause. This architecture ensures fast query execution, avoiding the gradual slowdowns that affect non-partitioned tables as they accumulate millions of rows. Chunk-local indexes keep indexing overhead minimal, ensuring index operations scans remain efficient regardless of dataset size. + +
+ +
+ +Hypertables are the foundation for all of TimescaleDB’s real-time analytics capabilities. They enable seamless data ingestion, high-throughput writes, optimized query execution, and chunk-based lifecycle management—including automated data retention (drop a chunk) and data tiering (move a chunk to object storage). + + + +### Row-columnar storage + +Traditional databases force a trade-off between fast inserts (row-based storage) and efficient analytics (columnar storage). Hypercore eliminates this trade-off, allowing real-time analytics without sacrificing transactional capabilities. + +Hypercore dynamically stores data in the most efficient format for its lifecycle: + + + +* **Row-based storage for recent data**: the most recent chunk (and possibly more) is always stored in the rowstore, ensuring fast inserts, updates, and low-latency single record queries. Additionally, row-based storage is used as a writethrough for inserts and updates to columnar storage. +* **Columnar storage for analytical performance**: chunks are automatically compressed into the columnstore, optimizing storage efficiency and accelerating analytical queries. + +Unlike traditional columnar databases, hypercore allows data to be inserted or modified at any stage, making it a flexible solution for both high-ingest transactional workloads and real-time analytics—within a single database. + + +### Columnar storage layout + +TimescaleDB’s columnar storage layout optimizes analytical query performance by structuring data efficiently on disk, reducing scan times, and maximizing compression rates. Unlike traditional row-based storage, where data is stored sequentially by row, columnar storage organizes and compresses data by column, allowing queries to retrieve only the necessary fields in batches rather than scanning entire rows. But unlike many column store implementations, Timescale’s columnstore supports full mutability—inserts, upserts, updates, and deletes, even at the individual record level—with transactional guarantees. Data is also immediately visible to queries as soon as it is written. + + +
+ +
+ + +#### Columnar batches + +TimescaleDB uses columnar collocation and columnar compression within row-based storage to optimize analytical query performance while maintaining full PostgreSQL compatibility. This approach ensures efficient storage, high compression ratios, and rapid query execution. + + +
+ +
+ +A rowstore chunk is converted to a columnstore chunk by successfully grouping together sets of rows (typically up to 1000) into a single batch, then converting the batch into columnar form. + +Each compressed batch does the following: + +* Encapsulates columnar data in compressed arrays of up to 1,000 values per column, stored as a single entry in the underlying compressed table +* Uses a column-major format within the batch, enabling efficient scans by co-locating values of the same column and allowing the selection of individual columns without reading the entire batch +* Applies advanced compression techniques at the column level, including run-length encoding, delta encoding, and Gorilla compression, to significantly reduce storage footprint (by up to 95%) and improve I/O performance. + +While the chunk interval of rowstore and columnstore batches usually remains the same, Timescale can also combine columnstore batches so they use a different chunk interval. + +This architecture provides the benefits of columnar storage—optimized scans, reduced disk I/O, and improved analytical performance—while seamlessly integrating with PostgreSQL’s row-based execution model. + + +#### Segmenting and ordering data + +To optimize query performance, TimescaleDB allows explicit control over how data is physically organized within columnar storage. By structuring data effectively, queries can minimize disk reads and execute more efficiently, using vectorized execution for parallel batch processing where possible. + + +
+ +
+ + + +* **Group related data together to improve scan efficiency**: organizing rows into logical segments ensures that queries filtering by a specific value only scan relevant data sections. For example, in the above, querying for a specific ID is particularly fast. *(Implemented with SEGMENTBY.)* +* **Sort data within segments to accelerate range queries**: defining a consistent order reduces the need for post-query sorting, making time-based queries and range scans more efficient. *(Implemented with ORDERBY.)* +* **Reduce disk reads and maximize vectorized execution**: a well-structured storage layout enables efficient batch processing (Single Instruction, Multiple Data, or SIMD vectorization) and parallel execution, optimizing query performance. + +By combining segmentation and ordering, Timescale ensures that columnar queries are not only fast but also resource-efficient, enabling high-performance real-time analytics. + + +### Data mutability + +Traditional databases force a trade-off between fast updates and efficient analytics. Fully immutable storage is impractical in real-world applications, where data needs to change. Asynchronous mutability—where updates only become visible after batch processing—introduces delays that break real-time workflows. In-place mutability, while theoretically ideal, is prohibitively slow in columnar storage, requiring costly decompression, segmentation, ordering, and recompression cycles. + +Hypercore navigates these trade-offs with a hybrid approach that enables immediate updates without modifying compressed columnstore data in place. By staging changes in an interim rowstore chunk, hypercore allows updates and deletes to happen efficiently while preserving the analytical performance of columnar storage. + + +
+ +
+ + +#### Real-time writes without delays + +All new data which is destined for a columnstore chunk is first written to an interim rowstore chunk, ensuring high-speed ingestion and immediate queryability. Unlike fully columnar systems that require ingestion to go through compression pipelines, hypercore allows fresh data to remain in a fast row-based structure before being later compressed into columnar format in ordered batches as normal. + +Queries transparently access both the rowstore and columnstore chunks, meaning applications always see the latest data instantly, regardless of its storage format. + + +#### Efficient updates and deletes without performance penalties + +When modifying or deleting existing data, hypercore avoids the inefficiencies of both asynchronous updates and in-place modifications. Instead of modifying compressed storage directly, affected batches are decompressed and staged in the interim rowstore chunk, where changes are applied immediately. + +These modified batches remain in row storage until they are recompressed and reintegrated into the columnstore (which happens automatically via a background process). This approach ensures updates are immediately visible, but without the expensive overhead of decompressing and rewriting entire chunks. This approach avoids: + + + +* The rigidity of immutable storage, which requires workarounds like versioning or copy-on-write strategies +* The delays of asynchronous updates, where modified data is only visible after batch processing +* The performance hit of in-place mutability, which makes compressed storage prohibitively slow for frequent updates +* The restrictions some databases have on not altering the segmentation or ordering keys + + +## Query optimizations + +Real-time analytics isn’t just about raw speed—it’s about executing queries efficiently, reducing unnecessary work, and maximizing performance. Timescale optimizes every step of the query lifecycle to ensure that queries scan only what’s necessary, make use of data locality, and execute in parallel for sub-second response times over large datasets. + + +### Skip unnecessary data + +Timescale minimizes the amount of data a query touches, reducing I/O and improving execution speed: + + +#### Primary partition exclusion (row and columnar) + +Queries automatically skip irrelevant partitions (chunks) based on the primary partitioning key (usually a timestamp), ensuring they only scan relevant data. + + +
+ +
+ + +#### Secondary partition exclusion (columnar) + +Min/max metadata allows queries filtering on correlated dimensions (e.g., `order_id` or secondary timestamps) to exclude chunks that don’t contain relevant data. + + +
+ +
+ + +#### PostgreSQL indexes (row and columnar) + +Unlike many databases, Timescale supports standard PostgreSQL indexes on columnstore data (B-tree and hash currently, when using the hypercore table access method), allowing queries to efficiently locate specific values within both row-based and compressed columnar storage. These indexes enable fast lookups, range queries, and filtering operations that further reduce unnecessary data scans. + + +
+ +
+ + +#### Batch-level filtering (columnar) + +Within each chunk, compressed columnar batches are organized using `SEGMENTBY` keys and ordered by `ORDERBY` columns. Indexes and min/max metadata can be used to quickly exclude batches that don’t match the query criteria. + + +
+ +
+ + +### Maximize locality + +Organizing data for efficient access ensures queries are read in the most optimal order, reducing unnecessary random reads and reducing scans of unneeded data. + + +
+ +
+ + + +* **Segmentation**: Columnar batches are grouped using `SEGMENTBY` to keep related data together, improving scan efficiency. +* **Ordering**: Data within each batch is physically sorted using `ORDERBY`, increasing scan efficiency (and reducing I/O operations), enabling efficient range queries, and minimizing post-query sorting. +* **Column selection**: Queries read only the necessary columns, reducing disk I/O, decompression overhead, and memory usage. + + +### Parallelize execution + +Once a query is scanning only the required columnar data in the optimal order, Timescale is able to maximize performance through parallel execution. As well as using multiple workers, Timescale accelerates columnstore query execution by using Single Instruction, Multiple Data (SIMD) vectorization, allowing modern CPUs to process multiple data points in parallel. + + + +
+ +
+ +The Timescale implementation of SIMD vectorization currently allows: + + + +* **Vectorized decompression**, which efficiently restores compressed data into a usable form for analysis. +* **Vectorized filtering**, which rapidly applies filter conditions across data sets. +* **Vectorized aggregation**, which performs aggregate calculations, such as sum or average, across multiple data points concurrently. + + +## Accelerating queries with continuous aggregates + +Aggregating large datasets in real time can be expensive, requiring repeated scans and calculations that strain CPU and I/O. While some databases attempt to brute-force these queries at runtime, compute and I/O are always finite resources—leading to high latency, unpredictable performance, and growing infrastructure costs as data volume increases. + +**Continuous aggregates**, Timescale’s implementation of incrementally updated materialized views, solve this by shifting computation from every query run to a single, asynchronous step after data is ingested. Only the time buckets that receive new or modified data are updated, and queries read precomputed results instead of scanning raw data—dramatically improving performance and efficiency. + + +
+ +
+ +When you know the types of queries you'll need ahead of time, continuous aggregates allow you to pre-aggregate data along meaningful time intervals—such as per-minute, hourly, or daily summaries—delivering instant results without on-the-fly computation. + +Continuous aggregates also avoid the time-consuming and error-prone process of maintaining manual rollups, while continuing to offer data mutability to support efficient updates, corrections, and backfills. Whenever new data is inserted or modified in chunks which have been materialized, Timescale stores invalidation records reflecting that these results are stale and need to be recomputed. Then, an asynchronous process re-computes regions that include invalidated data, and updates the materialized results. Timescale tracks the lineage and dependencies between continuous aggregates and their underlying data, to ensure the continuous aggregates are regularly kept up-to-date. This happens in a resource-efficient manner, and where multiple invalidations can be coalesced into a single refresh (as opposed to refreshing any dependencies at write time, such as via a trigger-based approach). + +Continuous aggregates themselves are stored in hypertables, and they can be converted to columnar storage for compression, and raw data can be dropped, reducing storage footprint and processing cost. Continuous aggregates also support hierarchical rollups (e.g., hourly to daily to monthly) and real-time mode, which merges precomputed results with the latest ingested data to ensure accurate, up-to-date analytics. + +This architecture enables scalable, low-latency analytics while keeping resource usage predictable—ideal for dashboards, monitoring systems, and any workload with known query patterns. + + +### Hyperfunctions for real-time analytics + +Real-time analytics requires more than basic SQL functions—efficient computation is essential as datasets grow in size and complexity. Hyperfunctions, available through the `timescaledb_toolkit` extension, provide high-performance, SQL-native functions tailored for time-series analysis. These include advanced tools for gap-filling, percentile estimation, time-weighted averages, counter correction, and state tracking, among others. + +A key innovation of hyperfunctions is their support for partial aggregation, which allows Timescale to store intermediate computational states rather than just final results. These partials can later be merged to compute rollups efficiently, avoiding expensive reprocessing of raw data and reducing compute overhead. This is especially effective when combined with continuous aggregates. + +Consider a real-world example: monitoring request latencies across thousands of application instances. You might want to compute p95 latency per minute, then roll that up into hourly and daily percentiles for dashboards or alerts. With traditional SQL, calculating percentiles requires a full scan and sort of all underlying data—making multi-level rollups computationally expensive. + +With Timescale, you can use the `percentile_agg` hyperfunction in a continuous aggregate to compute and store a partial aggregation state for each minute. This state efficiently summarizes the distribution of latencies for that time bucket, without storing or sorting all individual values. Later, to produce an hourly or daily percentile, you simply combine the stored partials—no need to reprocess the raw latency values. + +This approach provides a scalable, efficient solution for percentile-based analytics. By combining hyperfunctions with continuous aggregates, Timescale enables real-time systems to deliver fast, resource-efficient insights across high-ingest, high-resolution datasets—without sacrificing accuracy or flexibility. + + +## Cloud-native architecture + +Real-time analytics requires a scalable, high-performance, and cost-efficient database that can handle high-ingest rates and low-latency queries without overprovisioning. Timescale Cloud is designed for elasticity, enabling independent scaling of storage and compute, workload isolation, and intelligent data tiering. + + +### Independent storage and compute scaling + +Real-time applications generate continuous data streams while requiring instant querying of both fresh and historical data. Traditional databases force users to pre-provision fixed storage, leading to unnecessary costs or unexpected limits. Timescale Cloud eliminates this constraint by dynamically scaling storage based on actual usage: + + + +* Storage expands and contracts automatically as data is added or deleted, avoiding manual intervention. +* Usage-based billing ensures costs align with actual storage consumption, eliminating large upfront allocations. +* Compute can be scaled independently to optimize query execution, ensuring fast analytics across both recent and historical data. + +With this architecture, databases grow alongside data streams, enabling seamless access to real-time and historical insights while efficiently managing storage costs. + + +### Workload isolation for real-time performance + +Balancing high-ingest rates and low-latency analytical queries on the same system can create contention, slowing down performance. Timescale Cloud mitigates this by allowing read and write workloads to scale independently: + + + +* The primary database efficiently handles both ingestion and real-time rollups without disruption. +* Read replicas scale query performance separately, ensuring fast analytics even under heavy workloads. + + +
+ +
+ +This separation ensures that frequent queries on fresh data don’t interfere with ingestion, making it easier to support live monitoring, anomaly detection, interactive dashboards, and alerting systems. + + +### Intelligent data tiering for cost-efficient real-time analytics + +Not all real-time data is equally valuable—recent data is queried constantly, while older data is accessed less frequently. Timescale Cloud can be configured to automatically tier data to cheaper bottomless object storage, ensuring that hot data remains instantly accessible, while historical data is still available. + + +
+ +
+ + + +* **Recent, high-velocity data** stays in high-performance storage for ultra-fast queries. +* **Older, less frequently accessed data** is automatically moved to cost-efficient object storage but remains queryable and available for building continuous aggregates. + +While many systems support this concept of data cooling, Timescale ensures that the data can still be queried from the same hypertable regardless of its current location. For real-time analytics, this means applications can analyze live data streams without worrying about storage constraints, while still maintaining access to long-term trends when needed. + + +### Cloud-native database observability + +Real-time analytics doesn’t just require fast queries—it requires the ability to understand why queries are fast or slow, where resources are being used, and how performance changes over time. That’s why Timescale is built with deep observability features, giving developers and operators full visibility into their database workloads. + +At the core of this observability is Insights, Timescale’s built-in query monitoring tool. Insights captures per-query statistics from our whole fleet in real time, showing you exactly how your database is behaving under load. It tracks key metrics like execution time, planning time, number of rows read and returned, I/O usage, and buffer cache hit rates—not just for the database as a whole, but for each individual query. + +Insights lets you do the following: + + + +* Identify slow or resource-intensive queries instantly +* Spot long-term performance regressions or trends +* Understand query patterns and how they evolve over time +* See the impact of schema changes, indexes, or continuous aggregates on workload performance +* Monitor and compare different versions of the same query to optimize execution + +All this is surfaced through an intuitive interface, available directly in Timescale Cloud, with no instrumentation or external monitoring infrastructure required. + +Beyond query-level visibility, Timescale also exposes metrics around service resource consumption, compression, continuous aggregates, and data tiering, allowing you to track how data moves through the system—and how those background processes impact storage and query performance. + +Together, these observability features give you the insight and control needed to operate a real-time analytics database at scale, with confidence, clarity, and performance you can trust**.** + + +## Ensuring reliability and scalability + +Maintaining high availability, efficient resource utilization, and data durability is essential for real-time applications. Timescale provides robust operational features to ensure seamless performance under varying workloads. + +* **High-availability (HA) replicas**: deploy multi-AZ HA replicas to provide fault tolerance and ensure minimal downtime. In the event of a primary node failure, replicas are automatically promoted to maintain service continuity. +* **Connection pooling**: optimize database connections by efficiently managing and reusing them, reducing overhead and improving performance for high-concurrency applications. +* **Backup and recovery**: leverage continuous backups, Point-in-Time Recovery (PITR), and automated snapshotting to protect against data loss. Restore data efficiently to minimize downtime in case of failures or accidental deletions. + +These operational capabilities ensure Timescale remains reliable, scalable, and resilient, even under demanding real-time workloads. + + +## Conclusion + +Real-time analytics is critical for modern applications, but traditional databases struggle to balance high-ingest performance, low-latency queries, and flexible data mutability. Timescale extends PostgreSQL to solve this challenge, combining automatic partitioning, hybrid row-columnar storage, and intelligent compression to optimize both transactional and analytical workloads. + +With continuous aggregates, hyperfunctions, and advanced query optimizations, Timescale ensures sub-second queries even on massive datasets that combine current and historic data. Its cloud-native architecture further enhances scalability with independent compute and storage scaling, workload isolation, and cost-efficient data tiering—allowing applications to handle real-time and historical queries seamlessly. + +For developers, this means building high-performance, real-time analytics applications without sacrificing SQL compatibility, transactional guarantees, or operational simplicity. + +Timescale delivers the best of PostgreSQL, optimized for real-time analytics. + diff --git a/api/_hyperfunctions/candlestick_agg/examples.md b/api/_hyperfunctions/candlestick_agg/examples.md index 137562588b..bc579d0b13 100644 --- a/api/_hyperfunctions/candlestick_agg/examples.md +++ b/api/_hyperfunctions/candlestick_agg/examples.md @@ -17,7 +17,7 @@ SELECT low(candlestick_agg("time", price, volume)), close(candlestick_agg("time", price, volume)), volume(candlestick_agg("time", price, volume)) -FROM stocks_real_time +FROM crypto_ticks WHERE "time" > now() - '1 day'::interval GROUP BY ts, symbol ; @@ -28,7 +28,7 @@ WITH cs AS ( SELECT time_bucket('1 hour'::interval, "time") AS hourly_bucket, symbol, candlestick_agg("time", price, volume) AS candlestick - FROM stocks_real_time + FROM crypto_ticks WHERE "time" > now() - '1 day'::interval GROUP BY hourly_bucket, symbol ) @@ -53,7 +53,7 @@ WITH (timescaledb.continuous) AS SELECT time_bucket('1 minute'::interval, "time") AS ts, symbol, candlestick_agg("time", price, volume) AS candlestick -FROM stocks_real_time +FROM crypto_ticks GROUP BY ts, symbol ; ``` diff --git a/api/_hyperfunctions/count_min_sketch/approx_count.md b/api/_hyperfunctions/count_min_sketch/approx_count.md index a3c6955a47..5c2a8244da 100644 --- a/api/_hyperfunctions/count_min_sketch/approx_count.md +++ b/api/_hyperfunctions/count_min_sketch/approx_count.md @@ -41,7 +41,7 @@ api_details: code: | WITH t AS ( SELECT toolkit_experimental.count_min_sketch(symbol, 0.01, 0.01) AS symbol_sketch - FROM stocks_real_time + FROM crypto_ticks ) SELECT toolkit_experimental.approx_count('AAPL', symbol_sketch) FROM t; diff --git a/api/_hyperfunctions/state_agg/timeline_agg.md b/api/_hyperfunctions/state_agg/timeline_agg.md index 5805e07d2d..d4cd19c01d 100644 --- a/api/_hyperfunctions/state_agg/timeline_agg.md +++ b/api/_hyperfunctions/state_agg/timeline_agg.md @@ -17,7 +17,7 @@ hyperfunction: api_details: summary: > Aggregate state data into a state aggregate to track state transitions. - Unlike [`state_agg`](/api/latest/hyperfunctions/state-tracking/state_agg/), + Unlike [`compact_state_agg`](/api/latest/hyperfunctions/state-tracking/compact_state_agg/), which only stores durations, `state_agg` also stores the timestamps of state transitions. signatures: diff --git a/api/actions.md b/api/actions.md deleted file mode 100644 index b504c106f7..0000000000 --- a/api/actions.md +++ /dev/null @@ -1,21 +0,0 @@ ---- -title: User-defined actions -excerpt: Timescale Cloud API reference for user-defined actions. Includes SQL functions for adding, altering, deleting, and running a user-defined action -keywords: [actions, jobs] -tags: [user-defined actions, background jobs, scheduled jobs, automation framework] ---- - -# User-Defined Actions Community - -User-defined actions allow you to run functions and procedures implemented in a -language of your choice on a schedule within Timescale. This allows -automatic periodic tasks that are not covered by existing policies and -even enhancing existing policies with additional functionality. - -The following APIs and views allow you to manage the jobs that you create and -get details around automatic jobs used by other TimescaleDB functions like -continuous aggregation refresh policies and data retention policies. To view the -policies that you set or the policies that already exist, see -[informational views][informational-views]. - -[informational-views]: /api/:currentVersion:/informational-views/jobs/ diff --git a/api/add_job.md b/api/add_job.md deleted file mode 100644 index 559c6e2272..0000000000 --- a/api/add_job.md +++ /dev/null @@ -1,66 +0,0 @@ ---- -api_name: add_job() -excerpt: Add a job to run a user-defined action automatically -topics: [jobs] -keywords: [jobs] -tags: [user-defined actions, scheduled jobs, background jobs, automation framework] -api: - license: community - type: function ---- - -# add_job() Community - -Register an action for scheduling by the automation framework. For more information about scheduling, including example actions, see the [actions section][using-actions]. - -### Required arguments - -|Name|Type|Description| -|-|-|-| -|`proc`|REGPROC|Name of the function or procedure to register as job| -|`schedule_interval`|INTERVAL|Interval between executions of this job. Defaults to 24 hours| - -### Optional arguments - -|Name|Type|Description| -|-|-|-| -|`config`|JSONB|Job-specific configuration, passed to the function when it runs| -|`initial_start`|TIMESTAMPTZ|Time the job is first run. In the case of fixed schedules, this also serves as the origin on which job executions are aligned. If omitted, the current time is used as origin in the case of fixed schedules.| -|`scheduled`|BOOLEAN|Set to `FALSE` to exclude this job from scheduling. Defaults to `TRUE`. | -|`check_config`|`REGPROC`|A function that takes a single argument, the `JSONB` `config` structure. The function is expected to raise an error if the configuration is not valid, and return nothing otherwise. Can be used to validate the configuration when adding a job. Only functions, not procedures, are allowed as values for `check_config`.| -|`fixed_schedule`|BOOLEAN|Set to `FALSE` if you want the next start of a job to be determined as its last finish time plus the schedule interval. Set to `TRUE` if you want the next start of a job to begin `schedule_interval` after the last start. Defaults to `TRUE`| -|`timezone`|TEXT|A valid time zone. If fixed_schedule is `TRUE`, subsequent executions of the job are aligned on its initial start. However, daylight savings time (DST) changes may shift this alignment. Set to a valid time zone if you want to mitigate this issue. Defaults to `NULL`.| - -### Returns - -|Column|Type|Description| -|-|-|-| -|`job_id`|INTEGER|TimescaleDB background job ID| - -### Sample use - -Register the `user_defined_action` procedure to run every hour: - -```sql -CREATE OR REPLACE PROCEDURE user_defined_action(job_id int, config jsonb) LANGUAGE PLPGSQL AS -$$ -BEGIN - RAISE NOTICE 'Executing action % with config %', job_id, config; -END -$$; - -SELECT add_job('user_defined_action','1h'); -SELECT add_job('user_defined_action','1h', fixed_schedule => false); -``` - -Register the `user_defined_action` procedure to run at midnight every Sunday. -The `initial_start` provided must satisfy these requirements, so it must be a Sunday midnight: - -```sql --- December 4, 2022 is a Sunday -SELECT add_job('user_defined_action','1 week', initial_start => '2022-12-04 00:00:00+00'::timestamptz); --- if subject to DST -SELECT add_job('user_defined_action','1 week', initial_start => '2022-12-04 00:00:00+00'::timestamptz, timezone => 'Europe/Berlin'); -``` - -[using-actions]: /use-timescale/:currentVersion:/user-defined-actions diff --git a/api/alter_job.md b/api/alter_job.md deleted file mode 100644 index 252ea02610..0000000000 --- a/api/alter_job.md +++ /dev/null @@ -1,109 +0,0 @@ ---- -api_name: alter_job() -excerpt: Alter a job that is scheduled to run automatically -topics: [jobs] -keywords: [jobs] -tags: [scheduled jobs, user-defined actions, automation framework, background jobs, alter, change] -api: - license: community - type: function ---- - -# alter_job() Community - -Actions scheduled using the TimescaleDB automation framework run periodically in -a background worker. You can change the schedule of these jobs with the -`alter_job` function. To alter an existing job, refer to it by `job_id`. The -`job_id` runs a given action, and its current schedule can be found in the -`timescaledb_information.jobs` view, which lists information about every -scheduled action, as well as in `timescaledb_information.job_stats`. The -`job_stats` view also gives information about when each job was last run and -other useful statistics for deciding what the new schedule should be. - -### Required arguments - -|Name|Type|Description| -|-|-|-| -|`job_id`|`INTEGER`|The ID of the policy job being modified| - -### Optional arguments - -|Name|Type|Description| -|-|-|-| -|`schedule_interval`|`INTERVAL`|The interval at which the job runs. Defaults to 24 hours.| -|`max_runtime`|`INTERVAL`|The maximum amount of time the job is allowed to run by the background worker scheduler before it is stopped.| -|`max_retries`|`INTEGER`|The number of times the job is retried if it fails.| -|`retry_period`|`INTERVAL`|The amount of time the scheduler waits between retries of the job on failure.| -|`scheduled`|`BOOLEAN`|Set to `FALSE` to exclude this job from being run as background job.| -|`config`|`JSONB`|Job-specific configuration, passed to the function when it runs.| -|`next_start`|`TIMESTAMPTZ`|The next time at which to run the job. The job can be paused by setting this value to `infinity`, and restarted with a value of `now()`.| -|`if_exists`|`BOOLEAN`|Set to `true`to issue a notice instead of an error if the job does not exist. Defaults to false.| -|`check_config`|`REGPROC`|A function that takes a single argument, the `JSONB` `config` structure. The function is expected to raise an error if the configuration is not valid, and return nothing otherwise. Can be used to validate the configuration when updating a job. Only functions, not procedures, are allowed as values for `check_config`.| -|`fixed_schedule`|`BOOLEAN`|To enable fixed scheduled job runs, set to `TRUE`.| -|`initial_start`|`TIMESTAMPTZ`|Set the time when the `fixed_schedule` job run starts. For example, `19:10:25-07`.| -|`timezone`|`TEXT`|Address the 1-hour shift in start time when clocks change from [Daylight Saving Time to Standard Time](https://en.wikipedia.org/wiki/Daylight_saving_time). For example, `America/Sao_Paulo`.| - -When a job begins, the `next_start` parameter is set to `infinity`. This -prevents the job from attempting to be started again while it is running. When -the job completes, whether or not the job is successful, the parameter is -automatically updated to the next computed start time. - -Note that altering the `next_start` value is only effective for the next -execution of the job in case of fixed schedules. On the next execution, it will -automatically return to the schedule. - -### Returns - -|Column|Type|Description| -|-|-|-| -|`job_id`|`INTEGER`|The ID of the job being modified| -|`schedule_interval`|`INTERVAL`|The interval at which the job runs. Defaults to 24 hours| -|`max_runtime`|`INTERVAL`|The maximum amount of time the job is allowed to run by the background worker scheduler before it is stopped| -|`max_retries`|INTEGER|The number of times the job is retried if it fails| -|`retry_period`|`INTERVAL`|The amount of time the scheduler waits between retries of the job on failure| -|`scheduled`|`BOOLEAN`|Returns `true` if the job is executed by the TimescaleDB scheduler| -|`config`|`JSONB`|Job-specific configuration, passed to the function when it runs| -|`next_start`|`TIMESTAMPTZ`|The next time to run the job| -|`check_config`|`TEXT`|The function used to validate updated job configurations| - -### Sample usage - -Reschedules job ID `1000` so that it runs every two days: - -```sql -SELECT alter_job(1000, schedule_interval => INTERVAL '2 days'); -``` - -Disables scheduling of the compression policy on the `conditions` hypertable: - -```sql -SELECT alter_job(job_id, scheduled => false) -FROM timescaledb_information.jobs -WHERE proc_name = 'policy_compression' AND hypertable_name = 'conditions' -``` - -Reschedules continuous aggregate job ID `1000` so that it next runs at 9:00:00 on 15 March, 2020: - -```sql -SELECT alter_job(1000, next_start => '2020-03-15 09:00:00.0+00'); -``` - -### Calculation of next start on failure - -When a job run results in a runtime failure, the next start of the job is calculated taking into account both its `retry_period` and `schedule_interval`. -The `next_start` time is calculated using the following formula: -``` -next_start = finish_time + consecutive_failures * retry_period ± jitter -``` -where jitter (± 13%) is added to avoid the "thundering herds" effect. - - -To ensure that the `next_start` time is not put off indefinitely or produce timestamps so large they end up out of range, it is capped at 5*`schedule_interval`. -Also, more than 20 consecutive failures are not considered, so if the number of consecutive failures is higher, then it multiplies by 20. - -Additionally, in the case of jobs with fixed schedules, the system ensures that if the next start calculated as specified, surpasses the next scheduled execution, then the job is executed again at the next scheduled slot and not after that. This ensures that the job does not miss scheduled executions. - -Finally, there is a distinction between runtime failures that do not cause the job to crash and job crashes. -In the event of a job crash, the next start calculation still follows the above formula, -but it is always at least 5 minutes after the job's last finish, to give an operator enough time to disable it before another crash. - diff --git a/api/alter_table_compression.md b/api/alter_table_compression.md deleted file mode 100644 index c2990aa5ac..0000000000 --- a/api/alter_table_compression.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -api_name: ALTER TABLE (Compression) -excerpt: Change compression settings on a compressed hypertable -topics: [compression] -keywords: [compression] -tags: [settings, hypertables, alter, change] -api: - license: community - type: command ---- - -# ALTER TABLE (Compression) - -'ALTER TABLE' statement is used to turn on compression and set compression -options. - -By itself, this `ALTER` statement alone does not compress a hypertable. To do so, either create a -compression policy using the [add_compression_policy][add_compression_policy] function or manually -compress a specific hypertable chunk using the [compress_chunk][compress_chunk] function. - -The syntax is: - -``` sql -ALTER TABLE SET (timescaledb.compress, - timescaledb.compress_orderby = ' [ASC | DESC] [ NULLS { FIRST | LAST } ] [, ...]', - timescaledb.compress_segmentby = ' [, ...]', - timescaledb.compress_chunk_time_interval='interval' -); -``` - -## Required arguments - -|Name|Type|Description| -|-|-|-| -|`timescaledb.compress`|BOOLEAN|Enable or disable compression| - -## Optional arguments - -|Name|Type|Description| -|-|-|-| -|`timescaledb.compress_orderby`|TEXT|Order used by compression, specified in the same way as the ORDER BY clause in a SELECT query. The default is the descending order of the hypertable's time column.| -|`timescaledb.compress_segmentby`|TEXT|Column list on which to key the compressed segments. An identifier representing the source of the data such as `device_id` or `tags_id` is usually a good candidate. The default is no `segment by` columns.| -|`timescaledb.compress_chunk_time_interval`|TEXT|EXPERIMENTAL: Set compressed chunk time interval used to roll chunks into. This parameter compresses every chunk, and then irreversibly merges it into a previous adjacent chunk if possible, to reduce the total number of chunks in the hypertable. Note that chunks will not be split up during decompression. It should be set to a multiple of the current chunk interval. This option can be changed independently of other compression settings and does not require the `timescaledb.compress` argument.| - -## Parameters - -|Name|Type|Description| -|-|-|-| -|`table_name`|TEXT|Hypertable that supports compression| -|`column_name`|TEXT|Column used to order by or segment by| -|`interval`|TEXT|Time interval used to roll compressed chunks into| - -## Sample usage - -Configure a hypertable that ingests device data to use compression. Here, if the hypertable -is often queried about a specific device or set of devices, the compression should be -segmented using the `device_id` for greater performance. - -```sql -ALTER TABLE metrics SET (timescaledb.compress, timescaledb.compress_orderby = 'time DESC', timescaledb.compress_segmentby = 'device_id'); -``` - -You can also specify compressed chunk interval without changing other -compression settings: - -```sql -ALTER TABLE metrics SET (timescaledb.compress_chunk_time_interval = '24 hours'); -``` - -To disable the previously set option, set the interval to 0: - -```sql -ALTER TABLE metrics SET (timescaledb.compress_chunk_time_interval = '0'); -``` - -[add_compression_policy]: /api/:currentVersion:/compression/add_compression_policy/ -[compress_chunk]: /api/:currentVersion:/compression/compress_chunk/ diff --git a/api/add_compression_policy.md b/api/compression/add_compression_policy.md similarity index 91% rename from api/add_compression_policy.md rename to api/compression/add_compression_policy.md index 74210e4142..7a154443c3 100644 --- a/api/add_compression_policy.md +++ b/api/compression/add_compression_policy.md @@ -9,8 +9,12 @@ api: type: function --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # add_compression_policy() + Replaced by add_columnstore_policy(). + Allows you to set a policy by which the system compresses a chunk automatically in the background after it reaches a given age. @@ -50,6 +54,9 @@ on the type of the time column of the hypertable or continuous aggregate: |`initial_start`|TIMESTAMPTZ|Time the policy is first run. Defaults to NULL. If omitted, then the schedule interval is the interval from the finish time of the last execution to the next start. If provided, it serves as the origin with respect to which the next_start is calculated | |`timezone`|TEXT|A valid time zone. If `initial_start` is also specified, subsequent executions of the compression policy are aligned on its initial start. However, daylight savings time (DST) changes may shift this alignment. Set to a valid time zone if this is an issue you want to mitigate. If omitted, UTC bucketing is performed. Defaults to `NULL`.| |`if_not_exists`|BOOLEAN|Setting to `true` causes the command to fail with a warning instead of an error if a compression policy already exists on the hypertable. Defaults to false.| +| `hypercore_use_access_method` | BOOLEAN | `NULL` | Set to `true` to use hypercore table access metod. If set to `NULL` it will use the value from `timescaledb.default_hypercore_use_access_method`. | + + @@ -89,3 +96,4 @@ SELECT add_compression_policy('cpu_weekly', INTERVAL '8 weeks'); [compression_continuous-aggregate]: /api/:currentVersion:/continuous-aggregates/alter_materialized_view/ [set_integer_now_func]: /api/:currentVersion:/hypertable/set_integer_now_func [informational-views]: /api/:currentVersion:/informational-views/jobs/ + diff --git a/api/compression/alter_table_compression.md b/api/compression/alter_table_compression.md new file mode 100644 index 0000000000..ae133e2cb6 --- /dev/null +++ b/api/compression/alter_table_compression.md @@ -0,0 +1,83 @@ +--- +api_name: ALTER TABLE (Compression) +excerpt: Change compression settings on a compressed hypertable +topics: [compression] +keywords: [compression] +tags: [settings, hypertables, alter, change] +api: + license: community + type: command +--- + +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + +# ALTER TABLE (Compression) + + Replaced by ALTER TABLE (Hypercore). + +'ALTER TABLE' statement is used to turn on compression and set compression +options. + +By itself, this `ALTER` statement alone does not compress a hypertable. To do so, either create a +compression policy using the [add_compression_policy][add_compression_policy] function or manually +compress a specific hypertable chunk using the [compress_chunk][compress_chunk] function. + +The syntax is: + +``` sql +ALTER TABLE SET (timescaledb.compress, + timescaledb.compress_orderby = ' [ASC | DESC] [ NULLS { FIRST | LAST } ] [, ...]', + timescaledb.compress_segmentby = ' [, ...]', + timescaledb.compress_chunk_time_interval='interval' +); +``` + +## Required arguments + +|Name|Type|Description| +|-|-|-| +|`timescaledb.compress`|BOOLEAN|Enable or disable compression| + +## Optional arguments + +|Name|Type| Description | +|-|-|--| +|`timescaledb.compress_orderby`|TEXT| Order used by compression, specified in the same way as the ORDER BY clause in a SELECT query. The default is the descending order of the hypertable's time column. | +|`timescaledb.compress_segmentby`|TEXT| Column list on which to key the compressed segments. An identifier representing the source of the data such as `device_id` or `tags_id` is usually a good candidate. The default is no `segment by` columns. | +|`timescaledb.enable_segmentwise_recompression`|TEXT| Set to `OFF` to disable segmentwise recompression on compressed chunks. This can be beneficial for some user workloads where segmentwise recompression is slow, and full recompression is more performant. The default is `ON`. | +|`timescaledb.compress_chunk_time_interval`|TEXT| EXPERIMENTAL: Set compressed chunk time interval used to roll chunks into. This parameter compresses every chunk, and then irreversibly merges it into a previous adjacent chunk if possible, to reduce the total number of chunks in the hypertable. Note that chunks will not be split up during decompression. It should be set to a multiple of the current chunk interval. This option can be changed independently of other compression settings and does not require the `timescaledb.compress` argument. | + + +## Parameters + +|Name|Type|Description| +|-|-|-| +|`table_name`|TEXT|Hypertable that supports compression| +|`column_name`|TEXT|Column used to order by or segment by| +|`interval`|TEXT|Time interval used to roll compressed chunks into| + +## Sample usage + +Configure a hypertable that ingests device data to use compression. Here, if the hypertable +is often queried about a specific device or set of devices, the compression should be +segmented using the `device_id` for greater performance. + +```sql +ALTER TABLE metrics SET (timescaledb.compress, timescaledb.compress_orderby = 'time DESC', timescaledb.compress_segmentby = 'device_id'); +``` + +You can also specify compressed chunk interval without changing other +compression settings: + +```sql +ALTER TABLE metrics SET (timescaledb.compress_chunk_time_interval = '24 hours'); +``` + +To disable the previously set option, set the interval to 0: + +```sql +ALTER TABLE metrics SET (timescaledb.compress_chunk_time_interval = '0'); +``` + +[add_compression_policy]: /api/:currentVersion:/compression/add_compression_policy/ +[compress_chunk]: /api/:currentVersion:/compression/compress_chunk/ diff --git a/api/chunk_compression_stats.md b/api/compression/chunk_compression_stats.md similarity index 92% rename from api/chunk_compression_stats.md rename to api/compression/chunk_compression_stats.md index 5f7b995937..2c3feb6ff4 100644 --- a/api/chunk_compression_stats.md +++ b/api/compression/chunk_compression_stats.md @@ -9,8 +9,12 @@ api: type: function --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # chunk_compression_stats() Community + Replaced by chunk_columnstore_stats(). + Get chunk-specific statistics related to hypertable compression. All sizes are in bytes. @@ -20,13 +24,13 @@ the chunk. An insert into a compressed chunk does not update the compressed sizes. For more information about how to compute chunk sizes, see the `chunks_detailed_size` section. -### Required arguments +## Required arguments |Name|Type|Description| |-|-|-| |`hypertable`|REGCLASS|Name of the hypertable| -### Returns +## Returns |Column|Type|Description| |-|-|-| @@ -43,7 +47,7 @@ sizes. For more information about how to compute chunk sizes, see the |`after_compression_total_bytes`|BIGINT|Size of the entire chunk table (table+indexes+toast) after compression (NULL if currently uncompressed)| |`node_name`|TEXT|nodes on which the chunk is located, applicable only to distributed hypertables| -### Sample usage +## Sample usage ```sql SELECT * FROM chunk_compression_stats('conditions') diff --git a/api/compress_chunk.md b/api/compression/compress_chunk.md similarity index 68% rename from api/compress_chunk.md rename to api/compression/compress_chunk.md index d8c4962439..82d781ec78 100644 --- a/api/compress_chunk.md +++ b/api/compression/compress_chunk.md @@ -9,8 +9,12 @@ api: type: function --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # compress_chunk() Community + Replaced by convert_to_columnstore(). + The `compress_chunk` function is used to compress (or recompress, if necessary) a specific chunk. This is most often used instead of the [`add_compression_policy`][add_compression_policy] function, when a user @@ -27,25 +31,26 @@ You can get a list of chunks belonging to a hypertable using the [`show_chunks` function](/api/latest/hypertable/show_chunks/). -### Required arguments +## Required arguments |Name|Type|Description| |---|---|---| | `chunk_name` | REGCLASS | Name of the chunk to be compressed| -### Optional arguments +## Optional arguments |Name|Type|Description| |---|---|---| | `if_not_compressed` | BOOLEAN | Disabling this will make the function error out on chunks that are already compressed. Defaults to true.| +| `hypercore_use_access_method` | BOOLEAN | `NULL` |✖| Set to `true` to use hypercore table access metod. If set to `NULL` it will use the value from `timescaledb.default_hypercore_use_access_method`. | -### Returns +## Returns -|Column|Description| -|---|---| -| `compress_chunk` | (REGCLASS) Name of the chunk that was compressed| +|Column|Type|Description| +|---|---|---| +| `compress_chunk` | REGCLASS | Name of the chunk that was compressed| -### Sample usage +## Sample usage Compress a single chunk. @@ -54,4 +59,4 @@ SELECT compress_chunk('_timescaledb_internal._hyper_1_2_chunk'); ``` [add_compression_policy]: /api/:currentVersion:/compression/add_compression_policy/ -[run-job]: /api/:currentVersion:/actions/run_job/ +[run-job]: /api/:currentVersion:/jobs-automation/run_job/ diff --git a/api/decompress_chunk.md b/api/compression/decompress_chunk.md similarity index 83% rename from api/decompress_chunk.md rename to api/compression/decompress_chunk.md index ef5c833fff..e503cf2e05 100644 --- a/api/decompress_chunk.md +++ b/api/compression/decompress_chunk.md @@ -8,39 +8,45 @@ api: type: function --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # decompress_chunk() Community + Replaced by convert_to_rowstore(). + If you need to modify or add a lot of data to a chunk that has already been compressed, you should decompress the chunk first. This is especially useful for backfilling old data. + Before decompressing chunks, stop any compression policy on the hypertable you are decompressing. You can use `SELECT alter_job(JOB_ID, scheduled => false);` to prevent scheduled execution. When you finish backfilling or updating data, turn the policy back on. The database automatically recompresses your chunks in the next scheduled job. + -### Required arguments +## Required arguments |Name|Type|Description| |---|---|---| |`chunk_name`|`REGCLASS`|Name of the chunk to be decompressed.| -### Optional arguments +## Optional arguments |Name|Type|Description| |---|---|---| |`if_compressed`|`BOOLEAN`|Disabling this will make the function error out on chunks that are not compressed. Defaults to true.| -### Returns +## Returns |Column|Type|Description| |---|---|---| |`decompress_chunk`|`REGCLASS`|Name of the chunk that was decompressed.| -### Sample usage +## Sample usage Decompress a single chunk: diff --git a/api/hypertable_compression_stats.md b/api/compression/hypertable_compression_stats.md similarity index 90% rename from api/hypertable_compression_stats.md rename to api/compression/hypertable_compression_stats.md index af91649f01..8c515ff3a3 100644 --- a/api/hypertable_compression_stats.md +++ b/api/compression/hypertable_compression_stats.md @@ -9,8 +9,12 @@ api: type: function --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # hypertable_compression_stats() Community + Replaced by hypertable_columnstore_stats(). + Get statistics related to hypertable compression. All sizes are in bytes. For more information about using hypertables, including chunk size partitioning, @@ -19,13 +23,13 @@ see the [hypertable section][hypertable-docs]. For more information about compression, see the [compression section][compression-docs]. -### Required arguments +## Required arguments |Name|Type|Description| |-|-|-| |`hypertable`|REGCLASS|Hypertable to show statistics for| -### Returns +## Returns |Column|Type|Description| |-|-|-| @@ -42,10 +46,12 @@ For more information about compression, see the |`node_name`|TEXT|nodes on which the hypertable is located, applicable only to distributed hypertables| + Returns show `NULL` if the data is currently uncompressed. + -### Sample usage +## Sample usage ```sql SELECT * FROM hypertable_compression_stats('conditions'); diff --git a/api/compression.md b/api/compression/index.md similarity index 88% rename from api/compression.md rename to api/compression/index.md index 430bc86699..3df9a04737 100644 --- a/api/compression.md +++ b/api/compression/index.md @@ -5,7 +5,13 @@ keywords: [compression] tags: [hypertables] --- -# Compression Community +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + +# Compression (Old API, replaced by Hypercore) Community + + Replaced by Hypercore. + +Compression functionality is included in Hypercore. Before you set up compression, you need to [configure the hypertable for compression][configure-compression] and then @@ -59,3 +65,4 @@ You can also use advanced insert statements like `ON CONFLICT` and `RETURNING`. [compress_chunk]: /api/:currentVersion:/compression/compress_chunk/ [configure-compression]: /api/:currentVersion:/compression/alter_table_compression/ [skipscan]: /use-timescale/:currentVersion:/query-data/skipscan/ +[hypercore]: /api/:currentVersion:/hypercore/ diff --git a/api/recompress_chunk.md b/api/compression/recompress_chunk.md similarity index 90% rename from api/recompress_chunk.md rename to api/compression/recompress_chunk.md index 85d6274881..a20282acf0 100644 --- a/api/recompress_chunk.md +++ b/api/compression/recompress_chunk.md @@ -9,8 +9,12 @@ api: type: function --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # recompress_chunk() + Replaced by convert_to_columnstore(). + Recompresses a compressed chunk that had more data inserted after compression. ```sql @@ -92,4 +96,4 @@ END $$; ``` -[run-job]: /api/:currentVersion:/actions/run_job/ +[run-job]: /api/:currentVersion:/jobs-automation/run_job/ diff --git a/api/remove_compression_policy.md b/api/compression/remove_compression_policy.md similarity index 80% rename from api/remove_compression_policy.md rename to api/compression/remove_compression_policy.md index 489d5ec052..3af46e54ea 100644 --- a/api/remove_compression_policy.md +++ b/api/compression/remove_compression_policy.md @@ -9,25 +9,30 @@ api: type: function --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # remove_compression_policy() + + Replaced by remove_columnstore_policy(). + If you need to remove the compression policy. To restart policy-based compression you need to add the policy again. To view the policies that already exist, see [informational views][informational-views]. -### Required arguments +## Required arguments |Name|Type|Description| |-|-|-| |`hypertable`|REGCLASS|Name of the hypertable or continuous aggregate the policy should be removed from| -### Optional arguments +## Optional arguments |Name|Type|Description| |---|---|---| | `if_exists` | BOOLEAN | Setting to true causes the command to fail with a notice instead of an error if a compression policy does not exist on the hypertable. Defaults to false.| -### Sample usage +## Sample usage Remove the compression policy from the 'cpu' table: diff --git a/api/add_continuous_aggregate_policy.md b/api/continuous-aggregates/add_continuous_aggregate_policy.md similarity index 72% rename from api/add_continuous_aggregate_policy.md rename to api/continuous-aggregates/add_continuous_aggregate_policy.md index 0dabce5df1..919e3b7c2e 100644 --- a/api/add_continuous_aggregate_policy.md +++ b/api/continuous-aggregates/add_continuous_aggregate_policy.md @@ -15,7 +15,7 @@ Create a policy that automatically refreshes a continuous aggregate. To view the policies that you set or the policies that already exist, see [informational views][informational-views]. -### Required arguments +## Required arguments |Name|Type|Description| |-|-|-| @@ -40,25 +40,32 @@ While setting `end_offset` to `NULL` is possible, it is not recommended. By default, querying a continuous aggregate returns data between `end_offset` and the current time. There is no need to set `end_offset` to `NULL`. To learn more about how continuous aggregates use real-time aggregation, see the -[real-time aggregation section](/use-timescale/latest/continuous-aggregates/real-time-aggregates/). +[real-time aggregation section](/use-timescale/:currentVersion:/continuous-aggregates/real-time-aggregates/). -### Optional arguments +## Optional arguments |Name|Type|Description| |-|-|-| |`if_not_exists`|BOOLEAN|Set to `true` to issue a notice instead of an error if the job already exists. Defaults to false.| |`timezone`|TEXT|A valid time zone. If you specify `initial_start`, subsequent executions of the refresh policy are aligned on `initial_start`. However, daylight savings time (DST) changes may shift this alignment. If this is an issue you want to mitigate, set `timezone` to a valid time zone. Default is `NULL`, [UTC bucketing](https://docs.timescale.com/use-timescale/latest/time-buckets/about-time-buckets/) is performed.| | `include_tiered_data` | BOOLEAN | Enable/disable reading tiered data. This setting helps override the current settings for the`timescaledb.enable_tiered_reads` GUC. The default is NULL i.e we use the current setting for `timescaledb.enable_tiered_reads` GUC | | +| `buckets_per_batch` | INTEGER | Number of buckets to be refreshed by a _batch_. This value is multiplied by the CAgg bucket width to determine the size of the batch range. Default value is `0`, single batch execution. Values of less than `0` are not allowed. | | +| `max_batches_per_execution` | INTEGER | Limit the maximum number of batches to run when a policy executes. If some batches remain, they are processed the next time the policy runs. Default value is `10`, each job processes a maximum of 10 batches. Set to `0` for the number of batches to be unlimited. Values of less than `0` are not allowed. | | + + +Setting `buckets_per_batch` greater than zero means that the refresh window is split in batches of `bucket width` * `buckets per batch`. For example, a given Continuous Aggregate with `bucket width` of `1 day` and `buckets_per_batch` of 10 has a batch size of `10 days` to process the refresh. +Because each `batch` is an individual transaction, executing a policy in batches make the data visible for the users before the entire job is executed. Batches are processed from the most recent data to the oldest. + -### Returns +## Returns |Column|Type|Description| |-|-|-| |`job_id`|INTEGER|TimescaleDB background job ID created to implement this policy| -### Sample use +## Sample use Add a policy that refreshes the last month once an hour, excluding the latest hour from the aggregate. For performance reasons, we recommend that you diff --git a/api/add_policies.md b/api/continuous-aggregates/add_policies.md similarity index 91% rename from api/add_policies.md rename to api/continuous-aggregates/add_policies.md index 3f13169887..1870c07d58 100644 --- a/api/add_policies.md +++ b/api/continuous-aggregates/add_policies.md @@ -25,7 +25,8 @@ timescaledb_experimental.add_policies( refresh_start_offset "any" = NULL, refresh_end_offset "any" = NULL, compress_after "any" = NULL, - drop_after "any" = NULL + drop_after "any" = NULL, + hypercore_use_access_method BOOL = NULL) ) RETURNS BOOL ``` @@ -52,6 +53,7 @@ If you would like to set this add your policies manually (see [`add_continuous_a |`refresh_end_offset`|`INTERVAL` or `INTEGER`|The end of the continuous aggregate refresh window, expressed as an offset from the policy run time. Must be greater than `refresh_start_offset`.| |`compress_after`|`INTERVAL` or `INTEGER`|Continuous aggregate chunks are compressed if they exclusively contain data older than this interval.| |`drop_after`|`INTERVAL` or `INTEGER`|Continuous aggregate chunks are dropped if they exclusively contain data older than this interval.| +| `hypercore_use_access_method` | BOOLEAN | `NULL` | Set to `true` to use hypercore table access metod. If set to `NULL` it will use the value from `timescaledb.default_hypercore_use_access_method`. | For arguments that could be either an `INTERVAL` or an `INTEGER`, use an `INTERVAL` if your time bucket is based on timestamps. Use an `INTEGER` if your @@ -59,7 +61,7 @@ time bucket is based on integers. ## Returns -Returns true if successful. +Returns `true` if successful. ## Sample usage diff --git a/api/alter_materialized_view.md b/api/continuous-aggregates/alter_materialized_view.md similarity index 98% rename from api/alter_materialized_view.md rename to api/continuous-aggregates/alter_materialized_view.md index 57eb5de81f..2471768b39 100644 --- a/api/alter_materialized_view.md +++ b/api/continuous-aggregates/alter_materialized_view.md @@ -28,13 +28,13 @@ continuous aggregate view: ALTER MATERIALIZED VIEW SET ( timescaledb. -## Cloud SQL editor licenses +## Cloud $SQL_EDITOR licenses -* **SQL editor in the ops mode**: free for anyone with a [Timescale Cloud account][create-cloud-account]. -* **Data mode**: the number of seats you are allocated depends on your [Pricing Plan][pricing-plan-features]. - [SQL Assistant][sql-assistant] is currently free for all users. In the future, limits or paid options may be +* **$SQL_EDITOR in the $OPS_MODE**: free for anyone with a [$CLOUD_LONG account][create-cloud-account]. +* **Data mode**: the number of seats you are allocated depends on your [$PRICING_PLAN][pricing-plan-features]. + [$SQL_ASSISTANT_SHORT][sql-assistant] is currently free for all users. In the future, limits or paid options may be introduced as we work to build the best experience. -* **PopSQL standalone**: there is a free plan available to everyone, as well as paid plans. See [PopSQL Pricing](https://popsql.com/pricing) for full - details. +* **$POPSQL standalone**: there is a free plan available to everyone, as well as paid plans. See [$POPSQL Pricing][popsql-pricing] for full details. + - -[readreplica]: /use-timescale/:currentVersion:/ha-replicas/read-scaling/ -[run-popsql]: /getting-started/:currentVersion:/run-queries-from-console/#data-mode -[run-sqleditor]: /getting-started/:currentVersion:/run-queries-from-console/#sql-editor -[integrations]: /use-timescale/:currentVersion:/integrations/ -[timescale-console]: https://console.cloud.timescale.com/ -[portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services -[pricing-plan-features]: https://www.timescale.com/pricing#features -[install-psql]: /use-timescale/:currentVersion:/integrations/psql/ +[bridge-connector]: https://docs.popsql.com/docs/bridge-connector +[charts]: https://docs.popsql.com/docs/creating-charts +[connection-info]: /integrations/:currentVersion:/find-connection-details/ +[console]: https://console.cloud.timescale.com +[create-chart]: https://docs.popsql.com/docs/creating-charts [create-cloud-account]: /getting-started/:currentVersion:/services/#create-a-timescale-cloud-account +[create-dashboard]: https://docs.popsql.com/docs/creating-dashboards +[create-schedule]: https://docs.popsql.com/docs/scheduled-queries +[install-psql]: /integrations/:currentVersion:/psql/ +[integrations]: /integrations/:currentVersion:/ +[popsql-connections]: https://popsql.com/connections +[popsql-desktop]: https://popsql.com/download +[popsql-pricing]: https://popsql.com/pricing [portal-data-mode]: https://console.cloud.timescale.com/dashboard/services?popsql -[services-portal]: https://console.cloud.timescale.com/dashboard/services +[portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services +[pricing-plan-features]: https://www.timescale.com/pricing#features [project-members]: /use-timescale/:currentVersion:/members/ -[console]: https://console.cloud.timescale.com -[sql-editor-settings]: https://console.cloud.timescale.com/dashboard/settings?popsql=%2Fpreferences%2Fai -[portal-data-mode]: https://console.cloud.timescale.com/dashboard/services?popsql +[query-variables]: https://docs.popsql.com/docs/query-variables +[read-replica]: /use-timescale/:currentVersion:/ha-replicas/read-scaling/ +[run-popsql]: /getting-started/:currentVersion:/run-queries-from-console/#data-mode +[run-sqleditor]: /getting-started/:currentVersion:/run-queries-from-console/#ops-mode-sql-editor +[schedules]: https://docs.popsql.com/docs/scheduled-queries +[schema-browser]: https://docs.popsql.com/docs/schema +[services-portal]: https://console.cloud.timescale.com/dashboard/services +[share-query]: https://docs.popsql.com/docs/sharing-a-link-to-your-query-and-results [sql-assistant]: /getting-started/:currentVersion:/run-queries-from-console/#sql-assistant -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ \ No newline at end of file +[sql-editor-settings]: https://console.cloud.timescale.com/dashboard/settings?popsql=%2Fpreferences%2Fai +[timescale-console]: https://console.cloud.timescale.com/ +[version-history]: https://docs.popsql.com/docs/version-history +[write-query]: https://docs.popsql.com/docs/writing-a-query \ No newline at end of file diff --git a/getting-started/services.md b/getting-started/services.md index 714df524b9..c11362ae31 100644 --- a/getting-started/services.md +++ b/getting-started/services.md @@ -1,16 +1,15 @@ --- title: Create your first Timescale service excerpt: Timescale Cloud offers a range of capabilities to accommodate your real-time analytics and AI and vector workloads. Learn more about each of them and create your first service in Timescale Console -products: [cloud] -layout_components: [next_prev_large] +products: [cloud, mst, self_hosted] content_group: Getting started --- import Install from "versionContent/_partials/_cloud-installation.mdx"; -import Connect from "versionContent/_partials/_cloud-connect.mdx"; -import CreateAHypertable from "versionContent/_partials/_create-hypertable.mdx"; +import Connect from "versionContent/_partials/_cloud-connect-service.mdx"; import ServiceOverview from "versionContent/_partials/_service-overview.mdx"; import CloudIntro from "versionContent/_partials/_cloud-intro.mdx"; +import WhereNext from "versionContent/_partials/_where-to-next.mdx"; # Create your first $SERVICE_LONG @@ -20,54 +19,48 @@ import CloudIntro from "versionContent/_partials/_cloud-intro.mdx"; To start using $CLOUD_LONG for your data: -1. [Create a $COMPANY account][create-an-account]: register in $CONSOLE to get a centralized point to administer and interact with your data. -1. [Create a $SERVICE_LONG][create-a-service]: that is, a PostgreSQL database instance, powered by [$TIMESCALE_DB][timescaledb], built for production, and extended with cloud features like transparent data tiering to object storage. +1. [Create a $COMPANY account][create-an-account]: register to get access to $CONSOLE as a centralized point to administer and interact with your data. +1. [Create a $SERVICE_LONG][create-a-service]: that is, a $PG database instance, powered by [$TIMESCALE_DB][timescaledb], built for production, and extended with cloud features like transparent data tiering to object storage. 1. [Connect to your $SERVICE_LONG][connect-to-your-service]: to run queries, add and migrate your data from other sources. -1. [Create a hypertable][create-a-hypertable]: create a standard PostgreSQL table in your service, then convert it into a [hypertable][hypertables]. - - Anything you can do with regular PostgreSQL tables, you can do with hypertables, just with better performance and improved user experience for time-series data. ## Create a $SERVICE_LONG -Now that you have an active $COMPANY account, you create and manage your $SERVICE_SHORTs in $CONSOLE. When you create a service, you give a structure for your future data, which you then add manually or migrate from other services. All relevant $CLOUD_LONG features under your pricing plan are automatically available when you create a service. +Now that you have an active $CLOUD_LONG account, you create and manage your $SERVICE_SHORTs in $CONSOLE. When you create a $SERVICE_SHORT, you effectively create a blank $PG database with additional $CLOUD_LONG features available under your $PRICING_PLAN. You then add or migrate your data into this database. -1. In the [service creation page][create-service], choose the PostgreSQL service or add `Time-series and analytics` and `AI and Vector` capabilities. Click `Save and continue`. +1. In the [$SERVICE_SHORT creation page][create-service], choose the $PG capability or add `Time-series and analytics` and `AI and Vector` capabilities. Click `Save and continue`. - ![Create Timescale Cloud service](https://assets.timescale.com/docs/images/create-timescale-service.png) + ![Create a $SERVICE_LONG](https://assets.timescale.com/docs/images/create-timescale-service.png) -1. Follow the next steps in `Create a service` to configure the compute size, environment, availability, region, and service name. Then click `Create service`. +1. Follow the next steps in `Create a service` to configure the region, compute size, environment, availability, connectivity, and $SERVICE_SHORT name. Then click `Create service`. - Your service is constructed immediately and is ready to use. + Your $SERVICE_SHORT is constructed and ready to use in a few seconds. -1. Click `Download the config` and store the configuration information you need to connect to this service in a secure location. +1. Click `Download the config` and store the configuration information you need to connect to this $SERVICE_SHORT in a secure location. - This file contains the passwords and configuration information you need to connect to your service using the - $CONSOLE data mode, from the command line, or using third-party database administration tools. + This file contains the passwords and configuration information you need to connect to your $SERVICE_SHORT using the + $CONSOLE $DATA_MODE, from the command line, or using third-party database administration tools. -If you choose to go directly to the service overview, [Check your service and connect to it][connect-to-your-service] +If you choose to go directly to the $SERVICE_SHORT overview, [Connect to your $SERVICE_SHORT][connect-to-your-service] shows you how to connect. -## Connect to your service +## Connect to your $SERVICE_SHORT -A Timescale service comes with access control to its data. To be able to run queries and perform other operations, connect to the service with credentials generated during its creation. +To run queries and perform other operations, connect to your $SERVICE_SHORT: -## Create a hypertable - - + -And that is it, you are up and running. Enjoy developing with $COMPANY. [tsc-portal]: https://console.cloud.timescale.com/ [services-how-to]: /use-timescale/:currentVersion:/services/ -[install-psql]: /use-timescale/:currentVersion:/integrations/psql/ +[install-psql]: /integrations/:currentVersion:/psql/ [create-an-account]: /getting-started/:currentVersion:/services/#create-a-timescale-cloud-account [create-a-service]: /getting-started/:currentVersion:/services/#create-a-timescale-cloud-service [connect-to-your-service]: /getting-started/:currentVersion:/services/#connect-to-your-service diff --git a/getting-started/start-coding-with-timescale.md b/getting-started/start-coding-with-timescale.md new file mode 100644 index 0000000000..252bcae7ee --- /dev/null +++ b/getting-started/start-coding-with-timescale.md @@ -0,0 +1,56 @@ +--- +title: Start coding with Timescale +excerpt: Integrate Timescale Cloud with your app using your preferred programming language. Connect to a service, + create and manage hypertables, then and ingest and query data +products: [cloud, mst, self_hosted] +content_group: Getting started +--- + +import StartCodingRuby from "versionContent/_partials/_start-coding-ruby.mdx"; +import StartCodingPython from "versionContent/_partials/_start-coding-python.mdx"; +import StartCodingNode from "versionContent/_partials/_start-coding-node.mdx"; +import StartCodingGoLang from "versionContent/_partials/_start-coding-golang.mdx"; +import StartCodingJava from "versionContent/_partials/_start-coding-java.mdx"; + +# Start coding with Timescale + +Easily integrate your app with $CLOUD_LONG. Use your favorite programming language to connect to your +$SERVICE_LONG, create and manage hypertables, then ingest and query data. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +You are not limited to these languages. $CLOUD_LONG is based on $PG, you can interface +with $TIMESCALE_DB and $CLOUD_LONG using any [$PG client driver][postgres-drivers]. + + +[postgres-drivers]: https://wiki.postgresql.org/wiki/List_of_drivers diff --git a/getting-started/tables-hypertables.md b/getting-started/tables-hypertables.md deleted file mode 100644 index 43589aaf13..0000000000 --- a/getting-started/tables-hypertables.md +++ /dev/null @@ -1,119 +0,0 @@ ---- -title: Tables and hypertables -excerpt: Hypertables are PostgreSQL tables designed to boost the performance of your real-time analytical queries. Create a table and then convert it into a hypertable in Timescale Console -products: [cloud] -keywords: [hypertables, create] -layout_components: [next_prev_large] -content_group: Getting started ---- - -import HypertableIntro from "versionContent/_partials/_hypertables-intro.mdx"; - -# Tables and hypertables - - - -Databases are made up of tables that contain your data. In PostgreSQL, these -tables are relational, so the data in one table relates to the data in another -table. In Timescale, you use regular PostgreSQL relational tables, in addition -to special time-series hypertables. - -Hypertables are designed specifically for time-series data, so they have a few -special qualities that makes them different to a regular PostgreSQL table. A -hypertable is always partitioned on time, but can also be partitioned on -additional columns as well. The other special thing about hypertables is that -they are broken down into smaller tables called chunks. - -In this section, you create a hypertable for time-series data, and regular -PostgreSQL tables for relational data. You also create an index on your -hypertable, which isn't required, but can help your queries run more efficiently. -One of the other special qualities of hypertables is that you can also create -indexes later on, if you need to. - -For more information, see -[the hypertables section][hypertable-how-to]. - -## Create your first hypertable - -For the financial dataset used in this guide, create a hypertable named -`stocks_real_time` that contains second-by-second stock-trade data for the top -100 most-traded symbols. - - - -### Creating your first hypertable - -1. At the command prompt, use the `psql` connection string from the cheat sheet - you downloaded to connect to your database. -1. Create a regular PostgreSQL table to store the real-time stock trade data - using `CREATE TABLE`: - - ```sql - CREATE TABLE stocks_real_time ( - time TIMESTAMPTZ NOT NULL, - symbol TEXT NOT NULL, - price DOUBLE PRECISION NULL, - day_volume INT NULL - ); - ``` - -1. Convert the regular table into a hypertable partitioned on the `time` column - using the `create_hypertable()` function provided by Timescale. You must - provide the name of the table (`stocks_real_time`) and the column in that - table that holds the timestamp data to use for partitioning (`time`): - - - -1. Create an index to support efficient queries on the `symbol` and `time` - columns: - - - - - -## Create regular PostgreSQL tables for relational data - -Timescale isn't just for hypertables. When you have other relational data that -enhances your time-series data, you can create regular PostgreSQL tables just as -you would normally. For this dataset, there is one other table of data called -`company`. - - - -### Creating regular PostgreSQL tables - -1. Add a table to store the company name and symbol for the stock trade data: - - ```sql - CREATE TABLE company ( - symbol TEXT NOT NULL, - name TEXT NOT NULL - ); - ``` - -1. You now have two tables within your Timescale database. One hypertable named - `stocks_real_time`, and one normal PostgreSQL table named `company`. You can - check this by running this command at the `psql` prompt: - - - - This command returns information about your tables, like this: - - - - - -[hypertable-how-to]: /use-timescale/:currentVersion:/hypertables/ diff --git a/getting-started/time-series-data.md b/getting-started/time-series-data.md deleted file mode 100644 index b4d7aff625..0000000000 --- a/getting-started/time-series-data.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -title: Time-series data -excerpt: Find out what time-series data is, how it is generated, and how you can run real-time analytics on it with Timescale Cloud -products: [cloud] -keywords: [ingest] -tags: [add, data, time-series] -layout_components: [next_prev_large] -content_group: Getting started ---- - -import TimeseriesIntro from "versionContent/_partials/_timeseries-intro.mdx"; - -# Time-series data - - - -To explore Timescale's features, you need some sample data. This guide -uses real-time stock trade data, also known as tick data, from -[Twelve Data][twelve-data]. - -## About the dataset - -The dataset contains second-by-second stock-trade data for the top 100 -most-traded symbols, in a hypertable named `stocks_real_time`. It also includes -a separate table of company symbols and company names, in a regular PostgreSQL -table named `company`. - -The dataset is updated on a nightly basis and contains data from the last four -weeks, typically ~8 million rows of data. Stock trades are recorded in real-time -Monday through Friday, during normal trading hours of the New York Stock -Exchange (9:30 AM - 4:00 PM EST). - -## Ingest the dataset - -To ingest data into the tables that you created, you need to download the -dataset and copy the data to your database. - - - -1. Unzip [real_time_stock_data.zip](https://assets.timescale.com/docs/downloads/get-started/real_time_stock_data.zip) to your local device. - This archive one `.csv` file with company information, and one with real-time stock trades for - the past month. - -1. At the `psql` prompt, use the `COPY` command to transfer data into your - Timescale instance. If the `.csv` files aren't in your current directory, - specify the file paths in the following commands: - - - - - - Because there are millions of rows of data, the `COPY` process may take a few - minutes depending on your internet connection and local client resources. - - - -[twelve-data]: https://twelvedata.com/ -[console-services]: https://console.cloud.timescale.com/dashboard/services diff --git a/getting-started/try-key-features-timescale-products.md b/getting-started/try-key-features-timescale-products.md new file mode 100644 index 0000000000..2e4986dc4b --- /dev/null +++ b/getting-started/try-key-features-timescale-products.md @@ -0,0 +1,439 @@ +--- +title: Try the key Timescale features +excerpt: Improve database performance with hypertables, time bucketing, compression and continuous aggregates. +products: [cloud, mst, self_hosted] +content_group: Getting started +--- + +import HASetup from 'versionContent/_partials/_high-availability-setup.mdx'; +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; + +# Try the key $COMPANY features + +$CLOUD_LONG scales $PG to ingest and query vast amounts of live data. $CLOUD_LONG +provides a range of features and optimizations that supercharge your queries while keeping the +costs down. For example: +* The $HYPERCORE row-columnar engine makes queries up to 350x faster, ingests 44% faster, and reduces storage by 90%. +* Tiered storage seamlessly moves your data from high performance storage for frequently accessed data to low cost bottomless storage for rarely accessed data. + +The following figure shows how $CLOUD_LONG optimizes your data for superfast real-time analytics and reduced +costs: + +![Main features and tiered data](https://assets.timescale.com/docs/images/timescalecloud-service-architecture.png ) + +This page shows you how to rapidly implement the features in $CLOUD_LONG that enable you to +ingest and query data faster while keeping the costs low. + +## Prerequisites + + + +## Optimize time-series data in $HYPERTABLEs + +Time-series data represents how a system, process, or behavior changes over time. $HYPERTABLE_CAPs are $PG tables +that help you improve insert and query performance by automatically partitioning your data by time. Each $HYPERTABLE +is made up of child tables called $CHUNKs. Each $CHUNK is assigned a range of time, and only +contains data from that range. When you run a query, $CLOUD_LONG identifies the correct $CHUNK and runs the query on it, instead of going through the entire table. You can also tune $HYPERTABLEs to increase performance +even more. + +![Main features and tiered data](https://assets.timescale.com/docs/images/hypertable-structure.png) + +$HYPERTABLE_CAPs exist alongside regular $PG tables. +You use regular $PG tables for relational data, and interact with $HYPERTABLEs +and regular $PG tables in the same way. + +This section shows you how to create regular tables and $HYPERTABLEs, and import +relational and time-series data from external files. + + + +1. **Import some time-series data into your $HYPERTABLE** + + 1. Unzip [crypto_sample.zip](https://assets.timescale.com/docs/downloads/candlestick/crypto_sample.zip) to a ``. + + This test dataset contains second-by-second trade data for the most-traded crypto-assets + and a regular table of asset symbols and company names. + + To import up to 100GB of data directly from your current $PG-based database, + [migrate with downtime][migrate-with-downtime] using native $PG tooling. To seamlessly import 100GB-10TB+ + of data, use the [live migration][migrate-live] tooling supplied by $COMPANY. To add data from non-$PG data sources, see [Import and ingest data][data-ingest]. + + 1. Upload data from the CSVs to your $SERVICE_SHORT: + + + + + + The $CONSOLE data upload creates the tables for you from the data you are uploading: + 1. In [$CONSOLE][portal-ops-mode], select the $SERVICE_SHORT to add data to, then click `Actions` > `Upload CSV`. + 1. Drag `/tutorial_sample_tick.csv` to `Upload .CSV` and change `New table name` to `crypto_ticks`. + 1. Enable `hypertable partition` for the `time` column and click `Upload CSV`. + + The upload wizard creates a $HYPERTABLE containing the data from the CSV file. + 1. When the data is uploaded, close `Upload .CSV`. + + If you want to have a quick look at your data, press `Run` . + 1. Repeat the process with `/tutorial_sample_assets.csv` and rename to `crypto_assets`. + + There is no time-series data in this table, so you don't see the `hypertable partition` option. + + + + + + 1. In Terminal, navigate to `` and connect to your $SERVICE_SHORT. + ```bash + psql -d "postgres://:@:/" + ``` + You use your [connection details][connection-info] to fill in this $PG connection string. + + 2. Create tables for the data to import: + + - For the time-series data: + + 1. In your sql client, create a normal $PG table: + + ```sql + CREATE TABLE crypto_ticks ( + "time" TIMESTAMPTZ, + symbol TEXT, + price DOUBLE PRECISION, + day_volume NUMERIC + ); + ``` + 1. Convert `crypto_ticks` to a $HYPERTABLE: + ```sql + SELECT create_hypertable('crypto_ticks', by_range('time')); + ``` + To more fully understand how $HYPERTABLEs work, and how to optimize them for performance by + tuning $CHUNK intervals and enabling chunk skipping, see [the $HYPERTABLEs documentation][hypertables-section]. + + - For the relational data: + + In your sql client, create a normal $PG table: + ```sql + CREATE TABLE crypto_assets ( + symbol TEXT NOT NULL, + name TEXT NOT NULL + ); + ``` + + 3. Upload the dataset to your $SERVICE_SHORT: + + ```sql + \COPY crypto_ticks from './tutorial_sample_tick.csv' DELIMITER ',' CSV HEADER; + ``` + + ```sql + \COPY crypto_assets from './tutorial_sample_assets.csv' DELIMITER ',' CSV HEADER; + ``` + + + + + + To more fully understand how $HYPERTABLEs work, and how to optimize them for performance by + tuning $CHUNK intervals and enabling chunk skipping, see [the $HYPERTABLEs documentation][hypertables-section]. + +1. **Have a quick look at your data** + + You query $HYPERTABLEs in exactly the same way as you would a relational $PG table. + Use one of the following SQL editors to run a query and see the data you uploaded: + - **Data mode**: write queries, visualize data, and share your results in [$CONSOLE][portal-data-mode] for all your $SERVICE_LONGs. + - **SQL editor**: write, fix, and organize SQL faster and more accurately in [$CONSOLE][portal-ops-mode] for a $SERVICE_LONG. + - **psql**: easily run queries on your $SERVICE_LONGs or self-hosted TimescaleDB deployment from Terminal. + + + + + +## Write fast and efficient analytical queries + +Aggregation is a way of combing data to get insights from it. Average, sum, and count are all +examples of simple aggregates. However, with large amounts of data, aggregation slows things down, quickly. +$CAGG_CAPs are a kind of $HYPERTABLE that is refreshed automatically in +the background as new data is added, or old data is modified. Changes to your dataset are tracked, +and the $HYPERTABLE behind the $CAGG is automatically updated in the background. + +![Reduced data calls with $CAGGs](https://assets.timescale.com/docs/images/continuous-aggregate.png ) + +You create $CAGGs on uncompressed data in high-performance storage. They continue to work +on [data in the $COLUMNSTORE][test-drive-enable-compression] +and [rarely accessed data in tiered storage][test-drive-tiered-storage]. You can even +create [$CAGGs on top of your $CAGGs][hierarchical-caggs]. + +You use $TIME_BUCKETs to create a $CAGG. $TIME_BUCKET_CAPs aggregate data in $HYPERTABLEs by time +interval. For example, a 5-minute, 1-hour, or 3-day bucket. The data grouped in a $TIME_BUCKET uses a single +timestamp. $CAGG_CAPs minimize the number of records that you need to look up to perform your +query. + +This section shows you how to run fast analytical queries using $TIME_BUCKETs and $CAGG in +$CONSOLE. You can also do this using psql. + + + + + + + +1. **Connect to your $SERVICE_SHORT** + + In [$CONSOLE][portal-data-mode], select your $SERVICE_SHORT in the connection drop-down in the top right. + +1. **Create a $CAGG** + + For a $CAGG, data grouped using a $TIME_BUCKET is stored in a + $PG `MATERIALIZED VIEW` in a $HYPERTABLE. `timescaledb.continuous` ensures that this data + is always up to date. + In data mode, use the following code to create a $CAGG on the real-time data in + the `crypto_ticks` table: + + ```sql + CREATE MATERIALIZED VIEW assets_candlestick_daily + WITH (timescaledb.continuous) AS + SELECT + time_bucket('1 day', "time") AS day, + symbol, + max(price) AS high, + first(price, time) AS open, + last(price, time) AS close, + min(price) AS low + FROM crypto_ticks srt + GROUP BY day, symbol; + ``` + + This $CAGG creates the [candlestick chart][charts] data you use to visualize + the price change of an asset. + +1. **Create a policy to refresh the view every hour** + + ```sql + SELECT add_continuous_aggregate_policy('assets_candlestick_daily', + start_offset => INTERVAL '3 weeks', + end_offset => INTERVAL '24 hours', + schedule_interval => INTERVAL '3 hours'); + ``` + +1. **Have a quick look at your data** + + You query $CAGGs exactly the same way as your other tables. To query the `assets_candlestick_daily` + $CAGG for all assets: + + + + + + + + + + + + +1. **In [$CONSOLE][portal-ops-mode], select the $SERVICE_SHORT you uploaded data to** +1. **Click `Operations` > `Continuous aggregates`, select `crypto_ticks`, then click `Create a Continuous Aggregate`** + ![$CAGG wizard](https://assets.timescale.com/docs/images/continuous-aggregate-wizard.png ) +1. **Create a view called `assets_candlestick_daily` on the `time` column with an interval of `1 day`, then click `Next step`** +1. **Update the view SQL with the following functions, then click `Run`** + ```sql + CREATE MATERIALIZED VIEW assets_candlestick_daily + WITH (timescaledb.continuous) AS + SELECT + time_bucket('1 day', "time") AS bucket, + symbol, + max(price) AS high, + first(price, time) AS open, + last(price, time) AS close, + min(price) AS low + FROM "public"."crypto_ticks" srt + GROUP BY bucket, symbol; + ``` +1. **When the view is created, click `Next step`** +1. **Define a refresh policy with the following values:** + - `How far back do you want to materialize?`: `3 weeks` + - `What recent data to exclude?`: `24 hours` + - `How often do you want the job to run?`: `3 hours` +1. **Click `Next step`, then click `Run`** + +$CLOUD_LONG creates the $CAGG and displays the aggregate ID in $CONSOLE. Click `DONE` to close the wizard. + + + + + + + +To see the change in terms of query time and data returned between a regular query and +a $CAGG, run the query part of the $CAGG +( `SELECT ...GROUP BY day, symbol;` ) and compare the results. + + +## Enhance query performance for analytics + +$HYPERCORE_CAP is the $CLOUD_LONG hybrid row-columnar storage engine, designed specifically for real-time analytics and +powered by time-series data. The advantage of $HYPERCORE is its ability to seamlessly switch between row-oriented and +column-oriented storage. This flexibility enables $CLOUD_LONG to deliver the best of both worlds, solving the key +challenges in real-time analytics. + +![Move from rowstore to columstore in hypercore](https://assets.timescale.com/docs/images/hypercore.png ) + +When you convert $CHUNKs from the $ROWSTORE to the $COLUMNSTORE, multiple records are grouped into a single row. +The columns of this row hold an array-like structure that stores all the data. Because a single row takes up less disk +space, you can reduce your $CHUNK size by more than 90%, and can also speed up your queries. This helps you save on storage costs, +and keeps your queries operating at lightning speed. + +Best practice is to compress data that is no longer needed for highest performance queries, but is still accessed +regularly. For example, last week's market data. + + + +1. **Enable $HYPERCORE on a $HYPERTABLE** + + Create a [job][job] that automatically moves $CHUNKs in a $HYPERTABLE to the $COLUMNSTORE at a specific time interval. + + ```sql + ALTER TABLE crypto_ticks SET ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'symbol'); + ``` + You [segmentby][alter-table-arguments] to speed up queries. + + +1. **Add a policy to convert $CHUNKs to the $COLUMNSTORE at a specific time interval** + + For example, yesterday's data: + ``` sql + CALL add_columnstore_policy('crypto_ticks', after => INTERVAL '1d'); + ``` + See [add_columnstore_policy][add_columnstore_policy]. + +1. **View your data space saving** + + When you convert data to the $COLUMNSTORE, as well as being optimized for analytics, it is compressed by more than + 90%. This helps you save on storage costs and keeps your queries operating at lightning speed. To see the amount of space + saved: + ``` sql + SELECT + pg_size_pretty(before_compression_total_bytes) as before, + pg_size_pretty(after_compression_total_bytes) as after + FROM hypertable_compression_stats('crypto_ticks'); + ``` + You see something like: + + | Before | After | + |--------|---------| + | 32 MB | 3808 KB | + + + + + +## Slash storage charges + +In the previous sections, you used $CAGGs to make fast analytical queries, and +$HYPERCORE to reduce storage costs on frequently accessed data. To reduce storage costs even more, +you create tiering policies to move rarely accessed data to the object store. The object store is +low-cost bottomless data storage built on Amazon S3. However, no matter the tier, you can +[query your data when you need][querying-tiered-data]. $CLOUD_LONG seamlessly accesses the correct storage +tier and generates the response. + +![Tiered storage](https://assets.timescale.com/docs/images/tiered-storage.png ) + +Data tiering is available in the [$SCALE and $ENTERPRISE][pricing-plans] $PRICING_PLANs for $CLOUD_LONG. + +To set up data tiering: + + + +1. **Enable data tiering** + + 1. In [$CONSOLE][portal-ops-mode], select the $SERVICE_SHORT to modify. + + You see the `Overview` section. + + 1. Scroll down, then click `Enable tiered storage`. + + ![Enable tiered storage](https://assets.timescale.com/docs/images/console-enable-tiered-storage.png) + + When tiered storage is enabled, you see the amount of data in the tiered object storage. + +1. **Set the time interval when data is tiered** + + In $CONSOLE, click `Data` to switch to the data mode, then enable data tiering on a $HYPERTABLE with the following query: + ```sql + SELECT add_tiering_policy('assets_candlestick_daily', INTERVAL '3 weeks'); + ``` + +1. **Query tiered data** + + You enable reads from tiered data for each query, for a session or for all future + sessions. To run a single query on tiered data: + + 1. Enable reads on tiered data: + ```sql + set timescaledb.enable_tiered_reads = true + ``` + 1. Query the data: + ```sql + SELECT * FROM crypto_ticks srt LIMIT 10 + ``` + 1. Disable reads on tiered data: + ```sql + set timescaledb.enable_tiered_reads = false; + ``` + For more information, see [Querying tiered data][querying-tiered-data]. + + + +## Reduce the risk of downtime and data loss + +By default, all $SERVICE_LONGs have rapid recovery enabled. However, if your app has very low tolerance +for downtime, $CLOUD_LONG offers $HA_REPLICAs. HA replicas are exact, up-to-date copies +of your database hosted in multiple AWS availability zones (AZ) within the same region as your primary node. +HA replicas automatically take over operations if the original primary data node becomes unavailable. +The primary node streams its write-ahead log (WAL) to the replicas to minimize the chances of +data loss during failover. + +![$SERVICE_LONG replicas](https://assets.timescale.com/docs/images/ha-read-replica.png ) + +High availability is available in the [$SCALE and $ENTERPRISE][pricing-plans] $PRICING_PLANs for $CLOUD_LONG. + + + +For more information, see [High availability][high-availability]. + +What next? See the [use case tutorials][tutorials], interact with the data in your $SERVICE_LONG using +[your favorite programming language][connect-with-code], integrate your $SERVICE_LONG with a range of +[third-party tools][integrations], plain old [Use Timescale][use-timescale], or dive into [the API][use-the-api]. + +[tutorials]: /tutorials/:currentVersion:/ +[connect-with-code]: /getting-started/:currentVersion:/start-coding-with-timescale/ +[integrations]: /integrations/:currentVersion:/ +[use-the-api]: /api/:currentVersion:/ +[use-timescale]: /use-timescale/:currentVersion:/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ +[create-a-service]: /getting-started/:currentVersion:/services/ +[deploy-self-hosted]: /self-hosted/:currentVersion:/install/ +[connect-to-your-service]: /getting-started/:currentVersion:/run-queries-from-console/ +[portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services +[portal-data-mode]: https://console.cloud.timescale.com/dashboard/services?popsql +[migrate-with-downtime]: /migrate/:currentVersion:/pg-dump-and-restore/ +[migrate-live]: /migrate/:currentVersion:/live-migration/ +[data-ingest]: /use-timescale/:currentVersion:/ingest-data/ +[hypertables-section]: /use-timescale/:currentVersion:/hypertables/ +[test-drive-enable-compression]: /getting-started/:currentVersion:/try-key-features-timescale-products/#enhance-query-performance-for-analytics +[test-drive-tiered-storage]: /getting-started/:currentVersion:/try-key-features-timescale-products/#slash-storage-charges +[data-tiering]: /use-timescale/:currentVersion:/data-tiering/ +[compression]: /use-timescale/:currentVersion:/compression/ +[hierarchical-caggs]: /use-timescale/:currentVersion:/continuous-aggregates/hierarchical-continuous-aggregates/ +[charts]: https://www.investopedia.com/terms/c/candlestick.asp +[hierarchical-storage]: https://en.wikipedia.org/wiki/Hierarchical_storage_management +[querying-tiered-data]: /use-timescale/:currentVersion:/data-tiering/querying-tiered-data/ +[data-tiering]: /use-timescale/:currentVersion:/data-tiering/ +[pricing-plans]: /about/:currentVersion:/pricing-and-account-management +[querying-tiered-data]: /use-timescale/:currentVersion:/data-tiering/querying-tiered-data/ +[high-availability]: /use-timescale/:currentVersion:/ha-replicas/high-availability/ +[sign-up]: https://console.cloud.timescale.com/signup +[job]: /api/:currentVersion:/actions/add_job/ +[alter-table-arguments]: /api/:currentVersion:/hypercore/alter_table/#arguments +[add_columnstore_policy]: /api/:currentVersion:/hypercore/add_columnstore_policy/ diff --git a/integrations/airbyte.md b/integrations/airbyte.md new file mode 100644 index 0000000000..529a9a4869 --- /dev/null +++ b/integrations/airbyte.md @@ -0,0 +1,160 @@ +--- +title: Integrate Airbyte with Timescale Cloud +excerpt: Airbyte is an open-source data integration platform. Integrate Airbyte with Timescale Cloud to enable seamless data movement and analytics. +products: [cloud, mst, self_hosted] +keywords: [Airbyte, integration] +--- + +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; + +# Integrate Airbyte with $CLOUD_LONG + +[Airbyte][airbyte] is an open-source data integration platform that enables you to move and consolidate +data from various sources to a destination of your choice. Airbyte provides pre-built connectors, +simplifying ETL (Extract, Transform, Load) processes. + +[Ubidots][ubidots] is an IoT platform that enables real-time IoT data collection and management. + +This page explains how to ingest IoT sensor data from Ubidots into $CLOUD_LONG with Airbyte. + +## Prerequisites + + + +- Install [Airbyte self-managed][airbyte-server] or sign up for [Airbyte Cloud][airbyte-cloud]. +- Sign up for [Ubidots][ubidots-signup]. + +## Simulate IoT sensor data in Ubidots + +To generate simulated IoT data in Ubidots: + + + +1. **Log in to [Ubidots][ubidots-signup]** + +1. **Create a new device** + + 1. Navigate to `Devices` > `Create a device`. + 1. Select `Blank Device` and name it `airbyte-iot-sensor`. + +1. **Add variables to the device** + + 1. Click `airbyte-iot-sensor` and select `Add Variable`. + 1. Create the following variables: + - `temperature` + - `humidity` + - `pressure` + 1. Note the `variable-id` for each + +1. **Simulate data for the device** + + 1. In a terminal window, run the following command separately for each variable using the `variable-id` and [Default Token][ubidots-token]: + + ```bash + curl -X POST 'https://industrial.api.ubidots.com/api/v1.6/variables//values' \ + -H 'Content-Type: application/json' \ + -H 'X-Auth-Token: ' \ + -d '[ + {"value": 25.4}, + {"value": 26.1}, + {"value": 24.9} + ]' + ``` + + You may change the values for `pressure` and `humidity`. + + + +## Create a table to store IoT sensor data in $CLOUD_LONG + +To store IoT data in $CLOUD_LONG, create a table: + + + +1. **Connect to your $SERVICE_LONG** + + For $CLOUD_LONG, open an [SQL editor][run-queries] in [$CONSOLE][console]. For self-hosted, use `psql`. + +1. **Create a table** + + Run the following query: + + ```sql + CREATE TABLE iot_sensor_data ( + time TIMESTAMPTZ DEFAULT now(), + device_id TEXT, + temperature DOUBLE PRECISION, + humidity DOUBLE PRECISION, + pressure DOUBLE PRECISION + ); + SELECT create_hypertable('iot_sensor_data', 'time'); + ``` + + + +## Connect Ubidots to $CLOUD_LONG using Airbyte + +To connect Ubidots to $CLOUD_LONG: + + + +1. **Log in to [Airbyte][airbyte-cloud]** + +1. **Configure the Ubidots source** + + 1. Click `Sources` > `New source` and select `Ubidots`. + 2. Enter the `API Key` which is your Ubidots `airbyte-iot-sensor` device [Token][ubidots-device-token]. + 3. Click `Set up source`. + +1. **Configure the $CLOUD_LONG destination** + + 1. Click `Destinations` > `New destination` and select `Postgres`. + 2. Enter the Destination name as $CLOUD_LONG and populate the required [connection details][connection-info]. + 3. Click `Set up destination`. + +1. **Create a connection between Ubidots and $CLOUD_LONG** + + 1. Click `Connections` > `New connection`. + 2. Select Ubidots as the source and $CLOUD_LONG as the destination. + 3. Select the sync mode and select the schema to sync with $CLOUD_LONG and click on `Next`. + 4. Configure the connection for with desired settings. + 5. Click on `Finish & Sync`. + + **// PLEASE TEST FROM HERE.** + + 6. Map Ubidots variables `temperature`, `humidity`, `pressure` to the corresponding columns in `iot_sensor_data`. + + + +## Check the connection + +To check if the data is successfully ingested: + + + +1. **Connect to your $SERVICE_LONG** +1. **Query data from the table** + + Run the following SQL query: + + ```sql + SELECT * FROM iot_sensor_data ORDER BY time DESC LIMIT 10; + ``` + + You should now see simulated IoT data from Ubidots appear in the table. + + + +You have successfully ingested data from Ubidots into $CLOUD_LONG using Airbyte! + +[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[airbyte]: https://airbyte.com/ +[airbyte-server]: https://docs.airbyte.com/deploying-airbyte/ +[airbyte-cloud]: https://cloud.airbyte.com/signup +[ubidots]: https://ubidots.com/ +[ubidots-signup]: https://ubidots.com/#signup-modal +[ubidots-token]: https://docs.ubidots.com/v1.6/reference/authentication?utm_source=api_docs&utm_medium=internal_referral&utm_campaign=Send%20data%20to%20a%20Variable +[run-queries]: /getting-started/:currentVersion:/run-queries-from-console/ +[ubidots-device-token]: https://help.ubidots.com/en/articles/3832305-security-managing-device-tokens +[console]: https://console.cloud.timescale.com/dashboard/services +[ubidots-api-key]: https://help.ubidots.com/en/articles/570026-api-authentication \ No newline at end of file diff --git a/use-timescale/integrations/amazon-sagemaker.md b/integrations/amazon-sagemaker.md similarity index 94% rename from use-timescale/integrations/amazon-sagemaker.md rename to integrations/amazon-sagemaker.md index 4e6ddb870d..3cb8494567 100644 --- a/use-timescale/integrations/amazon-sagemaker.md +++ b/integrations/amazon-sagemaker.md @@ -31,6 +31,8 @@ Create a table in $SERVICE_LONG to store model predictions generated by SageMake For $CLOUD_LONG, open an [SQL editor][run-queries] in [$CONSOLE][open-console]. For self-hosted, use [`psql`][psql]. +1. **Create a table to store model predictions** + ```sql CREATE TABLE model_predictions ( time TIMESTAMPTZ NOT NULL, @@ -38,7 +40,7 @@ Create a table in $SERVICE_LONG to store model predictions generated by SageMake prediction DOUBLE PRECISION NOT NULL ); ``` - + 1. **For better performance and easier real-time analytics, convert the table to a hypertable** [Hypertables][about-hypertables] are PostgreSQL tables that automatically partition your data by time. You interact @@ -136,12 +138,12 @@ machine learning models. You can also untegrate visualization tools like [Grafan [install-python]: https://www.python.org/downloads/ [install-postgresql]: https://www.postgresql.org/download/ [console]: https://console.cloud.timescale.com/ -[grafana-integration]: /use-timescale/:currentVersion:/integrations/grafana/ -[tableau-integration]: /use-timescale/:currentVersion:/integrations/tableau/ +[grafana-integration]: /integrations/:currentVersion:/grafana/ +[tableau-integration]: /integrations/:currentVersion:/tableau/ [run-queries]: /getting-started/:currentVersion:/run-queries-from-console/ [open-console]: https://console.cloud.timescale.com/dashboard/services -[psql]: /use-timescale/:currentVersion:/integrations/psql/ +[psql]: /integrations/:currentVersion:/psql/ [about-hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/ [aws-notebooks-git-repos]:https://console.aws.amazon.com/sagemaker/home#/notebooks-and-git-repos [secure-vpc-aws]: /use-timescale/:currentVersion:/vpc/ -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ diff --git a/use-timescale/integrations/apache-airflow.md b/integrations/apache-airflow.md similarity index 90% rename from use-timescale/integrations/apache-airflow.md rename to integrations/apache-airflow.md index e675e70af9..dfbd325a9a 100644 --- a/use-timescale/integrations/apache-airflow.md +++ b/integrations/apache-airflow.md @@ -26,7 +26,7 @@ This page shows you how to use a Python connector in a DAG to integrate Apache A Ensure that your Airflow instance has network access to $CLOUD_LONG. -This example DAG uses the `company` table you create in [Create regular PostgreSQL tables for relational data][create-a-table-in-timescale] +This example DAG uses the `company` table you create in [Optimize time-series data in hypertables][create-a-table-in-timescale] ## Install python connectivity libraries @@ -98,10 +98,10 @@ To exchange data between Airflow and your $SERVICE_LONG: This could be any query. This example inserts data into the table you create in: - https://docs.timescale.com/getting-started/latest/tables-hypertables/#create-regular-postgresql-tables-for-relational-data + https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables """ - cursor.execute("INSERT INTO company (symbol, name) VALUES (%s, %s)", - ('new_company_symbol', 'New Company Name')) + cursor.execute("INSERT INTO crypto_assets (symbol, name) VALUES (%s, %s)", + ('NEW/Asset','New Asset Name')) conn.commit() cursor.close() conn.close() @@ -138,13 +138,13 @@ To exchange data between Airflow and your $SERVICE_LONG: You have successfully integrated Apache Airflow with $CLOUD_LONG and created a data pipeline. -[create-a-table-in-timescale]: /getting-started/:currentVersion:/tables-hypertables/#create-regular-postgresql-tables-for-relational-data +[create-a-table-in-timescale]: /getting-started/:currentVersion:/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables [install-apache-airflow]: https://airflow.apache.org/docs/apache-airflow/stable/start.html [install-python-pip]: https://docs.python.org/3/using/index.html [console]: https://console.cloud.timescale.com/ [create-service]: /getting-started/:currentVersion:/services/ [enable-timescaledb]: /self-hosted/:currentVersion:/install/ -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ [Airflow-DAG]: https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/dags.html#dags [Airflow-Task]:https://airflow.apache.org/docs/apache-airflow/stable/core-concepts/tasks.html [Airflow_UI]: localhost:8080 diff --git a/use-timescale/integrations/apache-kafka.md b/integrations/apache-kafka.md similarity index 98% rename from use-timescale/integrations/apache-kafka.md rename to integrations/apache-kafka.md index 570522a4f1..014a8f344c 100644 --- a/use-timescale/integrations/apache-kafka.md +++ b/integrations/apache-kafka.md @@ -238,7 +238,7 @@ To test this integration, send some messages onto the `accounts` topic. You can You have successfully integrated Apache Kafka with $CLOUD_LONG. -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ [apache-kafka]: https://kafka.apache.org/documentation/ [install-kafka]: https://kafka.apache.org/quickstart [java-installers]: https://www.oracle.com/java/technologies/downloads/ diff --git a/use-timescale/integrations/aws-lambda.md b/integrations/aws-lambda.md similarity index 97% rename from use-timescale/integrations/aws-lambda.md rename to integrations/aws-lambda.md index 6b4430cab1..cfe34e5697 100644 --- a/use-timescale/integrations/aws-lambda.md +++ b/integrations/aws-lambda.md @@ -203,8 +203,8 @@ You can now seamlessly ingest time-series data from AWS Lambda into $CLOUD_LONG. [install-postgresql]: https://www.postgresql.org/download/ [console]: https://console.cloud.timescale.com/ [run-queries]: /getting-started/:currentVersion:/run-queries-from-console/ -[psql]: /use-timescale/:currentVersion:/integrations/psql/ +[psql]: /integrations/:currentVersion:/psql/ [about-hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/ [aws-iam-role]: https://docs.aws.amazon.com/IAM/latest/UserGuide/access-keys-admin-managed.html#admin-list-access-key [open-console]: https://console.cloud.timescale.com/dashboard/services -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ diff --git a/integrations/aws.md b/integrations/aws.md new file mode 100644 index 0000000000..62df7a6ff4 --- /dev/null +++ b/integrations/aws.md @@ -0,0 +1,37 @@ +--- +title: Integrate Amazon Web Services with Timescale Cloud +excerpt: AWS enables you to build, run, and manage applications across cloud, hybrid, and edge environments with AI, analytics, security, and scalable infrastructure. Integrate AWS with Timescale Cloud using AWS Transit Gateway +products: [cloud] +keywords: [AWS, integrations] +--- + +import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; +import TransitGateway from "versionContent/_partials/_transit-gateway.mdx"; + +# Integrate Amazon Web Services with $CLOUD_LONG + +[Amazon Web Services (AWS)][aws] is a comprehensive cloud computing platform that provides on-demand infrastructure, storage, databases, AI, analytics, and security services to help businesses build, deploy, and scale applications in the cloud. + +This page explains how to integrate your AWS infrastructure with $CLOUD_LONG using [AWS Transit Gateway][aws-transit-gateway]. + +## Prerequisites + + + +- Set up [AWS Transit Gateway][gtw-setup]. + +## Connect your AWS infrastructure to your $SERVICE_LONGs + +To connect to $CLOUD_LONG: + + + + + + + +You have successfully integrated your AWS infrastructure with $CLOUD_LONG. + +[aws]: https://aws.amazon.com/ +[aws-transit-gateway]: https://aws.amazon.com/transit-gateway/ +[gtw-setup]: https://docs.aws.amazon.com/vpc/latest/tgw/tgw-getting-started.html diff --git a/use-timescale/integrations/azure-data-studio.md b/integrations/azure-data-studio.md similarity index 92% rename from use-timescale/integrations/azure-data-studio.md rename to integrations/azure-data-studio.md index 328869dd09..7e3466e950 100644 --- a/use-timescale/integrations/azure-data-studio.md +++ b/integrations/azure-data-studio.md @@ -48,8 +48,8 @@ You have successfully integrated Azure Data Studio with $CLOUD_LONG. [ms-azure-data-studio]: https://docs.microsoft.com/en-us/sql/azure-data-studio/download-azure-data-studio?view=sql-server-ver16#install-azure-data-studio [postgresql-azure-data-studio]: https://docs.microsoft.com/en-us/sql/azure-data-studio/extensions/postgres-extension?view=sql-server-ver16 -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ [azure-data-studio]: https://azure.microsoft.com/en-us/products/data-studio [ssl-mode]: /use-timescale/:currentVersion:/security/strict-ssl/ -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ diff --git a/use-timescale/integrations/cloudwatch.md b/integrations/cloudwatch.md similarity index 98% rename from use-timescale/integrations/cloudwatch.md rename to integrations/cloudwatch.md index a3848874d0..40b6fb36d4 100644 --- a/use-timescale/integrations/cloudwatch.md +++ b/integrations/cloudwatch.md @@ -7,7 +7,7 @@ keywords: [integrate] import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; import CloudWatchExporter from "versionContent/_partials/_cloudwatch-data-exporter.mdx"; -import ManageDataExporter from "versionContent/_partials/_manage-data-exporter.mdx"; +import ManageDataExporter from "versionContent/_partials/_manage-a-data-exporter.mdx"; # Integrate Amazon CloudWatch with $CLOUD_LONG diff --git a/integrations/corporate-data-center.md b/integrations/corporate-data-center.md new file mode 100644 index 0000000000..a6af17ad5c --- /dev/null +++ b/integrations/corporate-data-center.md @@ -0,0 +1,39 @@ +--- +title: Integrate your data center with Timescale Cloud +excerpt: Integrate your on-premise data center with Timescale Cloud using AWS Transit Gateway +products: [cloud] +keywords: [on-premise, integrations] +--- + +import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; +import TransitGateway from "versionContent/_partials/_transit-gateway.mdx"; + +# Integrate your data center with $CLOUD_LONG + +This page explains how to integrate your corporate on-premise infrastructure with $CLOUD_LONG using [AWS Transit Gateway][aws-transit-gateway]. + +## Prerequisites + + + +- Set up [AWS Transit Gateway][gtw-setup]. + +## Connect your on-premise infrastructure to your $SERVICE_LONGs + +To connect to $CLOUD_LONG: + + + +1. **Connect your infrastructure to AWS Transit Gateway** + + Establish connectivity between your on-premise infrastructure and AWS. See the [Centralize network connectivity using AWS Transit Gateway][aws-onprem]. + + + + + +You have successfully integrated your Microsoft Azure infrastructure with $CLOUD_LONG. + +[aws-transit-gateway]: https://aws.amazon.com/transit-gateway/ +[gtw-setup]: https://docs.aws.amazon.com/vpc/latest/tgw/tgw-getting-started.html +[aws-onprem]: https://docs.aws.amazon.com/prescriptive-guidance/latest/patterns/centralize-network-connectivity-using-aws-transit-gateway.html diff --git a/integrations/datadog.md b/integrations/datadog.md new file mode 100644 index 0000000000..45ac3d197a --- /dev/null +++ b/integrations/datadog.md @@ -0,0 +1,149 @@ +--- +title: Integrate Datadog with Timescale Cloud +excerpt: Datadog is a cloud-based monitoring and analytics platform. Export telemetry data from your Timescale Cloud service with time-series and analytics capability to Datadog +products: [cloud, mst, self_hosted] +keywords: [integrate] +--- + +import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; +import DataDogExporter from "versionContent/_partials/_datadog-data-exporter.mdx"; +import ManageDataExporter from "versionContent/_partials/_manage-a-data-exporter.mdx"; + +# Integrate Datadog with $CLOUD_LONG + +[Datadog][datadog] is a cloud-based monitoring and analytics platform that provides comprehensive visibility into +applications, infrastructure, and systems through real-time monitoring, logging, and analytics. + +This page explains how to: + +- [Monitor $SERVICE_LONG metrics with Datadog][datadog-monitor-cloud] + + This integration is available for [Scale and Enterprise][pricing-plan-features] pricing plans. + +- Configure Datadog Agent to collect metrics for your $SERVICE_LONG + + This integration is available for all pricing plans. + + +## Prerequisites + + + +- Sign up for [Datadog][datadog-signup]. + + You need your [Datadog API key][datadog-api-key] to follow this procedure. + +- Install [Datadog Agent][datadog-agent-install]. + +## Monitor Timescale Cloud service metrics with Datadog + +Export telemetry data from your $SERVICE_LONGs with the time-series and analytics capability enabled to +Datadog using a $CLOUD_LONG data exporter. The available metrics include CPU usage, RAM usage, and storage. + +### Create a data exporter + +A $CLOUD_LONG data exporter sends telemetry data from a $SERVICE_LONG to a third-party monitoring +tool. You create an exporter on the [project level][projects], in the same AWS region as your $SERVICE_SHORT: + + + +### Manage a data exporter + +This section shows you how to attach, monitor, edit, and delete a data exporter. + + + +## Configure Datadog Agent to collect metrics for your $SERVICE_LONGs + +Datadog Agent includes a [$PG integration][datadog-postgres] that you use to collect detailed $PG database +metrics about your $SERVICE_LONGs. + +1. **Connect to your $SERVICE_LONG** + + For $CLOUD_LONG, open an [SQL editor][run-queries] in [$CONSOLE][open-console]. For self-hosted, use [`psql`][psql]. + +1. **Add the `datadog` user to your $SERVICE_LONG** + + ```sql + create user datadog with password ''; + ``` + + ```sql + grant pg_monitor to datadog; + ``` + + ```sql + grant SELECT ON pg_stat_database to datadog; + ``` + +1. **Test the connection and rights for the datadog user** + + Update the following command with your [connection details][connection-info], then run it from the command line: + + ```bash + psql "postgres://datadog:@:/tsdb?sslmode=require" -c \ + "select * from pg_stat_database LIMIT(1);" \ + && echo -e "\e[0;32mPostgres connection - OK\e[0m" || echo -e "\e[0;31mCannot connect to Postgres\e[0m" + ``` + You see the output from the `pg_stat_database` table, which means you have given the correct rights to `datadog`. + +1. **Connect Datadog to your $SERVICE_LONG** + + 1. Open the datadog agent $PG configuration file, usually located at: + - **Linux**: `/etc/datadog-agent/conf.d/postgres.d/conf.yaml` + - **MacOS**: `/opt/datadog-agent/etc/conf.d/postgres.d/conf.yaml` + - **Windows**: `C:\ProgramData\Datadog\conf.d\postgres.d\conf.yaml` + + 1. Integrate Datadog Agent with your $SERVICE_LONG + + Use your [connection details][connection-info] to update the following and add it to the datadog agent $PG + configuration file: + + ```yaml + init_config: + + instances: + - host: + port: + username: datadog + password: > + dbname: tsdb + disable_generic_tags: true + ``` + +1. **Add $CLOUD_LONG metrics** + + Tags to make it easier for build Datadog dashboards that combine metrics from the $CLOUD_LONG data exporter and + Datadog Agent. Use your [connection details][connection-info] to update the following and add it to + `/datadog.yaml`: + + ```yaml + tags: + - project-id: + - service-id: + - region: + ``` + +1. **Restart Datadog Agent** + + See how to [Start, stop, and restart Datadog Agent][datadog-agent-restart]. + +Metrics for your $SERVICE_LONG are now visible in Datadog. Check the Datadog $PG integration documentation for a +comprehensive list of [metrics][datadog-postgres-metrics] collected. + +[datadog]: https://www.datadoghq.com/ +[datadog-agent-install]: https://docs.datadoghq.com/getting_started/agent/#installation +[datadog-postgres]: https://docs.datadoghq.com/integrations/postgres/ +[datadog-postgres-metrics]:https://docs.datadoghq.com/integrations/postgres/?tab=host#metrics +[datadog-postgres-setup]: https://docs.datadoghq.com/integrations/postgres/?tab=host#configuration +[datadog-signup]: https://www.datadoghq.com/ +[datadog-monitor-cloud]: /use-timescale/:currentVersion:/integrations/datadog/#monitor-timescale-cloud-service-metrics-with-datadog +[datadog-agent]: /use-timescale/:currentVersion:/integrations/datadog/#configure-datadog-agent-to-collect-metrics-for-your-timescale-cloud-services +[datadog-agent-restart]: https://docs.datadoghq.com/agent/configuration/agent-commands/#start-stop-and-restart-the-agent +[projects]: /use-timescale/:currentVersion:/members/ +[datadog-api-key]: https://docs.datadoghq.com/account_management/api-app-keys/#add-an-api-key-or-client-token +[pricing-plan-features]: /about/:currentVersion:/pricing-and-account-management/#features-included-in-each-plan +[run-queries]: /getting-started/:currentVersion:/run-queries-from-console/ +[open-console]: https://console.cloud.timescale.com/dashboard/services +[psql]: /use-timescale/:currentVersion:/integrations/psql/ +[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ diff --git a/use-timescale/integrations/dbeaver.md b/integrations/dbeaver.md similarity index 95% rename from use-timescale/integrations/dbeaver.md rename to integrations/dbeaver.md index cc7af9434b..12473e1896 100644 --- a/use-timescale/integrations/dbeaver.md +++ b/integrations/dbeaver.md @@ -48,5 +48,5 @@ You have successfully integrated DBeaver with $CLOUD_LONG. [dbeaver]: https://dbeaver.io/ [dbeaver-downloads]: https://dbeaver.io/download/ -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ [ssl-mode]: /use-timescale/:currentVersion:/security/strict-ssl/ diff --git a/integrations/decodable.md b/integrations/decodable.md new file mode 100644 index 0000000000..b71e5553b0 --- /dev/null +++ b/integrations/decodable.md @@ -0,0 +1,75 @@ +--- +title: Integrate Decodable with Timescale Cloud +excerpt: Decodable enables you to build, run, and manage data pipelines effortlessly. Seamlessly integrate Decodable with Timescale Cloud to unlock real-time data processing capabilities +products: [cloud, mst, self_hosted] +keywords: [Decodable, Timescale Cloud] +--- + +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; + +# Integrate Decodable with $CLOUD_LONG + +[Decodable][decodable] is a real-time data platform that allows you to build, run, and manage data pipelines effortlessly. + +![Decodable workflow](https://assets.timescale.com/docs/images/integrations-decodable-configuration.png) + +This page explains how to integrate Decodable with your $SERVICE_LONG to enable efficient real-time streaming and analytics. + +## Prerequisites + + + +- Sign up for [Decodable][sign-up-decodable]. + + This page uses the pipeline you create using the [Decodable Quickstart Guide][decodable-quickstart]. + +## Connect Decodable to your $SERVICE_LONG + +To stream data gathered in Decodable to a $SERVICE_LONG: + + + +1. **Create the sync to pipe a Decodable data stream into your $SERVICE_LONG** + + 1. Log in to your [Decodable account][decodable-app]. + 1. Click `Connections`, then click `New Connection`. + 1. Select a `PostgreSQL sink` connection type, then click `Connect`. + 1. Using your [connection details][connection-info], fill in the connection information. + + Leave `schema` and `JDBC options` empty. + 1. Select the `http_events` source stream, then click `Next`. + + Decodable creates the table in your $SERVICE_LONG and starts streaming data. + + + +1. **Test the connection** + + 1. Connect to your $SERVICE_LONG. + + For $CLOUD_LONG, open an [SQL editor][run-queries] in [$CONSOLE][open-console]. For self-hosted, use [`psql`][psql]. + + 1. Check the data from Decodable is streaming into your $SERVICE_LONG. + + ```sql + SELECT * FROM http_events; + ``` + You see something like: + + ![Decodable workflow](https://assets.timescale.com/docs/images/integrations-decodable-data-in-service.png) + + + + +You have successfully integrated Decodable with $CLOUD_LONG. + + +[decodable]: https://www.decodable.co/ +[decodable-app]:https://app.decodable.co/-/accounts +[sign-up-decodable]: https://auth.decodable.co/u/signup/ +[decodable-quickstart]: https://docs.decodable.co/get-started/quickstart.html +[connection-info]: /integrations/:currentVersion:/find-connection-details/ +[run-queries]: /getting-started/:currentVersion:/run-queries-from-console/ +[open-console]: https://console.cloud.timescale.com/dashboard/services +[psql]: /integrations/:currentVersion:/psql/ +[about-hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/ diff --git a/use-timescale/integrations/find-connection-details.md b/integrations/find-connection-details.md similarity index 76% rename from use-timescale/integrations/find-connection-details.md rename to integrations/find-connection-details.md index 16c9d83a68..9871a4ab90 100644 --- a/use-timescale/integrations/find-connection-details.md +++ b/integrations/find-connection-details.md @@ -21,6 +21,8 @@ Find the connection details based on your deployment type: +## Connect to your service + Retrieve the connection details for your $SERVICE_LONG: - **In `-credentials.txt`**: @@ -33,6 +35,24 @@ Retrieve the connection details for your $SERVICE_LONG: ![Timescale service connection details](https://assets.timescale.com/docs/images/timescale-service-connection-details.png) +## Find your project and service ID + +To retrieve the connection details for your $CLOUD_LONG project and $SERVICE_LONG: + + + +1. **Retreive your project ID**: + + In [$CONSOLE][console-services], click `Timescale Project`, then click `Copy` next to the project ID. + ![Retrive the project id in console](https://assets.timescale.com/docs/images/console-project-id.png) + +1. **Retrieve your service ID**: + + Click the dots next to the service, then click `Copy` next to the service ID. + ![Retrive the service id in console](https://assets.timescale.com/docs/images/console-service-id.png) + + + diff --git a/use-timescale/integrations/fivetran.md b/integrations/fivetran.md similarity index 97% rename from use-timescale/integrations/fivetran.md rename to integrations/fivetran.md index a1003c6eed..78f554bae2 100644 --- a/use-timescale/integrations/fivetran.md +++ b/integrations/fivetran.md @@ -85,7 +85,7 @@ To see data injected by Fivetran into your $SERVICE_LONG: You have successfully integrated Fivetran with $CLOUD_LONG. -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ [fivetran]: https://www.fivetran.com/docs [sign-up-fivetran]: https://www.fivetran.com/ [fivetran-dashboard-destinations]: https://fivetran.com/dashboard/destinations diff --git a/integrations/google-cloud.md b/integrations/google-cloud.md new file mode 100644 index 0000000000..486f5cb878 --- /dev/null +++ b/integrations/google-cloud.md @@ -0,0 +1,42 @@ +--- +title: Integrate Google Cloud with Timescale Cloud +excerpt: Google Cloud enables you to deploy, manage, and scale cloud-based applications, databases, and data processing workflows. Integrate Google Cloud with Timescale Cloud using AWS Transit Gateway +products: [cloud] +keywords: [Google Cloud, integrations] +--- + +import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; +import TransitGateway from "versionContent/_partials/_transit-gateway.mdx"; + +# Integrate Google Cloud with $CLOUD_LONG + +[Google Cloud][google-cloud] is a suite of cloud computing services, offering scalable infrastructure, AI, analytics, databases, security, and developer tools to help businesses build, deploy, and manage applications. + +This page explains how to integrate your Google Cloud infrastructure with $CLOUD_LONG using [AWS Transit Gateway][aws-transit-gateway]. + +## Prerequisites + + + +- Set up [AWS Transit Gateway][gtw-setup]. + +## Connect your Google Cloud infrastructure to your $SERVICE_LONGs + +To connect to $CLOUD_LONG: + + + +1. **Connect your infrastructure to AWS Transit Gateway** + + Establish connectivity between Google Cloud and AWS. See [Connect HA VPN to AWS peer gateways][gcp-aws]. + + + + + +You have successfully integrated your Google Cloud infrastructure with $CLOUD_LONG. + +[google-cloud]: https://cloud.google.com/?hl=en +[aws-transit-gateway]: https://aws.amazon.com/transit-gateway/ +[gtw-setup]: https://docs.aws.amazon.com/vpc/latest/tgw/tgw-getting-started.html +[gcp-aws]: https://cloud.google.com/network-connectivity/docs/vpn/how-to/connect-ha-vpn-aws-peer-gateway diff --git a/use-timescale/integrations/grafana.md b/integrations/grafana.md similarity index 94% rename from use-timescale/integrations/grafana.md rename to integrations/grafana.md index a42b3556a4..102f6fbe70 100644 --- a/use-timescale/integrations/grafana.md +++ b/integrations/grafana.md @@ -4,7 +4,7 @@ excerpt: Grafana enables you to query, visualize, alert on, and explore your met products: [cloud] keywords: [Grafana, visualizations, analytics, monitoring] --- - +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; # Integrate Grafana and Timescale Cloud @@ -13,6 +13,12 @@ import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; This page shows you how to integrate Grafana with a $SERVICE_LONG, create a dashboard and panel, then visualize geospatial data. +## Prerequisites + + + +* Install [self-managed Grafana][grafana-self-managed] or sign up for [Grafana Cloud][grafana-cloud]. + ## Create a Grafana dashboard and panel @@ -177,4 +183,5 @@ tutorial as a starting point. [nyc-taxi]: /tutorials/:currentVersion:/nyc-taxi-cab [grafana-website]: https://www.grafana.com [time-buckets]: /use-timescale/:currentVersion:/time-buckets/ - +[grafana-self-managed]: https://grafana.com/get/?tab=self-managed +[grafana-cloud]: https://grafana.com/get/ diff --git a/use-timescale/integrations/index.md b/integrations/index.md similarity index 79% rename from use-timescale/integrations/index.md rename to integrations/index.md index aa09fc9ff7..454a086e1f 100644 --- a/use-timescale/integrations/index.md +++ b/integrations/index.md @@ -45,15 +45,17 @@ Some of the most in-demand integrations for $CLOUD_LONG are listed below, with l | [Terraform][terraform] | Safely and predictably provision and manage infrastructure in any cloud. | -## Data engineering and extract, transform, load -| Name | Description | -|:--------------------------------:|--------------------------------------------------------------------------------------------------------| -| [Amazon SageMaker][amazon-sagemaker]| Build, train, and deploy ML models into a production-ready hosted environment. | -| [Apache Airflow][apache-airflow] | Programmatically author, schedule, and monitor workflows. | -| [Apache Kafka][kafka] | Stream high-performance data pipelines, analytics, and data integration. | -| [AWS Lambda][aws-lambda]| Run code without provisioning or managing servers, scaling automatically as needed. | +## Data engineering and extract, transform, load + +| Name | Description | +|:--------------------------------:|-------------------------------------------------------------------------------------| +| [Amazon SageMaker][amazon-sagemaker]| Build, train, and deploy ML models into a production-ready hosted environment. | +| [Apache Airflow][apache-airflow] | Programmatically author, schedule, and monitor workflows. | +| [Apache Kafka][kafka] | Stream high-performance data pipelines, analytics, and data integration. | +| [AWS Lambda][aws-lambda]| Run code without provisioning or managing servers, scaling automatically as needed. | +| [Decodable][decodable] | Build, run, and manage data pipelines effortlessly. | ## Data ingestion and streaming @@ -68,6 +70,16 @@ Some of the most in-demand integrations for $CLOUD_LONG are listed below, with l | [Power BI][power-bi] | Visualize data, build interactive dashboards, and share insights. | +## Secure connectivity to $CLOUD_LONG + +| Name | Description | +|:------------------------------------:|-----------------------------------------------------------------------------| +| [Amazon Web Services][aws] | Connect your other services and applications running in AWS to $CLOUD_LONG. | +| [Google Cloud][google-cloud] | Connect your Google Cloud infrastructure to $CLOUD_LONG. | +| [Microsoft Azure][azure] | Connect your Microsoft Azure infrastructure to $CLOUD_LONG. | +| [Corporate data center][data-center] | Connect your on-premise data center to $CLOUD_LONG. | + + [psql]: /use-timescale/:currentVersion:/integrations/psql/ [qstudio]: /use-timescale/:currentVersion:/integrations/qstudio/ [cloudwatch]: /use-timescale/:currentVersion:/integrations/cloudwatch/ @@ -81,10 +93,15 @@ Some of the most in-demand integrations for $CLOUD_LONG are listed below, with l [apache-airflow]: /use-timescale/:currentVersion:/integrations/apache-airflow [aws-lambda]: /use-timescale/:currentVersion:/integrations/aws-lambda [postgresql-integrations]: https://slashdot.org/software/p/PostgreSQL/integrations/ +[decodable]: /use-timescale/:currentVersion:/integrations/decodable [power-bi]: /use-timescale/:currentVersion:/integrations/power-bi [fivetran]: /use-timescale/:currentVersion:/integrations/fivetran [prometheus]: /use-timescale/:currentVersion:/integrations/prometheus +[airbyte]: /use-timescale/:currentVersion:/integrations/airbyte [amazon-sagemaker]: /use-timescale/:currentVersion:/integrations/amazon-sagemaker [postgresql]: /use-timescale/:currentVersion:/integrations/postgresql [kafka]: /use-timescale/:currentVersion:/integrations/apache-kafka - +[aws]: /use-timescale/:currentVersion:/integrations/aws +[google-cloud]: /use-timescale/:currentVersion:/integrations/google-cloud +[azure]: /use-timescale/:currentVersion:/integrations/microsoft-azure +[data-center]: /use-timescale/:currentVersion:/integrations/corporate-data-center diff --git a/integrations/kubernetes.md b/integrations/kubernetes.md new file mode 100644 index 0000000000..8cfe08fc8c --- /dev/null +++ b/integrations/kubernetes.md @@ -0,0 +1,312 @@ +--- +title: Integrate Kubernetes with Timescale Cloud +excerpt: Learn how to integrate Kubernetes with Timescale Cloud to enable seamless deployment and scaling of your PostgreSQL workloads +products: [cloud, mst, self_hosted] +keywords: [Kubernetes, Timescale Cloud, PostgreSQL, container orchestration] +--- + +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; + +# Integrate Kubernetes with $CLOUD_LONG + +[Kubernetes][kubernetes] is an open-source container orchestration system that automates the deployment, scaling, and management of containerized applications. You can deploy $CLOUD_LONG within your Kubernetes clusters. + +This guide explains how to connect a Kubernetes cluster to $CLOUD_LONG, configure persistent storage, and manage deployments effectively. + +## Prerequisites + + + +- Install [self-managed Kubernetes][kubernetes-install] or sign up for a Kubernetes [Turnkey Cloud Solution][kubernetes-managed]. +- Install [kubectl][kubectl] for command-line interaction with your cluster. + +## Integrate Timescale in a Kubernetes cluster + + + + + +To connect your Kubernetes cluster to your $SERVICE_LONG: + + + +1. **Create a default namespace for your $CLOUD_LONG components** + + 1. Create the Timescale namespace: + + ```shell + kubectl create namespace timescale + ``` + + 1. Set this namespace as the default for your session: + + ```shell + kubectl config set-context --current --namespace=timescale + ``` + + For more information, see [Kubernetes Namespaces][kubernetes-namespace]. + +1. **Create a Kubernetes secret that stores your $SERVICE_LONG credentials** + + Update the following command with your [connection details][connection-info], then run it: + + ```shell + kubectl create secret generic timescale-secret \ + --from-literal=PGHOST= \ + --from-literal=PGPORT= \ + --from-literal=PGDATABASE= \ + --from-literal=PGUSER= \ + --from-literal=PGPASSWORD= + ``` + +1. **Configure network access to $CLOUD_LONG** + + - **Managed Kubernetes**: outbound connections to external databases like $CLOUD_LONG work by default. + Make sure your cluster’s security group or firewall rules allow outbound traffic to $CLOUD_LONG IP. + + - **Self-hosted Kubernetes**: If your cluster is behind a firewall or running on-premise, you may need to allow + egress traffic to $CLOUD_LONG. Test connectivity using your [connection details][connection-info]: + + ```shell + nc -zv + ``` + + If the connection fails, check your firewall rules. + +1. **Create a Kubernetes deployment that can access your $CLOUD_LONG** + + Run the following command to apply the deployment: + + ```shell + kubectl apply -f - < + + + + + +To connect your Kubernetes cluster to self-hosted $TIMESCALE_DB running in the cluster: + + + +1. **Create a default namespace for $COMPANY components** + + 1. Create the $COMPANY namespace: + + ```shell + kubectl create namespace timescale + ``` + + 1. Set this namespace as the default for your session: + + ```shell + kubectl config set-context --current --namespace=timescale + ``` + + For more information, see [Kubernetes Namespaces][kubernetes-namespace]. + +1. **Set up a persistent volume claim (PVC) storage** + + To manually set up a persistent volume and claim for self-hosted Kubernetes, run the following command: + + ```yaml + kubectl apply -f - < + + + + + +You have successfully integrated Kubernetes with $CLOUD_LONG. + +[connection-info]: /integrations/:currentVersion:/find-connection-details/ +[kubernetes]: https://kubernetes.io/ +[kubectl]: https://kubernetes.io/docs/tasks/tools/ +[kubernetes-namespace]: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/ +[kubernetes-install]: https://kubernetes.io/docs/setup/ +[kubernetes-managed]: https://kubernetes.io/docs/setup/production-environment/turnkey-solutions/ +[timescale-docker-image]: https://hub.docker.com/r/timescale/timescaledb diff --git a/integrations/microsoft-azure.md b/integrations/microsoft-azure.md new file mode 100644 index 0000000000..1582970653 --- /dev/null +++ b/integrations/microsoft-azure.md @@ -0,0 +1,42 @@ +--- +title: Integrate Microsoft Azure with Timescale Cloud +excerpt: Microsoft Azure enables you to build, deploy, and manage applications across cloud, hybrid, and edge environments. Integrate Microsoft Azure with Timescale Cloud using AWS Transit Gateway +products: [cloud] +keywords: [Azure, integrations] +--- + +import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; +import TransitGateway from "versionContent/_partials/_transit-gateway.mdx"; + +# Integrate Microsoft Azure with $CLOUD_LONG + +[Microsoft Azure][azure] is a cloud computing platform and services suite, offering infrastructure, AI, analytics, security, and developer tools to help businesses build, deploy, and manage applications. + +This page explains how to integrate your Microsoft Azure infrastructure with $CLOUD_LONG using [AWS Transit Gateway][aws-transit-gateway]. + +## Prerequisites + + + +- Set up [AWS Transit Gateway][gtw-setup]. + +## Connect your Microsoft Azure infrastructure to your $SERVICE_LONGs + +To connect to $CLOUD_LONG: + + + +1. **Connect your infrastructure to AWS Transit Gateway** + + Establish connectivity between Azure and AWS. See the [AWS architectural documentation][azure-aws] for details. + + + + + +You have successfully integrated your Microsoft Azure infrastructure with $CLOUD_LONG. + +[aws-transit-gateway]: https://aws.amazon.com/transit-gateway/ +[gtw-setup]: https://docs.aws.amazon.com/vpc/latest/tgw/tgw-getting-started.html +[azure]: https://azure.microsoft.com/en-gb/ +[azure-aws]: https://aws.amazon.com/blogs/modernizing-with-aws/designing-private-network-connectivity-aws-azure/ diff --git a/integrations/page-index/page-index.js b/integrations/page-index/page-index.js new file mode 100644 index 0000000000..e86988f8ea --- /dev/null +++ b/integrations/page-index/page-index.js @@ -0,0 +1,152 @@ +module.exports = [ + { + title: "Integrations", + href: "integrations", + filePath: "index.md", + excerpt: + "Integrate third-party solutions with Timescale Cloud", + children: [ + { + title: "Find your connection details", + href: "find-connection-details", + excerpt: "Find connection information for your Timescale Cloud service", + }, + { + title: "Airbyte", + href: "airbyte", + excerpt: "Integrate Airbyte with Timescale Cloud", + }, + { + title: "Amazon CloudWatch", + href: "cloudwatch", + excerpt: "Integrate Amazon Cloudwatch with Timescale Cloud", + }, + { + title: "Amazon SageMaker", + href: "amazon-sagemaker", + excerpt: "Integrate Amazon SageMaker with Timescale Cloud", + }, + { + title: "Amazon Web Services", + href: "aws", + excerpt: "Integrate AWS with Timescale Cloud", + }, + { + title: "Apache Airflow", + href: "apache-airflow", + excerpt: "Integrate Apache Airflow with Timescale Cloud", + }, + { + title: "Apache Kafka", + href: "apache-kafka", + excerpt: "Integrate Apache Kafka with Timescale Cloud", + }, + { + title: "AWS Lambda", + href: "aws-lambda", + excerpt: "Integrate AWS Lambda with Timescale Cloud", + }, + { + title: "Azure Data Studio", + href: "azure-data-studio", + excerpt: "Integrate Azure Data Studio with Timescale Cloud", + }, + { + title: "Corporate data center", + href: "corporate-data-center", + excerpt: "Integrate your corporate data center with Timescale Cloud", + }, + { + title: "Datadog", + href: "datadog", + excerpt: "Integrate Datadog with Timescale Cloud", + }, + { + title: "DBeaver", + href: "dbeaver", + excerpt: "Integrate DBeaver with Timescale Cloud", + }, + { + title: "Decodable", + href: "decodable", + excerpt: "Integrate Decodable with Timescale Cloud", + }, + { + title: "Fivetran", + href: "fivetran", + excerpt: "Integrate Fivetran with Timescale Cloud", + }, + { + title: "Google Cloud", + href: "google-cloud", + excerpt: "Integrate Google Cloud with Timescale Cloud", + }, + { + title: "Grafana", + href: "grafana", + excerpt: "Integrate Grafana with Timescale Cloud", + }, + { + title: "Kubernetes", + href: "kubernetes", + excerpt: "Integrate Kubernetes with Timescale Cloud", + }, + { + title: "Microsoft Azure", + href: "microsoft-azure", + excerpt: "Integrate Microsoft Azure with Timescale Cloud", + }, + { + title: "pgAdmin", + href: "pgadmin", + excerpt: "Integrate pgAdmin with Timescale Cloud", + }, + { + title: "PostgreSQL", + href: "postgresql", + excerpt: "Integrate PostgreSQL with Timescale Cloud", + }, + { + title: "Prometheus", + href: "prometheus", + excerpt: "Integrate Prometheus with Timescale Cloud", + }, + { + title: "Power BI", + href: "power-bi", + excerpt: "Integrate Power BI with Timescale Cloud", + }, + { + title: "psql", + href: "psql", + excerpt: "Connect to Timescale products with psql", + }, + { + title: "qStudio", + href: "qstudio", + excerpt: "Integrate qstudio with Timescale Cloud", + }, + { + title: "Supabase", + href: "supabase", + excerpt: "Integrate Supabase with Timescale Cloud", + }, + { + title: "Tableau", + href: "tableau", + excerpt: "Integrate Tableau with Timescale Cloud", + }, + { + title: "Terraform", + href: "terraform", + excerpt: "Integrate Terraform with Timescale Cloud", + }, + { + title: "Troubleshooting Timescale integrations", + href: "troubleshooting", + type: "placeholder", + excerpt: "Resolve integration issues", + }, + ], + }, +]; diff --git a/use-timescale/integrations/pgadmin.md b/integrations/pgadmin.md similarity index 95% rename from use-timescale/integrations/pgadmin.md rename to integrations/pgadmin.md index b2cba48513..effbfe3576 100644 --- a/use-timescale/integrations/pgadmin.md +++ b/integrations/pgadmin.md @@ -42,5 +42,5 @@ You have successfully integrated pgAdmin with $CLOUD_LONG. [pgadmin]: https://www.pgadmin.org/ [download-pgadmin]: https://www.pgadmin.org/download/ -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ -[ssl-mode]: /use-timescale/:currentVersion:/security/strict-ssl/ \ No newline at end of file +[connection-info]: /integrations/:currentVersion:/find-connection-details/ +[ssl-mode]: /use-timescale/:currentVersion:/security/strict-ssl/ diff --git a/use-timescale/integrations/postgresql.md b/integrations/postgresql.md similarity index 100% rename from use-timescale/integrations/postgresql.md rename to integrations/postgresql.md diff --git a/use-timescale/integrations/power-bi.md b/integrations/power-bi.md similarity index 96% rename from use-timescale/integrations/power-bi.md rename to integrations/power-bi.md index 656b435564..3b0b76b5d8 100644 --- a/use-timescale/integrations/power-bi.md +++ b/integrations/power-bi.md @@ -67,7 +67,7 @@ Establish a connection and import data from your $SERVICE_LONG into Power BI: You have successfully integrated Power BI with $CLOUD_LONG. [timescale-on-windows]: https://docs.timescale.com/self-hosted/latest/install/installation-windows/ -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ [power-bi]: https://powerbi.microsoft.com/ [power-bi-install]: https://www.microsoft.com/en-us/power-platform/products/power-bi/downloads [postgresql-odbc-driver]: https://www.postgresql.org/ftp/odbc/releases/ diff --git a/use-timescale/integrations/prometheus.md b/integrations/prometheus.md similarity index 100% rename from use-timescale/integrations/prometheus.md rename to integrations/prometheus.md diff --git a/use-timescale/integrations/psql.md b/integrations/psql.md similarity index 97% rename from use-timescale/integrations/psql.md rename to integrations/psql.md index 98faa3e57d..0057f99b8c 100644 --- a/use-timescale/integrations/psql.md +++ b/integrations/psql.md @@ -262,6 +262,6 @@ edited query by pressing `↑`, and press `Enter` to run it. [homebrew]: https://docs.brew.sh/Installation [macports]: https://guide.macports.org/#installing.macports [windows-installer]: https://www.postgresql.org/download/windows/ -[connect-database]:/use-timescale/:currentVersion:/integrations/psql/#connect-to-your-service -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connect-database]:/integrations/:currentVersion:/psql/#connect-to-your-service +[connection-info]: /integrations/:currentVersion:/find-connection-details/ diff --git a/use-timescale/integrations/qstudio.md b/integrations/qstudio.md similarity index 94% rename from use-timescale/integrations/qstudio.md rename to integrations/qstudio.md index d52dea19a2..fc2b82e142 100644 --- a/use-timescale/integrations/qstudio.md +++ b/integrations/qstudio.md @@ -50,4 +50,4 @@ You have successfully integrated qStudio with $CLOUD_LONG. [qstudio]: https://www.timestored.com/qstudio [qstudio-downloads]: https://www.timestored.com/qstudio/download -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ diff --git a/integrations/supabase.md b/integrations/supabase.md new file mode 100644 index 0000000000..056c0a29df --- /dev/null +++ b/integrations/supabase.md @@ -0,0 +1,272 @@ +--- +title: Integrate Supabase with Timescale Cloud +excerpt: Supabase is an open source Firebase alternative. Integrate Supabase with Timescale Cloud +products: [cloud, mst, self_hosted] +keywords: [integrate] +--- + +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; + +# Integrate Supabase with Timescale Cloud + +[Supabase][supabase] is an open source Firebase alternative. This page shows how to run real-time analytical queries +against a $SERVICE_LONG through Supabase using a foreign data wrapper (fdw) to bring aggregated data from your +$SERVICE_LONG. + +## Prerequisites + + + +- Create a [Supabase project][supabase-new-project] + +## Setup your $SERVICE_LONG + +To setup a $SERVICE_LONG optimized for analytics to receive data from Supabase: + + + +1. **Optimize time-series data in hypertables** + + Time-series data represents how a system, process, or behavior changes over time. [Hypertables][hypertables-section] + are PostgreSQL tables that help you improve insert and query performance by automatically partitioning your data by + time. + + 1. [Connect to your $SERVICE_LONG][connect] and create a table that will point to a Supabase database: + + ```sql + CREATE TABLE signs ( + time timestamptz NOT NULL DEFAULT now(), + origin_time timestamptz NOT NULL, + name TEXT + ); + ``` + + 1. Turn the table to a hypertable: + + ```sql + SELECT create_hypertable('signs', by_range('time')); + ``` +1. **Optimize cooling data for analytics** + + Hypercore is the $TIMESCALE_DB hybrid row-columnar storage engine, designed specifically for real-time analytics + and powered by time-series data. The advantage of Hypercore is its ability to seamlessly switch between row-oriented + and column-oriented storage. This flexibility enables $CLOUD_LONG to deliver the best of both worlds, solving the + key challenges in real-time analytics. + + ```sql + ALTER TABLE signs SET ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'name'); + ``` + +1. **Create optimized analytical queries** + + Continuous aggregates are designed to make queries on very large datasets run + faster. Continuous aggregates in $CLOUD_LONG use PostgreSQL [materialized views][postgres-materialized-views] to + continuously, and incrementally refresh a query in the background, so that when you run the query, + only the data that has changed needs to be computed, not the entire dataset. + + 1. Create a continuous aggregate pointing to the Supabase database. + + ```sql + CREATE MATERIALIZED VIEW IF NOT EXISTS signs_per_minute + WITH (timescaledb.continuous) + AS + SELECT time_bucket('1 minute', time) as ts, + name, + count(*) as total + FROM signs + GROUP BY 1, 2 + WITH NO DATA; + ``` + + 1. Setup a delay stats comparing `origin_time` to `time`. + + ```sql + CREATE MATERIALIZED VIEW IF NOT EXISTS _signs_per_minute_delay + WITH (timescaledb.continuous) + AS + SELECT time_bucket('1 minute', time) as ts, + stats_agg(extract(epoch from origin_time - time)::float8) as delay_agg, + candlestick_agg(time, extract(epoch from origin_time - time)::float8, 1) as delay_candlestick + FROM signs GROUP BY 1 + WITH NO DATA; + ``` + + 1. Setup a view to recieve the data from Supabase. + + ```sql + CREATE VIEW signs_per_minute_delay + AS + SELECT ts, + average(delay_agg) as avg_delay, + stddev(delay_agg) as stddev_delay, + open(delay_candlestick) as open, + high(delay_candlestick) as high, + low(delay_candlestick) as low, + close(delay_candlestick) as close + FROM _signs_per_minute_delay + ``` + +1. **Add refresh policies for your analytical queries** + + You use `start_offset` and `end_offset` to define the time range that the continuous aggregate will cover. Assuming + that the data is being inserted without any delay, set the `start_offset` to `5 minutes` and the `end_offset` to + `1 minute`. This means that the continuous aggregate is refreshed every minute, and the refresh covers the last 5 + minutes. + You set `schedule_interval` to `INTERVAL '1 minute'` so the continuous aggregate refreshes on your $SERVICE_LONG + every minute. The data is accessed from Supabase, and the continuous aggregate is refreshed every minute in + the other side. + + ```sql + SELECT add_continuous_aggregate_policy('signs_per_minute', + start_offset => INTERVAL '5 minutes', + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 minute'); + ``` + Do the same thing for data inserted with a delay: + ```sql + SELECT add_continuous_aggregate_policy('_signs_per_minute_delay', + start_offset => INTERVAL '5 minutes', + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 minute'); + ``` + + + + +## Setup a Supabase database + +To setup a Supabase database that injects data into your $SERVICE_LONG: + + + +1. **Connect a foreign server in Supabase to your $SERVICE_LONG** + + 1. Connect to your Supabase project using Supabase dashboard or psql. + 1. Enable the `postgres_fdw` extension. + + ```sql + CREATE EXTENSION postgres_fdw; + ``` + 1. Create a foreign server that points to your $SERVICE_LONG. + + Update the following command with your [connection details][connection-info], then run it + in the Supabase database: + + ```sql + CREATE SERVER timescale + FOREIGN DATA WRAPPER postgres_fdw + OPTIONS ( + host '', + port '', + dbname '', + sslmode 'require', + extensions 'timescaledb' + ); + ``` + +1. **Create the user mapping for the foreign server** + + Update the following command with your [connection details][connection-info], the run it + in the Supabase database: + + ```sql + CREATE USER MAPPING FOR CURRENT_USER + SERVER timescale + OPTIONS ( + user '', + password '' + ); + ``` + +1. **Create a foreign table that points to a table in your $SERVICE_LONG.** + + This query introduced the following columns: + - `time`: with a default value of `now()`. This is because the `time` column is used by $CLOUD_LONG to optimize data + in the columnstore. + - `origin_time`: store the original timestamp of the data. + + Using both columns, you understand the delay between Supabase (`origin_time`) and the time the data is + inserted into your $SERVICE_LONG (`time`). + + ```sql + CREATE FOREIGN TABLE signs ( + TIME timestamptz NOT NULL DEFAULT now(), + origin_time timestamptz NOT NULL, + NAME TEXT) + SERVER timescale OPTIONS ( + schema_name 'public', + table_name 'signs' + ); + ``` + +1. **Create a foreign table in Supabase** + + 1. Create a foreign table that matches the `signs_per_minute` view in your $SERVICE_LONG. It represents a top level + view of the data. + + ```sql + CREATE FOREIGN TABLE signs_per_minute ( + ts timestamptz, + name text, + total int + ) + SERVER timescale OPTIONS (schema_name 'public', table_name 'signs_per_minute'); + ``` + + 1. Create a foreign table that matches the `signs_per_minute_delay` view in your $SERVICE_LONG. + + ```sql + CREATE FOREIGN TABLE signs_per_minute_delay ( + ts timestamptz, + avg_delay float8, + stddev_delay float8, + open float8, + high float8, + low float8, + close float8 + ) SERVER timescale OPTIONS (schema_name 'public', table_name 'signs_per_minute_delay'); + ``` + + + +## Test the integration + +To inject data into your $SERVICE_LONG from a Supabase database using a foreign table: + + + +1. **Insert data into your Supabase database** + + Connect to Supabase and run the following query: + + ```sql + INSERT INTO signs (origin_time, name) VALUES (now(), 'test') + ``` + +1. **Check the data in your $SERVICE_LONG** + + [Connect to your $SERVICE_LONG][connect] and run the following query: + + ```sql + SELECT * from signs; + ``` + You see something like: + + | origin_time | time | name | + |-------------|------|------| + | 2025-02-27 16:30:04.682391+00 | 2025-02-27 16:30:04.682391+00 | test | + + + +You have successfully integrated Supabase with your $SERVICE_LONG. + +[supabase]: https://supabase.com/ +[supabase-new-project]: https://supabase.com/dashboard/new + +[hypertables-section]: /use-timescale/:currentVersion:/hypertables/ +[connect]: /getting-started/:currentVersion:/run-queries-from-console/ +[hypercore]: /use-timescale/:currentVersion:/hypercore/ +[postgres-materialized-views]: https://www.postgresql.org/docs/current/rules-materializedviews.html +[connection-info]: /integrations/:currentVersion:/find-connection-details/ diff --git a/use-timescale/integrations/tableau.md b/integrations/tableau.md similarity index 95% rename from use-timescale/integrations/tableau.md rename to integrations/tableau.md index 25f2b7e001..a2dfa021f1 100644 --- a/use-timescale/integrations/tableau.md +++ b/integrations/tableau.md @@ -49,5 +49,5 @@ You have successfully integrated Tableau with $CLOUD_LONG. [tableau-server]: https://www.tableau.com/support/releases/server/2024.2.6#esdalt [tableau-login]: http://online.tableau.com/ [cloud-login]: https://console.cloud.timescale.com/ -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ [tableau]: https://www.tableau.com/ diff --git a/use-timescale/integrations/terraform.md b/integrations/terraform.md similarity index 97% rename from use-timescale/integrations/terraform.md rename to integrations/terraform.md index f0b092a5a2..d16ed3ca53 100644 --- a/use-timescale/integrations/terraform.md +++ b/integrations/terraform.md @@ -147,7 +147,7 @@ You can now manage your database with Terraform. [terraform]: https://www.terraform.io/ [console]: https://console.cloud.timescale.com/dashboard/services [terraform-provider]: https://registry.terraform.io/providers/timescale/timescale/latest/docs -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ [terraform-resources]: https://registry.terraform.io/providers/timescale/timescale/latest/docs/resources/peering_connection [terraform-data-sources]: https://registry.terraform.io/providers/timescale/timescale/latest/docs/data-sources/products -[pg-provider]: https://registry.terraform.io/providers/cyrilgdn/postgresql/latest \ No newline at end of file +[pg-provider]: https://registry.terraform.io/providers/cyrilgdn/postgresql/latest diff --git a/lambda/redirects.js b/lambda/redirects.js index 3393be2af9..d3fca3fa2a 100644 --- a/lambda/redirects.js +++ b/lambda/redirects.js @@ -145,7 +145,7 @@ module.exports = [ }, { from: "/cloud/latest/integrations", - to: "https://docs.timescale.com/use-timescale/latest/metrics-logging/integrations/", + to: "https://docs.timescale.com/integrations/latest/", }, { from: "/cloud/latest/metrics", @@ -233,23 +233,23 @@ module.exports = [ }, { from: "/getting-started/exploring-cloud", - to: "https://docs.timescale.com/mst/latest/about-mst/", + to: "https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/", }, { from: "/getting-started/installation/timescale-cloud/installation-timescale-cloud", - to: "https://docs.timescale.com/install/latest/installation-mst/", + to: "https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/", }, { from: "/getting-started/latest/add-data/", - to: "https://docs.timescale.com/getting-started/latest/time-series-data/", + to: "https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables", }, { - from: "/getting-started/latest/compress-data/", - to: "https://docs.timescale.com/use-timescale/latest/compression/", + from: "/getting-started/latest/aggregation/", + to: "https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/#write-fast-analytical-queries-on-frequently-access-data-using-time-buckets-and-continuous-aggregates", }, { from: "/getting-started/latest/create-cagg/", - to: "https://docs.timescale.com/getting-started/latest/aggregation/", + to: "https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/#write-fast-analytical-queries-on-frequently-access-data-using-time-buckets-and-continuous-aggregates", }, { from: "/getting-started/latest/data-retention", @@ -264,8 +264,16 @@ module.exports = [ to: "https://docs.timescale.com/self-hosted/latest/install/installation-macos/", }, { - from: "/getting-started/latest/query-data/", - to: "https://docs.timescale.com/getting-started/latest/queries/", + from: "/getting-started/latest/queries/", + to: "https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables", + }, + { + from: "/getting-started/latest/tables-hypertables/", + to: "https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables", + }, + { + from: "/getting-started/latest/time-series-data/", + to: "https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables", }, { from: "/install/latest", @@ -336,6 +344,14 @@ module.exports = [ from: "/latest/tutorials/tutorial-setup-timescale-prometheus/", to: "https://docs.timescale.com/tutorials/latest/", }, + { + from: "/migrate/latest/livesync", + to: "https://docs.timescale.com/migrate/latest/livesync-for-postgresql/", + }, + { + from: "/migrate/latest/livesync", + to: "https://docs.timescale.com/migrate/latest/livesync-for-postgresql/", + }, { from: "/mst/latest/create-a-service", to: "https://docs.timescale.com/mst/latest/about-mst", @@ -372,7 +388,7 @@ module.exports = [ }, { from: "/timescaledb/latest/contribute-to-docs", - to: "https://docs.timescale.com/about/latest/contribute-to-docs/", + to: "https://docs.timescale.com/about/latest/contribute-to-timescale/", }, { from: "/timescaledb/latest/contribute-to-timescaledb", @@ -484,7 +500,7 @@ module.exports = [ }, { from: "/timescaledb/latest/overview/core-concepts/user-defined-actions", - to: "https://docs.timescale.com/use-timescale/latest/user-defined-actions/", + to: "https://docs.timescale.com/use-timescale/latest/jobs/", }, { from: "/timescaledb/latest/overview/deployment-options", @@ -536,7 +552,7 @@ module.exports = [ }, { from: "/timescaledb/latest/using-timescaledb/actions", - to: "https://docs.timescale.com/use-timescale/latest/user-defined-actions/", + to: "https://docs.timescale.com/use-timescale/latest/jobs/", }, { from: "/timescaledb/latest/using-timescaledb/telemetry", @@ -558,13 +574,29 @@ module.exports = [ from: "/tutorials/latest/aws-lambda/", to: "https://docs.timescale.com/tutorials/latest/", }, + { + from: "/tutorials/latest/energy-data/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-energy-consumption/", + }, + { + from: "/tutorials/latest/energy-data/dataset-energy/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-energy-consumption/", + }, + { + from: "/tutorials/latest/energy-data/query-energy/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-energy-consumption/", + }, + { + from: "/tutorials/latest/energy-data/compress-energy/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-energy-consumption/", + }, { from: "/tutorials/latest/financial-candlestick-tick-data/create-candlestick-aggregates/", to: "https://docs.timescale.com/tutorials/latest/financial-tick-data/financial-tick-query/", }, { from: "/tutorials/latest/grafana/", - to: "https://docs.timescale.com/use-timescale/latest/integrations/grafana/", + to: "https://docs.timescale.com/integrations/latest/grafana/", }, { from: "/tutorials/latest/howto-monitor-django-prometheus", @@ -578,6 +610,30 @@ module.exports = [ from: "/tutorials/latest/nfl-analytics/", to: "https://docs.timescale.com/tutorials/latest/", }, + { + from: "/tutorials/latest/nyc-taxi-cab/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-transport/", + }, + { + from: "/tutorials/latest/nyc-taxi-cab/dataset-nyc/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-transport/", + }, + { + from: "/tutorials/latest/nyc-taxi-cab/query-nyc/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-transport/", + }, + { + from: "/tutorials/latest/nyc-taxi-cab/compress-nyc/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-transport/", + }, + { + from: "/tutorials/latest/nyc-taxi-geospatial/dataset-nyc/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-transport/", + }, + { + from: "/tutorials/latest/nyc-taxi-geospatial/plot-nyc", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-transport/", + }, { from: "/tutorials/latest/prometheus-adapter/", to: "https://docs.timescale.com/tutorials/latest/", @@ -601,27 +657,27 @@ module.exports = [ { from: "/tutorials", to: "https://docs.timescale.com/tutorials/latest/" }, { from: "/use-timescale/latest/connecting/", - to: "https://docs.timescale.com/use-timescale/latest/integrations/", + to: "https://docs.timescale.com/integrations/latest/", }, { from: "/use-timescale/latest/connecting/about-connecting", - to: "https://docs.timescale.com/use-timescale/latest/integrations/find-connection-details/", + to: "https://docs.timescale.com/integrations/latest/find-connection-details/", }, { from: "/use-timescale/latest/connecting/about-psql/", - to: "https://docs.timescale.com/use-timescale/latest/integrations/psql/", + to: "https://docs.timescale.com/integrations/latest/psql/", }, { from: "/use-timescale/latest/connecting/dbeaver/", - to: "https://docs.timescale.com/use-timescale/latest/integrations/dbeaver/", + to: "https://docs.timescale.com/integrations/latest/dbeaver/", }, { from: "/use-timescale/latest/connecting/pgadmin/", - to: "https://docs.timescale.com/use-timescale/latest/integrations/pgadmin/", + to: "https://docs.timescale.com/integrations/latest/pgadmin/", }, { from: "/use-timescale/latest/connecting/psql/", - to: "https://docs.timescale.com/use-timescale/latest/integrations/psql/", + to: "https://docs.timescale.com/integrations/latest/psql/", }, { from: "/use-timescale/latest/data-tiering/move-data/", @@ -717,7 +773,7 @@ module.exports = [ }, { from: "/using-timescaledb/visualizing-data", - to: "https://docs.timescale.com/use-timescale/latest/integrations/grafana/", + to: "https://docs.timescale.com/integrations/latest/grafana/", }, { from: "/use-timescale/latest/account-management/", @@ -755,7 +811,7 @@ module.exports = [ }, { from: "/timescaledb/latest/tutorials/grafana/visualizations/pie-chart/", - to: "https://docs.timescale.com/use-timescale/latest/integrations/grafana/", + to: "https://docs.timescale.com/integrations/latest/grafana/", }, { from: "/promscale/latest/about-promscale/#promscale-schema-for-metric-data", @@ -763,7 +819,7 @@ module.exports = [ }, { from: "/promscale/latest/visualize-data/grafana", - to: "https://docs.timescale.com/use-timescale/latest/integrations/grafana/", + to: "https://docs.timescale.com/integrations/latest/grafana/", }, { from: "/install/latest/self-hosted/installation-redhat/#where-to-next", @@ -779,7 +835,7 @@ module.exports = [ }, { from: "/timescaledb/latest/tutorials/grafana/grafana-variables/", - to: "https://docs.timescale.com/use-timescale/latest/integrations/grafana/", + to: "https://docs.timescale.com/integrations/latest/grafana/", }, { from: "/clustering/using-timescaledb/update-db", @@ -819,50 +875,106 @@ module.exports = [ }, { from: '/use-timescale/latest/integrations/observability-alerting/grafana/installation/', - to: 'https://docs.timescale.com/use-timescale/latest/integrations/grafana/', + to: 'https://docs.timescale.com/integrations/latest/grafana/', }, { from: '/use-timescale/latest/integrations/observability-alerting/grafana/geospatial-dashboards/', - to: 'https://docs.timescale.com/use-timescale/latest/integrations/grafana/', + to: 'https://docs.timescale.com/integrations/latest/grafana/', }, { from: '/use-timescale/latest/integrations/observability-alerting/grafana/create-dashboard-and-panel/', - to: 'https://docs.timescale.com/use-timescale/latest/integrations/grafana/', + to: 'https://docs.timescale.com/integrations/latest/grafana/', }, { from: '/use-timescale/latest/integrations/query-admin/about-psql/', - to: 'https://docs.timescale.com/use-timescale/latest/integrations/psql/', + to: 'https://docs.timescale.com/integrations/latest/psql/', }, { from: '/use-timescale/latest/integrations/query-admin/psql/', - to: 'https://docs.timescale.com/use-timescale/latest/integrations/psql/', + to: 'https://docs.timescale.com/integrations/latest/psql/', }, { from: '/use-timescale/latest/integrations/query-admin/about-connecting/', - to: 'https://docs.timescale.com/use-timescale/latest/integrations/find-connection-details/', + to: 'https://docs.timescale.com/integrations/latest/find-connection-details/', }, { from: '/use-timescale/latest/integrations/query-admin/azure-data-studio/', - to: 'https://docs.timescale.com/use-timescale/latest/integrations/azure-data-studio/', + to: 'https://docs.timescale.com/integrations/latest/azure-data-studio/', }, { from: '/use-timescale/latest/integrations/query-admin/dbeaver/', - to: 'https://docs.timescale.com/use-timescale/latest/integrations/dbeaver/', + to: 'https://docs.timescale.com/integrations/latest/dbeaver/', }, { from: '/use-timescale/latest/integrations/query-admin/pgadmin/', - to: 'https://docs.timescale.com/use-timescale/latest/integrations/pgadmin/', + to: 'https://docs.timescale.com/integrations/latest/pgadmin/', }, { from: '/use-timescale/latest/integrations/query-admin/qstudio/', - to: 'https://docs.timescale.com/use-timescale/latest/integrations/qstudio/', + to: 'https://docs.timescale.com/integrations/latest/qstudio/', }, { from: '/use-timescale/latest/integrations/config-deploy/terraform/', - to: 'https://docs.timescale.com/use-timescale/latest/integrations/terraform/', + to: 'https://docs.timescale.com/integrations/latest/terraform/', }, { from: '/use-timescale/latest/integrations/observability-alerting/tableau/', - to: 'https://docs.timescale.com/use-timescale/latest/integrations/tableau/', + to: 'https://docs.timescale.com/integrations/latest/tableau/', + }, + { + from: '/use-timescale/latest/user-defined-actions/troubleshooting/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/troubleshooting/', + }, + { + from: '/use-timescale/latest/user-defined-actions/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/', + }, + { + from: '/use-timescale/latest/user-defined-actions/about-user-defined-actions/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/create-and-manage-jobs/', + }, + { + from: '/use-timescale/latest/user-defined-actions/create-and-register/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/create-and-manage-jobs/', + }, + { + from: '/use-timescale/latest/user-defined-actions/test-and-debug/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/create-and-manage-jobs/', + }, + { + from: '/use-timescale/latest/user-defined-actions/alter-and-delete/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/create-and-manage-jobs/', + }, + { + from: '/use-timescale/latest/user-defined-actions/example-generic-retention/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/example-generic-retention/', + }, + { + from: '/use-timescale/latest/user-defined-actions/example-tiered-storage/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/example-tiered-storage/', + }, + { + from: '/use-timescale/latest/user-defined-actions/example-downsample-and-compress/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/example-downsample-and-compress/', + }, + { + from: '/api/latest/actions/add_job/', + to: 'https://docs.timescale.com/api/latest/jobs-automation/add_job/', + }, + { + from: '/api/latest/actions/alter_job/', + to: 'https://docs.timescale.com/api/latest/jobs-automation/alter_job/', + }, + { + from: '/api/latest/actions/delete_job/', + to: 'https://docs.timescale.com/api/latest/jobs-automation/delete_job/', + }, + { + from: '/api/latest/actions/run_job/', + to: 'https://docs.timescale.com/api/latest/jobs-automation/run_job/', + }, + { + from: '/api/latest/actions/', + to: 'https://docs.timescale.com/api/latest/jobs-automation/', }, ]; diff --git a/lambda/regexRedirects.js b/lambda/regexRedirects.js index e0e3d4005c..07aec09cb8 100644 --- a/lambda/regexRedirects.js +++ b/lambda/regexRedirects.js @@ -1,14 +1,23 @@ module.exports = [ { from: /^\/v[0-1]\.[0-9]\/api/, - to: 'https://docs.timescale.com/api/latest/', + to: "https://docs.timescale.com/api/latest/" }, { from: /^\/v[0-1]\.[0-9]\/getting-started/, - to: 'https://docs.timescale.com/getting-started/latest/', + to: "https://docs.timescale.com/getting-started/latest/" }, { - from: /^\/v[0-1]\.[0-9]\/tutorials/, - to: 'https://docs.timescale.com/tutorials/latest/', + from: /^\/use-timescale\/latest\/integrations(\/.*)?$/, + to: (match) => + `https://docs.timescale.com/integrations/latest${match[1] || ""}` + }, + { + from: /^\/quick-start\/latest\/(\/.*)?$/, + to: `https://docs.timescale.com/getting-started/latest/start-coding-with-timescale/` }, + { + from: /^\/v[0-1]\.[0-9]\/tutorials/, + to: "https://docs.timescale.com/tutorials/latest/" + } ]; diff --git a/migrate/dual-write-and-backfill/dual-write-from-other.md b/migrate/dual-write-and-backfill/dual-write-from-other.md index 9725cef8d4..be376ca5a6 100644 --- a/migrate/dual-write-and-backfill/dual-write-from-other.md +++ b/migrate/dual-write-and-backfill/dual-write-from-other.md @@ -79,7 +79,7 @@ continuous aggregate over the whole time range to ensure that there are no holes in the aggregated data. -[tables and hypertables]: /getting-started/:currentVersion:/tables-hypertables/ +[tables and hypertables]: /getting-started/:currentVersion:/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables @@ -137,4 +137,4 @@ your production workload. -[dual-write-and-backfill]: /migrate/:currentVersion:/dual-write-and-backfill/ \ No newline at end of file +[dual-write-and-backfill]: /migrate/:currentVersion:/dual-write-and-backfill/ diff --git a/migrate/dual-write-and-backfill/dual-write-from-postgres.md b/migrate/dual-write-and-backfill/dual-write-from-postgres.md index 4299536479..5442383b1c 100644 --- a/migrate/dual-write-and-backfill/dual-write-from-postgres.md +++ b/migrate/dual-write-and-backfill/dual-write-from-postgres.md @@ -130,7 +130,7 @@ features, such as: - [compression] to reduce the size of your hypertables - [continuous aggregates] to write blisteringly fast aggregate queries on your data -[time-series data]: /getting-started/:currentVersion:/time-series-data/ +[time-series data]: /getting-started/:currentVersion:/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables [create_table API reference]: /api/:currentVersion:/hypertable/create_hypertable/ [hypertable documentation]: /use-timescale/:currentVersion:/hypertables/ [retention policies]: /use-timescale/:currentVersion:/data-retention/ diff --git a/migrate/index.md b/migrate/index.md index ef8ffbae1a..dfe76d523e 100644 --- a/migrate/index.md +++ b/migrate/index.md @@ -8,11 +8,14 @@ tags: [ingest, migrate, RDS] import OpenSupportRequest from "versionContent/_partials/_migrate_open_support_request.mdx" -# Migrate your data to Timescale Cloud +# Migrate and sync data to Timescale Cloud -You have chosen to migrate your data to Timescale Cloud, thank you. Depending on the amount of data -you need to migrate, and the amount of downtime you can afford, we offer the following ways to migrate -your data: +You have chosen to migrate or sync your data to $CLOUD_LONG, thank you. + +## Migrate your data + +Depending on the amount of data you need to migrate, and the amount of downtime you can afford. $COMPANY offers +the following ways to migrate your data: | Migration strategy | Use when | Downtime requirements | |--------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|-----------------------| @@ -31,12 +34,25 @@ If you're migrating your data from another source database type, best practice i a `.csv` file, then import to your Timescale Cloud Service using [timescaledb-parallel-copy][parallel-copy]. For other ingestion methods, see [Ingest data from other sources][data-ingest]. +## Livesync your data + +You use $LIVESYNC to synchronize all or some of your data to your $SERVICE_LONG in real-time. You run $LIVESYNC +continuously, using your data as a primary database and your $SERVICE_LONG as a logical replica. This enables you +to leverage $CLOUD_LONG’s real-time analytics capabilities on your replica data. + +| $LIVESYNC options | Downtime requirements | +|----------------------------------------|-----------------------| +| [$LIVESYNC for $PG][livesync-postgres] | None | +| [$LIVESYNC for S3][livesync-s3] | None | + [data-ingest]: /use-timescale/:currentVersion:/ingest-data/ [dual-write]: /migrate/:currentVersion:/dual-write-and-backfill/ [pg-dump-restore]: /migrate/:currentVersion:/pg-dump-and-restore/ [parallel-copy]: /use-timescale/:currentVersion:/ingest-data/import-csv/ +[livesync-postgres]: /migrate/:currentVersion:/livesync-for-postgresql/ +[livesync-s3]: /migrate/:currentVersion:/livesync-for-s3/ [troubleshooting]: /migrate/:currentVersion:/troubleshooting/ [live-migration]: /migrate/:currentVersion:/live-migration/ [pgcopydb]: https://github.com/dimitri/pgcopydb diff --git a/migrate/livesync-for-postgresql.md b/migrate/livesync-for-postgresql.md new file mode 100644 index 0000000000..4135b81ab0 --- /dev/null +++ b/migrate/livesync-for-postgresql.md @@ -0,0 +1,73 @@ +--- +title: Livesync from Postgres to Timescale Cloud +excerpt: Synchronize updates to your primary PostgreSQL database with the corresponding Timescale Cloud service in real time +products: [cloud] +keywords: [migration, low-downtime, backup] +tags: [recovery, logical backup, replication] +--- + +import MigrationPrerequisites from "versionContent/_partials/_migrate_prerequisites.mdx"; +import SetupConnectionStrings from "versionContent/_partials/_migrate_live_setup_connection_strings.mdx"; +import LivesyncConsole from "versionContent/_partials/_livesync-console.mdx" +import LivesyncTerminal from "versionContent/_partials/_livesync-terminal.mdx" +import EarlyAccessNoRelease from "versionContent/_partials/_early_access.mdx"; + +# Livesync from PostgreSQL to Timescale Cloud + +You use $LIVESYNC to synchronize all the data, or specific tables, from a PostgreSQL database instance to your +$SERVICE_LONG in real-time. You run $LIVESYNC continuously, turning PostgreSQL into a primary database with your +$SERVICE_LONG as a logical replica. This enables you to leverage $CLOUD_LONG’s real-time analytics capabilities on +your replica data. + +![livesync view status](https://assets.timescale.com/docs/images/livesync-view-status.png) + +$LIVESYNC_CAP leverages the well-established PostgreSQL logical replication protocol. By relying on this protocol, +$LIVESYNC ensures compatibility, familiarity, and a broader knowledge base. Making it easier for you to adopt $LIVESYNC +and integrate your data. + +You use $LIVESYNC for data synchronization, rather than migration. Livesync can: +* Copy existing data from a PostgreSQL instance to a $SERVICE_LONG: + - Copy data at up to 150 GB/hr. + + You need at least a 4 CPU/16GB source database, and a 4 CPU/16GB target $SERVICE_SHORT. + - Copy the publication tables in parallel. + + Large tables are still copied using a single connection. Parallel copying is in the backlog. + - Forget foreign key relationships. + + $LIVESYNC_CAP disables foreign key validation during the sync. For example, if a `metrics` table refers to + the `id` column on the `tags` table, you can still sync only the `metrics` table without worrying about their + foreign key relationships. + - Track progress. PostgreSQL exposes `COPY` progress under `pg_stat_progress_copy`. + +* Synchronize real-time changes from a PostgreSQL instance to a $SERVICE_LONG. +* Add and remove tables on demand using the [PostgreSQL PUBLICATION interface][postgres-publication-interface]. +* Enable features such as [hypertables][about-hypertables], [columnstore][compression], and + [continuous aggregates][caggs] on your logical replica. + +: livesync is not supported for production use. If you have an questions or feedback, talk to us in #livesync in Timescale Community. + + + + + + + + + + + + + + + + +[create-publication]: https://www.postgresql.org/docs/current/sql-createpublication.html +[alter-publication]: https://www.postgresql.org/docs/current/sql-alterpublication.html +[install-docker]: https://docs.docker.com/engine/install/ +[about-hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/ +[lives-sync-specify-tables]: /migrate/:currentVersion:/livesync-for-postgresql/#specify-the-tables-to-synchronize +[compression]: /use-timescale/:currentVersion:/compression/about-compression +[caggs]: /use-timescale/:currentVersion:/continuous-aggregates/about-continuous-aggregates/ +[join-livesync-on-slack]: https://app.slack.com/client/T4GT3N2JK/C086NU9EZ88 +[postgres-publication-interface]: https://www.postgresql.org/docs/current/sql-createpublication.html diff --git a/migrate/livesync-for-s3.md b/migrate/livesync-for-s3.md new file mode 100644 index 0000000000..81a39d2e74 --- /dev/null +++ b/migrate/livesync-for-s3.md @@ -0,0 +1,165 @@ +--- +title: Livesync S3 to Timescale Cloud +excerpt: Synchronize data from S3 to Timescale Cloud service in real time +products: [cloud] +keywords: [migration, low-downtime, backup] +tags: [recovery, logical backup, replication] +--- + +import PrereqCloud from "versionContent/_partials/_prereqs-cloud-only.mdx"; +import EarlyAccessNoRelease from "versionContent/_partials/_early_access.mdx"; + +# Livesync from S3 to Timescale Cloud + +You use $LIVESYNC to synchronize tabular data, from an S3 bucket to your +$SERVICE_LONG in real time. You run $LIVESYNC continuously, turning S3 into a primary database with your +$SERVICE_LONG as a logical replica. This enables you to leverage $CLOUD_LONG’s real-time analytics capabilities on +your replica data. + +![$LIVESYNC_CAP view status](https://assets.timescale.com/docs/images/livesync-s3-view-status.png) + +You use $LIVESYNC for data synchronization, rather than migration. Livesync can: + +* Sync data from an S3 bucket instance to a $SERVICE_LONG: + - $LIVESYNC uses Glob patterns to identify the objects to sync. + - $LIVESYNC uses the objects returned for subsequent queries. This efficient approach means files are synced in + [lexicographical order][lex-order]. + - $LIVESYNC watches an S3 bucket for new files and imports them automatically. $LIVESYNC runs on a configurable + schedule and tracks processed files. + - For large backlogs, $LIVESYNC checks every minute until caught up. + +* Sync data from multiple file formats: + + * CSV: checked for compression in `.gz` and `.zip` format, then processing using [timescaledb-parallel-copy][parallel-copy] + + * Parquet: converted to CSV, then processed using [timescaledb-parallel-copy][parallel-copy] + +* Enable features such as [hypertables][about-hypertables], [columnstore][compression], and + [continuous aggregates][caggs] on your logical replica. + +$LIVESYNC for S3 continuously imports data from an Amazon S3 bucket into your database. It monitors your S3 bucket for new +files matching a specified pattern and automatically imports them into your designated database table. + +: livesync is not supported for production use. If you have any questions or feedback, talk to us in #livesync in Timescale Community. + +## Prerequisites + + + +- Access to a standard Amazon S3 bucket containing your data files. + Directory buckets are not supported. +- Access credentials for the S3 bucket. + - The following credentials are supported: + - [IAM Role][credentials-iam]. + + - Configure the trust policy. Set the: + + - `Principal`: `arn:aws:iam::142548018081:role/timescale-s3-connections`. + - `ExternalID`: set to the [$CLOUD_LONG project and $SERVICE_LONG ID][connection-project-service-id] of the + $SERVICE_SHORT you are syncing to in the format `/`. + + This is to avoid the [confused deputy problem][confused-deputy-problem]. + - Give the following access permissions: + + - `s3:GetObject`. + - `s3:ListBucket`. + + - [Public anonymous user][credentials-public]. + +## Limitations + +- **CSV**: + - Maximum file size: 1GB + To increase these limits, contact sales@timescale.com + - Maximum row size: 2MB + - Supported compressed formats: + - `.gz` + - `.zip` + - Advanced settings: + - Delimiter: the default character is `,`, you can choose a different delimiter + - Skip Header: skip the first row if your file has headers +- **Parquet**: + - Maximum file size: 1GB + - Maximum row group uncompressed size: 200MB + - Maximum row size: 2MB +- **Sync iteration**: + To prevent system overload, $LIVESYNC tracks up to 100 files for each sync iteration. Additional checks only fill + empty queue slots. + +## Synchronize data to your $SERVICE_LONG + +To sync data from your S3 bucket to your $SERVICE_LONG using $CONSOLE: + + + +1. **Connect to your $SERVICE_LONG** + + In [$CONSOLE][portal-ops-mode], select the service to sync live data to. +1. **Start livesync** + 1. Click `Actions` > `livesync for S3`. + 2. Click `New Livesync for S3` + +1. **Connect the source S3 bucket to the target $SERVICE_SHORT** + + ![Livesync connect to bucket](https://assets.timescale.com/docs/images/livesync-s3-wizard.png) + + 1. In `Livesync for S3`, set the `Bucket name` and `Authentication method`, then click `Continue`. + + For instruction on creating the IAM role you need to connect your S3 bucket, click `Learn how`: + ![Livesync connect to bucket](https://assets.timescale.com/docs/images/livesync-s3-create-credentials.png) + $CONSOLE connects to the source bucket. + 1. In `Define files to sync`, choose the `File type` and set the `Glob pattern`. + + Use the following patterns: + - `/*`: match all files in a folder. Also, any pattern ending with `/` is treated as `/*`. + - `/**`: match all recursively. + - `/**/*.csv`: match a specific file type. + + $LIVESYNC uses prefix filters where possible, place patterns carefully at the end of your glob expression. + AWS S3 doesn't support complex filtering. If your expression filters too many files, the list operation may timeout. + + 1. Click the search icon, you see files to sync. Click `Continue`. + +1. **Optimize the data to synchronize in hypertables** + + $CONSOLE checks the file schema and, if possible, suggests the column to use as the time dimension in a + [hypertable][about-hypertables]. + + ![Livesync choose table](https://assets.timescale.com/docs/images/livesync-s3-create-tables.png) + + 1. Choose the `Data type` for each column, then click `Continue`. + 1. Choose the interval. This can be a minute, an hour or use a [cron expression][cron-expression]. + 1. Repeat this step for each table you want to sync. + 1. Press `Start Livesync`. + + $CONSOLE starts $LIVESYNC between the source database and the target $SERVICE_SHORT and displays the progress. + +1. **Monitor syncronization** + 1. To view the progress of the livesync, click the name of the $LIVESYNC process: + You see the status of the file being synced. Only one file runs at a time. + ![livesync view status](https://assets.timescale.com/docs/images/livesync-s3-view-status.png) + 1. To pause and restart livesync, click the buttons on the right of the $LIVESYNC process and select an action: + During pauses, you can edit the configuration before resuming. + ![livesync start stop](https://assets.timescale.com/docs/images/livesync-s3-start-stop.png) + + + +And that is it, you are using $LIVESYNC to synchronize all the data, or specific files, from an S3 bucket to your +$SERVICE_LONG in real time. + + +[about-hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/ +[lives-sync-specify-tables]: /migrate/:currentVersion:/livesync-for-postgresql/#specify-the-tables-to-synchronize +[compression]: /use-timescale/:currentVersion:/compression/about-compression +[caggs]: /use-timescale/:currentVersion:/continuous-aggregates/about-continuous-aggregates/ +[join-livesync-on-slack]: https://app.slack.com/client/T4GT3N2JK/C086NU9EZ88 +[parallel-copy]: https://github.com/timescale/timescaledb-parallel-copy +[deputy-problem]: https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html +[lex-order]:https://en.wikipedia.org/wiki/Lexicographic_order +[credentials-iam]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user.html#roles-creatingrole-user-console +[credentials-public]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/example-bucket-policies.html#example-bucket-policies-anonymous-user +[portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services +[hypertable-docs]: /use-timescale/:currentVersion:/hypertables/ +[cron-expression]: https://en.wikipedia.org/wiki/Cron#Cron_expression +[confused-deputy-problem]: https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html +[connection-project-service-id]: /integrations/:currentVersion:/find-connection-details/#find-your-project-and-service-id diff --git a/migrate/page-index/page-index.js b/migrate/page-index/page-index.js index 29dc6c3d96..eb8abfdd35 100644 --- a/migrate/page-index/page-index.js +++ b/migrate/page-index/page-index.js @@ -1,6 +1,6 @@ module.exports = [ { - title: "Migrate your data to Timescale Cloud", + title: "Migrate and sync data to Timescale Cloud", href: "migrate", excerpt: "Migrating your data to Timescale", children: [ @@ -16,10 +16,15 @@ module.exports = [ excerpt: "Migrate a large database with low downtime", }, { - title: "Livesync from Postgres to Timescale Cloud", - href: "livesync", + title: "Livesync from PostgreSQL", + href: "livesync-for-postgresql", excerpt: "Synchronize updates to a primary postgres database instance to Timescale Cloud service in real-time", }, + { + title: "Livesync from S3", + href: "livesync-for-s3", + excerpt: "Synchronize data from S3 to Timescale Cloud service in real time", + }, { title: "Dual-write and backfill", href: "dual-write-and-backfill", diff --git a/migrate/troubleshooting.md b/migrate/troubleshooting.md index 372cf9c17f..61b5db8780 100644 --- a/migrate/troubleshooting.md +++ b/migrate/troubleshooting.md @@ -235,7 +235,7 @@ pg_restore -d "$TARGET" \ ## Ownership of background jobs The `_timescaledb_config.bgw_jobs` table is used to manage background jobs. -This includes both user-defined actions, compression policies, retention +This includes custom $JOBs, compression policies, retention policies, and continuous aggregate refresh policies. On Timescale, this table has a trigger which ensures that no database user can create or modify jobs owned by another database user. This trigger can provide an obstacle for migrations. diff --git a/mlc_config.json b/mlc_config.json index e6d39aa44c..9eca629fe9 100644 --- a/mlc_config.json +++ b/mlc_config.json @@ -78,6 +78,9 @@ }, { "pattern": "^https://www.tableau.com/support/releases/server/2024.2.6#esdalt" + }, + { + "pattern": "^https://azure.microsoft.com/en-us/products/data-studio" } ] } diff --git a/mst/ingest-data.md b/mst/ingest-data.md index 279cec75f3..e64a29ebe5 100644 --- a/mst/ingest-data.md +++ b/mst/ingest-data.md @@ -154,7 +154,7 @@ recommend that you use the connector with Kafka and Kafka Connect. See [these instructions][gh-kafkaconnector] for using the Kafka connector. -[code-qs]: /quick-start/:currentVersion:/ +[code-qs]: /getting-started/:currentVersion:/start-coding-with-timescale/ [gh-kafkaconnector]: https://github.com/debezium/debezium/tree/master/debezium-connector-postgres [github-parallel-copy]: https://github.com/timescale/timescaledb-parallel-copy [postgres-odbc]: https://odbc.postgresql.org/ diff --git a/mst/installation-mst.md b/mst/installation-mst.md index 61cacd4595..3b0a446f59 100644 --- a/mst/installation-mst.md +++ b/mst/installation-mst.md @@ -122,8 +122,8 @@ You can always [contact us][contact] if you need help working something out, or if you want to have a chat. [contact]: https://www.timescale.com/contact -[install-psql]: /use-timescale/:currentVersion:/integrations/psql/ +[install-psql]: /integrations/:currentVersion:/psql/ [mst-docs]: /mst/:currentVersion:/ [tutorials]: /tutorials/:currentVersion:/ [mst-signup]: https://www.timescale.com/mst-signup -[mst-login]:https://portal.managed.timescale.com/login \ No newline at end of file +[mst-login]:https://portal.managed.timescale.com/login diff --git a/mst/migrate-to-mst.md b/mst/migrate-to-mst.md index 777814d58f..9c11f94c7e 100644 --- a/mst/migrate-to-mst.md +++ b/mst/migrate-to-mst.md @@ -129,7 +129,7 @@ them. The migration still occurs successfully. [install-mst]: /mst/:currentVersion:/installation-mst/#create-your-first-service [pg_dump]: https://www.postgresql.org/docs/current/app-pgdump.html [pg_restore]: https://www.postgresql.org/docs/current/app-pgrestore.html -[psql]: /use-timescale/:currentVersion:/integrations/psql/ +[psql]: /integrations/:currentVersion:/psql/ [upgrading-postgresql]: https://kb-managed.timescale.com/en/articles/5368016-perform-a-postgresql-major-version-upgrade [upgrading-postgresql-self-hosted]: /self-hosted/:currentVersion:/upgrades/upgrade-pg/ -[upgrading-timescaledb]: /self-hosted/:currentVersion:/upgrades/major-upgrade/ \ No newline at end of file +[upgrading-timescaledb]: /self-hosted/:currentVersion:/upgrades/major-upgrade/ diff --git a/page-index/page-index.js b/page-index/page-index.js index 41bf674685..f01047c462 100644 --- a/page-index/page-index.js +++ b/page-index/page-index.js @@ -5,16 +5,16 @@ const gsgPageIndex = require("../getting-started/page-index/page-index"); const timescaleUsingPageIndex = require("../use-timescale/page-index/page-index"); const navigationPageIndex = require("../navigation/page-index/page-index"); const tutorialsPageIndex = require("../tutorials/page-index/page-index.js"); -const codeQuickStartsPageIndex = require("../quick-start/page-index/page-index.js"); const migrationPageIndex = require("../migrate/page-index/page-index.js"); const timescaleAboutPageIndex = require("../about/page-index/page-index"); +const integrationsIndex = require("../integrations/page-index/page-index"); const AIPageIndex = require("../ai/page-index/page-index"); module.exports = [ ...gsgPageIndex, ...timescaleUsingPageIndex, ...tutorialsPageIndex, - ...codeQuickStartsPageIndex, + ...integrationsIndex, ...apiReferencePageIndex, ...selfHostedPageIndex, ...timescaleMSTPageIndex, diff --git a/quick-start/index.md b/quick-start/index.md deleted file mode 100644 index e1e25ed346..0000000000 --- a/quick-start/index.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -title: Code quick starts -excerpt: Get started with Timescale Cloud or TimescaleDB in your preferred programming language. Learn to connect to a database, create and manage hypertables, and ingest and query data ---- - -# Start coding with Timescale - -These pages help you to quickly integrate Timescale Cloud and TimescaleDB into your apps using code: - -* [Node.js](/quick-start/:currentVersion:/node/) -* [Python](/quick-start/:currentVersion:/python/) -* [Ruby](/quick-start/:currentVersion:/ruby/) -* [Golang](/quick-start/:currentVersion:/golang/) -* [Java](/quick-start/:currentVersion:/java/) - -You are not limited to these languages. Timescale Cloud is based on PostgreSQL, you can interface -with TimescaleDB and Timescale Cloud using any [PostgreSQL client driver](https://wiki.postgresql.org/wiki/List_of_drivers). diff --git a/quick-start/page-index/page-index.js b/quick-start/page-index/page-index.js deleted file mode 100644 index 92661c627a..0000000000 --- a/quick-start/page-index/page-index.js +++ /dev/null @@ -1,34 +0,0 @@ -module.exports = [ - { - title: "Start coding with Timescale", - href: "quick-start", - excerpt: "A collection of quick starts for common programming languages.", - children: [ - { - title: "Node.js", - href: "node", - excerpt: "Learn about using TimescaleDB with node.js", - }, - { - title: "Python", - href: "python", - excerpt: "Learn about using TimescaleDB with Python", - }, - { - title: "Ruby", - href: "ruby", - excerpt: "Learn about using TimescaleDB with Ruby", - }, - { - title: "Golang", - href: "golang", - excerpt: "Learn about using TimescaleDB with Golang", - }, - { - title: "Java", - href: "java", - excerpt: "Learn about using TimescaleDB with Java", - }, - ], - }, -]; diff --git a/self-hosted/configuration/docker-config.md b/self-hosted/configuration/docker-config.md index 71b65e71cc..8e7092f0c1 100644 --- a/self-hosted/configuration/docker-config.md +++ b/self-hosted/configuration/docker-config.md @@ -59,4 +59,4 @@ using the `-c` option. For example: docker run -i -t timescale/timescaledb:latest-pg10 postgres -c max_wal_size=2GB ``` -[docker]: /self-hosted/latest/install/installation-docker/ +[docker]: /self-hosted/:currentVersion:/install/installation-docker/ diff --git a/self-hosted/configuration/timescaledb-config.md b/self-hosted/configuration/timescaledb-config.md index 54c89e1da6..c357981adf 100644 --- a/self-hosted/configuration/timescaledb-config.md +++ b/self-hosted/configuration/timescaledb-config.md @@ -7,6 +7,7 @@ tags: [tune] --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; +import ConfigCloudSelf from "versionContent/_partials/_cloud_self_configuration.mdx"; # TimescaleDB configuration and tuning @@ -15,14 +16,6 @@ settings that may be useful to your specific installation and performance needs. also be set within the `postgresql.conf` file or as command-line parameters when starting PostgreSQL. -## Policies - -### `timescaledb.max_background_workers (int)` - -Max background worker processes allocated to TimescaleDB. Set to at least 1 + -the number of databases loaded with a TimescaleDB extension in a PostgreSQL -instance. Default value is 16. - ## Query Planning and Execution ### `timescaledb.enable_chunkwise_aggregation (bool)` @@ -31,7 +24,6 @@ planning. The first part of the aggregation is executed on a per-chunk basis. Then, these partial results are combined and finalized. Splitting aggregations decreases the size of the created hash tables and increases data locality, which speeds up queries. - ### `timescaledb.vectorized_aggregation (bool)` Enables or disables the vectorized optimizations in the query executor. For @@ -44,6 +36,22 @@ Set to `ON` to dramatically decrease the amount of data written on a continuous in the presence of a small number of changes, reduce the i/o cost of refreshing a [continuous aggregate][continuous-aggregates], and generate fewer Write-Ahead Logs (WAL). Only works for continuous aggregates that don't have compression enabled. +## Policies + +### `timescaledb.max_background_workers (int)` + +Max background worker processes allocated to TimescaleDB. Set to at least 1 + +the number of databases loaded with a TimescaleDB extension in a PostgreSQL +instance. Default value is 16. + + + +## Hypercore features + +### `timescaledb.default_hypercore_use_access_method (bool)` + +The default value for `hypercore_use_access_method` for functions that have this parameter. This function is in `user` context, meaning that any user can set it for the session. The default value is `false`. + ## Distributed hypertables diff --git a/self-hosted/distributed-hypertables/about-distributed-hypertables.md b/self-hosted/distributed-hypertables/about-distributed-hypertables.md index 0a13985bd1..886970e9ee 100644 --- a/self-hosted/distributed-hypertables/about-distributed-hypertables.md +++ b/self-hosted/distributed-hypertables/about-distributed-hypertables.md @@ -3,6 +3,8 @@ title: About distributed hypertables excerpt: Sunsetted v2.14.x. Distributed hypertables are hypertables that span multiple nodes products: [self_hosted] keywords: [distributed hypertables, multi-node] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; @@ -203,7 +205,7 @@ for several reasons: it isn't guaranteed to have a consistent result across each node. An example non-immutable function is [`random()`][random-func], which depends on the current seed. -* The query includes a user-defined function. The access node assumes the +* The query includes a $JOB function. The access node assumes the function doesn't exist on the data nodes, and doesn't push it down. Timescale uses several optimizations to avoid these limitations, and push down diff --git a/self-hosted/distributed-hypertables/alter-drop-distributed-hypertables.md b/self-hosted/distributed-hypertables/alter-drop-distributed-hypertables.md index 829ce73156..88596aa977 100644 --- a/self-hosted/distributed-hypertables/alter-drop-distributed-hypertables.md +++ b/self-hosted/distributed-hypertables/alter-drop-distributed-hypertables.md @@ -4,6 +4,8 @@ excerpt: Sunsetted v2.14.x. Alter and drop distributed hypertables in your self- products: [self_hosted] keywords: [distributed hypertables, alter, delete] tags: [change, delete] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/distributed-hypertables/create-distributed-hypertables.md b/self-hosted/distributed-hypertables/create-distributed-hypertables.md index 6cb6f9e827..b094deaea4 100644 --- a/self-hosted/distributed-hypertables/create-distributed-hypertables.md +++ b/self-hosted/distributed-hypertables/create-distributed-hypertables.md @@ -3,6 +3,8 @@ title: Create distributed hypertables excerpt: Sunsetted v2.14.x. Create a distributed hypertable in a self-hosted multi-node TimescaleDB instance products: [self_hosted] keywords: [distributed hypertables, multi-node, create] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/distributed-hypertables/foreign-keys.md b/self-hosted/distributed-hypertables/foreign-keys.md index 5a013fe574..ffe77b00bd 100644 --- a/self-hosted/distributed-hypertables/foreign-keys.md +++ b/self-hosted/distributed-hypertables/foreign-keys.md @@ -4,6 +4,8 @@ excerpt: Sunsetted v2.14.x. Add foreign keys to the nodes of a distributed hyper products: [self_hosted] keywords: [distributed hypertable, foreign keys] tags: [constraints] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/distributed-hypertables/index.md b/self-hosted/distributed-hypertables/index.md index be1af846e9..3554ecb303 100644 --- a/self-hosted/distributed-hypertables/index.md +++ b/self-hosted/distributed-hypertables/index.md @@ -3,6 +3,8 @@ title: Distributed hypertables excerpt: Sunsetted v2.14.x. Distributed hypertables help you horizontally scale your data storage in multi-node clusters products: [self_hosted] keywords: [distributed hypertables, multi-node] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/distributed-hypertables/insert.md b/self-hosted/distributed-hypertables/insert.md index 7b23b4bf6a..418d50234f 100644 --- a/self-hosted/distributed-hypertables/insert.md +++ b/self-hosted/distributed-hypertables/insert.md @@ -4,6 +4,8 @@ excerpt: Sunsetted v2.14.x. Insert data into distributed hypertables in your sel products: [self_hosted] keywords: [write, distributed hypertables] tags: [ingest, insert] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/distributed-hypertables/query.md b/self-hosted/distributed-hypertables/query.md index aea21f251f..08fb1c3d04 100644 --- a/self-hosted/distributed-hypertables/query.md +++ b/self-hosted/distributed-hypertables/query.md @@ -3,6 +3,8 @@ title: Query data in distributed hypertables excerpt: Sunsetted v2.14.x. Query data in distributed hypertables in your self-hosted TimescaleDB installation products: [self_hosted] keywords: [distributed hypertables, multi-node, queries] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/distributed-hypertables/triggers.md b/self-hosted/distributed-hypertables/triggers.md index 8020fcb0a0..1f44c6d55b 100644 --- a/self-hosted/distributed-hypertables/triggers.md +++ b/self-hosted/distributed-hypertables/triggers.md @@ -3,6 +3,8 @@ title: Use triggers on distributed hypertables excerpt: Sunsetted v2.14.x. Set up triggers on a distributed hypertable in your self-hosted TimescaleDB installation products: [self_hosted] keywords: [distributed hypertables, triggers, multi-node] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/install/installation-macos.md b/self-hosted/install/installation-macos.md index 8d67781e72..21dce30f25 100644 --- a/self-hosted/install/installation-macos.md +++ b/self-hosted/install/installation-macos.md @@ -80,7 +80,7 @@ And that is it! You have TimescaleDB running on a database on a self-hosted inst For the latest functionality, install MacOS 14 Sanoma. The oldest supported version is macOS 10.15 Catalina [homebrew]: https://docs.brew.sh/Installation -[install-psql]: /use-timescale/:currentVersion:/integrations/psql/ +[install-psql]: /integrations/:currentVersion:/psql/ [macports]: https://guide.macports.org/#installing.macports [install-from-source]: /self-hosted/:currentVersion:/install/installation-source/ -[install-postgresql]: https://www.postgresql.org/download/macosx/ \ No newline at end of file +[install-postgresql]: https://www.postgresql.org/download/macosx/ diff --git a/self-hosted/install/installation-source.md b/self-hosted/install/installation-source.md index 6a317c3b2e..7ade529b57 100644 --- a/self-hosted/install/installation-source.md +++ b/self-hosted/install/installation-source.md @@ -73,7 +73,7 @@ And that is it! You have TimescaleDB running on a database on a self-hosted inst -[install-psql]: /use-timescale/:currentVersion:/integrations/psql/ +[install-psql]: /integrations/:currentVersion:/psql/ [config]: /self-hosted/:currentVersion:/configuration/ [postgres-download]: https://www.postgresql.org/download/ [cmake-download]: https://cmake.org/download/ diff --git a/self-hosted/manage-storage.md b/self-hosted/manage-storage.md index 219b14f027..c4d993dad5 100644 --- a/self-hosted/manage-storage.md +++ b/self-hosted/manage-storage.md @@ -139,8 +139,8 @@ Alternatively, you can set up a third tablespace called `history_indexes`, and move the data to `history` and the indexes to `history_indexes`. In Timescale 2.0 and later, you can use `move_chunk` with the job scheduler -framework. For more information, see the [user-defined actions section][actions]. +framework. For more information, see the [$JOBs section][jobs]. -[actions]: /use-timescale/:currentVersion:/user-defined-actions/ +[jobs]: /use-timescale/:currentVersion:/jobs/ [api-move-chunk]: /api/:currentVersion:/hypertable/move_chunk [api-reorder-chunk]: /api/:currentVersion:/hypertable/reorder_chunk diff --git a/self-hosted/migration/entire-database.md b/self-hosted/migration/entire-database.md index 5abbcf6ab4..beb4c9ed59 100644 --- a/self-hosted/migration/entire-database.md +++ b/self-hosted/migration/entire-database.md @@ -118,10 +118,10 @@ information about compression and decompression, see [Compression][compression]. [migrate-separately]: /self-hosted/:currentVersion:/migration/schema-then-data/ [pg_dump]: https://www.postgresql.org/docs/current/app-pgdump.html [pg_restore]: https://www.postgresql.org/docs/current/app-pgrestore.html -[psql]: /use-timescale/:currentVersion:/integrations/psql/ +[psql]: /integrations/:currentVersion:/psql/ [timescaledb_pre_restore]: /api/:currentVersion:/administration/#timescaledb_pre_restore [timescaledb_post_restore]: /api/:currentVersion:/administration/#timescaledb_post_restore [upgrading-postgresql-self-hosted]: /self-hosted/:currentVersion:/upgrades/upgrade-pg/ [upgrading-timescaledb]: /self-hosted/:currentVersion:/upgrades/major-upgrade/ [live-migration]: /migrate/:currentVersion:/live-migration/ -[compression]: /use-timescale/:currentVersion:/compression/ \ No newline at end of file +[compression]: /use-timescale/:currentVersion:/compression/ diff --git a/self-hosted/migration/migrate-influxdb.md b/self-hosted/migration/migrate-influxdb.md index 0c123b65f0..5edd7b32fd 100644 --- a/self-hosted/migration/migrate-influxdb.md +++ b/self-hosted/migration/migrate-influxdb.md @@ -43,8 +43,6 @@ and MacOS. -### Installing Outflux - 1. Go to the [releases section][outflux-releases] of the Outflux repository. 1. Download the latest compressed tarball for your platform. 1. Extract it to a preferred location. @@ -167,4 +165,4 @@ outflux migrate --help [discover-validate-and-transfer-schema]: /self-hosted/:currentVersion:/migration/migrate-influxdb/#discover-validate-and-transfer-schema [migrate-data-to-timescale]: /self-hosted/:currentVersion:/migration/migrate-influxdb/#migrate-data-to-timescale [outflux-gitbuh]: https://github.com/timescale/outflux#connection -[outflux-readme]: https://github.com/timescale/outflux/blob/master/README.md \ No newline at end of file +[outflux-readme]: https://github.com/timescale/outflux/blob/master/README.md diff --git a/self-hosted/migration/same-db.md b/self-hosted/migration/same-db.md index 563aaa281a..2070007e3c 100644 --- a/self-hosted/migration/same-db.md +++ b/self-hosted/migration/same-db.md @@ -81,4 +81,4 @@ Migrate your data into Timescale from within the same database. [create_hypertable]: /api/:currentVersion:/hypertable/create_hypertable/ -[install]: /getting-started/latest/ +[install]: /self-hosted/:currentVersion:/install/ diff --git a/self-hosted/migration/schema-then-data.md b/self-hosted/migration/schema-then-data.md index 31e7f0cf60..0a4fffcc9e 100644 --- a/self-hosted/migration/schema-then-data.md +++ b/self-hosted/migration/schema-then-data.md @@ -210,7 +210,7 @@ the [compression section](https://docs.timescale.com/use-timescale/latest/compre [install-selfhosted]: /self-hosted/:currentVersion:/install/ [pg_dump]: https://www.postgresql.org/docs/current/app-pgdump.html [pg_restore]: https://www.postgresql.org/docs/current/app-pgrestore.html -[psql]: /use-timescale/:currentVersion:/integrations/psql/ +[psql]: /integrations/:currentVersion:/psql/ [timescaledb-parallel-copy]: https://github.com/timescale/timescaledb-parallel-copy [upgrading-postgresql]: https://kb-managed.timescale.com/en/articles/5368016-perform-a-postgresql-major-version-upgrade [upgrading-postgresql-self-hosted]: /self-hosted/:currentVersion:/upgrades/upgrade-pg/ diff --git a/self-hosted/multinode-timescaledb/about-multinode.md b/self-hosted/multinode-timescaledb/about-multinode.md index 66bcf2240a..53c8f8f9c8 100644 --- a/self-hosted/multinode-timescaledb/about-multinode.md +++ b/self-hosted/multinode-timescaledb/about-multinode.md @@ -3,6 +3,8 @@ title: About multi-node excerpt: Sunsetted in v2.14.x. Multi-node TimescaleDB enables you to run and manage a cluster of databases, which can result in faster data ingest, and more responsive and efficient queries for large workloads products: [self_hosted] keywords: [multi-node] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/multinode-timescaledb/index.md b/self-hosted/multinode-timescaledb/index.md index 600e990e7e..114443cd10 100644 --- a/self-hosted/multinode-timescaledb/index.md +++ b/self-hosted/multinode-timescaledb/index.md @@ -3,6 +3,8 @@ title: Multi-node excerpt: Learn about using multi-node TimescaleDB to scale your database horizontally, including setup, authentication, configuration, and managing your cluster products: [self_hosted] keywords: [multi-node, scaling] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; @@ -32,4 +34,4 @@ giving you faster data ingest, and more responsive and efficient queries. [multi-node-grow-shrink]: /self-hosted/:currentVersion:/multinode-timescaledb/multinode-grow-shrink/ [multi-node-ha]: /self-hosted/:currentVersion:/multinode-timescaledb/multinode-ha/ [multi-node-maintenance]: /self-hosted/:currentVersion:/multinode-timescaledb/multinode-maintenance/ -[setup-selfhosted]: /self-hosted/:currentVersion:/multinode-timescaledb/multinode-setup/ \ No newline at end of file +[setup-selfhosted]: /self-hosted/:currentVersion:/multinode-timescaledb/multinode-setup/ diff --git a/self-hosted/multinode-timescaledb/multinode-administration.md b/self-hosted/multinode-timescaledb/multinode-administration.md index 1f0161b539..149e02cf7b 100644 --- a/self-hosted/multinode-timescaledb/multinode-administration.md +++ b/self-hosted/multinode-timescaledb/multinode-administration.md @@ -4,6 +4,8 @@ excerpt: Sunsetted v2.14.x. Administer your multi-node TimescaleDB cluster direc products: [self_hosted] keywords: [multi-node, admin] tags: [manage] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/multinode-timescaledb/multinode-auth.md b/self-hosted/multinode-timescaledb/multinode-auth.md index 28bee57df8..47d7e4f38b 100644 --- a/self-hosted/multinode-timescaledb/multinode-auth.md +++ b/self-hosted/multinode-timescaledb/multinode-auth.md @@ -4,6 +4,8 @@ excerpt: Sunsetted v2.14.x. Configure authentication between access nodes and da products: [self_hosted] keywords: [multi-node, authenticate] tags: [admin] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/multinode-timescaledb/multinode-config.md b/self-hosted/multinode-timescaledb/multinode-config.md index 3059c60608..4e8504af9a 100644 --- a/self-hosted/multinode-timescaledb/multinode-config.md +++ b/self-hosted/multinode-timescaledb/multinode-config.md @@ -3,6 +3,8 @@ title: Multi-node configuration excerpt: Sunsetted v2.14.x. Configure the main settings in your multi-node TimescaleDB instance products: [self_hosted] keywords: [configuration, settings, multi-node] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/multinode-timescaledb/multinode-grow-shrink.md b/self-hosted/multinode-timescaledb/multinode-grow-shrink.md index 4e3dbdfda7..0ddf99cd6e 100644 --- a/self-hosted/multinode-timescaledb/multinode-grow-shrink.md +++ b/self-hosted/multinode-timescaledb/multinode-grow-shrink.md @@ -4,6 +4,8 @@ excerpt: Sunsetted v2.14.x. Add and remove data nodes from your self-hosted mult products: [self_hosted] keywords: [multi-node, data nodes] tags: [add, remove] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/multinode-timescaledb/multinode-ha.md b/self-hosted/multinode-timescaledb/multinode-ha.md index 49467a4001..ed04821abb 100644 --- a/self-hosted/multinode-timescaledb/multinode-ha.md +++ b/self-hosted/multinode-timescaledb/multinode-ha.md @@ -3,6 +3,8 @@ title: High availability with multi-node excerpt: Sunsetted v2.14.x. Configure your self-hosted multi-node TimescaleDB for high availability products: [self_hosted] keywords: [multi-node, high availability] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/multinode-timescaledb/multinode-maintenance.md b/self-hosted/multinode-timescaledb/multinode-maintenance.md index cd82026cd3..b7f884af0f 100644 --- a/self-hosted/multinode-timescaledb/multinode-maintenance.md +++ b/self-hosted/multinode-timescaledb/multinode-maintenance.md @@ -4,6 +4,8 @@ excerpt: Sunsetted v2.14.x. Run maintenance on your self-hosted multi-node Times products: [self_hosted] keywords: [multi-node, maintenance] tags: [manage] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; @@ -28,9 +30,7 @@ completed their part of the distributed transaction can complete it later when they become available. This transaction log requires regular cleanup to remove transactions that have completed, and complete those that haven't. We highly recommended that you configure the access node to run a maintenance -job that regularly cleans up any unfinished distributed transactions. - -The custom maintenance job can be run as a user-defined action. For example: +job that regularly cleans up any unfinished distributed transactions. For example: diff --git a/self-hosted/multinode-timescaledb/multinode-setup.md b/self-hosted/multinode-timescaledb/multinode-setup.md index 448bea394d..ec0746a6a1 100644 --- a/self-hosted/multinode-timescaledb/multinode-setup.md +++ b/self-hosted/multinode-timescaledb/multinode-setup.md @@ -3,6 +3,8 @@ title: Set up multi-node on self-hosted TimescaleDB excerpt: Sunsetted v2.14.x. Learn how to set up a self-hosted multi-node TimescaleDB instance products: [self_hosted] keywords: [multi-node, self-hosted] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/page-index/page-index.js b/self-hosted/page-index/page-index.js index f1947f3816..27faa96424 100644 --- a/self-hosted/page-index/page-index.js +++ b/self-hosted/page-index/page-index.js @@ -150,7 +150,89 @@ module.exports = [ excerpt: "Manage storage by moving data between tablespaces", }, { - title: "Multi-node", + title: "Replication and High Availability", + href: "replication-and-ha", + children: [ + { + title: "About high availability", + href: "about-ha", + excerpt: "High availability in self-hosted TimescaleDB", + }, + { + title: "Configure replication", + href: "configure-replication", + excerpt: "Configure replication", + }, + ], + }, + { + title: "Additional tooling", + href: "tooling", + children: [ + { + title: "TimescaleDB Tune", + href: "about-timescaledb-tune", + }, + { + title: "Install and update Timescale Toolkit", + href: "install-toolkit", + excerpt: "Install and update the Timescale Toolkit", + }, + ], + }, + { + title: "Upgrade self-hosted TimescaleDB", + href: "upgrades", + children: [ + { + title: "Upgrade TimescaleDB to a minor version", + href: "minor-upgrade", + excerpt: + "Upgrade self-hosted TimescaleDB to a new minor version", + }, + { + title: "Upgrade TimescaleDB to a major version", + href: "major-upgrade", + excerpt: + "Upgrade self-hosted TimescaleDB to a new major version", + }, + { + title: "Upgrade TimescaleDB running in Docker", + href: "upgrade-docker", + excerpt: + "Upgrade self-hosted TimescaleDB running in a Docker container to a new minor version", + }, + { + title: "Upgrade PostgreSQL", + href: "upgrade-pg", + excerpt: "Upgrade PostgreSQL to a new version", + }, + { + title: "Downgrade TimescaleDB to a minor version", + href: "downgrade", + excerpt: "Downgrade self-hosted TimescaleDB to the previous minor version", + }, + ], + }, + { + title: "Uninstall self-hosted TimescaleDB", + href: "uninstall", + excerpt: "Uninstalling self-hosted TimescaleDB", + children: [ + { + title: "Uninstall self-hosted TimescaleDB on macOS", + href: "uninstall-timescaledb", + excerpt: "Uninstall self-hosted TimescaleDB on macOS", + }, + ], + }, + { + title: "Troubleshooting self-hosted TimescaleDB", + href: "troubleshooting", + type: "placeholder", + }, + { + title: "Multi-node (Sunsetted v2.14.x)", href: "multinode-timescaledb", children: [ { @@ -196,7 +278,7 @@ module.exports = [ ], }, { - title: "Distributed hypertables", + title: "Distributed hypertables (Sunsetted v2.14.x)", href: "distributed-hypertables", excerpt: "Distributed hypertables for multi-node Timescale", children: [ @@ -239,88 +321,6 @@ module.exports = [ }, ], }, - { - title: "Replication and High Availability", - href: "replication-and-ha", - children: [ - { - title: "About high availability", - href: "about-ha", - excerpt: "High availability in self-hosted TimescaleDB", - }, - { - title: "Configure replication", - href: "configure-replication", - excerpt: "Configure replication", - }, - ], - }, - { - title: "Additional tooling", - href: "tooling", - children: [ - { - title: "TimescaleDB Tune", - href: "about-timescaledb-tune", - }, - { - title: "Install and update Timescale Toolkit", - href: "install-toolkit", - excerpt: "Install and update the Timescale Toolkit", - }, - ], - }, - { - title: "Upgrade self-hosted TimescaleDB", - href: "upgrades", - children: [ - { - title: "Upgrade TimescaleDB to a minor version", - href: "minor-upgrade", - excerpt: - "Upgrade self-hosted TimescaleDB to a new minor version", - }, - { - title: "Upgrade TimescaleDB to a major version", - href: "major-upgrade", - excerpt: - "Upgrade self-hosted TimescaleDB to a new major version", - }, - { - title: "Upgrade TimescaleDB running in Docker", - href: "upgrade-docker", - excerpt: - "Upgrade self-hosted TimescaleDB running in a Docker container to a new minor version", - }, - { - title: "Upgrade PostgreSQL", - href: "upgrade-pg", - excerpt: "Upgrade PostgreSQL to a new version", - }, - { - title: "Downgrade TimescaleDB to a minor version", - href: "downgrade", - excerpt: "Downgrade self-hosted TimescaleDB to the previous minor version", - }, - ], - }, - { - title: "Uninstall self-hosted TimescaleDB", - href: "uninstall", - excerpt: "Uninstalling self-hosted TimescaleDB", - children: [ - { - title: "Uninstall self-hosted TimescaleDB on macOS", - href: "uninstall-timescaledb", - excerpt: "Uninstall self-hosted TimescaleDB on macOS", - }, - ], - }, - { - title: "Troubleshooting self-hosted TimescaleDB", - href: "troubleshooting", - type: "placeholder", - }, ], }, ]; diff --git a/self-hosted/tooling/install-toolkit.md b/self-hosted/tooling/install-toolkit.md index be1c9d0239..65b9d7525b 100644 --- a/self-hosted/tooling/install-toolkit.md +++ b/self-hosted/tooling/install-toolkit.md @@ -248,8 +248,8 @@ developer documentation][toolkit-gh-docs]. [brew-install]: https://brew.sh [cloud]: /use-timescale/:currentVersion:/services/ -[debian-install]: /self-hosted/latest/install/installation-linux/ -[docker-install]: /self-hosted/latest/install/installation-docker/ +[debian-install]: /self-hosted/:currentVersion:/install/installation-linux/ +[docker-install]: /self-hosted/:currentVersion:/install/installation-docker/ [mst]: /mst/:currentVersion:/ -[red-hat-install]: /self-hosted/latest/install/installation-linux/ +[red-hat-install]: /self-hosted/:currentVersion:/install/installation-linux/ [toolkit-gh-docs]: https://github.com/timescale/timescaledb-toolkit#-installing-from-source diff --git a/self-hosted/upgrades/major-upgrade.md b/self-hosted/upgrades/major-upgrade.md index b6e2aa2d6e..d3017050d0 100644 --- a/self-hosted/upgrades/major-upgrade.md +++ b/self-hosted/upgrades/major-upgrade.md @@ -127,11 +127,12 @@ notice is shown. - + To upgrade TimescaleDB in a Docker container, see the [Docker container upgrades](/self-hosted/latest/upgrades/upgrade-docker) section. + ## Verify the updated policy settings and jobs diff --git a/self-hosted/upgrades/upgrade-pg.md b/self-hosted/upgrades/upgrade-pg.md index beb8282847..d68f8b9e25 100644 --- a/self-hosted/upgrades/upgrade-pg.md +++ b/self-hosted/upgrades/upgrade-pg.md @@ -6,6 +6,7 @@ keywords: [upgrades, PostgreSQL, versions, compatibility] --- import PlanUpgrade from "versionContent/_partials/_plan_upgrade.mdx"; +import SupportMatrix from "versionContent/_partials/_migrate_self_postgres_timescaledb_compatibility.mdx"; import ConsiderCloud from "versionContent/_partials/_consider-cloud.mdx"; import PlanMigrationPath from "versionContent/_partials/_migrate_self_postgres_plan_migration_path.mdx"; @@ -15,21 +16,7 @@ TimescaleDB is a PostgreSQL extension. Ensure that you upgrade to compatible ver - -||PostgreSQL 17|PostgreSQL 16|PostgreSQL 15|PostgreSQL 14|PostgreSQL 13|PostgreSQL 12|PostgreSQL 11|PostgreSQL 10|PostgreSQL 9.6| -|-|-|-|-|-|-|-|-|-| -|TimescaleDB 2.17 and higher|✅|✅|✅|✅|❌|❌|❌|❌|❌| -|TimescaleDB 2.16 and higher|❌|✅|✅|✅|❌|❌|❌|❌|❌|❌| -|TimescaleDB 2.15 and higher|❌|✅|✅|✅|✅|❌|❌|❌|❌|❌| -|TimescaleDB 2.14 and higher|❌|✅|✅|✅|✅|❌|❌|❌|❌|❌| -|TimescaleDB 2.13 and higher|❌|✅|✅|✅|✅|❌|❌|❌|❌| -|TimescaleDB 2.12 and higher|❌|❌|✅|✅|✅|❌|❌|❌|❌| -|TimescaleDB 2.10 and higher|❌|❌|✅|✅|✅|✅|❌|❌|❌| -|TimescaleDB 2.5 to 2.9|❌|❌|❌|✅|✅|✅|❌|❌|❌| -|TimescaleDB 2.4|❌|❌|❌|❌|✅|✅|❌|❌|❌| -|TimescaleDB 2.1 to 2.3|❌|❌|❌|❌|✅|✅|✅|❌|❌| -|TimescaleDB 2.0|❌|❌|❌|❌|❌|✅|✅|❌|❌ -|TimescaleDB 1.7|❌|❌|❌|❌|❌|✅|✅|✅|✅| +## Prerequisites diff --git a/tutorials/OLD-financial-candlestick-tick-data/index.md b/tutorials/OLD-financial-candlestick-tick-data/index.md index bce50ffab8..da42d2fffb 100644 --- a/tutorials/OLD-financial-candlestick-tick-data/index.md +++ b/tutorials/OLD-financial-candlestick-tick-data/index.md @@ -73,4 +73,4 @@ Follow this tutorial and see how to set up your TimescaleDB database to consume [create]: /tutorials/:currentVersion:/financial-candlestick-tick-data/create-candlestick-aggregates [query]: /tutorials/:currentVersion:/financial-candlestick-tick-data/query-candlestick-views [manage]: /tutorials/:currentVersion:/financial-candlestick-tick-data/advanced-data-management -[psql]: /use-timescale/:currentVersion:/integrations/psql/ +[psql]: /integrations/:currentVersion:/psql/ diff --git a/tutorials/OLD_analyze-bitcoin-blockchain/index.md b/tutorials/OLD_analyze-bitcoin-blockchain/index.md index a56596e2aa..a7d4e94c96 100644 --- a/tutorials/OLD_analyze-bitcoin-blockchain/index.md +++ b/tutorials/OLD_analyze-bitcoin-blockchain/index.md @@ -47,4 +47,4 @@ this tutorial is to [install-timescale]: /getting-started/latest/ -[psql-install]: /use-timescale/:currentVersion:/integrations/psql/ +[psql-install]: /integrations/:currentVersion:/psql/ diff --git a/tutorials/OLD_analyze-intraday-stocks/index.md b/tutorials/OLD_analyze-intraday-stocks/index.md index 7a75121641..30ab61296b 100644 --- a/tutorials/OLD_analyze-intraday-stocks/index.md +++ b/tutorials/OLD_analyze-intraday-stocks/index.md @@ -51,4 +51,4 @@ pip install pandas [explore]: /tutorials/:currentVersion:/ [fetch-ingest]: /tutorials/:currentVersion:/ [install-timescale]: /getting-started/latest/ -[psql-install]: /use-timescale/:currentVersion:/integrations/psql/ +[psql-install]: /integrations/:currentVersion:/psql/ diff --git a/tutorials/OLD_grafana/visualizations/pie-chart.md b/tutorials/OLD_grafana/visualizations/pie-chart.md index 41268df2c4..d7c8ef0656 100644 --- a/tutorials/OLD_grafana/visualizations/pie-chart.md +++ b/tutorials/OLD_grafana/visualizations/pie-chart.md @@ -166,4 +166,4 @@ Pie charts are a great tool for comparing categorized data. They're especially g for visualizing percentages. But they don't work as well if you have too many categories with similar percentages or large amount of data. -[gsg-data]: https://docs.timescale.com/getting-started/latest/time-series-data/ +[gsg-data]: https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables diff --git a/tutorials/OLD_nfl-analytics/index.md b/tutorials/OLD_nfl-analytics/index.md index 4912ba2857..d7615d9676 100644 --- a/tutorials/OLD_nfl-analytics/index.md +++ b/tutorials/OLD_nfl-analytics/index.md @@ -60,6 +60,6 @@ This tutorial has a few sections to help you on your journey: [install-timescale]: /getting-started/latest/ [join-data]: /tutorials/:currentVersion:/nfl-analytics/join-with-relational [kaggle-download]: https://www.kaggle.com/c/nfl-big-data-bowl-2021/data -[psql-install]: /use-timescale/:currentVersion:/integrations/psql/ +[psql-install]: /integrations/:currentVersion:/psql/ [toolkit]: /self-hosted/:currentVersion:/tooling/install-toolkit/ [visualize-plays]: /tutorials/:currentVersion:/nfl-analytics/play-visualization/ diff --git a/tutorials/OLD_nfl-fantasy-league.md b/tutorials/OLD_nfl-fantasy-league.md index 85447b696a..3d7f4f77eb 100644 --- a/tutorials/OLD_nfl-fantasy-league.md +++ b/tutorials/OLD_nfl-fantasy-league.md @@ -588,4 +588,4 @@ draw_play(game_id=2018112900, [extra-download]: https://assets.timescale.com/docs/downloads/nfl_2018.zip [install-timescale]: /getting-started/latest/ [kaggle-download]: https://www.kaggle.com/c/nfl-big-data-bowl-2021/data -[psql-install]: /use-timescale/:currentVersion:/integrations/psql/ +[psql-install]: /integrations/:currentVersion:/psql/ diff --git a/tutorials/OLD_nyc-taxi-cab.md b/tutorials/OLD_nyc-taxi-cab.md index cabd0c1f17..da21463b09 100644 --- a/tutorials/OLD_nyc-taxi-cab.md +++ b/tutorials/OLD_nyc-taxi-cab.md @@ -828,5 +828,5 @@ querying by time and location. [migrate]: /use-timescale/:currentVersion:/migration/ [parallel-copy]: https://github.com/timescale/timescaledb-parallel-copy [postgis]: http://postgis.net/documentation -[setup-psql]: /use-timescale/:currentVersion:/integrations/psql/ +[setup-psql]: /integrations/:currentVersion:/psql/ [time-series-forecasting]: /tutorials/:currentVersion:/time-series-forecast/ diff --git a/tutorials/blockchain-analyze/blockchain-dataset.md b/tutorials/blockchain-analyze/blockchain-dataset.md index 0c1189424f..d57ceeb7f1 100644 --- a/tutorials/blockchain-analyze/blockchain-dataset.md +++ b/tutorials/blockchain-analyze/blockchain-dataset.md @@ -7,43 +7,14 @@ layout_components: [next_prev_large] content_group: Analyze the Bitcoin blockchain --- -import CreateAndConnect from "versionContent/_partials/_cloud-create-connect-tutorials.mdx"; -import CreateHypertableBlockchain from "versionContent/_partials/_create-hypertable-blockchain.mdx"; -import AddDataBlockchain from "versionContent/_partials/_add-data-blockchain.mdx"; +import IngestData from "versionContent/_partials/_use-case-setup-blockchain-dataset.mdx"; import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; -# Set up the database - -This tutorial uses a dataset that contains Bitcoin blockchain data for -the past five days, in a hypertable named `transactions`. - - - - - - - - - -The dataset contains around 1.5 million Bitcoin transactions, the trades for five days. It includes -information about each transaction, along with the value in [satoshi][satoshi-def]. It also states if a -trade is a [coinbase][coinbase-def] transaction, and the reward a coin miner receives for mining the coin. - - - - - - - - - -The queries in this tutorial are suitable for graphing in Grafana. If you want -to visualize the results of your queries, connect your Grafana account to the -Bitcoin blockchain dataset. + - + [satoshi-def]: https://www.pcmag.com/encyclopedia/term/satoshi [coinbase-def]: https://www.pcmag.com/encyclopedia/term/coinbase-transaction diff --git a/tutorials/blockchain-analyze/index.md b/tutorials/blockchain-analyze/index.md index 7aee648ba8..b039df0c7a 100644 --- a/tutorials/blockchain-analyze/index.md +++ b/tutorials/blockchain-analyze/index.md @@ -7,16 +7,19 @@ layout_components: [next_prev_large] content_group: Analyze the Bitcoin blockchain --- +import FinancialIndustry from "versionContent/_partials/_financial-industry-data-analysis.mdx"; + # Analyze the Bitcoin blockchain -[Blockchains][blockchain-def] are, at their essence, a distributed database. The -[transactions][transactions-def] in a blockchain are an example of time-series -data. You can use Timescale to query transactions on a blockchain, in exactly the -same way as you might query time-series transactions in any other database. + -In this tutorial, you use Timescale hyperfunctions to analyze transactions -on the Bitcoin blockchain. You can use these instructions to query any type of data on a -blockchain, including other cryptocurrencies, smart contracts, or health data. +In this tutorial, you use Timescale to ingest, store, and analyze transactions +on the Bitcoin blockchain. + +[Blockchains][blockchain-def] are, at their essence, a distributed database. The +[transactions][transactions-def] in a blockchain are an example of time-series data. You can use +Timescale to query transactions on a blockchain, in exactly the same way as you +might query time-series transactions in any other database. ## Prerequisites @@ -56,4 +59,4 @@ to graph the output in Grafana. [blockchain-query]: /tutorials/:currentVersion:/blockchain-query/beginner-blockchain-query/ [blockchain-def]: https://www.pcmag.com/encyclopedia/term/blockchain [transactions-def]: https://www.pcmag.com/encyclopedia/term/bitcoin-transaction -[grafana-setup]: /use-timescale/:currentVersion:/integrations/grafana/ +[grafana-setup]: /integrations/:currentVersion:/grafana/ diff --git a/tutorials/blockchain-query/blockchain-dataset.md b/tutorials/blockchain-query/blockchain-dataset.md index d19b09a721..76033d0afd 100644 --- a/tutorials/blockchain-query/blockchain-dataset.md +++ b/tutorials/blockchain-query/blockchain-dataset.md @@ -7,32 +7,7 @@ layout_components: [next_prev_large] content_group: Query the Bitcoin blockchain --- -import CreateAndConnect from "versionContent/_partials/_cloud-create-connect-tutorials.mdx"; -import CreateHypertableBlockchain from "versionContent/_partials/_create-hypertable-blockchain.mdx"; -import AddDataBlockchain from "versionContent/_partials/_add-data-blockchain.mdx"; +import IngestData from "versionContent/_partials/_use-case-setup-blockchain-dataset.mdx"; -# Set up the database -This tutorial uses a dataset that contains Bitcoin blockchain data for -the past five days, in a hypertable named `transactions`. - - - - - - - - - -The dataset contains around 1.5 million Bitcoin transactions, the trades for five days. It includes -information about each transaction, along with the value in [satoshi][satoshi-def]. It also states if a -trade is a [coinbase][coinbase-def] transaction, and the reward a coin miner receives for mining the coin. - - - - - - - -[satoshi-def]: https://www.pcmag.com/encyclopedia/term/satoshi -[coinbase-def]: https://www.pcmag.com/encyclopedia/term/coinbase-transaction + diff --git a/tutorials/blockchain-query/index.md b/tutorials/blockchain-query/index.md index 133f10cb5d..159f3da678 100644 --- a/tutorials/blockchain-query/index.md +++ b/tutorials/blockchain-query/index.md @@ -7,17 +7,20 @@ layout_components: [next_prev_large] content_group: Query the Bitcoin blockchain --- +import FinancialIndustry from "versionContent/_partials/_financial-industry-data-analysis.mdx"; + # Query the Bitcoin blockchain + + +In this tutorial, you use Timescale to ingest, store, and analyze transactions +on the Bitcoin blockchain. + [Blockchains][blockchain-def] are, at their essence, a distributed database. The [transactions][transactions-def] in a blockchain are an example of time-series data. You can use Timescale to query transactions on a blockchain, in exactly the same way as you might query time-series transactions in any other database. -In this tutorial, you use Timescale to ingest, store, and analyze transactions -on the Bitcoin blockchain. You can use these skills to query any data on a -blockchain, including other cryptocurrencies, smart contracts, or health data. - ## Prerequisites Before you begin, make sure you have: diff --git a/tutorials/energy-data/dataset-energy.md b/tutorials/energy-data/dataset-energy.md index 812b5fa05b..88f1b3e095 100644 --- a/tutorials/energy-data/dataset-energy.md +++ b/tutorials/energy-data/dataset-energy.md @@ -8,115 +8,35 @@ layout_components: [next_prev_large] content_group: Analyze energy consumption data --- -import CreateAndConnect from "versionContent/_partials/_cloud-create-connect-tutorials.mdx"; +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; import CreateHypertableEnergy from "versionContent/_partials/_create-hypertable-energy.mdx"; import AddDataEnergy from "versionContent/_partials/_add-data-energy.mdx"; import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; import CreateCaggs from "versionContent/_partials/_caggs-intro.mdx"; +import CreateCaggsOnIOTData from "versionContent/_partials/_use-case-iot-create-cagg.mdx"; -# Set up the database +# Ingest data into a $SERVICE_LONG This tutorial uses the energy consumption data for over a year in a hypertable named `metrics`. - +## Prerequisites - - - - - - -This tutorial uses the energy consumption data for over a year in a typical -household. You can use this data to analyze the energy consumption pattern. + - - - +## Create continuous aggregates -## Create continuous aggregates - -### Creating continuous aggregates for energy consumption by day and hour - -1. Create a continuous aggregate `kwh_day_by_day` for energy consumption on a - day to day basis: - - ```sql - CREATE MATERIALIZED VIEW kwh_day_by_day(time, value) - with (timescaledb.continuous) as - SELECT time_bucket('1 day', created, 'Europe/Berlin') AS "time", - round((last(value, created) - first(value, created)) * 100.) / 100. AS value - FROM metrics - WHERE type_id = 5 - GROUP BY 1; - ``` - -1. Add a refresh policy to keep the continuous aggregate up-to-date: - - ```sql - SELECT add_continuous_aggregate_policy('kwh_day_by_day', - start_offset => NULL, - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); - ``` - -1. Create a continuous aggregate `kwh_hour_by_hour` for energy consumption on - an hourly basis: - - ```sql - CREATE MATERIALIZED VIEW kwh_hour_by_hour(time, value) - with (timescaledb.continuous) as - SELECT time_bucket('01:00:00', metrics.created, 'Europe/Berlin') AS "time", - round((last(value, created) - first(value, created)) * 100.) / 100. AS value - FROM metrics - WHERE type_id = 5 - GROUP BY 1; - ``` - -1. Add a refresh policy to keep the continuous aggregate up-to-date: - - ```sql - SELECT add_continuous_aggregate_policy('kwh_hour_by_hour', - start_offset => NULL, - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); - ``` - -1. You can confirm that the continuous aggregates were created: - - ```sql - SELECT view_name, format('%I.%I', materialization_hypertable_schema,materialization_hypertable_name) AS materialization_hypertable - FROM timescaledb_information.continuous_aggregates; - ``` - - You should see this: - - ```sql - view_name | materialization_hypertable - ------------------+-------------------------------------------------- - kwh_day_by_day | _timescaledb_internal._materialized_hypertable_2 - kwh_hour_by_hour | _timescaledb_internal._materialized_hypertable_3 - - ``` + - - - - -The queries in this tutorial are suitable for visualizing in Grafana. If you -want to visualize the results of your queries, connect your Grafana account to -the energy consumption dataset. - - diff --git a/tutorials/energy-data/index.md b/tutorials/energy-data/index.md index f951d215c7..6d6b7285e3 100644 --- a/tutorials/energy-data/index.md +++ b/tutorials/energy-data/index.md @@ -10,15 +10,13 @@ content_group: Analyze energy consumption data # Analyze energy consumption data -When you are planning to switch to a rooftop solar system it isn't easy, even +When you are planning to switch to a rooftop solar system, it isn't easy, even with a specialist at hand. You need details of your power consumption, typical -usage hours, or distribution over a year. Collecting consumption data at the -granularity of a few seconds is key to finding all the answers for more -precision. This tutorial uses energy consumption data from a typical household -for over a year. Because nearly all of this data is time-series data, proper -analysis requires a purpose-built time-series database, like Timescale. +usage hours, distribution over a year, and other information. Collecting consumption data at the +granularity of a few seconds and then getting insights on it is key - and this is what Timescale is best at. -In this tutorial you can construct queries that look at how many watts were +This tutorial uses energy consumption data from a typical household +for over a year. You construct queries that look at how many watts were consumed, and when. Additionally, you can visualize the energy consumption data in Grafana. @@ -58,4 +56,4 @@ you through the steps to visualize the results in Grafana. [query-energy]: /tutorials/:currentVersion:/energy-data/query-energy/ [compress-energy]: /tutorials/:currentVersion:/energy-data/compress-energy/ [cloud-install]: /getting-started/:currentVersion:/#create-your-timescale-account -[grafana-setup]: /use-timescale/:currentVersion:/integrations/grafana/ +[grafana-setup]: /integrations/:currentVersion:/grafana/ diff --git a/tutorials/financial-ingest-real-time/financial-ingest-dataset.md b/tutorials/financial-ingest-real-time/financial-ingest-dataset.md index d75ba5fe11..0997c010b8 100644 --- a/tutorials/financial-ingest-real-time/financial-ingest-dataset.md +++ b/tutorials/financial-ingest-real-time/financial-ingest-dataset.md @@ -8,40 +8,36 @@ layout_components: [next_prev_large] content_group: Ingest real-time financial websocket data --- -import CreateAndConnect from "versionContent/_partials/_cloud-create-connect-tutorials.mdx"; +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; import CreateHypertable from "versionContent/_partials/_create-hypertable-twelvedata-stocks.mdx"; -import CreateHypertableStocks from "versionContent/_partials/_create-hypertable-twelvedata-stocks.mdx"; +import CreateHypertableCrypo from "versionContent/_partials/_create-hypertable-twelvedata-crypto.mdx"; import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; -# Set up the database +# Ingest data into a $SERVICE_LONG This tutorial uses a dataset that contains second-by-second stock-trade data for the top 100 most-traded symbols, in a hypertable named `stocks_real_time`. It also includes a separate table of company symbols and company names, in a regular PostgreSQL table named `company`. - +## Prerequisites - + - - - +## Connect to the websocket server When you connect to the Twelve Data API through a websocket, you create a persistent connection between your computer and the websocket server. You set up a Python environment, and pass two arguments to create a websocket object and establish the connection. -## Set up a new Python environment +### Set up a new Python environment Create a new Python virtual environment for this project and activate it. All the packages you need to complete for this tutorial are installed in this environment. -### Setting up a new Python environment - 1. Create and activate a Python virtual environment: ```bash @@ -67,13 +63,13 @@ the packages you need to complete for this tutorial are installed in this enviro -## Create the websocket connection +### Create the websocket connection A persistent connection between your computer and the websocket server is used to receive data for as long as the connection is maintained. You need to pass two arguments to create a websocket object and establish connection. -### Websocket arguments +#### Websocket arguments * `on_event` @@ -99,7 +95,7 @@ two arguments to create a websocket object and establish connection. -### Connecting to the websocket server +### Connect to the websocket server 1. Create a new Python file called `websocket_test.py` and connect to the Twelve Data servers using the ``: @@ -163,22 +159,8 @@ two arguments to create a websocket object and establish connection. - - - - -To ingest the data into your Timescale service, you need to implement the -`on_event` function. - -After the websocket connection is set up, you can use the `on_event` function -to ingest data into the database. This is a data pipeline that ingests real-time -financial data into your Timescale service. - -Stock trades are ingested in real-time Monday through Friday, typically during -normal trading hours of the New York Stock Exchange (9:30 AM to -4:00 PM EST). - + When you ingest data into a transactional database like Timescale, it is more efficient to insert data in batches rather than inserting data row-by-row. Using @@ -195,6 +177,13 @@ universal, but you can experiment with different batch sizes Using batching is a fairly common pattern when ingesting data into TimescaleDB from Kafka, Kinesis, or websocket connections. +To ingest the data into your Timescale service, you need to implement the +`on_event` function. + +After the websocket connection is set up, you can use the `on_event` function +to ingest data into the database. This is a data pipeline that ingests real-time +financial data into your Timescale service. + You can implement a batching solution in Python with Psycopg2. You can implement the ingestion logic within the `on_event` function that you can then pass over to the websocket object. @@ -207,7 +196,7 @@ This function needs to: 1. Add it to the in-memory batch, which is a list in Python. 1. If the batch reaches a certain size, insert the data, and reset or empty the list. -## Ingesting data in real-time +## Ingest data in real-time @@ -323,17 +312,8 @@ If you see an error message similar to this: Then check that you use a proper API key received from Twelve Data. - - - - -The queries in this tutorial are suitable for visualizing in Grafana. If you -want to visualize the results of your queries, connect your Grafana account to -the energy consumption dataset. - - [twelve-wrapper]: https://github.com/twelvedata/twelvedata-python [psycopg2]: https://www.psycopg.org/docs/ diff --git a/tutorials/financial-ingest-real-time/financial-ingest-query.md b/tutorials/financial-ingest-real-time/financial-ingest-query.md index 116668b4ea..c24909c34c 100644 --- a/tutorials/financial-ingest-real-time/financial-ingest-query.md +++ b/tutorials/financial-ingest-real-time/financial-ingest-query.md @@ -38,7 +38,7 @@ the last two hours' worth of data. MIN(price) AS low, LAST(price, time) AS "close", LAST(day_volume, time) AS day_volume - FROM stocks_real_time + FROM crypto_ticks GROUP BY bucket, symbol; ``` diff --git a/tutorials/financial-ingest-real-time/index.md b/tutorials/financial-ingest-real-time/index.md index cff6156daf..8b7231c798 100644 --- a/tutorials/financial-ingest-real-time/index.md +++ b/tutorials/financial-ingest-real-time/index.md @@ -1,5 +1,5 @@ --- -title: Ingest real-time financial websocket data +title: Ingest real-time financial data using WebSocket excerpt: Ingest time-series data into Timescale Cloud using a websocket connection products: [cloud] keywords: [finance, analytics, websockets, data pipeline] @@ -9,8 +9,11 @@ content_group: Ingest real-time financial websocket data --- import CandlestickIntro from "versionContent/_partials/_candlestick_intro.mdx"; +import FinancialIndustry from "versionContent/_partials/_financial-industry-data-analysis.mdx"; -# Ingest real-time financial websocket data +# Ingest real-time financial data using WebSocket + + This tutorial shows you how to ingest real-time time-series data into TimescaleDB using a websocket connection. The tutorial sets up a data pipeline diff --git a/tutorials/financial-tick-data/financial-tick-compress.md b/tutorials/financial-tick-data/financial-tick-compress.md index e8493cd746..d4f29c8388 100644 --- a/tutorials/financial-tick-data/financial-tick-compress.md +++ b/tutorials/financial-tick-data/financial-tick-compress.md @@ -49,7 +49,7 @@ memory. order-by column using the `ALTER TABLE` command: ```sql - ALTER TABLE stocks_real_time + ALTER TABLE crypto_ticks SET ( timescaledb.compress, timescaledb.compress_segmentby='symbol', @@ -64,7 +64,7 @@ memory. `compress_chunk` in this manner: ```sql - SELECT compress_chunk(c) from show_chunks('stocks_real_time') c; + SELECT compress_chunk(c) from show_chunks('crypto_ticks') c; ``` You can also [automate compression][automatic-compression] by adding a [compression policy][add_compression_policy] which will @@ -76,7 +76,7 @@ memory. SELECT pg_size_pretty(before_compression_total_bytes) as before, pg_size_pretty(after_compression_total_bytes) as after - FROM hypertable_compression_stats('stocks_real_time'); + FROM hypertable_compression_stats('crypto_ticks'); ``` This shows a significant improvement in data usage: @@ -97,7 +97,7 @@ allows you to compress data that is older than a particular age, for example, to compress all chunks that are older than 8 days: ```sql -SELECT add_compression_policy('stocks_real_time', INTERVAL '8 days'); +SELECT add_compression_policy('crypto_ticks', INTERVAL '8 days'); ``` Compression policies run on a regular schedule, by default once every @@ -141,7 +141,7 @@ timing query times in psql by running: To decompress the whole dataset, run: ```sql - SELECT decompress_chunk(c) from show_chunks('stocks_real_time') c; + SELECT decompress_chunk(c) from show_chunks('crypto_ticks') c; ``` On an example setup, speedup performance observed was significant, diff --git a/tutorials/financial-tick-data/financial-tick-dataset.md b/tutorials/financial-tick-data/financial-tick-dataset.md index d654ae50bc..d6dab2ce25 100644 --- a/tutorials/financial-tick-data/financial-tick-dataset.md +++ b/tutorials/financial-tick-data/financial-tick-dataset.md @@ -8,42 +8,28 @@ layout_components: [next_prev_large] content_group: Analyze financial tick data --- -import CreateAndConnect from "versionContent/_partials/_cloud-create-connect-tutorials.mdx"; -import CreateHypertable from "versionContent/_partials/_create-hypertable-twelvedata-stocks.mdx"; -import AddData from "versionContent/_partials/_add-data-twelvedata-stocks.mdx"; +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; +import CreateHypertableCrypto from "versionContent/_partials/_create-hypertable-twelvedata-crypto.mdx"; +import AddDataCrypto from "versionContent/_partials/_add-data-twelvedata-crypto.mdx"; +import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; -# Set up the database +# Ingest data into a $SERVICE_LONG -This tutorial uses a dataset that contains second-by-second stock-trade data for -the top 100 most-traded symbols, in a hypertable named `stocks_real_time`. It -also includes a separate table of company symbols and company names, in a -regular PostgreSQL table named `company`. - - - - - - - - +This tutorial uses a dataset that contains second-by-second trade data for +the most-traded crypto-assets. You optimize this time-series data in a a hypertable called `assets_real_time`. +You also create a separate table of asset symbols in a regular PostgreSQL table named `assets`. The dataset is updated on a nightly basis and contains data from the last four -weeks, typically around 8 million rows of data. Stock trades are recorded in -real-time Monday through Friday, typically during normal trading hours of the -New York Stock Exchange (9:30 AM - 4:00 PM EST). - - +weeks, typically around 8 million rows of data. Trades are recorded in +real-time from 180+ cryptocurrency exchanges. - +## Prerequisites - + - + -The queries in this tutorial are suitable for visualizing in Grafana. If you -want to visualize the results of your queries, connect your Grafana account to -the energy consumption dataset. + - diff --git a/tutorials/financial-tick-data/financial-tick-query.md b/tutorials/financial-tick-data/financial-tick-query.md index 1fa9f7b78d..8a71337c50 100644 --- a/tutorials/financial-tick-data/financial-tick-query.md +++ b/tutorials/financial-tick-data/financial-tick-query.md @@ -61,7 +61,7 @@ the last two days' worth of data. MIN(price) AS low, LAST(price, time) AS "close", LAST(day_volume, time) AS day_volume - FROM stocks_real_time + FROM crypto_ticks GROUP BY bucket, symbol; ``` diff --git a/tutorials/financial-tick-data/index.md b/tutorials/financial-tick-data/index.md index 8fcc4fceb0..e2733e90a3 100644 --- a/tutorials/financial-tick-data/index.md +++ b/tutorials/financial-tick-data/index.md @@ -8,9 +8,13 @@ layout_components: [next_prev_large] content_group: Analyze financial tick data --- +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; import CandlestickIntro from "versionContent/_partials/_candlestick_intro.mdx"; +import FinancialIndustry from "versionContent/_partials/_financial-industry-data-analysis.mdx"; -# Analyze financial tick data with TimescaleDB +# Analyze financial tick data + + To analyze financial data, you can chart the open, high, low, close, and volume (OHLCV) information for a financial asset. Using this data, you can create @@ -24,25 +28,7 @@ aggregated data, and visualize the data in Grafana. ## Prerequisites -Before you begin, make sure you have: - -* Signed up for a [free Timescale account][cloud-install]. - -## Steps in this tutorial - -This tutorial covers: - -1. [Setting up your dataset][financial-tick-dataset]: Load data from - [Twelve Data][twelve-data] into your TimescaleDB database. -1. [Querying your dataset][financial-tick-query]: Create candlestick views, query - the aggregated data, and visualize the data in Grafana. -1. [Bonus: Store data efficiently][financial-tick-compress]: Learn how to store and query -your financial tick data more efficiently using compression feature of Timescale. - - This tutorial shows you how to ingest real-time time-series data into a Timescale - database. To create candlestick views, query the - aggregated data, and visualize the data in Grafana, see the - [ingest real-time websocket data section][advanced-websocket]. + ## About OHLCV data and candlestick charts @@ -58,6 +44,23 @@ these stories from some Timescale community members: * [How Messari uses data to open the cryptoeconomy to everyone][messari] * [How I power a (successful) crypto trading bot with TimescaleDB][bot] +## Steps in this tutorial + +This tutorial shows you how to ingest real-time time-series data into a Timescale +database: + +1. [Setting up your dataset][financial-tick-dataset]: Load data from + [Twelve Data][twelve-data] into your TimescaleDB database. +1. [Querying your dataset][financial-tick-query]: Create candlestick views, query + the aggregated data, and visualize the data in Grafana. +1. [Bonus: Store data efficiently][financial-tick-compress]: Learn how to store and query +your financial tick data more efficiently using compression feature of Timescale. + + +To create candlestick views, query the aggregated data, and visualize the data in Grafana, see the +[ingest real-time websocket data section][advanced-websocket]. + + [advanced-websocket]: /tutorials/:currentVersion:/financial-ingest-real-time/ [cloud-install]: /getting-started/:currentVersion:/#create-your-timescale-account [financial-tick-dataset]: /tutorials/:currentVersion:/financial-tick-data/financial-tick-dataset/ diff --git a/tutorials/index.md b/tutorials/index.md index 53adf0ac44..69e22acf85 100644 --- a/tutorials/index.md +++ b/tutorials/index.md @@ -9,14 +9,26 @@ products: [cloud, mst, self_hosted] Timescale tutorials are designed to help you get up and running with Timescale fast. They walk you through a variety of scenarios using example datasets, to teach you how to construct interesting queries, find out what information your -database has hidden in it, and even gives you options for visualizing and +database has hidden in it, and even give you options for visualizing and graphing your results. -|🔐 Cryptocurrency|🔋 Energy|💰 Finance|🚘 Transport| 💡 IoT | -|-|-|-|-|------------------------------------------------------------| -|🟢 [Part 1][beginner-crypto]
Do your own research on the Bitcoin blockchain|🟢 [Part 1][beginner-energy]
Optimize your energy consumption for a rooftop solar PV system|🟢 [Part 1][beginner-finance]
Chart the trading highs and lows for your favorite stock|🟢 [Part 1][beginner-fleet]
Find out about taxi rides taken in and around NYC| 🟢 [Part 1][iot]
Simulate an IoT sensor dataset | -|⭐ [Part 2][intermediate-crypto]
Discover the relationship between transactions, blocks, fees, and miner revenue|⭐ *Coming Soon!*|⭐ [Part 2][advanced-finance]
Use a websocket connection to visualize the trading highs and lows for your favorite stock|⭐ [Part 2][intermediate-fleet]
Map the longest taxi rides in NYC| | +- **Real-time analytics** + - [Analytics on energy consumption][rta-energy]: make data-driven decisions using energy consumption data. + - [Analytics on transport and geospatial data][rta-transport]: optimize profits using geospatial transport data. +- **Cryptocurrency** + - [Query the Bitcoin blockchain][beginner-crypto]: do your own research on the Bitcoin blockchain. + - [Analyze the Bitcoin blockchain][intermediate-crypto]: discover the relationship between transactions, blocks, fees, and miner revenue. +- **Finance** + - [Analyze financial tick data][beginner-finance]: chart the trading highs and lows for your favorite stock. + - [Ingest real-time financial data using WebSocket][advanced-finance]: use a websocket connection to visualize the trading highs and lows for your favorite stock. +- **IoT** + - [Simulate an IoT sensor dataset][iot]: simulate an IoT sensor dataset and run simple queries on it. +- **Cookbooks** + - [Timescale community cookbook][cookbooks]: get suggestions from the TimescaleDB Community about how to resolve common issues. + +[rta-energy]: /tutorials/:currentVersion:/real-time-analytics-energy-consumption +[rta-transport]: /tutorials/:currentVersion:/real-time-analytics-transport [beginner-fleet]: /tutorials/:currentVersion:/nyc-taxi-cab/ [beginner-finance]: /tutorials/:currentVersion:/financial-tick-data/ [beginner-crypto]: /tutorials/:currentVersion:/blockchain-query/ @@ -25,3 +37,4 @@ graphing your results. [intermediate-crypto]: /tutorials/:currentVersion:/blockchain-analyze/ [advanced-finance]: /tutorials/:currentVersion:/financial-ingest-real-time/ [iot]: /tutorials/:currentVersion:/simulate-iot-sensor-data/ +[cookbooks]: /tutorials/:currentVersion:/cookbook/ diff --git a/tutorials/ingest-real-time-websocket-data.md b/tutorials/ingest-real-time-websocket-data.md index ce4f604d95..c06d4450df 100644 --- a/tutorials/ingest-real-time-websocket-data.md +++ b/tutorials/ingest-real-time-websocket-data.md @@ -1,5 +1,5 @@ --- -title: Ingest real-time financial websocket data +title: Ingest real-time financial data using WebSocket excerpt: Set up a data pipeline to get data from different financial APIs products: [cloud, mst, self_hosted] keywords: [finance, analytics, websockets, data pipeline] @@ -8,7 +8,7 @@ keywords: [finance, analytics, websockets, data pipeline] import CreateHypertableStocks from "versionContent/_partials/_create-hypertable-twelvedata-stocks.mdx"; import GraphOhlcv from "versionContent/_partials/_graphing-ohlcv-data.mdx"; -# Ingest real-time financial websocket data +# Ingest real-time financial data using WebSocket This tutorial shows you how to ingest real-time time-series data into TimescaleDB using a websocket connection. The tutorial sets up a data pipeline diff --git a/tutorials/nyc-taxi-cab/dataset-nyc.md b/tutorials/nyc-taxi-cab/dataset-nyc.md index fceb651151..06219aaa41 100644 --- a/tutorials/nyc-taxi-cab/dataset-nyc.md +++ b/tutorials/nyc-taxi-cab/dataset-nyc.md @@ -11,31 +11,22 @@ import CreateAndConnect from "versionContent/_partials/_cloud-create-connect-tut import CreateHypertableNyc from "versionContent/_partials/_create-hypertable-nyctaxis.mdx"; import AddDataNyc from "versionContent/_partials/_add-data-nyctaxis.mdx"; import PreloadedData from "versionContent/_partials/_preloaded-data.mdx"; +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; -# Set up the database -This tutorial uses a dataset that contains historical data from New York's -yellow taxi network, in a hypertable named `rides`. It also includes a separate +# Ingest data into a $SERVICE_LONG + +This tutorial uses a dataset that contains historical data from the New York City Taxi and Limousine +Commission [NYC TLC][nyc-tlc], in a hypertable named `rides`. It also includes a separate tables of payment types and rates, in a regular PostgreSQL table named `payment_types`, and `rates`. - - - - - +## Prerequisites - - - - -This tutorial uses historical data from New York's yellow taxi network, provided -by the New York City Taxi and Limousine Commission [NYC TLC][nyc-tlc]. + - - [nyc-tlc]: https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page diff --git a/tutorials/nyc-taxi-geospatial/dataset-nyc.md b/tutorials/nyc-taxi-geospatial/dataset-nyc.md index eb7e335897..8a6d82e93a 100644 --- a/tutorials/nyc-taxi-geospatial/dataset-nyc.md +++ b/tutorials/nyc-taxi-geospatial/dataset-nyc.md @@ -12,39 +12,25 @@ import CreateAndConnect from "versionContent/_partials/_cloud-create-connect-tut import CreateHypertableNyc from "versionContent/_partials/_create-hypertable-nyctaxis.mdx"; import AddDataNyc from "versionContent/_partials/_add-data-nyctaxis.mdx"; import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; -# Set up the database +# Ingest data into a $SERVICE_LONG -This tutorial uses a dataset that contains historical data from New York's -yellow taxi network, in a hypertable named `rides`. It also includes a separate +This tutorial uses a dataset that contains historical data from the New York City Taxi and Limousine +Commission [NYC TLC][nyc-tlc], in a hypertable named `rides`. It also includes a separate tables of payment types and rates, in a regular PostgreSQL table named `payment_types`, and `rates`. - +## Prerequisites - - - - - - -This tutorial uses historical data from New York's yellow taxi network, provided -by the New York City Taxi and Limousine Commission [NYC TLC][nyc-tlc]. + - - - - -The queries in this tutorial are suitable for visualizing in Grafana. If you -want to visualize the results of your queries, connect your Grafana account to -the NYC taxi cab dataset. - - + [nyc-tlc]: https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page diff --git a/tutorials/nyc-taxi-geospatial/index.md b/tutorials/nyc-taxi-geospatial/index.md index b26e068059..aa96cfaae0 100644 --- a/tutorials/nyc-taxi-geospatial/index.md +++ b/tutorials/nyc-taxi-geospatial/index.md @@ -63,4 +63,4 @@ information, and plotting the results in Grafana. [cloud-install]: /getting-started/:currentVersion:/#create-your-timescale-account [beginner-fleet]: /tutorials/:currentVersion:/nyc-taxi-cab/ [plot-nyc]: /tutorials/:currentVersion:/nyc-taxi-geospatial/plot-nyc/ -[grafana-setup]: /use-timescale/:currentVersion:/integrations/grafana/ +[grafana-setup]: /integrations/:currentVersion:/grafana/ diff --git a/tutorials/page-index/page-index.js b/tutorials/page-index/page-index.js index 8b2281dca2..a8a77f25df 100644 --- a/tutorials/page-index/page-index.js +++ b/tutorials/page-index/page-index.js @@ -5,18 +5,38 @@ module.exports = [ excerpt: "Learn about common scenarios and use cases for Timescale", children: [ { - title: "Cryptocurrency - part 1", + title: "Analytics on transport and geospatial data", + href: "real-time-analytics-transport", + excerpt: "Analyse your data in real-time", + }, + { + title: "Analytics on energy consumption", + href: "real-time-analytics-energy-consumption", + excerpt: "Analyse your data in real-time", + }, + { + title: "Simulate an IoT sensor dataset", + href: "simulate-iot-sensor-data", + excerpt: "Simulate and query an IoT sensor dataset", + }, + { + title: "Timescale community cookbook", + href: "cookbook", + excerpt: "Code examples from the community that help you with loads of common conundrums.", + }, + { + title: "Query the Bitcoin blockchain", href: "blockchain-query", excerpt: "Query the Bitcoin blockchain", children: [ { - title: "Set up", + title: "Ingest data into a service", href: "blockchain-dataset", excerpt: "Set up a dataset so you can query the Bitcoin blockchain", }, { - title: "Query data", + title: "Query the data", href: "beginner-blockchain-query", excerpt: "Query the Bitcoin blockchain dataset", }, @@ -29,58 +49,35 @@ module.exports = [ ], }, { - title: "Cryptocurrency - part 2", + title: "Analyze the Bitcoin blockchain", href: "blockchain-analyze", excerpt: "Analyze the Bitcoin blockchain with Timescale hyperfunctions", children: [ { - title: "Set up", + title: "Ingest data into a service", href: "blockchain-dataset", excerpt: "Set up a dataset so you can analyze the Bitcoin blockchain", }, { - title: "Query data", + title: "Analyse the data", href: "analyze-blockchain-query", excerpt: "Analyze the Bitcoin blockchain dataset with Timescale hyperfunctions", }, ], }, { - title: "Energy - part 1", - href: "energy-data", - excerpt: "Learn how to analyze energy consumption data", - children: [ - { - title: "Set up", - href: "dataset-energy", - excerpt: "Set up a dataset so you can analyze energy consumption data", - }, - { - title: "Query data", - href: "query-energy", - excerpt: "Queries energy consumption data", - }, - { - title: "Bonus: set up compression", - href: "compress-energy", - excerpt: - "Compress the dataset so you can store the data more efficiently", - }, - ] - }, - { - title: "Finance - part 1", + title: "Analyze financial tick data ", href: "financial-tick-data", excerpt: "Use Timescale to store financial tick data", children: [ { - title: "Set up", + title: "Ingest data into a service", href: "financial-tick-dataset", excerpt: "Set up a dataset so you can query financial tick data", }, { - title: "Query data", + title: "Query the data", href: "financial-tick-query", excerpt: "Query and visualize financial tick data", }, @@ -93,72 +90,22 @@ module.exports = [ ], }, { - title: "Finance - part 2", + title: "Ingest real-time financial data", href: "financial-ingest-real-time", excerpt: "Ingest real-time financial data with websocket", children: [ { - title: "Set up", + title: "Ingest data into a service", href: "financial-ingest-dataset", excerpt: "Set up a dataset so you can query the real-time data", }, { - title: "Query data", + title: "Query the data", href: "financial-ingest-query", excerpt: "Query and visualize real-time data", }, ], }, - { - title: "Transport - part 1", - href: "nyc-taxi-cab", - excerpt: "An introduction to time-series using NYC taxi data", - children: [ - { - title: "Set up", - href: "dataset-nyc", - excerpt: "Set up a dataset so you can query NYC data", - }, - { - title: "Query data", - href: "query-nyc", - excerpt: "Query NYC data", - }, - { - title: "Bonus: set up compression", - href: "compress-nyc", - excerpt: - "Compress the dataset so you can store the data more efficiently", - }, - ], - }, - { - title: "Transport - part 2", - href: "nyc-taxi-geospatial", - excerpt: "Learn how to plot geospatial time-series data with NYC taxi cabs", - children: [ - { - title: "Set up", - href: "dataset-nyc", - excerpt: "Set up a dataset so you can plot geospatial NYC taxi data", - }, - { - title: "Query data", - href: "plot-nyc", - excerpt: "Plot geospatial NYC taxi data", - }, - ], - }, - { - title: "Internet of things", - href: "simulate-iot-sensor-data", - excerpt: "Simulate and query an IoT sensor dataset", - }, - { - title: "Timescale community cookbook", - href: "cookbook", - excerpt: "Code examples from the community that help you with loads of common conundrums.", - }, ], }, ]; diff --git a/tutorials/real-time-analytics-energy-consumption.md b/tutorials/real-time-analytics-energy-consumption.md new file mode 100644 index 0000000000..371e719a63 --- /dev/null +++ b/tutorials/real-time-analytics-energy-consumption.md @@ -0,0 +1,226 @@ +--- +title: Real-time analytics with Timescale Cloud and Grafana +excerpt: Simulate an IOT dataset in your Timescale Cloud service +products: [cloud, mst, self_hosted] +keywords: [IoT, simulate] +--- + + +import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; +import ImportDataEnergy from "versionContent/_partials/_import-data-iot.mdx"; +import CreateCaggsOnIOTData from "versionContent/_partials/_use-case-iot-create-cagg.mdx"; +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; + +# Analytics on energy consumption + +Energy providers understand that customers tend to lose patience when there is not enough power for them +to complete day-to-day activities. Task one is keeping the lights on. If you are transitioning to renewable energy, +it helps to know when you need to produce energy so you can choose a suitable energy source. + +Real-time analytics refers to the process of collecting, analyzing, and interpreting data instantly as it is generated. +This approach enables you to track and monitor activity, make the decisions based on real-time insights on data stored in +a $SERVICE_LONG and keep those lights on. + + +[Grafana][grafana-docs] is a popular data visualization tool that enables you to create customizable dashboards +and effectively monitor your systems and applications. + +![Grafana real-time analytics](https://assets.timescale.com/docs/images/use-case-rta-grafana-timescale-energy-cagg.png) + +This page shows you how to integrate Grafana with a $SERVICE_LONG and make insights based on visualization of +data optimized for size and speed in the columnstore. + +## Prerequisites + + + +* Install and run [self-managed Grafana][grafana-self-managed], or sign up for [Grafana Cloud][grafana-cloud]. + +## Optimize time-series data in hypertables + + + +## Write fast analytical queries + +Aggregation is a way of combining data to get insights from it. Average, sum, and count are all examples of simple +aggregates. However, with large amounts of data aggregation slows things down, quickly. Continuous aggregates +are a kind of hypertable that is refreshed automatically in the background as new data is added, or old data is +modified. Changes to your dataset are tracked, and the hypertable behind the continuous aggregate is automatically +updated in the background. + +By default, querying continuous aggregates provides you with real-time data. Pre-aggregated data from the materialized +view is combined with recent data that hasn't been aggregated yet. This gives you up-to-date results on every query. + +You create continuous aggregates on uncompressed data in high-performance storage. They continue to work +on [data in the columnstore][test-drive-enable-compression] +and [rarely accessed data in tiered storage][test-drive-tiered-storage]. You can even +create [continuous aggregates on top of your continuous aggregates][hierarchical-caggs]. + + + + + + + +## Optimize your data for real-time analytics + +[Hypercore][hypercore] is the $COMPANY hybrid row-columnar storage engine used by hypertables. Hypertables partition your data in +chunks. Chunks stored in the rowstore use a row-oriented data format optimized for high-speed inserts and updates. +Chunks stored in the columnstore use a columnar data format optimized for analytics. You ingest `hot` data into the +rowstore. As data cools and becomes more suited for analytics, $CLOUD_LONG automatically converts these chunks of data +to the columnstore. You define the moment when data is converted using a columnstore policy. + +When $CLOUD_LONG converts a chunk to the columnstore, TimescaleDB automatically creates a different schema for your +data. $TIMESCALE_DB creates and uses custom indexes to incorporate the `segmentby` and `orderby` parameters when +you write to and read from the columstore. + +To increase the speed of your analytical queries by a factor of 10 and reduce storage costs by up to 90%, convert data +to the columnstore: + + + +1. **Connect to your $SERVICE_LONG** + + In [$CONSOLE][services-portal] open an [SQL editor][in-console-editors]. The in-Console editors display the query speed. + You can also connect to your service using [psql][connect-using-psql]. + +1. **Enable columnstore on a hypertable** + + Create a [job][job] that automatically moves chunks in a hypertable to the columnstore at a specific time interval. + By default, your table is `orderedby` the time column. For efficient queries on columnstore data, remember to + `segmentby` the column you will use most often to filter your data: + + ```sql + ALTER TABLE metrics SET ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'type_id', + timescaledb.orderby = 'created DESC' + ); + ``` + +1. **Add a policy to convert chunks to the columnstore at a specific time interval** + + For example, 60 days after the data was added to the table: + ``` sql + CALL add_columnstore_policy('metrics', INTERVAL '8 days'); + ``` + See [add_columnstore_policy][add_columnstore_policy]. + +1. **View your data space saving** + + When you convert data to the columnstore, as well as being optimized for analytics, it is compressed by more than 90%. + This saves on storage costs and keeps your queries operating at lightning speed. To see the amount of space saved: + ```sql + SELECT + pg_size_pretty(before_compression_total_bytes) as before, + pg_size_pretty(after_compression_total_bytes) as after + FROM hypertable_compression_stats('metrics'); + ``` + You see something like: + + | before | after | + |--------|-------| + | 181 MB | 16 MB | + +1. **Faster analytical queries on data in the columnstore** + + Now run the analytical query again: + ```sql + SELECT time_bucket('1 day', created, 'Europe/Berlin') AS "time", + round((last(value, created) - first(value, created)) * 100.) / 100. AS value + FROM metrics + WHERE type_id = 5 + GROUP BY 1; + ``` + On this amount of data, this analytical query on data in the columnstore takes about 0.8 seconds. + + + +Just to hit this one home, by converting cooling data to the columnstore, you have increased the speed of your analytical +queries by a factor of 10, and reduced storage by up to 90%. + + + +## Visualize energy consumption + +A Grafana dashboard represents a view into the performance of a system, and each dashboard consists of one or +more panels, which represent information about a specific metric related to that system. + +To visually monitor the volume of energy consumption over time: + + + +1. **Create the dashboard** + + 1. On the `Dashboards` page, click `New` and select `New dashboard`. + + 1. Click `Add visualization`, then select the data source that connects to your $SERVICE_LONG and the `Bar chart` + visualization. + + ![Grafana create dashboard](https://assets.timescale.com/docs/images/use-case-rta-grafana-timescale-configure-dashboard.png) + 1. In the `Queries` section, select `Code`, then run the following query based on your continuous aggregate: + + ```sql + WITH per_hour AS ( + SELECT + time, + value + FROM kwh_hour_by_hour + WHERE "time" at time zone 'Europe/Berlin' > date_trunc('month', time) - interval '1 year' + ORDER BY 1 + ), hourly AS ( + SELECT + extract(HOUR FROM time) * interval '1 hour' as hour, + value + FROM per_hour + ) + SELECT + hour, + approx_percentile(0.50, percentile_agg(value)) as median, + max(value) as maximum + FROM hourly + GROUP BY 1 + ORDER BY 1; + ``` + + This query averages the results for households in a specific time zone by hour and orders them by time. + Because you use a continuous aggregate, this data is always correct in real time. + + ![Grafana real-time analytics](https://assets.timescale.com/docs/images/use-case-rta-grafana-timescale-energy-cagg.png) + + You see that energy consumption is highest in the evening and at breakfast time. You also know that the wind + drops off in the evening. This data proves that you need to supply a supplementary power source for peak times, + or plan to store energy during the day for peak times. + +1. **Click `Save dashboard`** + + + + +You have integrated Grafana with a Timescale Cloud service and made insights based on visualization of your data. + +[grafana-docs]: https://grafana.com/docs/ +[grafana-self-managed]: https://grafana.com/get/?tab=self-managed +[grafana-cloud]: https://grafana.com/get/ +[use-time-buckets]: /use-timescale/:currentVersion:/time-buckets/use-time-buckets/ + +[test-drive-enable-compression]: /getting-started/:currentVersion:/try-key-features-timescale-products/#enhance-query-performance-for-analytics +[test-drive-tiered-storage]: /getting-started/:currentVersion:/try-key-features-timescale-products/#slash-storage-charges +[data-tiering]: /use-timescale/:currentVersion:/data-tiering/ +[compression]: /use-timescale/:currentVersion:/compression/ +[hierarchical-caggs]: /use-timescale/:currentVersion:/continuous-aggregates/hierarchical-continuous-aggregates/ +[job]: /api/:currentVersion:/actions/add_job/ +[alter_table_hypercore]: /api/:currentVersion:/hypercore/alter_table/ +[compression_continuous-aggregate]: /api/:currentVersion:/hypercore/alter_materialized_view/ +[convert_to_rowstore]: /api/:currentVersion:/hypercore/convert_to_rowstore/ +[convert_to_columnstore]: /api/:currentVersion:/hypercore/convert_to_columnstore/ +[informational-views]: /api/:currentVersion:/informational-views/jobs/ +[add_columnstore_policy]: /api/:currentVersion:/hypercore/add_columnstore_policy/ +[hypercore_workflow]: /api/:currentVersion:/hypercore/#hypercore-workflow +[alter_job]: /api/:currentVersion:/actions/alter_job/ +[remove_columnstore_policy]: /api/:currentVersion:/hypercore/remove_columnstore_policy/ +[in-console-editors]: /getting-started/:currentVersion:/run-queries-from-console/ +[services-portal]: https://console.cloud.timescale.com/dashboard/services +[connect-using-psql]: /integrations/:currentVersion:/psql#connect-to-your-service +[insert]: /use-timescale/:currentVersion:/write-data/insert/ +[hypercore]: /use-timescale/:currentVersion:/hypercore/ diff --git a/tutorials/real-time-analytics-transport.md b/tutorials/real-time-analytics-transport.md new file mode 100644 index 0000000000..015c3be256 --- /dev/null +++ b/tutorials/real-time-analytics-transport.md @@ -0,0 +1,189 @@ +--- +title: Analytics on transport and geospatial data +excerpt: Simulate and analyze a transport dataset in your Timescale Cloud service +products: [cloud, mst, self_hosted] +keywords: [IoT, simulate] +--- + +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; +import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; +import ImportData from "versionContent/_partials/_import-data-nyc-taxis.mdx"; +import GeolocationAnalytics from "versionContent/_partials/_use-case-transport-geolocation.mdx"; + +# Analytics on transport and geospatial data + +Real-time analytics refers to the process of collecting, analyzing, and interpreting data instantly as it +is generated. This approach enables you track and monitor activity, and make decisions based on real-time +insights on data stored in a $SERVICE_LONG. + +![Real-time analytics geolocation](https://assets.timescale.com/docs/images/use-case-rta-grafana-heatmap.png) + +This page shows you how to integrate [Grafana][grafana-docs] with a $SERVICE_LONG and make insights based on visualization +of data optimized for size and speed in the columnstore. + +## Prerequisites + + + +* Install and run [self-managed Grafana][grafana-self-managed], or sign up for [Grafana Cloud][grafana-cloud]. + +## Optimize time-series data in hypertables + + + +## Optimize your data for real-time analytics + +[Hypercore][hypercore] is the $COMPANY hybrid row-columnar storage engine used by hypertables. Hypertables partition your data in +chunks. Chunks stored in the rowstore use a row-oriented data format optimized for high-speed inserts and updates. +Chunks stored in the columnstore use a columnar data format optimized for analytics. You ingest `hot` data into the +rowstore. As data cools and becomes more suited for analytics, $CLOUD_LONG automatically converts these chunks of data +to the columnstore. You define the moment when data is converted using a columnstore policy. + +When $CLOUD_LONG converts a chunk to the columnstore, TimescaleDB automatically creates a different schema for your +data. $TIMESCALE_DB creates and uses custom indexes to incorporate the `segmentby` and `orderby` parameters when +you write to and read from the columstore. + +To increase the speed of your analytical queries by a factor of 10 and reduce storage costs by up to 90%, convert data +to the columnstore: + + + +1. **Connect to your $SERVICE_LONG** + + In [$CONSOLE][services-portal] open an [SQL editor][in-console-editors]. The in-Console editors display the query speed. + You can also connect to your service using [psql][connect-using-psql]. + +1. **Enable columnstore on a hypertable** + + Create a [job][job] that automatically moves chunks in a hypertable to the columnstore at a specific time interval. + By default, your table is `orderedby` the time column. For efficient queries on columnstore data, remember to + `segmentby` the column you will use most often to filter your data: + + ```sql + ALTER TABLE rides SET ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'vendor_id', + timescaledb.orderby = 'pickup_datetime DESC' + ); + ``` + +1. **Add a policy to convert chunks to the columnstore at a specific time interval** + + For example, convert data older than 8 days old to the columstore: + ``` sql + CALL add_columnstore_policy('rides', INTERVAL '8 days'); + ``` + See [add_columnstore_policy][add_columnstore_policy]. + +1. **View your data space saving** + + When you convert data to the columnstore, as well as being optimized for analytics, it is compressed by more than 90%. + This saves on storage costs and keeps your queries operating at lightning speed. To see the amount of space saved: + ```sql + SELECT + pg_size_pretty(before_compression_total_bytes) as before, + pg_size_pretty(after_compression_total_bytes) as after + FROM hypertable_compression_stats('rides'); + ``` + You see something like: + + | before | after | + |--|--| + |2818 MB | 673 MB | + +1. **Faster analytical queries on data in the columnstore** + + Now run the analytical query again: + ```sql + SELECT rates.description, COUNT(vendor_id) AS num_trips + FROM rides + JOIN rates ON rides.rate_code = rates.rate_code + WHERE pickup_datetime < '2016-01-08' + GROUP BY rates.description + ORDER BY LOWER(rates.description); + ``` + On this large amount of data, this analytical query on data in the columnstore takes about 6 seconds. + + + +Just to hit this one home, by converting cooling data to the columnstore, you have increased the speed of your analytical +queries by a factor of 10, and reduced storage by up to 90%. + + + + +## Monitor performance over time + +A Grafana dashboard represents a view into the performance of a system, and each dashboard consists of one or +more panels, which represent information about a specific metric related to that system. + +To visually monitor the volume of taxi rides over time: + + + +1. **Create the dashboard** + + 1. On the `Dashboards` page, click `New` and select `New dashboard`. + + 1. Click `Add visualization`. + 1. Select the data source that connects to your $SERVICE_LONG. + The `Time series` visualization is chosen by default. + ![Grafana create dashboard](https://assets.timescale.com/docs/images/use-case-rta-grafana-timescale-configure-dashboard.png) + 1. In the `Queries` section, select `Code`, then select `Time series` in `Format`. + 1. Select the data range for your visualization: + the data set is from 2016. Click the date range above the panel and set: + - From: ```2016-01-01 01:00:00``` + - To: ```2016-01-30 01:00:00``` + +1. **Combine $TIMESCALE_DB and Grafana functionality to analyze your data** + + Combine a $TIMESCALE_DB [time_bucket][use-time-buckets], with the Grafana `$__timefilter()` function to set the + `pickup_datetime` column as the filtering range for your visualizations. + ```sql + SELECT + time_bucket('1 day', pickup_datetime) AS "time", + COUNT(*) + FROM rides + WHERE $__timeFilter(pickup_datetime) + GROUP BY time + ORDER BY time; + ``` + This query groups the results by day and orders them by time. + + ![Grafana real-time analytics](https://assets.timescale.com/docs/images/use-case-rta-grafana-timescale-final-dashboard.png) + +1. **Click `Save dashboard`** + + + + +## Optimize revenue potential + +Having all this data is great but how do you use it? Monitoring data is useful to check what +has happened, but how can you analyse this information to your advantage? This section explains +how to create a visualization that shows how you can maximize potential revenue. + + + +You have integrated Grafana with a $SERVICE_LONG and made insights based on visualization of +your data. + +[grafana-docs]: https://grafana.com/docs/ +[grafana-self-managed]: https://grafana.com/get/?tab=self-managed +[grafana-cloud]: https://grafana.com/get/ +[use-time-buckets]: /use-timescale/:currentVersion:/time-buckets/use-time-buckets/ +[job]: /api/:currentVersion:/actions/add_job/ +[alter_table_hypercore]: /api/:currentVersion:/hypercore/alter_table/ +[compression_continuous-aggregate]: /api/:currentVersion:/hypercore/alter_materialized_view/ +[convert_to_rowstore]: /api/:currentVersion:/hypercore/convert_to_rowstore/ +[convert_to_columnstore]: /api/:currentVersion:/hypercore/convert_to_columnstore/ +[informational-views]: /api/:currentVersion:/informational-views/jobs/ +[add_columnstore_policy]: /api/:currentVersion:/hypercore/add_columnstore_policy/ +[hypercore_workflow]: /api/:currentVersion:/hypercore/#hypercore-workflow +[alter_job]: /api/:currentVersion:/actions/alter_job/ +[remove_columnstore_policy]: /api/:currentVersion:/hypercore/remove_columnstore_policy/ +[in-console-editors]: /getting-started/:currentVersion:/run-queries-from-console/ +[services-portal]: https://console.cloud.timescale.com/dashboard/services +[connect-using-psql]: /integrations/:currentVersion:/psql#connect-to-your-service +[insert]: /use-timescale/:currentVersion:/write-data/insert/ +[hypercore]: /use-timescale/:currentVersion:/hypercore/ diff --git a/use-timescale/OLD-cloud-multi-node.md b/use-timescale/OLD-cloud-multi-node.md index c6dd61f207..5aba352f99 100644 --- a/use-timescale/OLD-cloud-multi-node.md +++ b/use-timescale/OLD-cloud-multi-node.md @@ -6,7 +6,7 @@ keywords: [multi-node, scaling] tags: [cluster, distributed hypertables] --- -import EarlyAccess from "versionContent/_partials/_early_access.mdx"; +import EarlyAccess from "versionContent/_partials/_early_access_2_18_0.mdx"; # Multi-node diff --git a/use-timescale/alerting.md b/use-timescale/alerting.md index 1f711d6061..dafb4a11c2 100644 --- a/use-timescale/alerting.md +++ b/use-timescale/alerting.md @@ -7,6 +7,8 @@ keywords: [alert, integration, Grafana, Datadog, Nagios, Zabbix] # Alerting +Early issue detecting and prevention, ensuring high availability, and performance optimization are only a few of the reasons why alerting plays a major role for modern applications, databases, and services. + There are a variety of different alerting solutions you can use in conjunction with Timescale that are part of the PostgreSQL ecosystem. Regardless of whether you are creating custom alerts embedded in your applications, or using @@ -15,7 +17,7 @@ are a wide selection of tools available. ## Grafana -Grafana is a great way to visualize and explore time-series data and has a +Grafana is a great way to visualize your analytical queries, and it has a first-class integration with Timescale. Beyond data visualization, Grafana also provides alerting functionality to keep you notified of anomalies. @@ -33,7 +35,9 @@ it is designed to take advantage of the database's time-series capabilities. From there, proceed to your dashboard and set up alert rules as described above. + Alerting is only available in Grafana v4.0 and later. + ## Other alerting tools diff --git a/use-timescale/compression/about-compression.md b/use-timescale/compression/about-compression.md index 3a5acd4e71..3889de65c7 100644 --- a/use-timescale/compression/about-compression.md +++ b/use-timescale/compression/about-compression.md @@ -4,11 +4,13 @@ excerpt: When you compress data in a hypertable, multiple records are grouped in products: [self_hosted] keywords: [compression, hypertables] --- - +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; import CompressionIntro from 'versionContent/_partials/_compression-intro.mdx'; # About compression + Replaced by Hypercore. + This section explains how to enable native compression, and then goes into diff --git a/use-timescale/compression/compression-design.md b/use-timescale/compression/compression-design.md index 54100d2d3d..68fffa6163 100644 --- a/use-timescale/compression/compression-design.md +++ b/use-timescale/compression/compression-design.md @@ -5,8 +5,13 @@ products: [cloud, mst, self_hosted] keywords: [compression, schema, tables] --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # Designing for compression + Replaced by Hypercore. + + Time-series data can be unique, in that it needs to handle both shallow and wide queries, such as "What's happened across the deployment in the last 10 minutes," and deep and narrow, such as "What is the average CPU usage for this server diff --git a/use-timescale/compression/compression-methods.md b/use-timescale/compression/compression-methods.md index 5fe3449fcc..a15abfe536 100644 --- a/use-timescale/compression/compression-methods.md +++ b/use-timescale/compression/compression-methods.md @@ -5,8 +5,13 @@ products: [cloud, mst, self_hosted] keywords: [compression] --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # About compression methods + Replaced by Hypercore. + + TimescaleDB uses different compression algorithms, depending on the data type that is being compressed. diff --git a/use-timescale/compression/compression-policy.md b/use-timescale/compression/compression-policy.md index 578a8fdc7d..6fdc5c9526 100644 --- a/use-timescale/compression/compression-policy.md +++ b/use-timescale/compression/compression-policy.md @@ -4,11 +4,14 @@ excerpt: Timescale Cloud can compress your data automatically, based on the cond products: [cloud, mst, self_hosted] keywords: [compression, hypertables, policy] --- - +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; import CompressionIntro from 'versionContent/_partials/_compression-intro.mdx'; # Compression policy + Replaced by Optimize your data for real-time analytics. + + You can enable compression on individual hypertables, by declaring which column you want to segment by. diff --git a/use-timescale/compression/decompress-chunks.md b/use-timescale/compression/decompress-chunks.md index 6a5dff47a1..9f6412a183 100644 --- a/use-timescale/compression/decompress-chunks.md +++ b/use-timescale/compression/decompress-chunks.md @@ -6,8 +6,12 @@ keywords: [compression, hypertables, backfilling] tags: [decompression] --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # Decompression + Replaced by Modify your data in Hypercore. + Timescale automatically supports `INSERT`s into compressed chunks. But if you need to insert a lot of data, for example as part of a bulk backfilling operation, you should first decompress the chunk. Inserting data into a @@ -15,10 +19,12 @@ compressed chunk is more computationally expensive than inserting data into an uncompressed chunk. This adds up over a lot of rows. + When compressing your data, you can reduce the amount of storage space for your Timescale instance. But you should always leave some additional storage capacity. This gives you the flexibility to decompress chunks when necessary, for actions such as bulk inserts. + This section describes commands to use for decompressing chunks. You can filter diff --git a/use-timescale/compression/index.md b/use-timescale/compression/index.md index 7532f19960..c932d0bf28 100644 --- a/use-timescale/compression/index.md +++ b/use-timescale/compression/index.md @@ -4,10 +4,12 @@ excerpt: With data compression, you can achieve a significant improvement in the products: [cloud, mst, self_hosted] keywords: [compression, hypertables] --- - +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; import UsageBasedStorage from "versionContent/_partials/_usage-based-storage-intro.mdx"; -# Compression +# Compression (Replaced by [Hypercore][hypercore]) + + Replaced by Hypercore. Time-series data can be compressed to reduce the amount of storage required, and increase the speed of some queries. This is a cornerstone feature of @@ -17,3 +19,6 @@ data to the form of compressed columns. This occurs across chunks of Timescale hypertables. + + +[hypercore]: /use-timescale/:currentVersion:/hypercore/ diff --git a/use-timescale/compression/manual-compression.md b/use-timescale/compression/manual-compression.md index e8b45c75a9..a6832c02ee 100644 --- a/use-timescale/compression/manual-compression.md +++ b/use-timescale/compression/manual-compression.md @@ -4,9 +4,12 @@ excerpt: Timescale Cloud provides automated and manual data compression. Learn t products: [self_hosted] keywords: [compression, hypertables] --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; # Manually compress chunks + Replaced by Modify your data in Hypercore. + In most cases, an [automated compression policy][add_compression_policy] is sufficient to automatically compress your chunks. However, if you want more control over compression, you can also manually compress specific chunks. @@ -26,12 +29,11 @@ use a hypertable called `example`, and compress chunks older than three days. 1. This returns a list of chunks. Take note of the chunks' names: - ```sql ||show_chunks| |---|---| |1|_timescaledb_internal_hyper_1_2_chunk| |2|_timescaledb_internal_hyper_1_3_chunk| - ``` + @@ -58,12 +60,11 @@ manually compress each one. The results show the chunks for the given hypertable, their compression status, and some other statistics: - ```sql |chunk_schema|chunk_name|compression_status|before_compression_table_bytes|before_compression_index_bytes|before_compression_toast_bytes|before_compression_total_bytes|after_compression_table_bytes|after_compression_index_bytes|after_compression_toast_bytes|after_compression_total_bytes|node_name| |---|---|---|---|---|---|---|---|---|---|---|---| |_timescaledb_internal|_hyper_1_1_chunk|Compressed|8192 bytes|16 kB|8192 bytes|32 kB|8192 bytes|16 kB|8192 bytes|32 kB|| |_timescaledb_internal|_hyper_1_20_chunk|Uncompressed|||||||||| - ``` + 1. Repeat for all chunks you want to compress. diff --git a/use-timescale/compression/modify-a-schema.md b/use-timescale/compression/modify-a-schema.md index 35a482bb2e..5111132d0e 100644 --- a/use-timescale/compression/modify-a-schema.md +++ b/use-timescale/compression/modify-a-schema.md @@ -5,8 +5,12 @@ products: [cloud, mst, self_hosted] keywords: [compression, schemas, hypertables] --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # Schema modifications + Replaced by Modify your data in Hypercore. + You can modify the schema of compressed hypertables in recent versions of Timescale. diff --git a/use-timescale/compression/modify-compressed-data.md b/use-timescale/compression/modify-compressed-data.md index f6f25983a9..8e5cad4808 100644 --- a/use-timescale/compression/modify-compressed-data.md +++ b/use-timescale/compression/modify-compressed-data.md @@ -4,9 +4,12 @@ excerpt: In Timescale Cloud, compressed data can still be modified. Learn to ins products: [cloud, mst, self_hosted] keywords: [compression, backfilling, hypertables] --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; # Insert and modify compressed data + Replaced by Modify your data in Hypercore. + In TimescaleDB 2.11 and later, you can insert data into compressed chunks, and modify data in compressed rows. diff --git a/use-timescale/configuration/advanced-parameters.md b/use-timescale/configuration/advanced-parameters.md index a9b4e687e5..17f6d2812c 100644 --- a/use-timescale/configuration/advanced-parameters.md +++ b/use-timescale/configuration/advanced-parameters.md @@ -9,6 +9,8 @@ cloud_ui: - [services, :serviceId, operations, database_parameters] --- +import ConfigCloudSelf from "versionContent/_partials/_cloud_self_configuration.mdx"; + # Advanced parameters It is possible to configure a wide variety of service database parameters by @@ -23,9 +25,9 @@ prompting you to confirm changes before the service is modified. ## Multiple databases -To create more than one Timescale database, you need to create a new -service for each database. Timescale does not support multiple -databases within the same service. Having a separate service for each database +To create more than one database, you need to create a new +$SERVICE_SHORT for each database. $CLOUD_LONG does not support multiple +databases within the same service. Having a separate $SERVICE_SHORT for each database affords each database its own isolated resources. You can also use [schemas][schemas] to organize tables into logical groups. A @@ -33,6 +35,9 @@ single database can contain multiple schemas, which in turn contain tables. The main difference between isolating with databases versus schemas is that a user can access objects in any of the schemas in the database they are connected to, so long as they have the corresponding privileges. Schemas can help isolate -smaller use cases that do not warrant their own service. +smaller use cases that do not warrant their own $SERVICE_SHORT. + + + [schemas]: /use-timescale/:currentVersion:/schema-management/ diff --git a/use-timescale/continuous-aggregates/about-continuous-aggregates.md b/use-timescale/continuous-aggregates/about-continuous-aggregates.md index 85dab2c3d9..f6182ed439 100644 --- a/use-timescale/continuous-aggregates/about-continuous-aggregates.md +++ b/use-timescale/continuous-aggregates/about-continuous-aggregates.md @@ -212,30 +212,22 @@ Continuous aggregates consist of: ### Materialization hypertable Continuous aggregates take raw data from the original hypertable, aggregate it, -and store the intermediate state in a materialization hypertable. When you query -the continuous aggregate view, the state is returned to you as needed. +and store the aggregated data in a materialization hypertable. When you query +the continuous aggregate view, the aggregated data is returned to you as needed. Using the same temperature example, the materialization table looks like this: -|day|location|chunk|avg temperature partial| +|day|location|chunk|avg temperature| |-|-|-|-| -|2021/01/01|New York|1|{3, 219}| -|2021/01/01|Stockholm|1|{4, 280}| +|2021/01/01|New York|1|73| +|2021/01/01|Stockholm|1|70| |2021/01/02|New York|2|| -|2021/01/02|Stockholm|2|{5, 345}| +|2021/01/02|Stockholm|2|69| The materialization table is stored as a Timescale hypertable, to take advantage of the scaling and query optimizations that hypertables offer. Materialization tables contain a column for each group-by clause in the query, -a `chunk` column identifying which chunk in the raw data this entry came from, -and a `partial aggregate` column for each aggregate in the query. - -The partial column is used internally to calculate the output. In this example, -because the query looks for an average, the partial column contains the number -of rows seen, and the sum of all their values. The most important thing to know -about partials is that they can be combined to create new partials spanning all -of the old partials' rows. This is important if you combine groups that span -multiple chunks. +and an `aggregate` column for each aggregate in the query. For more information, see [materialization hypertables][cagg-mat-hypertables]. @@ -248,13 +240,6 @@ unblocks other transactions, and materializes the aggregates. The first transaction is very quick, and most of the work happens during the second transaction, to ensure that the work does not interfere with other operations. -When you query the continuous aggregate view, the materialization engine -combines the aggregate partials into a single partial for each time range, and -calculates the value that is returned. For example, to compute an average, each -partial sum is added up to a total sum, and each partial count is added up to a -total count, then the average is computed as the total sum divided by the total -count. - ### Invalidation engine Any change to the data in a hypertable could potentially invalidate some diff --git a/use-timescale/continuous-aggregates/create-a-continuous-aggregate.md b/use-timescale/continuous-aggregates/create-a-continuous-aggregate.md index 6b1228d27e..85d3787136 100644 --- a/use-timescale/continuous-aggregates/create-a-continuous-aggregate.md +++ b/use-timescale/continuous-aggregates/create-a-continuous-aggregate.md @@ -75,14 +75,6 @@ the hypertable. The time bucket allows you to define a time interval, instead of having to use specific timestamps. For example, you can define a time bucket as five minutes, or one day. -When the continuous aggregate is materialized, the materialization table stores -partials, which are then used to calculate the result of the query. This means a -certain amount of processing capacity is required for any query, and the amount -required becomes greater as the interval gets smaller. Because of this, if you -have very small intervals, it can be more efficient to run the aggregate query -on the raw data in the hypertable. You should test both methods to determine -what is best for your dataset and desired bucket interval. - You can't use [time_bucket_gapfill][api-time-bucket-gapfill] directly in a continuous aggregate. This is because you need access to previous data to determine the gapfill content, which isn't yet available when you create the diff --git a/use-timescale/continuous-aggregates/hierarchical-continuous-aggregates.md b/use-timescale/continuous-aggregates/hierarchical-continuous-aggregates.md index 985b2ce6cf..957edea425 100644 --- a/use-timescale/continuous-aggregates/hierarchical-continuous-aggregates.md +++ b/use-timescale/continuous-aggregates/hierarchical-continuous-aggregates.md @@ -6,9 +6,9 @@ keywords: [continuous aggregates, hierarchical, create] # Hierarchical continuous aggregates -You can create continuous aggregates on top of other continuous aggregates. This -allows you to summarize data at different levels of granularity. For example, -you might have an hourly continuous aggregate that summarizes minute-by-minute +The more data you have, the more likely you are to run a more sophisticated analysis on it. When a simple one-level aggregation is not enough, $CLOUD_LONG lets you create continuous aggregates on top of other continuous aggregates. This way, you summarize data at different levels of granularity, while still saving resources with precomputing. + +For example, you might have an hourly continuous aggregate that summarizes minute-by-minute data. To get a daily summary, you can create a new continuous aggregate on top of your hourly aggregate. This is more efficient than creating the daily aggregate on top of the original hypertable, because you can reuse the diff --git a/use-timescale/continuous-aggregates/index.md b/use-timescale/continuous-aggregates/index.md index 84f701c6dd..902cf0187e 100644 --- a/use-timescale/continuous-aggregates/index.md +++ b/use-timescale/continuous-aggregates/index.md @@ -7,22 +7,20 @@ keywords: [continuous aggregates] # Continuous aggregates -Continuous aggregates are designed to make queries on very large datasets run -faster. Timescale continuous aggregates use -PostgreSQL [materialized views][postgres-materialized-views] to continuously and -incrementally refresh a query in the background, so that when you run the query, -only the data that has changed needs to be computed, not the entire dataset. +From real-time dashboards to performance monitoring and historical trend analysis, data aggregation is a must-have for any sort of analytical application. To address this need, $CLOUD_LONG continuous aggregates precompute and store aggregate data for you. Using PostgreSQL [materialized views][postgres-materialized-views], continuous aggregates incrementally refresh the aggregation query in the background, so that when you do run it, only the data that has changed needs to be computed, not the entire dataset. This means you always have the latest aggregate data at your fingertips - and spend as little resources on it, as possible. + +In this section you: * [Learn about continuous aggregates][about-caggs] to understand how it works before you begin using it. * [Create a continuous aggregate][cagg-create] and query it. -* [Create a continuous aggregate on top of another continuous aggregate][cagg-on-cagg] +* [Create a continuous aggregate on top of another continuous aggregate][cagg-on-cagg]. * [Add refresh policies][cagg-autorefresh] to an existing continuous aggregate. * [Manage time][cagg-time] in your continuous aggregates. * [Drop data][cagg-drop] from your continuous aggregates. * [Manage materialized hypertables][cagg-mat-hypertables]. * [Use real-time aggregates][cagg-realtime]. -* [Compression with continuous aggregates][cagg-compression]. +* [Use compression with continuous aggregates][cagg-compression]. * [Migrate your continuous aggregates][cagg-migrate] from old to new format. Continuous aggregates created in Timescale 2.7 and later are in the new format, unless explicitly created in the old format. diff --git a/use-timescale/continuous-aggregates/materialized-hypertables.md b/use-timescale/continuous-aggregates/materialized-hypertables.md index 9a87388511..999d41b398 100644 --- a/use-timescale/continuous-aggregates/materialized-hypertables.md +++ b/use-timescale/continuous-aggregates/materialized-hypertables.md @@ -9,7 +9,7 @@ tags: [materialized views] # Materialized hypertables Continuous aggregates take raw data from the original hypertable, aggregate it, -and store the intermediate state in a materialization hypertable. You can modify +and store the aggregated data in a materialization hypertable. You can modify this materialized hypertable in the same way as any other hypertable. ## Discover the name of a materialized hypertable diff --git a/use-timescale/continuous-aggregates/real-time-aggregates.md b/use-timescale/continuous-aggregates/real-time-aggregates.md index cd861e20f5..4a7863b80b 100644 --- a/use-timescale/continuous-aggregates/real-time-aggregates.md +++ b/use-timescale/continuous-aggregates/real-time-aggregates.md @@ -7,35 +7,36 @@ keywords: [continuous aggregates, real-time aggregates] import CaggsRealTimeHistoricalDataRefreshes from 'versionContent/_partials/_caggs-real-time-historical-data-refreshes.mdx'; -# Real time aggregates +# Real-time aggregates -Continuous aggregates do not include the most recent data chunk from the -underlying hypertable. Real time aggregates use the aggregated data and add the -most recent raw data to it to provide accurate and up to date results, without -needing to aggregate data as it is being written. In Timescale versions 1.7 to 2.12, -real time aggregates are enabled by default; when you create a continuous +Rapidly growing data means you need more control over what to aggregate and how to aggregate it. With this in mind, $CLOUD_LONG equips you with tools for more fine-tuned data analysis. + +By default, continuous aggregates do not include the most recent data chunk from the +underlying hypertable. Real-time aggregates, however, use the aggregated data **and** add the +most recent raw data to it. This provides accurate and up-to-date results, without +needing to aggregate data as it is being written. + +In Timescale 2.13 and later real-time aggregates are *DISABLED* by default. In Timescale versions 1.7 to 2.12, real-time aggregates are enabled by default; when you create a continuous aggregate view, queries to that view include the most recent data, even if -it has not yet been aggregated. In Timescale 2.13 and later real time aggregates are *DISABLED* by default. +it has not yet been aggregated. -For more detail on the comparison between continuous and real time aggregates, +For more detail on the comparison between continuous and real-time aggregates, see our [real time aggregate blog post][blog-rtaggs]. -## Use real time aggregates +## Use real-time aggregates -You can enable and disable real time aggregation by setting the +You can enable and disable real-time aggregation by setting the `materialized_only` parameter when you create or alter the view. -### Using real time aggregation - -1. For an existing table, at the `psql` prompt, disable real time aggregation: +1. For an existing table, at the `psql` prompt, disable real-time aggregation: ```sql ALTER MATERIALIZED VIEW table_name set (timescaledb.materialized_only = true); ``` -1. Re-enable real time aggregation: +1. Re-enable real-time aggregation: ```sql ALTER MATERIALIZED VIEW table_name set (timescaledb.materialized_only = false); diff --git a/use-timescale/continuous-aggregates/refresh-policies.md b/use-timescale/continuous-aggregates/refresh-policies.md index f29d213490..b9be7e2a2e 100644 --- a/use-timescale/continuous-aggregates/refresh-policies.md +++ b/use-timescale/continuous-aggregates/refresh-policies.md @@ -31,18 +31,16 @@ Among others, `add_continuous_aggregate_policy` takes the following arguments: 24 hours. If you set the `start_offset` or `end_offset` to `NULL`, the range is open-ended -and extends to the beginning or end of time. However, it's recommended to set -the `end_offset` so that at least the most recent time bucket is excluded. For -time-series data that mostly contains writes that occur in time stamp order, the -time buckets that see lots of writes quickly have out-of-date aggregates. You -get better performance by excluding the time buckets that are getting a lot of -writes. - -In addition, materializing the most recent bucket might interfere with -[real-time aggregation][future-watermark]. - -See the [API reference][api-reference] -for the full list of required and optional arguments and use examples. +and extends to the beginning or end of time. + +If you set `end_offset` within the current time bucket, this bucket is excluded. This is done for the following reasons: + +- The current bucket is incomplete and can't be refreshed. +- The current bucket gets lots of writes in the time-stamp order and its aggregate becomes outdated very quickly. Excluding it improves performance. + +To include the current time bucket, enable [real-time aggregation][future-watermark]. In Timescale 2.13 and later, it is disabled by default. + +See the [API reference][api-reference] for the full list of required and optional arguments and use examples. diff --git a/use-timescale/data-retention/about-data-retention.md b/use-timescale/data-retention/about-data-retention.md index 331e4ccaeb..ee879d975b 100644 --- a/use-timescale/data-retention/about-data-retention.md +++ b/use-timescale/data-retention/about-data-retention.md @@ -9,15 +9,12 @@ import UsageBasedStorage from "versionContent/_partials/_usage-based-storage-int # About data retention -In time-series applications, data often becomes less useful as it gets older. If -you don't need your historical data, you can delete it once it reaches a certain -age. Timescale lets you set up -[automatic data retention policies][retention-policy] to discard old data. You -can also fine-tune data retention by [manually dropping chunks][manual-drop]. - -Often, you want to keep summaries of your historical data, but you don't need -the raw data. You can downsample your older data by -[combining data retention with continuous aggregates][retention-with-caggs]. +In modern applications, data grows exponentially. As data gets older, it often becomes less useful in day-to-day operations. +However, you still need it for analysis. Timescale elegantly solves this problem with +[automated data retention policies][retention-policy]. + +Data retention policies delete raw old data for you on a schedule that you define. +By [combining retention policies with continuous aggregates][retention-with-caggs], you can downsample your data and keep useful summaries of it instead. This lets you analyze historical data - while also saving on storage. diff --git a/use-timescale/data-tiering/about-data-tiering.md b/use-timescale/data-tiering/about-data-tiering.md index ec6aa3e656..5a6921eb8e 100644 --- a/use-timescale/data-tiering/about-data-tiering.md +++ b/use-timescale/data-tiering/about-data-tiering.md @@ -11,12 +11,16 @@ cloud_ui: # About the object storage tier -Timescale's tiered storage architecture includes a standard high-performance storage tier and a low-cost object storage tier built on Amazon S3. You can use the standard tier for data that requires quick access, and the object tier for rarely used historical data. Chunks from a single hypertable, including compressed chunks, can stretch across these two storage tiers. A compressed chunk uses a different storage representation after tiering. +$COMPANY's tiered storage architecture includes a standard high-performance storage tier and a low-cost object storage tier built on Amazon S3. You can use the standard tier for data that requires quick access, and the object tier for rarely used historical data. Chunks from a single hypertable, including compressed chunks, can stretch across these two storage tiers. -In the high-performance storage, chunks are stored in the block format. In the object storage, they are stored in a compressed, columnar format. For better interoperability across various platforms, this format is different from that of the internals of the database. It allows for more efficient columnar scans across longer time periods, and Timescale Cloud uses other metadata and query optimizations to reduce the amount of data that needs to be fetched from the object storage tier to satisfy a query. +In the high-performance storage, your data is stored in the block format. In the object storage, it is stored in [Apache Parquet][parquet]. The original size of the data in your $SERVICE_SHORT, compressed or uncompressed, does not correspond directly to its size in S3. A compressed hypertable may even take more space in S3 than it does in $CLOUD_LONG. + +Apache Parquet allows for more efficient scans across longer time periods, and $CLOUD_LONG uses other metadata and query optimizations to reduce the amount of data that needs to be fetched from the object storage tier to satisfy a query. Regardless of where your data is stored, you can still query it with standard SQL. A single SQL query transparently pulls data from the appropriate chunks using the chunk exclusion algorithms. You can `JOIN` against tiered data, build views, and even define continuous aggregates on it. In fact, because the implementation of continuous aggregates also uses hypertables, they can be tiered to low-cost storage as well. +$COMPANY charges only for the storage that your data occupies in S3, regardless of whether it was compressed in $CLOUD_LONG before tiering. There are no additional expenses, such as data transfer or compute. + ## Benefits of the object storage tier The object storage tier is more than an archiving solution. It is also: @@ -119,3 +123,4 @@ for `device_uuid`, `sensor_id`, and `observed_at` as the query needs only these [blog-data-tiering]: https://www.timescale.com/blog/expanding-the-boundaries-of-postgresql-announcing-a-bottomless-consumption-based-object-storage-layer-built-on-amazon-s3/ [querying-tiered-data]: /use-timescale/:currentVersion:/data-tiering/querying-tiered-data/ +[parquet]: https://parquet.apache.org/ \ No newline at end of file diff --git a/use-timescale/data-tiering/enabling-data-tiering.md b/use-timescale/data-tiering/enabling-data-tiering.md index 4ae589a219..ce4ad620a4 100644 --- a/use-timescale/data-tiering/enabling-data-tiering.md +++ b/use-timescale/data-tiering/enabling-data-tiering.md @@ -9,6 +9,8 @@ cloud_ui: - [services, :serviceId, overview] --- +import TieredStorageBilling from "versionContent/_partials/_tiered-storage-billing.mdx"; + # Manage automated and manual tiering You use tiered storage to save on storage costs. Specifically, you can migrate rarely used data from @@ -18,6 +20,8 @@ or [manually tier and untier data][manual-tier]. You can query the data on the object storage tier, but you cannot modify it. Make sure that you are not tiering data that needs to be **actively modified**. + + ## Enable tiered storage You enable tiered storage from the `Overview` tab in Console. @@ -34,14 +38,14 @@ You enable tiered storage from the `Overview` tab in Console. When tiered storage is enabled, you see the amount of data in the tiered object storage. - - - Data tiering is available in [Scale and Enterprise][pricing-plans] pricing plans only. - - - + + +Data tiering is available in [Scale and Enterprise][pricing-plans] pricing plans only. + + + ## Automate tiering with policies A tiering policy automatically moves any chunks that only contain data diff --git a/use-timescale/extensions/index.md b/use-timescale/extensions/index.md index ccfbb11fee..22d0e0ae0c 100644 --- a/use-timescale/extensions/index.md +++ b/use-timescale/extensions/index.md @@ -75,17 +75,18 @@ supported extensions: ## Third-party extensions -| Extension | Description | Enabled by default | -|--------------------------------------------------|------------------------------------------------------------|-----------------------| -| [pgaudit][pgaudit] | Detailed session and/or object audit logging | - | -| [pgpcre][pgpcre] | Perl-compatible RegEx | - | -| [pgrouting][pgrouting] | Geospatial routing functionality | - | -| [postgis][postgis] | PostGIS geometry and geography spatial types and functions | - | -| [postgis_raster][postgis-raster] | PostGIS raster types and functions | - | -| [postgis_sfcgal][postgis-sfcgal] | PostGIS SFCGAL functions | - | -| [postgis_tiger_geocoder][postgis-tiger-geocoder] | PostGIS tiger geocoder and reverse geocoder | - | -| [postgis_topology][postgis-topology] | PostGIS topology spatial types and functions | - | -| [unit][unit] | SI units for PostgreSQL | - | +| Extension | Description | Enabled by default | +|--------------------------------------------------|------------------------------------------------------------|--------------------| +| [h3][h3] | H3 bindings for PostgreSQL | - | +| [pgaudit][pgaudit] | Detailed session and/or object audit logging | - | +| [pgpcre][pgpcre] | Perl-compatible RegEx | - | +| [pgrouting][pgrouting] | Geospatial routing functionality | - | +| [postgis][postgis] | PostGIS geometry and geography spatial types and functions | - | +| [postgis_raster][postgis-raster] | PostGIS raster types and functions | - | +| [postgis_sfcgal][postgis-sfcgal] | PostGIS SFCGAL functions | - | +| [postgis_tiger_geocoder][postgis-tiger-geocoder] | PostGIS tiger geocoder and reverse geocoder | - | +| [postgis_topology][postgis-topology] | PostGIS topology spatial types and functions | - | +| [unit][unit] | SI units for PostgreSQL | - | [amcheck]: https://www.postgresql.org/docs/current/amcheck.html [autoinc]: https://www.postgresql.org/docs/current/contrib-spi.html#CONTRIB-SPI-AUTOINC @@ -100,6 +101,7 @@ supported extensions: [earthdistance]: https://www.postgresql.org/docs/current/earthdistance.html [fuzzystrmatch]: https://www.postgresql.org/docs/current/fuzzystrmatch.html [hstore]: https://www.postgresql.org/docs/current/hstore.html +[h3]: https://pgxn.org/dist/h3/ [insert-username]: https://www.postgresql.org/docs/current/contrib-spi.html#CONTRIB-SPI-INSERT-USERNAME [intagg]: https://www.postgresql.org/docs/current/intagg.html [intarray]: https://www.postgresql.org/docs/current/intarray.html diff --git a/use-timescale/ha-replicas/high-availability.md b/use-timescale/ha-replicas/high-availability.md index bfb3c56766..3834fe5749 100644 --- a/use-timescale/ha-replicas/high-availability.md +++ b/use-timescale/ha-replicas/high-availability.md @@ -9,6 +9,8 @@ cloud_ui: - [services, :serviceId, operations, replication] --- +import HASetup from 'versionContent/_partials/_high-availability-setup.mdx'; + # Manage high availability @@ -71,26 +73,12 @@ The `High` and `Highest` HA strategies are available with the [Scale and the Ent To enable HA for a Timescale Cloud Service: - - -1. In [Timescale Console][cloud-login], select the service to enable replication for. -1. Click `Operations`, then select `High availability`. -1. Choose your replication strategy, then click `Change configuration`. - Creating a database replica in Timescale - -1. In `Change high availability configuration`, click `Change config`. + To change your HA replica strategy, click `Change configuration`, choose a strategy and click `Change configuration`. To download the connection information for the HA replica, either click the link next to the replica `Active configuration`, or find the information in the `Overview` tab for this service. - - - ## Test failover for your HA replicas To test the failover mechanism, you can trigger a switchover. A switchover is a diff --git a/use-timescale/hypercore/best-practice-for-large-amounts-of-data.md b/use-timescale/hypercore/best-practice-for-large-amounts-of-data.md new file mode 100644 index 0000000000..e01b8e802f --- /dev/null +++ b/use-timescale/hypercore/best-practice-for-large-amounts-of-data.md @@ -0,0 +1,188 @@ +--- +title: Indexing data in Hypercore +excerpt: Use indexes on hybrid row-columnar data +products: [cloud,] +keywords: [hypertable, compression, row-columnar storage, hypercore] +--- + +import Prereq from "versionContent/_partials/_prereqs-cloud-and-self.mdx"; +import HypercoreManualWorkflow from "versionContent/_partials/_hypercore_manual_workflow.mdx"; + +# SUGGESTION: Best practice for large amounts of data + +IAIN: Hi Mats, I left this in here so you do no lose your work. + +Indexes are a central component to designing efficient and performant +databases: they allow unstructured data to be efficiently searched to +find the data of interests and in that way allow efficient execution +of queries. + +Indexes are similar to a alpabetical book index: you can quickly find +the right name in the index at the end of the book because it is +alphabetical and it will give you the page and maybe the paragraph +where the name is. + +Indexes uses a *key*—for example the pair first-last name "John, +Doe"—to search through the index and find the *page*. + +## Indexes and Table Access Methods in PostgreSQL + +The keys of an index can be either be a single column, or consist of +multiple colums. For example, using the example hypertable from the +previous chapter: + +```sql +create table metrics( + metric_id serial, + created_at timestamptz not null unique, + location_id smallint, + device_id bigint, + temp float8, + humidity float4 +); +``` + +You can then define an index with the name `metrics_device_id_idx` for +the devices, which will allow you to efficiently find the device +information in the `metrics` table. + +```sql +create index metrics_device_id_idx on metrics (device_id); +``` + +In the same way as indexes for books, indexes in PostgreSQL are data +structures that translate keys to a *disk page* and an *offset*, which +are in PostgreSQL called a *tuple identifier* (or just *TID*). The TID +corresponds to the page for the data in the book analogy. You can +imagine that the index is a sorted list of rows like this, with the +keys each in one column and the TID in a separate column with the page +and the offset: + +| `device_id` | TID | +|-------------|------| +| . | . | +| . | . | +| . | . | +| 2 | 5:32 | +| 3 | 6:19 | +| 3 | 7:23 | +| 3 | 4:99 | +| 4 | 1:79 | +| 4 | 4:51 | +| 5 | 2:45 | +| 6 | 1:23 | +| . | . | +| . | . | +| . | . | + +When PostgreSQL then want to use the index, it can locate the TIDs for +a particular key. For example, if you execute the query: + +```sql +select * from metrics where device_id = 3; +``` + +PostgreSQL will first use the index to locate all the TIDs for the key +`3` (that is, `6:19`, `7:23`, and `4:99`), but then it need to find +the tuples (the entire row with data) for those TIDs, so how is that +done? The answer is that it is the responsibility of the *table access +method* for the table to figure that out. + +Each table in PostgreSQL has an associated table access method. The +default one is called the "heap". The TID is passed to the table +access method in question, which then returns the entire tuple. + +You can see the table access method for a table in this manner: + +```sql +example=# \d+ metrics + Table "public.metrics" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +-------------+--------------------------+-----------+----------+--------------------------------------------+---------+-------------+--------------+------------- + metric_id | integer | | not null | nextval('metrics_metric_id_seq'::regclass) | plain | | | + created_at | timestamp with time zone | | not null | | plain | | | + location_id | smallint | | | | plain | | | + device_id | bigint | | | | plain | | | + temp | double precision | | | | plain | | | + humidity | real | | | | plain | | | +Indexes: + "metrics_created_at_key" UNIQUE CONSTRAINT, btree (created_at) +Access method: heap +``` + +Since data for the Hypercore columnstore is stored in a combined row +and columnar format (that is, some data is in row format and some are +in columnar format), Timescale has a dedicated table access method +called `hypercore` that allow chunks of a hypertable to maintain +indexes over the combined row-columnar format that Hypercore uses. + +## Creating a new hypertable using Hypercore TAM + +This case is normally used if you want to create a new hypertable +where all chunks uses the `hypercore` table access method. + +You can create a new hypertable using the Hypercore TAM using the +following steps: + +1. Create the table using `CREATE TABLE`, for example, the table + above. +2. Transform the table into a hypertable using `create_hypertable` and + use the `created_at` column for partitioning the hypertable. + + ```sql + select create_hypertable('readings', by_range('created_at')); + ``` + +3. Configure compression using `alter table` as in the previous + chapter, but set the table access method to `hypercore`. + + ```sql + alter table metrics + set access method hypercore, + set (timescaledb.compress_orderby = 'created_at', + timescaledb.compress_segmentby = 'location_id'); + ``` + + Setting the table access method in this manner for a hypertable + means that any new chunks created when inserting data will use the + `hypercore` table access method. + +4. Set up a policy as in the previous chapter, but tell the policy to + use the (table) access method to compress the chunks using the + `hypercore_use_access_method` parameter. + + ```sql + CALL add_columnstore_policy('metrics', interval '1 day', + hypercore_use_access_method => true); + ``` + +## Changing a single chunk to replaced by Hypercore TAM + +If you have an existing hypertable, and want to just change a single +chunk to use the `hypercore` table access method, you can change the +table access method for that chunk only either using `alter table` or +the procedure `convert_to_columnstore`. + +This can be useful if you want to experiment with indexing on +compressed data, or if you just want to index some chunks of the +hypertable. + +As an example, if you have a chunk +`_timescaledb_internal._hyper_1_21_chunk` (in this case from the +`metrics` table above), you can convert it to a columnstore chunk +using the table access method either by setting the access method: + +```sql +alter table _timescaledb_internal._hyper_1_21_chunk + set access method hypercore; +``` + +Or using the `convert_to_columnstore` procedure with +`hypercore_use_access_method`: + +```sql +call convert_to_columnstore('_timescaledb_internal._hyper_1_21_chunk', + hypercore_use_access_method => true); +``` + +This will also rebuild indexes for the chunk if it was compressed, s diff --git a/use-timescale/hypercore/efficient-data-access-in-hypercore.md b/use-timescale/hypercore/efficient-data-access-in-hypercore.md new file mode 100644 index 0000000000..308742d27a --- /dev/null +++ b/use-timescale/hypercore/efficient-data-access-in-hypercore.md @@ -0,0 +1,75 @@ +--- +title: Efficient data access in Hypercore +excerpt: Read and write data efficiently in Hypercore. +products: [cloud,self_hosted] +keywords: [hypertable, compression, row-columnar storage, hypercore] +--- +import Prereq from "versionContent/_partials/_prereqs-cloud-and-self.mdx"; + +# Access your data efficiently in Hypercore + + +Intro + + + +This page shows you how to ... . + +## Prerequisites + + + +- [Prepare your data for real-time analytics in Hypercore][setup-hypercore] + + +## Read data efficiently + +Explain why and how to read data efficiently. + + + +1. **Do this** + + Why and how to do this. + + ``` sql + Code example + ``` + +1. **Do that** + + Why and how to do that. + + ``` sql + Code example + ``` + + + +## Write data efficiently + + +Explain why and how to write data efficiently. + + + +1. **Do this** + + Why and how to do this. + + ``` sql + Code example + ``` + +1. **Do that** + + Why and how to do that. + + ``` sql + Code example + ``` + + + + +[setup-hypercore]: /use-timescale/:currentVersion:/hypercore/real-time-analytics-in-hypercore/ diff --git a/use-timescale/hypercore/index.md b/use-timescale/hypercore/index.md new file mode 100644 index 0000000000..2734840dc5 --- /dev/null +++ b/use-timescale/hypercore/index.md @@ -0,0 +1,59 @@ +--- +title: Hypercore +excerpt: The Timescale hybrid row-columnar storage engine for real-time analytics, powered by time-series data +products: [cloud,] +keywords: [hypercore, hypertable, compression, row-columnar storage] +--- + +import UsageBasedStorage from "versionContent/_partials/_usage-based-storage-intro.mdx"; + +# Hypercore + +Hypercore is the $TIMESCALE_DB hybrid row-columnar storage engine, designed specifically for +real-time analytics and powered by time-series data. The advantage of Hypercore is its ability +to seamlessly switch between row-oriented and column-oriented storage. This flexibility enables +$CLOUD_LONG to deliver the best of both worlds, solving the key challenges in real-time analytics: + +- High ingest throughput +- Low-latency ingestion +- Fast query performance +- Efficient handling of data updates and late-arriving data +- Streamlined data management + +Hypercore’s hybrid approach combines the benefits of row-oriented and column-oriented formats +in each $SERVICE_LONG: + +- **Fast ingest with rowstore**: new data is initially written to the rowstore, which is optimized for + high-speed inserts and updates. This process ensures that real-time applications easily handle + rapid streams of incoming data. Mutability—upserts, updates, and deletes happen seamlessly. + +- **Efficient analytics with columnstore**: as the data _cools_ and becomes more suited for + analytics, it is automatically migrated to the columnstore. This columnar format enables + fast scanning and aggregation, optimizing performance for analytical workloads while also + saving significant storage space. + +- **Faster queries on compressed data in columnstore**: in columnstore conversion, hypertable + chunks are compressed by more than 90%, and organized for efficient, large-scale queries. Combined with [chunk skipping][chunk-skipping], this helps you save on storage costs and keeps your queries operating at lightning speed. + +- **Full mutability with transactional semantics**: regardless of where data is stored, + Hypercore provides full ACID support. Like in a vanilla Postgres database, inserts and updates + to the rowstore and columnstore are always consistent, and available to queries as soon as they are + completed. + + +![Hypercore workflow](https://assets.timescale.com/docs/images/hypercore-overview.png) + +In Timescale Cloud you only pay for what you use. Data moved to the columnstore is compressed, which +immediately translates into cost savings. + + +This section shows you how to: + +* [Optimize your data for real-time analytics][setup-hypercore] +* [Modify data in the columnstore][modify-data-in-hypercore] + +[setup-hypercore]: /use-timescale/:currentVersion:/hypercore/real-time-analytics-in-hypercore/ +[modify-data-in-hypercore]: /use-timescale/:currentVersion:/hypercore/modify-data-in-hypercore/ +[indexing-data-in-hypercore]: /use-timescale/:currentVersion:/hypercore/best-practice-for-large-amounts-of-data/ +[compression]: /use-timescale/:currentVersion:/compression/ +[chunk-skipping]: /use-timescale/:currentVersion:/hypertables/improve-query-performance/ diff --git a/use-timescale/hypercore/modify-data-in-hypercore.md b/use-timescale/hypercore/modify-data-in-hypercore.md new file mode 100644 index 0000000000..b0162628ec --- /dev/null +++ b/use-timescale/hypercore/modify-data-in-hypercore.md @@ -0,0 +1,110 @@ +--- +title: Modify data in Hypercore +excerpt: Update data and the table schema in the columnstore +products: [cloud,] +keywords: [hyperscore, hypertable, compression, row-columnar storage, hypercore] +--- + +import PrereqCloud from "versionContent/_partials/_prereqs-cloud-and-self.mdx"; +import HypercoreManualWorkflow from "versionContent/_partials/_hypercore_manual_workflow.mdx"; + +# Modify your data in Hypercore + +You [set up Hypercore][setup-hypercore] to automatically convert data between the rowstore and columnstore +when it reaches a certain age. After you have optimized data in the columnstore, you may need to modify it. +For example, to make small changes, or backfill large amounts of data. You may even have to update the schema to +accommodate these changes to the data. + +This page shows you how to update small and large amounts of new data, and update the schema in columnstore. + +## Prerequisites + + + +- [Optimize your data for real-time analytics][setup-hypercore] + +## Modify small amounts of data + +You can [`INSERT` `UPDATE` and `DELETE`][write] data in the columnstore, even if the data you are +inserting has unique constraints. When you insert data into a chunk in the columnstore, a small amount +of data is decompressed to allow a speculative insertion, and block any inserts which could violate +constraints. + +When you `DELETE` whole segments of data, filter your deletes using the column you `segment_by` +instead of separate deletes. This considerably increases performance. + +## Modify large amounts of data + +If you need to modify or add a lot of data to a chunk in the columnstore, best practice is to stop +any [jobs][job] moving chunks to the columnstore, convert the chunk back to the rowstore, then modify the +data. After the update, [convert the chunk to the columnstore][convert_to_columnstore] and restart the jobs. +This workflow is especially useful if you need to backfill old data. + + + + + + + +## Modify a table schema for data in the columnstore + +You can modify the schema of a table in the columnstore. To do this, you need to: + + + +1. **Stop the jobs that are automatically adding chunks to the columnstore** + + Retrieve the list of jobs from the [timescaledb_information.jobs][informational-views] view + to find the job you need to [alter_job][alter_job]. + + ``` sql + SELECT alter_job(JOB_ID, scheduled => false); + ``` + +1. **Convert a chunk to update back to the rowstore** + + ``` sql + CALL convert_to_rowstore('_timescaledb_internal._hyper_2_2_chunk'); + ``` + +2. **Modify the schema**: + + Possible modifications are: + + - Add a nullable column: + + `ALTER TABLE ADD COLUMN ;` + - Add a column with a default value and a `NOT NULL` constraint: + + `ALTER TABLE ADD COLUMN NOT NULL DEFAULT ;` + - Rename a column: + + `ALTER TABLE RENAME TO ;` + - Drop a column: + + `ALTER TABLE DROP COLUMN ;` + + You cannot change the data type of an existing column. + +1. **Convert the updated chunks back to the columnstore** + + ``` sql + CALL convert_to_columnstore('_timescaledb_internal._hyper_1_2_chunk'); + ``` + +1. **Restart the jobs that are automatically converting chunks to the columnstore** + + ``` sql + SELECT alter_job(JOB_ID, scheduled => true); + ``` + + + +[write]: /use-timescale/:currentVersion:/write-data/ +[setup-hypercore]: /use-timescale/:currentVersion:/hypercore/real-time-analytics-in-hypercore/ +[job]: /api/:currentVersion:/actions/ +[alter_job]: /api/:currentVersion:/actions/alter_job/ +[convert_to_columnstore]: /api/:currentVersion:/hypercore/convert_to_columnstore/ +[informational-views]: /api/:currentVersion:/informational-views/jobs/ +[insert]: /use-timescale/:currentVersion:/write-data/insert/ +[setup-hypercore]: /use-timescale/:currentVersion:/hypercore/real-time-analytics-in-hypercore/ diff --git a/use-timescale/hypercore/real-time-analytics-in-hypercore.md b/use-timescale/hypercore/real-time-analytics-in-hypercore.md new file mode 100644 index 0000000000..d7b0c3296c --- /dev/null +++ b/use-timescale/hypercore/real-time-analytics-in-hypercore.md @@ -0,0 +1,83 @@ +--- +title: Optimize your data for real-time analytics +excerpt: Reduce your chunk size by more than 90% and speed up your queries by automatically converting data between the rowstore and columnstore. +products: [cloud,] +keywords: [hyperscore, hypertable, compression, row-columnar storage, hypercore] +--- + +import PrereqCloud from "versionContent/_partials/_prereqs-cloud-only.mdx"; +import HCConversionOverview from "versionContent/_partials/_hypercore-conversion-overview.mdx"; +import HCPolicyWorkflow from "versionContent/_partials/_hypercore_policy_workflow.mdx"; +import UsageBasedStorage from "versionContent/_partials/_usage-based-storage-intro.mdx"; +import EarlyAccess from "versionContent/_partials/_early_access_2_18_0.mdx"; + +# Optimize your data for real-time analytics + +Hypercore is the TimescaleDB hybrid row-columnar storage engine used by hypertables. +Hypertables partition your data in chunks. Chunks stored in the rowstore use a row-oriented data format optimized for high-speed inserts and updates. Chunks stored in the columnstore use a columnar data format optimized for analytics. +You ingest _hot_ data into the rowstore. As data _cools_ and becomes more suited for analytics, +$CLOUD_LONG automatically converts these chunks of data to the columnstore. You define the moment when data is converted +using a columnstore policy. + + + +This page shows you how to get the best results when you set a policy to automatically convert chunks in a hypertable +from the rowstore to the columnstore. + +## Prerequisites + + + +The code samples in this page use the [crypto_sample.zip](https://assets.timescale.com/docs/downloads/candlestick/crypto_sample.zip) data from [Try the key Timescale features][ingest-data]. + +## Optimize your data with columnstore policies + +The compression ratio and query performance of data in the columnstore is dependent on the order and structure of your +data. Rows that change over a dimension should be close to each other. With time-series data, you `orderby` the time +dimension. For example, `Timestamp`: + +| Timestamp | Device ID | Device Type | CPU |Disk IO| +|---|---|---|---|---| +|12:00:01|A|SSD|70.11|13.4| + +This ensures that records are compressed and accessed in the same order. However, you would always have to +access the data using the time dimension, then filter all the rows using other criteria. To make your queries more +efficient, you segment your data based on the way you want to access it. For example, to rapidly access data about a +single device, you `segmentby` the `Device ID` column. This enables you to run much faster analytical queries on +data in the columnstore. + +When $CLOUD_LONG converts a chunk to the columnstore, TimescaleDB automatically creates a different schema for your +data. $TIMESCALE_DB creates and uses custom indexes to incorporate the `segmentby` and `orderby` parameters when +you write to and read from the columstore. + +To set up your Hypercore automation: + + + + + +## Reference + +For integers, timestamps, and other integer-like types, data is compressed using [delta encoding][delta], +[delta-of-delta][delta-delta], [simple-8b][simple-8b], and [run-length encoding][run-length]. For columns with few +repeated values,[XOR-based][xor] and [dictionary compression][dictionary] is used. For all other types, +[dictionary compression][dictionary] is used. + + + +[create-hypertable]: /use-timescale/:currentVersion:/compression/ +[add_columnstore_policy]: /api/:currentVersion:/hypercore/add_columnstore_policy/ +[delta]: /use-timescale/:currentVersion:/compression/compression-methods/#delta-encoding +[delta-delta]: /use-timescale/:currentVersion:/compression/compression-methods/#delta-of-delta-encoding +[simple-8b]: /use-timescale/:currentVersion:/compression/compression-methods/#simple-8b +[run-length]: /use-timescale/:currentVersion:/compression/compression-methods/#run-length-encoding +[xor]: /use-timescale/:currentVersion:/compression/compression-methods/#xor-based-encoding +[dictionary]: /use-timescale/:currentVersion:/compression/compression-methods/#dictionary-compression +[ingest-data]: /getting-started/:currentVersion:/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables +[add_columnstore_policy]: /api/:currentVersion:/hypercore/add_columnstore_policy/ +[run-job]: /api/:currentVersion:/actions/run_job/ +[convert_to_rowstore]: /api/:currentVersion:/hypercore/convert_to_rowstore/ +[alter_job]: /api/:currentVersion:/actions/alter_job/ +[informational-views]: /api/:currentVersion:/informational-views/jobs/ +[insert]: /use-timescale/:currentVersion:/write-data/insert/ +[modify-data-in-hypercore]: /use-timescale/:currentVersion:/hypercore/modify-data-in-hypercore/ diff --git a/use-timescale/hypercore/secondary-indexes.md b/use-timescale/hypercore/secondary-indexes.md new file mode 100644 index 0000000000..08f774a545 --- /dev/null +++ b/use-timescale/hypercore/secondary-indexes.md @@ -0,0 +1,313 @@ +--- +title: Improve query and upsert performance using secondary indexes +excerpt: Using secondary indexes on data in the columnstore make lookup queries 1,185x and upserts 224x faster +products: [cloud, mst, self_hosted] +keywords: [hypertable, compression, row-columnar storage, hypercore] +--- + +import EarlyAccess from "versionContent/_partials/_early_access.mdx"; + +# Improve query and upsert performance using secondary indexes + +Real-time analytics applications require more than fast inserts and analytical queries. They also need high performance +when retrieving individual records, enforcing constraints, or performing upserts, something that OLAP/columnar databases +lack. + +$TIMESCALE_DB supports and accelerates real-time analytics using [Hypercore][hypercore] without missing out on important +PostgreSQL features, including support for standard PostgreSQL indexes. Hypercore is a hybrid storage engine +because it supports deep analytics while staying true to PostgreSQL. Full support for B-tree and hash indexes +on columnstore data enables you to perform point lookups 1,185x faster, enforce unique constraints, and execute +upserts 224x faster—all while maintaining columnstore compression and analytics performance. + + + +## Choose the best indexing method + +[Indexes are a fundamental part of database performance optimization][blog-perf-tuning], they enable queries to +quickly locate and retrieve data without scanning entire tables. B-tree and hash indexes are among PostgreSQL’s +most widely used index types. However, they are designed for different query types: + +- [B-tree indexes][b-tree-overview]: keep data sorted in a hierarchical structure ideal for queries that involve + range (>, <, BETWEEN) and equality (=) lookups. A B-tree index enables you to check unique constraints or for + range-based filtering. You quickly retrieve and filter the relevant rows without scanning all the data. + + When a query searches for a specific value or a range of values in an indexed column, the B-tree structure + enables the database to quickly traverse the tree and find the relevant records in logarithmic time (`O(log n)`), + significantly improving performance compared to a full table scan. + +- [Hash indexes][hash-overview]: designed for exact-match lookups (=) and use a hashing function to map values to + unique disk locations. When searching for a specific ID, a hash index allows direct access to the data with + minimal overhead, and provides the fastest results + + When a query searches for a single value, such as searching for transaction by transaction ID, hash + indexes can be even faster than B-tree indexes as they don’t require tree traversal and have an amortized constant + (`O(1)`) lookup. Hash indexes don’t support range queries, they are ideal specifically for cases with + frequently queried, unique keys. + +The performance advantage from these indexing methods comes from optimized data structures designed for efficient key +searching. This results in fewer disk page reads, which in turn reduces I/O spikes when locating specific data points +or enforcing uniqueness. + +## How B-tree and hash indexes work + +PostgreSQL offers [multiple index types][postgres-index-types], For example, the default B-tree, hash, GIN, and BRIN. +all implemented as Index Access Methods (IAMs). PostgreSQL supplies the [table access method (TAM)][postgres-tam-methods] +interface for table storage. + +![TAM architecture](https://assets.timescale.com/docs/images/tam_architecture.png) + +By default, $TIMESCALE_DB stores data in the rowstore in standard PostgreSQL row-oriented tables, using the default heap +TAM. To make the heap TAM work with the columnstore, $TIMESCALE_DB integrates PostgreSQL [TOAST][storage-toast] to store +columnar data as compressed arrays. However, querying columnized data returns compressed, opaque data. To support +normal queries, $TIMESCALE_DB adds the `DecompressChunk` scan node to the PostgreSQL query plan in order to decompress data +on-the-fly. However, the heap TAM only indexes the compressed values, not the original data. + +Hypercore TAM handles decompression behind the scenes. This enables PostgreSQL to use standard interfaces for +indexing, to collect statistics, enforce constraints and lock tuples by reference. This also allows PostgreSQL’s built-in +scan nodes, such as sequential and index scans, to operate on the columnstore. Custom scan nodes are used for +analytical query performance optimizations, including vectorized filtering and aggregation. + +Hypercore TAM supports B-tree and hash indexes, making point lookups, upserts, and unique constraint +enforcement more efficient on the columnstore. Our benchmarks demonstrate substantial performance improvements: + +* 1,185x faster point lookup queries to retrieve a single record. +* 224.3x faster inserts when checking unique constraints. +* 2.6x faster upserts. +* 4.5x faster range queries. + +## When to use B-tree and hash indexes + +Adding B-tree and hash indexes to compressed data enables dramatically faster lookups and inserts, but it comes with +a trade-off: increased storage usage due to additional indexing structures. + +B-tree and hash indexes are particularly helpful when: + +- You need fast lookups on non-`SEGMENTBY` keys. For example, querying specific records by UUID. +- Query latency on compressed data is a bottleneck for your application. +- You perform frequent updates to historical data and need efficient uniqueness enforcement. + +However, consider the storage trade-off when: + +- Your queries already benefit from columnstore min/max indexes or `SEGMENTBY` optimizations. +- Your workloads prioritize compression efficiency over lookup speed. +- You primarily run aggregations and range scans, where indexes may not provide meaningful speedups. + + +## Enable secondary indexing + +To speed up your queries using secondary indexes you enable hypercore TAM on your hypertable in the columnstore: + + + +1. **Create a table with the desired columns and constraints** + ```sql + create table readings ( + metric_uuid uuid default gen_random_uuid(), + created_at timestamptz not null, + uploaded_at timestamptz not null, + location_id integer references locations (location_id), + device_id integer references devices (device_id), + temperature float, + humidity float + ); + ``` + +1. **Convert the table to a [hypertable][convert-to-hypertable]** + + ```sql + select create_hypertable ( + 'readings', + by_range('uploaded_at') + ); + ``` + +1. **Enable hypercore TAM for the hypertable** + ```sql + alter table readings + set access method hypercore + set ( + timescaledb.orderby = 'created_at', + timescaledb.segmentby = 'location_id' + ); + ``` + This enables the columnstore on the table. Hypercore TAM is applied to chunks created after you set the access + method. Existing chunks continue to use the default `heap`. + + To return to the `heap` TAM, call `set access method heap`. You can also change the table access method for an + existing chunk with a call like `ALTER TABLE _timescaledb_internal._hyper_1_1_chunk SET ACCESS METHOD hypercore;` + +1. **Move chunks from rowstore to columnstore as they age** + + ```sql + CALL add_columnstore_policy( + readings, + interval '1 day' + ); + ``` + + + +Hypercore TAM is now active on all new chunks created in the hypertable. + +## Create b-tree and hash indexes + +Once you have enabled hypercore TAM in your hypertable, the indexes are rebuilt when the table chunks are converted from +the rowstore to the columnstore. When you query data, these indexes are used by the PostgreSQL query planner over the +rowstore and columnstore. + +You add hash and B-tree indexes to a hypertable the same way as a regular PostgreSQL table: + +- **Hash index** + ```sql + CREATE INDEX readings_metric_uuid_hash_idx ON readings USING hash (metric_uuid); + ``` +- **B-tree index** + ```sql + CREATE UNIQUE INDEX readings_metric_uuid_metric_uuid_uploaded_at_idx + ON readings (metric_uuid, uploaded_at); + ``` + +If you have existing chunks that have not been updated to use the hypercore TAM, to use B-tree and hash indexes, you +change the table access method for an existing chunk with a call like `ALTER TABLE _timescaledb_internal._hyper_1_1_chunk SET ACCESS METHOD hypercore;` + +## Point lookups + +Indexes are particularly useful for highly selective queries, such as retrieving a unique event by its identifier. +For example: + +```sql +SELECT + created_at, + device_id, + temperature +FROM readings +WHERE metric_uuid = 'dd19f5d3-d04b-4afc-a78f-9b231fb29e52'; +``` + +Without an index, a query scans and filters the entire dataset, leading to slow execution and high I/O usage. +Min/max sparse indexes help when with incremental numeric IDs but do not work well for random numeric values or UUIDs. + +Set a hash index on `metric_uuid` to enable a direct lookup. This significantly improves performance by decompressing +only the relevant data segment. + +```sql +CREATE INDEX readings_metric_uuid_hash_idx ON readings USING hash (metric_uuid); +``` + +With a hash index and hypercore TAM enabled, the same SELECT query performs 1,185x faster; hash comes in at 10.9 ms vs. +12,915 ms and B-tree at 12.57. + + +## Backfill and updates to historical data + +A common use case in real-time applications is backfilling or updating old data. For example, a sensor fails +during a batch upload or gets temporarily disconnected, it resends the data later. To avoid possible duplicate +records, you have to check if the data already exists in the database before storing it. + +To prevent duplicate entries, you enforce uniqueness using a primary key. Primary constraints are enforced through +unique indexes, making conflict checks fast. Without an index, verifying uniqueness involves scanning and decompressing +potentially large amounts of data. This significantly slows inserts and consuming excessive [IOPS][iops]. A `UNIQUE` +constraint on a hypertable must also include the hypertable partition key. + +The following `UNIQUE` uses a B-tree index. + +```sql +CREATE UNIQUE INDEX readings_metric_uuid_metric_uuid_created_at_idx + ON readings (metric_uuid, created_at); +``` + +Possible strategies for backfilling historic data include: + +- **Insert data when the row does not exist**: + + An insert statement ensuring no duplicates looks like: + + ```sql + INSERT INTO readings VALUES (...) ON CONFLICT (device_id, created_at) DO NOTHING; + ``` + Our benchmarks showed this makes inserts 224.3x faster, reducing the execution time from 289,139 ms to 1,289 ms. + +- **Insert missing data or update existing data**: + + An upsert is a database operation that inserts a new row if it does not already exist, or updates the existing row + if a conflict occurs. This enables you to re-ingest new versions of rows instead of performing separate update + statements. Without an index, the system needs to scan and decompress data, considerably slowing ingestion speed. + With a primary key index, conflicting rows are directly located within the compressed data segment in + the columnstore. + + The following query attempts to insert a new record. If a record for the same `metric_uuid` and `created_at` values + already exists, it updates `temperature` with the corresponding value from the new record. + + ```sql + INSERT INTO readings VALUES (...) ON CONFLICT DO UPDATE SET temperature = EXCLUDED.temperature; + ``` + + $COMPANY benchmarks showed this makes upserts 2.6x faster, reducing the execution time from 24,805 ms to 9,520 ms. + + +## Fast anomaly detection + +To regularly report the number of times a device has exceeded a critical temperature, you +count the number of times the temperature was exceeded, and group by device ID. + +```sql +SELECT + device_id, + COUNT(temperature) +FROM readings +WHERE temperature > 52.5 +GROUP BY device_id; +``` + +You see something like: + +| device_id | count | +| -- | -- | +68 | 1 | +| 258 | 1 | +| 192 | 1 | +| 276 | 1 | +| 114 | 1 | +| 227 | 1 | +| 153 | 1 | +| 210 | 1 | +| 266 | 1 | +| 165 | 1 | +| 296 | 1 | +| 144 | 1 | +| 93 | 1 | +| 285 | 1 | +| 221 | 1 | +| 167 | 1 | +| 14 | 1 | +| 123 | 1 | +| 152 | 1 | +| 206 | 1 | +| 230 | 1 | +| 136 | 1 | +| 256 | 2 +| 1 | 1 | + +To make this query faster, use a partial B-tree index. That is, a B-tree index that only includes rows satisfying a +specific WHERE condition. This makes a smaller index that is more efficient for queries that match that +condition. For example, to create a partial B-tree index for temperature readings over 52.5: + +```sql +CREATE INDEX ON readings (temperature) where temperature > 52.5; +``` + +Compared with using a sparse min/max index in columnstore, $COMPANY benchmarks show that the B-tree index query is +4.5x faster. + + + +[hypercore]: /use-timescale/:currentVersion:/hypercore/ +[blog-perf-tuning]: https://www.timescale.com/learn/postgresql-performance-tuning-optimizing-database-indexes +[create-hypertable]: /use-timescale/:currentVersion:/compression/ +[b-tree-overview]: https://www.timescale.com/learn/postgresql-performance-tuning-optimizing-database-indexes#:~:text=a%20quick%20summary%3A-,B%2DTree%20indexes%20(default%20index%20type%20in%20PostgreSQL),-CREATE%20INDEX%20index_product_id +[hash-overview]: https://www.timescale.com/learn/postgresql-performance-tuning-optimizing-database-indexes#:~:text=in%20ascending%20order.-,Hash%20indexes,-CREATE%20INDEX%20index_product_id +[storage-toast]: https://www.postgresql.org/docs/current/storage-toast.html +[postgres-index-types]: https://www.timescale.com/learn/database-indexes-in-postgres +[postgres-tam-methods]: https://www.postgresql.org/docs/current/tableam.html +[convert-to-hypertable]: /use-timescale/:currentVersion:/hypertables/create/ +[iops]: https://en.wikipedia.org/wiki/IOPS diff --git a/use-timescale/hyperfunctions/about-hyperfunctions.md b/use-timescale/hyperfunctions/about-hyperfunctions.md index 4e27f5d3e3..248e2a7dc2 100644 --- a/use-timescale/hyperfunctions/about-hyperfunctions.md +++ b/use-timescale/hyperfunctions/about-hyperfunctions.md @@ -11,25 +11,18 @@ import ExperimentalUpgrade from "versionContent/_partials/_experimental-schema-u # About Timescale hyperfunctions -Timescale hyperfunctions are a specialized set of functions that allow you to -analyze time-series data. You can use hyperfunctions to analyze anything you -have stored as time-series data, including IoT devices, IT systems, marketing -analytics, user behavior, financial metrics, and cryptocurrency. +Timescale hyperfunctions are a specialized set of functions that power real-time analytics on time series and events. +IoT devices, IT systems, marketing analytics, user behavior, financial metrics, cryptocurrency - these are only a few examples of domains where +hyperfunctions can make a huge difference. Hyperfunctions provide you with meaningful, actionable insights in real time. -Hyperfunctions allow you to perform critical time-series queries quickly, -analyze time-series data, and extract meaningful information. They aim to -identify, build, and combine all of the functionality SQL needs to perform -time-series analysis into a single extension. +$CLOUD_LONG includes all hyperfunctions by default, while self-hosted $TIMESCALE_DB includes a subset of them. For +additional hyperfunctions, install the [Timescale Toolkit][install-toolkit] PostgreSQL extension. -Some hyperfunctions are included in the default TimescaleDB product. For -additional hyperfunctions, you need to install the -[Timescale Toolkit][install-toolkit] PostgreSQL extension. - -## Hyperfunctions available with TimescaleDB and Timescale Toolkit +## Available hyperfunctions Here is a list of all the hyperfunctions provided by Timescale. Hyperfunctions -marked 'Toolkit' require an installation of Timescale Toolkit. Hyperfunctions -marked 'experimental' are still under development. +with a tick in the `Toolkit` column require an installation of Timescale Toolkit for self-hosted deployments. Hyperfunctions +with a tick in the `Experimental` column are still under development. diff --git a/use-timescale/hyperfunctions/heartbeat-agg.md b/use-timescale/hyperfunctions/heartbeat-agg.md index 6a51f8263a..f6279bf089 100644 --- a/use-timescale/hyperfunctions/heartbeat-agg.md +++ b/use-timescale/hyperfunctions/heartbeat-agg.md @@ -7,11 +7,12 @@ keywords: [hyperfunctions, Toolkit, heartbeat, liveness] # Heartbeat aggregation Given a series of timestamped health checks, it can be tricky to determine the -overall health of a system over a given interval. PostgresQL provides window -functions which can be used to get a sense of where unhealthy gaps are, but can -be somewhat awkward to use efficiently. The heartbeat aggregate is part of the -Timescale Toolkit, and can be used to solve this problem in a simpler, more -accessible manner. +overall health of a system over a given interval. PostgreSQL provides window +functions that you use to get a sense of where unhealthy gaps are, but they can +be somewhat awkward to use efficiently. + +This is one of the many cases where hyperfunctions provide an efficient, simple solution for +a frequently occurring problem. Heartbeat aggregation helps analyze event-based time-series data with intermittent or irregular signals. This example uses the [SustData public dataset][sustdata]. This dataset tracks the power usage of a small number of apartments and houses over four different @@ -19,7 +20,7 @@ deployment intervals. The data is collected in one minute samples from each unit. When you have loaded the data into hypertables, you can create a materialized -view containing weekly heartbeat aggregates for each of the units. +view containing weekly heartbeat aggregates for each of the units: ```sql CREATE MATERIALIZED VIEW weekly_heartbeat AS @@ -69,7 +70,7 @@ SELECT live_ranges(rollup(heartbeat_agg)) FROM weekly_heartbeat WHERE unit = 17; ("2012-03-25 03:00:51+00","2012-04-11 00:01:00+00") ``` -You can do also do more elaborate queries, like looking for the 5 units with the +You can construct more elaborate queries. For example, to return the 5 units with the lowest uptime during the third deployment: ```sql @@ -90,7 +91,7 @@ ORDER BY uptime LIMIT 5; 30 | 222 days 22:05:00 ``` -You can also combine aggregates from different units to get the combined +Combine aggregates from different units to get the combined coverage. This example queries the interval where any part of a deployment was active: @@ -115,11 +116,12 @@ FROM weekly_heartbeat group by deploy order by deploy; 4 | ("2014-03-30 03:00:01+00","2014-04-25 00:01:00+00") ``` -You can use this data to make some observations. First, it looks like the second -deployment had a lot more problems than the other ones. Second, it looks like -there were some readings from February 2013 that were incorrectly categorized as -a second deployment. And finally, it looks like the timestamps are given in a -local time without time zone, resulting in some missing hours around springtime +Then use this data to make observations and draw conclusions: + +- The second deployment had a lot more problems than the other ones. +- There were some readings from February 2013 that were incorrectly categorized as +a second deployment. +- The timestamps are given in a local time without time zone, resulting in some missing hours around springtime daylight savings time changes. For more information about heartbeat aggregation API calls, see the diff --git a/use-timescale/hyperfunctions/index.md b/use-timescale/hyperfunctions/index.md index 41168901f3..1887625014 100644 --- a/use-timescale/hyperfunctions/index.md +++ b/use-timescale/hyperfunctions/index.md @@ -7,12 +7,10 @@ keywords: [hyperfunctions, Toolkit, analytics] # Hyperfunctions -Hyperfunctions allow you to perform critical time-series queries quickly, -analyze time-series data, and extract meaningful information. +Timescale hyperfunctions are designed to efficiently process, aggregate, and analyze significant volumes of data while maintaining high performance. +They let you run sophisticated analytical queries and get meaningful data in real time. -Some hyperfunctions are included by default in Timescale. For -additional hyperfunctions, you need to install the -[Timescale Toolkit][install-toolkit] PostgreSQL extension. +$CLOUD_LONG includes all hyperfunctions by default, while self-hosted $TIMESCALE_DB includes a subset of them. To include all hyperfunctions with $TIMESCALE_DB, install the [Timescale Toolkit][install-toolkit] PostgreSQL extension on your self-hosted PostgreSQL deployment. For more information, read the [hyperfunctions blog post][hyperfunctions-blog]. @@ -21,7 +19,7 @@ For more information, read the [hyperfunctions blog post][hyperfunctions-blog]. * [Learn about hyperfunctions][about-hyperfunctions] to understand how they work before using them. * Install the [Toolkit extension][install-toolkit] to access more - hyperfunctions. + hyperfunctions on self-hosted $TIMESCALE_DB. ## Browse hyperfunctions and Toolkit features by category diff --git a/use-timescale/hypertables/about-hypertables.md b/use-timescale/hypertables/about-hypertables.md index 1c77c0bac3..5ca8132a3c 100644 --- a/use-timescale/hypertables/about-hypertables.md +++ b/use-timescale/hypertables/about-hypertables.md @@ -17,10 +17,7 @@ Inheritance is not supported for hypertables and may lead to unexpected behavior ## Hypertable partitioning -When you create and use a hypertable, it automatically partitions data by time, -and optionally by space. - -Each hypertable is made up of child tables called chunks. Each chunk is assigned +Each hypertable is partitioned into child tables called chunks. Each chunk is assigned a range of time, and only contains data from that range. If the hypertable is also partitioned by space, each chunk is also assigned a subset of the space values. @@ -36,7 +33,7 @@ it makes sense. ### Time partitioning Each chunk of a hypertable only holds data from a specific time range. When you -insert data from a time range that doesn't yet have a chunk, Timescale +insert data from a time range that doesn't yet have a chunk, $CLOUD_LONG automatically creates a chunk to store it. By default, each chunk covers 7 days. You can change this to better suit your @@ -50,7 +47,7 @@ alt="A normal table compared to a hypertable. The normal table holds data for 3 /> -Timescale divides time into potential chunk ranges, based on the +$CLOUD_LONG divides time into potential chunk ranges, based on the `chunk_time_interval`. If data exists for a potential chunk range, that chunk is created. @@ -70,7 +67,7 @@ affect query planning time and compression. Best practice is to set `chunk_time_interval` so that prior to processing, one chunk of data takes up 25% of main memory, including the indexes from each active hypertable. -For example, if your write approximately 2 GB of data per day to a database with 64 GB of +For example, if you write approximately 2 GB of data per day to a database with 64 GB of memory, set `chunk_time_interval` to 1 week. If you write approximately 10 GB of data per day on the same machine, set the time interval to 1 day. diff --git a/use-timescale/hypertables/improve-query-performance.md b/use-timescale/hypertables/improve-query-performance.md index e6d1d8219b..c266332cf7 100644 --- a/use-timescale/hypertables/improve-query-performance.md +++ b/use-timescale/hypertables/improve-query-performance.md @@ -7,6 +7,7 @@ keywords: [hypertables, indexes, chunks] # Improve query performance +One of the key purposes of hypertables is to make your analytical queries run with the lowest latency possible - and they have been designed accordingly. When you execute a query on a hypertable, you do not parse the whole table; you only access the chunks necessary to satisfy the query. This works well when the `WHERE` clause of a query uses the column by which a hypertable is partitioned. For example, in a hypertable where every day of the year is a separate chunk, a query for September 1 @@ -18,7 +19,7 @@ performance. To improve query performance, TimescaleDB enables you to skip chunks on non-partitioning columns in hypertables. -Chunk skipping can only be enabled for **compressed** hypertables. +Chunk skipping can only be enabled for **compressed** hypertables and applies to the chunks created **after** it as enabled. ## How chunk skipping works diff --git a/use-timescale/hypertables/index.md b/use-timescale/hypertables/index.md index d179cb74df..101225c528 100644 --- a/use-timescale/hypertables/index.md +++ b/use-timescale/hypertables/index.md @@ -7,10 +7,8 @@ keywords: [hypertables] # Hypertables -Hypertables are PostgreSQL tables with special features that make it easy to -handle time-series data. Anything you can do with regular PostgreSQL tables, you -can do with hypertables. In addition, you get the benefits of improved -performance and user experience for time-series data. +Hypertables are PostgreSQL tables designed to simplify and accelerate data analysis. Anything you can do with regular PostgreSQL tables, you +can do with hypertables - but much faster and more conveniently. In this section, you: * [Learn about hypertables][about-hypertables] * [Create a hypertable][create-hypertables] diff --git a/use-timescale/ingest-data/ingest-telegraf.md b/use-timescale/ingest-data/ingest-telegraf.md index 33392a99c9..756191a48f 100644 --- a/use-timescale/ingest-data/ingest-telegraf.md +++ b/use-timescale/ingest-data/ingest-telegraf.md @@ -157,6 +157,6 @@ see the [PostgreQL output plugin][output-plugin]. [output-plugin]: https://github.com/influxdata/telegraf/blob/release-1.24/plugins/outputs/postgresql/README.md [install-telegraf]: https://docs.influxdata.com/telegraf/v1.21/introduction/installation/ [create-service]: /getting-started/latest/ -[connect-timescaledb]: /use-timescale/:currentVersion:/integrations/find-connection-details/ -[grafana]: /use-timescale/:currentVersion:/integrations/grafana/ +[connect-timescaledb]: /integrations/:currentVersion:/find-connection-details/ +[grafana]: /integrations/:currentVersion:/grafana/ [about-hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/ diff --git a/use-timescale/integrations/datadog.md b/use-timescale/integrations/datadog.md deleted file mode 100644 index df76c78268..0000000000 --- a/use-timescale/integrations/datadog.md +++ /dev/null @@ -1,42 +0,0 @@ ---- -title: Integrate Datadog with Timescale Cloud -excerpt: Datadog is a cloud-based monitoring and analytics platform. Export telemetry data from your Timescale Cloud service with time-series and analytics capability to Datadog -products: [cloud, mst, self_hosted] -keywords: [integrate] ---- - -import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; -import DataDogExporter from "versionContent/_partials/_datadog-data-exporter.mdx"; -import ManageDataExporter from "versionContent/_partials/_manage-data-exporter.mdx"; - -# Integrate Datadog with $CLOUD_LONG - -[Datadog][datadog] is a cloud-based monitoring and analytics platform that provides comprehensive visibility into applications, infrastructure, and systems through real-time monitoring, logging, and analytics. - -You can export telemetry data from your $SERVICE_LONGs with the time-series and analytics capability enabled to Datadog. The available metrics include CPU usage, RAM usage, and storage. This integration is available for [Scale and Enterprise][pricing-plan-features] -pricing tiers. - -This pages explains how to export telemetry data from your $SERVICE_LONG into Datadog by creating a $CLOUD_LONG data exporter, then attaching it to the $SERVICE_SHORT. - -## Prerequisites - - - -- Sign up for [Datadog][datadog-signup]. - - You need your API key to follow this procedure. [Create one][datadog-api-key] if you haven't yet. - -## Create a data exporter - -A $CLOUD_LONG data exporter sends telemetry data from a $SERVICE_LONG to a third-party monitoring -tool. You create an exporter on the [project level][projects], in the same AWS region as your $SERVICE_SHORT: - - - - - -[datadog]: https://www.datadoghq.com/ -[datadog-signup]: https://www.datadoghq.com/ -[projects]: /use-timescale/:currentVersion:/members/ -[datadog-api-key]: https://docs.datadoghq.com/account_management/api-app-keys/#add-an-api-key-or-client-token -[pricing-plan-features]: /about/:currentVersion:/pricing-and-account-management/#features-included-in-each-plan \ No newline at end of file diff --git a/use-timescale/jobs/create-and-manage-jobs.md b/use-timescale/jobs/create-and-manage-jobs.md new file mode 100644 index 0000000000..9826c0fe81 --- /dev/null +++ b/use-timescale/jobs/create-and-manage-jobs.md @@ -0,0 +1,177 @@ +--- +title: Create and manage jobs +excerpt: Jobs are custom PostgreSQL functions and procedures that you set up to run on a schedule. Create, register, test, alter, and delete jobs in Timescale Cloud +products: [cloud, mst, self_hosted] +keywords: [jobs] +tags: [scheduled jobs, background jobs, automation framework] +--- + +import Prerequisites from "versionContent/_partials/_prereqs-cloud-and-self.mdx"; + +# Create and manage $JOBs + +$JOB_CAPs in Timescale are custom functions or procedures that run on a schedule that you define. This page explains how to create, test, alter, and delete a $JOB. + +## Prerequisites + + + +## Create a $JOB + +To create a $JOB, create a [function][postgres-createfunction] or [procedure][postgres-createprocedure] that you want your $SERVICE_SHORT to execute, then set it up to run on a schedule. + + + +1. **Define a function or procedure in the language of your choice** + + Wrap it in a `CREATE` statement: + + ```sql + CREATE FUNCTION (required arguments) + RETURNS AS $$ + DECLARE + ; + BEGIN + ; + RETURN { | value } + END; + $$ LANGUAGE ; + ``` + + For example, to create a function that returns the total row count of a table within a $SERVICE_SHORT: + + ```sql + CREATE FUNCTION totalRecords (job_id INT DEFAULT NULL, config JSONB DEFAULT NULL) + RETURNS integer AS $total$ + declare + total integer; + BEGIN + SELECT count(*) into total FROM fill_measurements; + RETURN total; + END; + $total$ LANGUAGE plpgsql; + ``` + + `job_id` and `config` are required arguments. This returns `CREATE FUNCTION` to indicate that the function has successfully been created. + +1. **Call the function to validate** + + For example: + + ```sql + select totalRecords(); + ``` + + The result looks like this: + + ```sql + totalrecords + -------------- + 48600500 + (1 row) + ``` + +1. **Register your $JOB with [`add_job`][api-add_job]** + + Pass the name of your $JOB, the schedule you want it to run on, and the content of your config. For the `config` value, if you don't need any special configuration parameters, set to `NULL`. For example, to run the `totalRecords` function every hour: + + ```sql + SELECT add_job('totalRecords', '1h', config => '{"hypertable":"metr"}'); + ``` + + The call returns a `job_id` and stores it along with `config` in the Timescale catalog. + + The $JOB runs on the schedule you set. You can also run it manually with [`run_job`][api-run_job] passing `job_id`. When the $JOB runs, `job_id` and `config` are passed as arguments. + +1. **Validate the job** + + List all currently registered $JOBs with [`timescaledb_information.jobs`][api-timescaledb_information-jobs]: + + ```sql + SELECT * FROM timescaledb_information.jobs; + ``` + + The result looks like this: + + ```sql + job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | config | next_start | hypertable_schema | hypertable_name + --------+----------------------------+-------------------+-------------+-------------+--------------+-----------------------+------------------+-----------+-----------+------------------------+-------------------------------+-------------------+----------------- + 1 | Telemetry Reporter [1] | 24:00:00 | 00:01:40 | -1 | 01:00:00 | _timescaledb_internal | policy_telemetry | postgres | t | | 2022-08-18 06:26:39.524065+00 | | + 1000 | User-Defined Action [1000] | 01:00:00 | 00:00:00 | -1 | 00:05:00 | public | totalrecords | tsdbadmin | t | {"hypertable": "metr"} | 2022-08-17 07:17:24.831698+00 | | + (2 rows) + ``` + + + +## Test and debug a $JOB + +To debug a $JOB, increase the log level and run the $JOB manually with [`run_job`][api-run_job] in the foreground. Because `run_job` is a stored procedure and not a function, run it with [`CALL`][postgres-call] instead of `SELECT`. + + + +1. **Set the minimum log level to `DEBUG1`** + + ```sql + SET client_min_messages TO DEBUG1; + ``` + +1. **Run the $JOB** + + Replace `1000` with your `job_id`: + + ```sql + CALL run_job(1000); + ``` + + + +## Alter and delete a $JOB + +Alter an existing $JOB with [`alter_job`][api-alter_job]. You can change both the config and the schedule on which the $JOB runs. + + + +1. **Change a $JOB's config** + + To replace the entire JSON config for a $JOB, call `alter_job` with a new `config` object. For example, replace the JSON config for a $JOB with ID `1000`: + + ```sql + SELECT alter_job(1000, config => '{"hypertable":"metrics"}'); + ``` + +1. **Turn off $JOB scheduling** + + To turn off automatic scheduling of a $JOB, call `alter_job` and set `scheduled`to `false`. You can still run the $JOB manually with `run_job`. For example, turn off the scheduling for a $JOB with ID `1000`: + + ```sql + SELECT alter_job(1000, scheduled => false); + ``` + +1. **Re-enable automatic scheduling of a $JOB** + + To re-enable automatic scheduling of a $JOB, call `alter_job` and set `scheduled` to `true`. For example, re-enable scheduling for a $JOB with ID `1000`: + + ```sql + SELECT alter_job(1000, scheduled => true); + ``` + +1. **Delete a $JOB with [`delete_job`][api-delete_job]** + + For example, to delete a $JOB with ID `1000`: + + ```sql + SELECT delete_job(1000); + ``` + + + +[postgres-call]: https://www.postgresql.org/docs/current/sql-call.html +[api-alter_job]: /api/:currentVersion:/jobs-automation/alter_job +[api-delete_job]: /api/:currentVersion:/jobs-automation/delete_job +[plpgsql]: https://www.postgresql.org/docs/current/plpgsql-overview.html +[api-add_job]: /api/:currentVersion:/jobs-automation/add_job +[api-run_job]: /api/:currentVersion:/jobs-automation/run_job +[api-timescaledb_information-jobs]: /api/:currentVersion:/informational-views/jobs/ +[postgres-createfunction]: https://www.postgresql.org/docs/current/xfunc.html +[postgres-createprocedure]: https://www.postgresql.org/docs/current/xproc.html +[plpgsql]: https://www.postgresql.org/docs/current/plpgsql-overview.html diff --git a/use-timescale/user-defined-actions/example-downsample-and-compress.md b/use-timescale/jobs/example-downsample-and-compress.md similarity index 72% rename from use-timescale/user-defined-actions/example-downsample-and-compress.md rename to use-timescale/jobs/example-downsample-and-compress.md index 55f7a20180..46dd83065d 100644 --- a/use-timescale/user-defined-actions/example-downsample-and-compress.md +++ b/use-timescale/jobs/example-downsample-and-compress.md @@ -1,29 +1,27 @@ --- -title: Use a user-defined action to downsample and compress chunks -excerpt: Downsample and compress your hypertable chunks by combining a continuous aggregate refresh policy with compression. Not quite what you need? Create a custom user-defined action +title: Use a job to downsample and compress chunks +excerpt: Downsample and compress your hypertable chunks by combining a continuous aggregate refresh policy with compression. Not quite what you need? Create a custom job products: [cloud, mst, self_hosted] -keywords: [actions, compression, downsample] +keywords: [jobs, compression, downsample] --- -# Use a user-defined action to downsample and compress chunks +# Use a $JOB to downsample and compress $CHUNKs -Timescale lets you downsample and compress chunks by combining a -[continuous aggregate refresh policy][cagg-refresh] with a +Timescale lets you downsample and compress $CHUNKs by combining a +[$CAGG refresh policy][cagg-refresh] with a [compression policy][compression]. If you want to implement features not supported by those policies, you can write -a user-defined action to downsample and compress chunks instead. The following +a $JOB to downsample and compress $CHUNKs instead. The following example downsamples raw data to an average over hourly data. This is an -illustrative example, which can be done more simply with a continuous aggregate +illustrative example, which can be done more simply with a $CAGG policy. But you can make the query arbitrarily complex. -## Using a user-defined action to downsample and compress chunks - -1. Create a procedure that first queries the chunks of a hypertable to - determine if they are older than the `lag` parameter. The hypertable in this - example is named `metrics`. If the chunk is not already compressed, +1. Create a procedure that first queries the $CHUNKs of a $HYPERTABLE to + determine if they are older than the `lag` parameter. The $HYPERTABLE in this + example is named `metrics`. If the $CHUNK is not already compressed, downsample it by taking the average of the raw data. Then compress it. A temporary table is used to store the data while calculating the average. @@ -80,8 +78,8 @@ policy. But you can make the query arbitrarily complex. $$; ``` -1. Register the job to run daily. In the `config`, set `lag` to 12 months - to drop chunks containing data older than 12 months. +1. Register the $JOB to run daily. In the `config`, set `lag` to 12 months + to drop $CHUNKs containing data older than 12 months. ```sql SELECT add_job('downsample_compress','1d', config => '{"lag":"12 month"}'); diff --git a/use-timescale/user-defined-actions/example-generic-retention.md b/use-timescale/jobs/example-generic-retention.md similarity index 64% rename from use-timescale/user-defined-actions/example-generic-retention.md rename to use-timescale/jobs/example-generic-retention.md index 7875dd19fe..4a7e613db5 100644 --- a/use-timescale/user-defined-actions/example-generic-retention.md +++ b/use-timescale/jobs/example-generic-retention.md @@ -1,23 +1,21 @@ --- -title: Use a user-defined action to create a generic retention policy +title: Use a job to create a generic retention policy excerpt: In Timescale Cloud, you can add a data retention policy to a hypertable, to store data more efficiently. Take it one step further by creating a generic data retention policy for your entire service products: [cloud, mst, self_hosted] -keywords: [actions, data retention] +keywords: [jobs, data retention] --- -# Use a user-defined action to create a generic retention policy +# Use a $JOB to create a generic retention policy Timescale natively supports adding a -[data retention policy][data-retention-policy] to a hypertable. If you want to -add a generic data retention policy to _all_ hypertables, you can write a -user-defined action. +[data retention policy][data-retention-policy] to a $HYPERTABLE. If you want to +add a generic data retention policy to all $HYPERTABLEs, you can create a custom +$JOB. -## Using a user-defined action to create a generic retention policy - -1. Create a procedure that drops chunks from any hypertable if they are older - than the `drop_after` parameter. To get all hypertables, the +1. Create a procedure that drops $CHUNKs from any $HYPERTABLE if they are older + than the `drop_after` parameter. To get all $HYPERTABLEs, the `timescaledb_information.hypertables` table is queried. ```sql @@ -42,17 +40,19 @@ user-defined action. $$; ``` -1. Register the job to run daily. In the `config`, set `drop_after` to 12 months - to drop chunks containing data older than 12 months. +1. Register the $JOB to run daily. In the `config`, set `drop_after` to 12 months + to drop $CHUNKs containing data older than 12 months. ```sql SELECT add_job('generic_retention','1d', config => '{"drop_after":"12 month"}'); ``` + You can further refine this policy by adding filters to your procedure. For -example, add a `WHERE` clause to the `PERFORM` query to only drop chunks from -particular hypertables. +example, add a `WHERE` clause to the `PERFORM` query to only drop $CHUNKs from +particular $HYPERTABLEs. + diff --git a/use-timescale/user-defined-actions/example-tiered-storage.md b/use-timescale/jobs/example-tiered-storage.md similarity index 68% rename from use-timescale/user-defined-actions/example-tiered-storage.md rename to use-timescale/jobs/example-tiered-storage.md index 895e05c8a6..d9ce80c3d0 100644 --- a/use-timescale/user-defined-actions/example-tiered-storage.md +++ b/use-timescale/jobs/example-tiered-storage.md @@ -1,29 +1,28 @@ --- -title: Use a user-defined action to implement automatic tablespace management -excerpt: Save on storage costs for old data. Timescale Cloud enables you to automatically move hypertable chunks between tablespaces with move_chunk. +title: Use a job to implement automatic tablespace management +excerpt: Save on storage costs for old data. Timescale Cloud enables you to automatically move hypertable chunks between tablespaces with move_chunk products: [cloud, mst, self_hosted] -keywords: [actions, tablespaces] +keywords: [jobs, tablespaces] --- -# Use a user-defined action to implement automatic tablespace management +# Use a $JOB to implement automatic tablespace management -[Moving older data to a different tablespace][moving-data] can help you save on +Moving older data to a different tablespace can help you save on storage costs. Timescale supports automatic tablespace management by providing the `move_chunk` function to move chunks between tablespaces. To schedule the -moves automatically, you can write a user-defined action. +moves automatically, you can write a custom $JOB. -On the Timescale cloud platform, use the [Tiered Storage](https://docs.timescale.com/use-timescale/latest/data-tiering/) -offering which handles this by providing a [tiering policy API](https://docs.timescale.com/use-timescale/latest/data-tiering/enabling-data-tiering/#add-a-tiering-policy) -to move data to low-cost object storage backed by Amazon S3. + +On $CLOUD_LONG, use [tiered storage](https://docs.timescale.com/use-timescale/latest/data-tiering/) which handles this by providing a [tiering policy API](https://docs.timescale.com/use-timescale/latest/data-tiering/enabling-data-tiering/#add-a-tiering-policy) to move data to low-cost object storage backed by Amazon S3. - +To implement automatic $CHUNK moving with a $JOB: -## Using a user-defined action to implement automatic chunk moving + -1. Create a procedure that moves chunks to a different tablespace if they +1. Create a procedure that moves $CHUNKs to a different tablespace if they contain data older than the `lag` parameter. ```sql @@ -73,9 +72,9 @@ to move data to low-cost object storage backed by Amazon S3. $$; ``` -1. Register the job to run daily. In the config, set `hypertable` to `metrics` - to implement automatic chunk moves on the `metrics` hypertable. Set `lag` to - 12 months to move chunks containing data older than 12 months. Set +1. Register the $JOB to run daily. In the config, set `hypertable` to `metrics` + to implement automatic $CHUNK moves on the `metrics` $HYPERTABLE. Set `lag` to + 12 months to move $CHUNKs containing data older than 12 months. Set `tablespace` to the destination tablespace. ```sql @@ -88,5 +87,3 @@ to move data to low-cost object storage backed by Amazon S3. - -[moving-data]: /use-timescale/:currentVersion:/user-defined-actions/example-tiered-storage/ diff --git a/use-timescale/jobs/index.md b/use-timescale/jobs/index.md new file mode 100644 index 0000000000..d43f9e88a2 --- /dev/null +++ b/use-timescale/jobs/index.md @@ -0,0 +1,36 @@ +--- +title: Jobs in Timescale Cloud +excerpt: Increase efficiency and reduce overhead by automating routine tasks. Set up custom jobs on top of Timescale Cloud policies. Learn how to create them in your service +products: [cloud, mst, self_hosted] +keywords: [jobs] +tags: [jobs, scheduled jobs, background jobs, automation framework] +--- + +# $JOB_CAPs + +Timescale natively includes some $JOB-scheduling policies, such as: + +* [$CAGG_CAP policies][caggs] to automatically refresh $CAGGs +* [Compression policies][compressing] to compress historical data +* [Retention policies][retention] to drop historical data +* [Reordering policies][reordering] to reorder data within $CHUNKs + +If these don't cover your use case, you can create and schedule custom-defined $JOBs to run within +your $SERVICE_LONG. They help you automate periodic tasks that aren't covered by the native policies. + +In this section you: + +* [Create and manage $JOBs][create-jobs] +* See the following examples: + * Set up a [generic data retention][generic-retention] policy that applies across all $HYPERTABLEs + * Implement [automatic moving of $CHUNKs between tablespaces][manage-storage] + * Automatically [downsample and compress][downsample-compress] older $CHUNKs + +[create-jobs]: /use-timescale/:currentVersion:/jobs/create-and-manage-jobs/ +[downsample-compress]: /use-timescale/:currentVersion:/jobs/example-downsample-and-compress +[generic-retention]: /use-timescale/:currentVersion:/jobs/example-generic-retention +[manage-storage]: /use-timescale/:currentVersion:/jobs/example-tiered-storage/ +[caggs]: /use-timescale/:currentVersion:/continuous-aggregates/refresh-policies/ +[compressing]: /use-timescale/:currentVersion:/compression/ +[reordering]: /api/:currentVersion:/hypertable/add_reorder_policy/ +[retention]: /use-timescale/:currentVersion:/data-retention/create-a-retention-policy/ \ No newline at end of file diff --git a/use-timescale/metrics-logging/integrations.md b/use-timescale/metrics-logging/integrations.md index dfc2dfbb48..b54eb1a2f1 100644 --- a/use-timescale/metrics-logging/integrations.md +++ b/use-timescale/metrics-logging/integrations.md @@ -11,7 +11,7 @@ cloud_ui: --- import DataDogExporter from "versionContent/_partials/_datadog-data-exporter.mdx"; -import ManageDataExporter from "versionContent/_partials/_manage-data-exporter.mdx"; +import ManageDataExporter from "versionContent/_partials/_manage-a-data-exporter.mdx"; import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; import CloudWatchExporter from "versionContent/_partials/_cloudwatch-data-exporter.mdx"; @@ -50,6 +50,10 @@ tool. You create an exporter on the [project level][projects], in the same AWS r +## Manage a data exporter + +This section shows you how to attach, monitor, edit, and delete a data exporter. + [attach-exporter]: /use-timescale/:currentVersion:/metrics-logging/integrations/#attach-a-data-exporter-to-a-timescale-cloud-service diff --git a/use-timescale/metrics-logging/service-metrics.md b/use-timescale/metrics-logging/service-metrics.md index 8d5e1aac84..159f3c153a 100644 --- a/use-timescale/metrics-logging/service-metrics.md +++ b/use-timescale/metrics-logging/service-metrics.md @@ -9,7 +9,7 @@ cloud_ui: - [services, :serviceId, metrics] --- -import EarlyAccess from "versionContent/_partials/_early_access.mdx"; +import EarlyAccess from "versionContent/_partials/_early_access_2_18_0.mdx"; # Service metrics @@ -136,4 +136,4 @@ performance bottlenecks with `pg_stat_statements`][blog-pg_stat_statements]. [metrics-dashboard]: /use-timescale/:currentVersion:/metrics-logging/service-metrics/ [pg-stat]: /use-timescale/:currentVersion:/metrics-logging/service-metrics/#query-level-statistics-with-pg_stat_statements [blog-pg_stat_statements]: -[psql]: /use-timescale/:currentVersion:/integrations/psql/ +[psql]: /integrations/:currentVersion:/psql/ diff --git a/use-timescale/page-index/page-index.js b/use-timescale/page-index/page-index.js index c17875cb58..fe8fa933b5 100644 --- a/use-timescale/page-index/page-index.js +++ b/use-timescale/page-index/page-index.js @@ -2,11 +2,155 @@ module.exports = [ { title: "Use Timescale", href: "use-timescale", + defaultOpen: true, filePath: "index.md", pageComponents: ["content-list"], excerpt: "How to connect to Timescale, administer, and configure the database.", children: [ + { + title: "Hypertables", + href: "hypertables", + children: [ + { + title: "About hypertables", + href: "about-hypertables", + excerpt: "Learn about hypertables in Timescale", + }, + { + title: "Create hypertables", + href: "create", + excerpt: "Create hypertables", + }, + { + title: "Change hypertable chunk intervals", + href: "change-chunk-intervals", + excerpt: "Change and view chunk time intervals for a hypertable", + }, + { + title: "Alter hypertables", + href: "alter", + excerpt: "Alter hypertables", + }, + { + title: "Create unique indexes on hypertables", + href: "hypertables-and-unique-indexes", + excerpt: "Create hypertables with unique indexes", + }, + { + title: "Improve query performance", + href: "improve-query-performance", + excerpt: "Skip chunks", + }, + { + title: "Drop hypertables", + href: "drop", + excerpt: "Drop hypertables", + }, + { + title: "Troubleshoot hypertables", + href: "troubleshooting", + type: "placeholder", + excerpt: "Troubleshooting and error fixes for hypertables", + }, + ], + }, + { + title: "Hypercore", + href: "hypercore", + excerpt: + "Seamlessly switch between row-oriented and column-oriented storage", + children: [ + { + title: "Optimize your data for real-time analytics", + href: "real-time-analytics-in-hypercore", + excerpt: "Automate", + }, + { + title: "Modify data in Hypercore", + href: "modify-data-in-hypercore", + excerpt: "Update data stored in the columnstore", + }, + { + title: "Improve query and upsert performance", + href: "secondary-indexes", + excerpt: "Automate", + }, + { + title: "Troubleshooting", + href: "troubleshooting", + type: "placeholder", + }, + ], + }, + { + title: "Continuous aggregates", + href: "continuous-aggregates", + children: [ + { + title: "About continuous aggregates", + href: "about-continuous-aggregates", + excerpt: "About continuous aggregates", + }, + { + title: "Create a continuous aggregate", + href: "create-a-continuous-aggregate", + excerpt: "Create continuous aggregates", + }, + { + title: "Hierarchical continuous aggregates", + href: "hierarchical-continuous-aggregates", + }, + { + title: "Refresh policies for continuous aggregates", + href: "refresh-policies", + excerpt: "Manage refresh policies for continuous aggregates", + }, + { + title: "Create an index on a continuous aggregate", + href: "create-index", + excerpt: + "Manage automatic index creation and manually create additional indexes", + }, + { + title: "Time in continuous aggregates", + href: "time", + excerpt: "Manage time in continuous aggregates", + }, + { + title: "Drop data from continuous aggregates", + href: "drop-data", + excerpt: "Drop data from continuous aggregates", + }, + { + title: "Manage materialized hypertables", + href: "materialized-hypertables", + excerpt: "Manage materialized hypertables in continuous aggregates", + }, + { + title: "Real time aggregates", + href: "real-time-aggregates", + excerpt: "Manage real time aggregates in continuous aggregates", + }, + { + title: "Compress continuous aggregates", + href: "compression-on-continuous-aggregates", + excerpt: "Compress continuous aggregates", + }, + { + title: "Migrate a continuous aggregate to the new form", + href: "migrate", + excerpt: + "Migrate old continuous aggregates to the new form introduced in Timescale 2.7", + }, + { + title: "Troubleshoot continuous aggregates", + href: "troubleshooting", + type: "placeholder", + excerpt: "Troubleshoot continuous aggregates", + }, + ], + }, { title: "Timescale Cloud regions", href: "regions", @@ -30,8 +174,7 @@ module.exports = [ { title: "Service management", href: "service-management", - excerpt: - "Timescale services operations, Service management tab", + excerpt: "Timescale services operations, Service management tab", }, { title: "Manually change resources", @@ -41,8 +184,7 @@ module.exports = [ { title: "Connection pooling", href: "connection-pooling", - excerpt: - "Using a connection pool with your Timescale services", + excerpt: "Using a connection pool with your Timescale services", }, { title: "I/O boost", @@ -145,53 +287,6 @@ module.exports = [ }, ], }, - { - title: "Hypertables", - href: "hypertables", - children: [ - { - title: "About hypertables", - href: "about-hypertables", - excerpt: "Learn about hypertables in Timescale", - }, - { - title: "Create hypertables", - href: "create", - excerpt: "Create hypertables", - }, - { - title: "Change hypertable chunk intervals", - href: "change-chunk-intervals", - excerpt: "Change and view chunk time intervals for a hypertable", - }, - { - title: "Alter hypertables", - href: "alter", - excerpt: "Alter hypertables", - }, - { - title: "Create unique indexes on hypertables", - href: "hypertables-and-unique-indexes", - excerpt: "Create hypertables with unique indexes", - }, - { - title: "Improve query performance", - href: "improve-query-performance", - excerpt: "Skip chunks", - }, - { - title: "Drop hypertables", - href: "drop", - excerpt: "Drop hypertables", - }, - { - title: "Troubleshoot hypertables", - href: "troubleshooting", - type: "placeholder", - excerpt: "Troubleshooting and error fixes for hypertables", - }, - ], - }, { title: "Schema management", href: "schema-management", @@ -247,57 +342,6 @@ module.exports = [ }, ], }, - { - title: "Compression", - href: "compression", - children: [ - { - title: "About compression", - href: "about-compression", - excerpt: "Learn about how compression works", - }, - { - title: "Compression design", - href: "compression-design", - excerpt: "The design of TimescaleDB compression", - }, - { - title: "About compression methods", - href: "compression-methods", - excerpt: "Learn about the different compression methods", - }, - { - title: "Enable a compression policy", - href: "compression-policy", - excerpt: "Create a compression policy on a hypertable", - }, - { - title: "Manual compression", - href: "manual-compression", - excerpt: "Compress data chunks", - }, - { - title: "Modify compressed data", - href: "modify-compressed-data", - excerpt: "Insert and modify data in compressed chunks", - }, - { - title: "Decompress chunks", - href: "decompress-chunks", - excerpt: "Decompress data chunks", - }, - { - title: "Modify a schema", - href: "modify-a-schema", - excerpt: "Change the data schema in compressed chunks", - }, - { - title: "Troubleshooting", - href: "troubleshooting", - type: "placeholder", - }, - ], - }, { title: "Configuration", href: "configuration", @@ -330,7 +374,8 @@ module.exports = [ { title: "Import and ingest data", href: "ingest-data", - excerpt: "Ingest data into a Timescale Cloud service from third-party sources", + excerpt: + "Ingest data into a Timescale Cloud service from third-party sources", children: [ { title: "Import data from CSV", @@ -353,80 +398,14 @@ module.exports = [ { title: "Ingest data with Kafka", href: "ingest-kafka", - excerpt: "Import data into a Timescale Cloud service using the PostgreSQL Kafka connector", + excerpt: + "Import data into a Timescale Cloud service using the PostgreSQL Kafka connector", }, { title: "Ingest metrics with Telegraf", href: "ingest-telegraf", - excerpt: "Ingest metrics into a Timescale Cloud service using the Telegraf plugin", - }, - ], - }, - { - title: "Continuous aggregates", - href: "continuous-aggregates", - children: [ - { - title: "About continuous aggregates", - href: "about-continuous-aggregates", - excerpt: "About continuous aggregates", - }, - { - title: "Create a continuous aggregate", - href: "create-a-continuous-aggregate", - excerpt: "Create continuous aggregates", - }, - { - title: "Hierarchical continuous aggregates", - href: "hierarchical-continuous-aggregates", - }, - { - title: "Refresh policies for continuous aggregates", - href: "refresh-policies", - excerpt: "Manage refresh policies for continuous aggregates", - }, - { - title: "Create an index on a continuous aggregate", - href: "create-index", - excerpt: - "Manage automatic index creation and manually create additional indexes", - }, - { - title: "Time in continuous aggregates", - href: "time", - excerpt: "Manage time in continuous aggregates", - }, - { - title: "Drop data from continuous aggregates", - href: "drop-data", - excerpt: "Drop data from continuous aggregates", - }, - { - title: "Manage materialized hypertables", - href: "materialized-hypertables", - excerpt: "Manage materialized hypertables in continuous aggregates", - }, - { - title: "Real time aggregates", - href: "real-time-aggregates", - excerpt: "Manage real time aggregates in continuous aggregates", - }, - { - title: "Compress continuous aggregates", - href: "compression-on-continuous-aggregates", - excerpt: "Compress continuous aggregates", - }, - { - title: "Migrate a continuous aggregate to the new form", - href: "migrate", excerpt: - "Migrate old continuous aggregates to the new form introduced in Timescale 2.7", - }, - { - title: "Troubleshoot continuous aggregates", - href: "troubleshooting", - type: "placeholder", - excerpt: "Troubleshoot continuous aggregates", + "Ingest metrics into a Timescale Cloud service using the Telegraf plugin", }, ], }, @@ -471,7 +450,8 @@ module.exports = [ { title: "Tiered storage", href: "data-tiering", - excerpt: "Save on storage costs by tiering older data to a low-cost bottomless object storage tier", + excerpt: + "Save on storage costs by tiering older data to a low-cost bottomless object storage tier", children: [ { title: "About the object storage tier", @@ -482,20 +462,17 @@ module.exports = [ { title: "Manage tiering", href: "enabling-data-tiering", - excerpt: - "How to enable the object storage tier", + excerpt: "How to enable the object storage tier", }, { title: "Querying tiered data", href: "querying-tiered-data", - excerpt: - "How to query tiered data", + excerpt: "How to query tiered data", }, { title: "Replicas and forks with tiered data", href: "tiered-data-replicas-forks", - excerpt: - "How tiered data works on replicas and forks", + excerpt: "How tiered data works on replicas and forks", }, { title: "Troubleshooting", @@ -656,8 +633,7 @@ module.exports = [ { title: "Export to Prometheus", href: "metrics-to-prometheus", - excerpt: - "Export telemetry data to Prometheus", + excerpt: "Export telemetry data to Prometheus", }, ], }, @@ -703,7 +679,7 @@ module.exports = [ href: "postgis", excerpt: "Using the postgis PostgreSQL extension", }, - ] + ], }, { title: "Backup, restore, and PITR", @@ -717,49 +693,34 @@ module.exports = [ { title: "Point-in-time recovery", href: "point-in-time-recovery", - excerpt: "PITR on Timescale services" - } - ] + excerpt: "PITR on Timescale services", + }, + ], }, { - title: "User-defined actions", - href: "user-defined-actions", + title: "Jobs", + href: "jobs", children: [ { - title: "About user-defined actions", - href: "about-user-defined-actions", - excerpt: "Learn about user-defined actions", - }, - { - title: "Create and register a user-defined action", - href: "create-and-register", - excerpt: "Create a user-defined action", + title: "Create and manage jobs", + href: "create-and-manage-jobs", + excerpt: "Create, test, alter, and delete jobs", }, { - title: "Test and debug a user-defined action", - href: "test-and-debug", - excerpt: "Test and debug user-defined actions", - }, - { - title: "Alter and delete a user-defined action", - href: "alter-and-delete", - excerpt: "Edit and delete user-defined actions", - }, - { - title: "Use an action for generic retention", + title: "Use a job for generic retention", href: "example-generic-retention", - excerpt: "Example user-defined action for a retention policy", + excerpt: "A job example for a retention policy", }, { - title: "Use an action for tablespace management", + title: "Use a job for tablespace management", href: "example-tiered-storage", excerpt: - "Example user-defined action for automatically moving chunks between tablespaces", + "A job example for automatically moving chunks between tablespaces", }, { - title: "Use an action for downsampling and compression", + title: "Use a job for downsampling and compression", href: "example-downsample-and-compress", - excerpt: "Example user-defined action for downsample and compress", + excerpt: "A job example for downsampling and compressing data", }, { title: "Troubleshooting", @@ -768,114 +729,6 @@ module.exports = [ }, ], }, - { - title: "Integrations", - href: "integrations", - excerpt: "Integrate third-party solutions with Timescale Cloud", - children: [ - { - title: "Find your connection details", - href: "find-connection-details", - excerpt: "Find connection information for your Timescale Cloud service", - }, - { - title: "Amazon CloudWatch", - href: "cloudwatch", - excerpt: "Integrate Amazon Cloudwatch with Timescale Cloud", - }, - { - title: "Amazon SageMaker", - href: "amazon-sagemaker", - excerpt: "Integrate Amazon SageMaker with Timescale Cloud", - }, - { - title: "Apache Airflow", - href: "apache-airflow", - excerpt: "Integrate Apache Airflow with Timescale products", - }, - { - title: "Apache Kafka", - href: "apache-kafka", - excerpt: "Integrate Apache Kafka with Timescale Cloud", - }, - { - title: "AWS Lambda", - href: "aws-lambda", - excerpt: "Integrate AWS Lambda with Timescale Cloud", - }, - { - title: "Azure Data Studio", - href: "azure-data-studio", - excerpt: "Integrate Azure Data Studio with Timescale Cloud", - }, - { - title: "Datadog", - href: "datadog", - excerpt: "Integrate Datadog with Timescale Cloud", - }, - { - title: "DBeaver", - href: "dbeaver", - excerpt: "Integrate DBeaver with Timescale Cloud", - }, - { - title: "Fivetran", - href: "fivetran", - excerpt: "Integrate Fivetran with Timescale Cloud", - }, - { - title: "Grafana", - href: "grafana", - excerpt: "Integrate Grafana with Timescale products", - }, - { - title: "pgAdmin", - href: "pgadmin", - excerpt: "Integrate pgAdmin with Timescale Cloud", - }, - { - title: "PostgreSQL", - href: "postgresql", - excerpt: "Integrate PostgreSQL with Timescale Cloud", - }, - { - title: "Prometheus", - href: "prometheus", - excerpt: "Integrate Prometheus with Timescale Cloud", - }, - { - title: "Power BI", - href: "power-bi", - excerpt: "Integrate Power BI with Timescale Cloud", - }, - { - title: "psql", - href: "psql", - excerpt: "Connect to Timescale products with psql", - }, - { - title: "qStudio", - href: "qstudio", - excerpt: "Integrate qstudio with Timescale Cloud", - }, - { - title: "Tableau", - href: "tableau", - excerpt: "Integrate Tableau with Timescale Cloud", - }, - { - title: "Terraform", - href: "terraform", - excerpt: "Integrate Terraform with Timescale Cloud", - }, - { - title: "Troubleshooting Timescale integrations", - href: "troubleshooting", - type: "placeholder", - excerpt: "Resolve integration issues", - }, - ], - }, { title: "Security", href: "security", @@ -899,7 +752,8 @@ module.exports = [ { title: "Client credentials", href: "client-credentials", - excerpt: "Client credentials to programmatically access your Timescale account", + excerpt: + "Client credentials to programmatically access your Timescale account", }, { title: "Read only role", @@ -913,14 +767,16 @@ module.exports = [ "Connect to Timescale with a stricter SSL mode of verify-ca or verify-full", }, { - title: "VPC peering and AWS PrivateLink", - href: "vpc", - excerpt: "Secure your Timescale Service with VPC peering and AWS PrivateLink", + title: "Connect securely from any cloud", + href: "transit-gateway", + excerpt: + "Peer your Timescale Cloud service with AWS Transit Gateway", }, { - title: "AWS Transit Gateway", - href: "transit-gateway", - excerpt: "Peer your Timescale Cloud service with AWS Transit Gateway", + title: "VPC peering and AWS PrivateLink", + href: "vpc", + excerpt: + "Secure your Timescale Service with VPC peering and AWS PrivateLink", }, { title: "IP allow list", @@ -940,6 +796,57 @@ module.exports = [ href: "troubleshoot-timescaledb", excerpt: "Troubleshooting Timescale", }, + { + title: "Compression (Old API, replaced by Hypercore)", + href: "compression", + children: [ + { + title: "About compression", + href: "about-compression", + excerpt: "Learn about how compression works", + }, + { + title: "Compression design", + href: "compression-design", + excerpt: "The design of TimescaleDB compression", + }, + { + title: "About compression methods", + href: "compression-methods", + excerpt: "Learn about the different compression methods", + }, + { + title: "Enable a compression policy", + href: "compression-policy", + excerpt: "Create a compression policy on a hypertable", + }, + { + title: "Manual compression", + href: "manual-compression", + excerpt: "Compress data chunks", + }, + { + title: "Modify compressed data", + href: "modify-compressed-data", + excerpt: "Insert and modify data in compressed chunks", + }, + { + title: "Decompress chunks", + href: "decompress-chunks", + excerpt: "Decompress data chunks", + }, + { + title: "Modify a schema", + href: "modify-a-schema", + excerpt: "Change the data schema in compressed chunks", + }, + { + title: "Troubleshooting", + href: "troubleshooting", + type: "placeholder", + }, + ], + }, ], }, ]; diff --git a/use-timescale/security/index.md b/use-timescale/security/index.md index ad4c204028..1471ba1a00 100644 --- a/use-timescale/security/index.md +++ b/use-timescale/security/index.md @@ -14,12 +14,11 @@ privacy. * Set up [multi-factor][mfa] and [SAML][saml] authentication * Generate multiple [client credentials][client-credentials] instead of using your username and password * Grant [read-only access][read-only] to your $SERVICE_LONGs -* Learn how to [connect with a stricter SSL mode][ssl] -* Secure your Timescale Cloud services with [VPC peering][vpc-peering] -* Peer your $SERVICE_LONGs with [AWS Transit Gateway][transit-gateway] +* Connect with a [stricter SSL mode][ssl] +* Secure your $SERVICE_LONGs with [VPC peering][vpc-peering] +* Connect to your $SERVICE_SHORTs from any cloud with [AWS Transit Gateway][transit-gateway] * Restrict access with an [IP address allow list][ip-allowlist] - [overview]: /use-timescale/:currentVersion:/security/overview/ [ssl]: /use-timescale/:currentVersion:/security/strict-ssl/ [mfa]: /use-timescale/:currentVersion:/security/multi-factor-authentication/ diff --git a/use-timescale/security/transit-gateway.md b/use-timescale/security/transit-gateway.md index a5ebc5a07c..2011eaa3b3 100644 --- a/use-timescale/security/transit-gateway.md +++ b/use-timescale/security/transit-gateway.md @@ -9,83 +9,87 @@ cloud_ui: - [services, :serviceId, operations, vpc] --- -# Peer your $SERVICE_LONGs with AWS Transit Gateway +import TransitGateway from "versionContent/_partials/_transit-gateway.mdx"; -You use [AWS Transit Gateway][aws-transit-gateway] as a traffic controller for your network. Instead of setting up lots of direct connections to virtual private clouds, on-premise data centers, and other AWS services, you connect everything to Transit Gateway. This simplifies your network and makes it easier to manage and scale. +# Securely connect to $CLOUD_LONG using AWS Transit Gateway -You can create a peering connection between your $SERVICE_LONGs and AWS Transit Gateway in $CLOUD_LONG. This means that, no matter how big or complex your infrastructure is, you can connect securely to your $SERVICE_LONGs. +[AWS Transit Gateway][aws-transit-gateway] enables you to securely connect to your $CLOUD_LONG from AWS, Google Cloud, Microsoft Azure, or any other cloud or on-premise environment. + +You use AWS Transit Gateway as a traffic controller for your network. Instead of setting up multiple direct connections to different clouds, on-premise data centers, and other AWS services, you connect everything to AWS Transit Gateway. This simplifies your network and makes it easier to manage and scale. + +You can then create a peering connection between your $SERVICE_LONGs and AWS Transit Gateway in $CLOUD_LONG. This means that, no matter how big or complex your infrastructure is, you can connect securely to your $SERVICE_LONGs. To configure this secure connection, you: -1. Create a $CLOUD_LONG Peering $VPC with a peering connection to your AWS Transit Gateway. +1. Connect your infrastructure to AWS Transit Gateway. +1. Create a $CLOUD_LONG Peering $VPC with a peering connection to AWS Transit Gateway. 1. Accept and configure the peering connection on your side. 1. Attach individual $SERVICE_SHORTs to the Peering $VPC. -Transit Gateway feature is available for Scale and Enterprise [pricing plans][pricing-plans]. +The AWS Transit Gateway feature is available for Scale and Enterprise [pricing plans][pricing-plans]. + +AWS Transit Gateway enable you to connect from almost any environment, this page provides examples for the most common use cases. -## Create a Peering $VPC + -To create a Peering $VPC: + -1. **In [$CONSOLE][console-login] > `Security` > `VPC`, click `Create a VPC`** + - ![$CLOUD_LONG new $VPC](https://assets.timescale.com/docs/images/add-peering-vpc.png) + -1. **Choose your region and IP range, name your VPC, then click `Create VPC`** + - ![Create a new VPC in $CLOUD_LONG](https://assets.timescale.com/docs/images/configure-peering-vpc.png) + - Your $SERVICE_SHORT and Peering $VPC must be in the same AWS region. The number of Peering $VPCs you can create in your project depends on your [pricing plan][pricing-plans]. If you need another Peering $VPC, either contact [support@timescale.com](mailto:support@timescale.com) or change your pricing plan in [$CONSOLE][console-login]. + -1. **Add a peering connection** +1. **Connect your infrastructure to AWS Transit Gateway** - 1. In the `VPC Peering` column, click `Add`. - 1. Provide your AWS account ID, VPC ID or Transit Gateway ID, CIDR ranges, and AWS region. - 1. Click `Add connection`. + Establish connectivity between Azure and AWS. See the [AWS architectural documentation][azure-aws] for details. - ![Add peering](https://assets.timescale.com/docs/images/add-peering.png) + -## Accept and configure peering connection + -Once your peering connection appears as `Processing`, you can accept and configure it in AWS: + -1. **Accept the peering request** +1. **Connect your infrastructure to AWS Transit Gateway** - In your AWS account, accept the peering request coming from $CLOUD_LONG. The request can take up to 5 min to arrive. Within 5 more minutes after accepting, the peering should appear as `Connected` in $CONSOLE. + Establish connectivity between Google Cloud and AWS. See [Connect HA VPN to AWS peer gateways][gcp-aws]. -1. **Configure networking in your AWS account** - - Configure at least the following: - - 1. Your subnet route table to route traffic to your Transit Gateway for the Peering VPC CIDRs. - 1. Your Transit Gateway route table to route traffic to the newly created Transit Gateway peering attachment for the Peering VPC CIDRs. - 1. Security groups to allow outbound TCP 5432. + -## Attach a $CLOUD_LONG service to the Peering VPC + -To attach a $SERVICE_SHORT to the Peering VPC: + -1. **In [$CONSOLE][console-services], select the $SERVICE_SHORT you want to connect to the Peering VPC** -1. **Click `Operations` > `Security` > `VPC`** -1. **Select the VPC, then click `Attach VPC`** +1. **Connect your infrastructure to AWS Transit Gateway** + + Establish connectivity between your on-premise infrastructure and AWS. See the [Centralize network connectivity using AWS Transit Gateway][aws-onprem]. - You cannot attach a $SERVICE_LONG to multiple $CLOUD_LONG $VPCs at the same time. + -You can now securely access your $SERVICE_SHORTs from any private cloud or on-premise data center connected to AWS Transit Gateway. + + + + +You can now securely access your $SERVICE_SHORTs in $CLOUD_LONG. [aws-transit-gateway]: https://aws.amazon.com/transit-gateway/ [pricing-plans]: /about/:currentVersion:/pricing-and-account-management/ -[console-login]: https://console.cloud.timescale.com/ -[console-services]: https://console.cloud.timescale.com/dashboard/services \ No newline at end of file +[azure-aws]: https://aws.amazon.com/blogs/modernizing-with-aws/designing-private-network-connectivity-aws-azure/ +[gcp-aws]: https://cloud.google.com/network-connectivity/docs/vpn/how-to/connect-ha-vpn-aws-peer-gateway +[aws-onprem]: https://docs.aws.amazon.com/prescriptive-guidance/latest/patterns/centralize-network-connectivity-using-aws-transit-gateway.html diff --git a/use-timescale/security/vpc.md b/use-timescale/security/vpc.md index e84d1ac780..302e011f07 100644 --- a/use-timescale/security/vpc.md +++ b/use-timescale/security/vpc.md @@ -9,7 +9,9 @@ cloud_ui: - [services, :serviceId, operations, vpc] --- -# Secure your $CLOUD_LONG services with VPC Peering and AWS PrivateLink +import VpcLimitations from "versionContent/_partials/_vpc-limitations.mdx"; + +# Secure your $CLOUD_LONG services with $VPC Peering and AWS PrivateLink You use Virtual Private Cloud ($VPC) peering to ensure that your $SERVICE_LONGs are only accessible through your secured AWS infrastructure. This reduces the potential @@ -20,215 +22,197 @@ $CLOUD_LONG is: ![$CLOUD_LONG isolation architecture](https://assets.timescale.com/docs/images/tsc-vpc-architecture.svg) -Your customer apps run inside your AWS Customer VPC, your $CLOUD_LONG services always run +Your customer apps run inside your AWS Customer $VPC, your $SERVICE_LONGs always run inside the secure $CLOUD_LONG $VPC. You control secure communication between apps in -your VPC and your $SERVICE_SHORTs using a dedicated Peering $VPC. The AWS PrivateLink connecting +your $VPC and your $SERVICE_SHORTs using a dedicated Peering $VPC. The AWS PrivateLink connecting $CLOUD_LONG $VPC to the dedicated Peering $VPC gives the same level of protection as using a direct AWS PrivateLink connection. It only enables communication to be initiated from your Customer $VPC -to services running in the $CLOUD_LONG $VPC. $CLOUD_LONG cannot initiate communication with your $VPC. +to $SERVICE_SHORTs running in the $CLOUD_LONG $VPC. $CLOUD_LONG cannot initiate communication with your Customer $VPC. -To configure this secure connection, you first create the $CLOUD_LONG Peering $VPC with +To configure this secure connection, you first create a Peering $VPC with AWS PrivateLink in $CONSOLE. After you have accepted and configured the peering connection to your Customer $VPC, you use AWS Security Groups to -restrict the services in your Customer $VPC that are visible to the Peering $VPC. -The last step is to attach individual services to the Peering $VPC. - -* You define each $CLOUD_LONG $VPC on a [$CLOUD_LONG project level][project-members]. -* You can attach: - * Up to 50 Customer $VPCs to a $CLOUD_LONG $VPC. - * A $SERVICE_LONG to a single $CLOUD_LONG $VPC at a time. - - The $SERVICE_SHORT and $VPC must be in the same AWS region. However, you can peer a Customer $VPC and a - $CLOUD_LONG $VPC that are in different regions. - * Multiple $SERVICE_LONGs to the same $CLOUD_LONG $VPC. -* You cannot attach a $SERVICE_LONG to multiple $CLOUD_LONG $VPCs at the same time. - -The number of $CLOUD_LONG $VPCs you can create in your project depends on your [pricing plan][pricing-plans]. -If you need another $CLOUD_LONG $VPC, either contact [support@timescale.com](mailto:support@timescale.com) or change your -pricing plan in [$CONSOLE][console-login]. +restrict the apps in your Customer $VPC that are visible to the Peering $VPC. +The last step is to attach individual $SERVICE_SHORTs to the Peering $VPC in $CONSOLE. + +* You create each Peering $VPC on a [$CLOUD_LONG project level][project-members]. + + ## Prerequisites -In order to set up VPC peering you need the following permissions in your AWS account: +To set up $VPC peering, you need the following permissions in your AWS account: -* Accept VPC peering requests +* Accept $VPC peering requests * Configure route table rules * Configure security group and firewall rules ## Set up a secured connection between $CLOUD_LONG and AWS -To connect to a $SERVICE_LONG using VPC peering, your apps and infrastructure must be already -running in an Amazon Web Services (AWS) VPC. You can peer your VPC from any AWS region. -However, your $CLOUD_LONG $VPC must be within one of the [Cloud-supported regions][tsc-regions]. +To connect to a $SERVICE_LONG using $VPC peering, your apps and infrastructure must be already +running in an Amazon Web Services (AWS) $VPC. You can peer your $VPC from any AWS region. +However, your Peering $VPC must be within one of the [Cloud-supported regions][tsc-regions]. -The stages to create a secured connection between $CLOUD_LONG services and your AWS infrastructure are: +The stages to create a secured connection between $SERVICE_LONGs and your AWS infrastructure are: -1. [Create a Peering VPC in $CONSOLE][aws-vpc-setup-vpc] -1. [Complete the VPC connection in your AWS][aws-vpc-complete] +1. [Create a Peering $VPC in $CONSOLE][aws-vpc-setup-vpc] +1. [Complete the $VPC connection in your AWS][aws-vpc-complete] 1. [Set up security groups in your AWS][aws-vpc-security-groups] -1. [Attach a Timescale Service to the Peering VPC][aws-vpc-connect-vpcs] +1. [Attach a $SERVICE_LONG to the Peering $VPC][aws-vpc-connect-vpcs] -### Create a Peering VPC in $CONSOLE +### Create a Peering $VPC in $CONSOLE -Create the VPC and the peering connection that enables you to securely route traffic -between $CLOUD_LONG and your own VPC in a logically isolated virtual network. +Create the $VPC and the peering connection that enables you to securely route traffic +between $CLOUD_LONG and your Customer $VPC in a logically isolated virtual network. -1. In [$CONSOLE > Security > VPC][console-vpc], click `Create a VPC`. +1. **In [$CONSOLE > Security > VPC][console-vpc], click `Create a VPC`** ![$CLOUD_LONG new $VPC](https://assets.timescale.com/docs/images/add-peering-vpc.png) - * You can attach: - * Up to 50 Customer $VPCs to a $CLOUD_LONG $VPC. - * A $SERVICE_LONG to a single $CLOUD_LONG $VPC at a time. - - The $SERVICE_SHORT and $VPC must be in the same AWS region. However, you can peer a Customer $VPC and a - $CLOUD_LONG $VPC that are in different regions. - * Multiple $SERVICE_LONGs to the same $CLOUD_LONG $VPC. - * You cannot attach a $SERVICE_LONG to multiple $CLOUD_LONG $VPCs at the same time. - - The number of $CLOUD_LONG $VPCs you can create in your project depends on your [pricing plan][pricing-plans]. - If you need another $CLOUD_LONG $VPC, either contact [support@timescale.com](mailto:support@timescale.com) or change your - pricing plan in [$CONSOLE][console-login]. - - -1. Choose your region and IP range, name your VPC, then click `Create VPC`. +1. **Choose your region and IP range, name your $VPC, then click `Create VPC`** ![Create a new VPC in $CLOUD_LONG](https://assets.timescale.com/docs/images/configure-peering-vpc.png) - The IP ranges of the Peering VPC and Customer VPC should not overlap. + The IP ranges of the Peering $VPC and Customer VPC should not overlap. -1. For as many peering connections as you need: +1. **For as many peering connections as you need**: 1. In the `VPC Peering` column, click `Add`. - 2. Enter information about your existing AWS VPC, then click `Add Connection`. + 2. Enter information about your existing Customer VPC, then click `Add Connection`. ![Add peering](https://assets.timescale.com/docs/images/add-peering.png) + + $CLOUD_LONG sends a peering request to your AWS account so you can [complete the VPC connection in AWS][aws-vpc-complete]. + -### Complete the VPC connection in AWS +### Complete the $VPC connection in AWS When you receive the $CLOUD_LONG peering request in AWS, edit your routing table to match -the `IP Range` and `CIDR block` between your AWS and $CLOUD_LONG VPCs. +the `IP Range` and `CIDR block` between your Customer and Peering VPCs. -When you peer a VPC with multiple CIDRs, all CIDRs are added to the $CLOUD_LONG rules automatically. -After you have finished peering, further changes in your VPC's CIDRs are not detected automatically. -If you need to refresh the CIDRs, please recreate the peering connection. +When you peer a $VPC with multiple CIDRs, all CIDRs are added to the $CLOUD_LONG rules automatically. +After you have finished peering, further changes in your $VPC's CIDRs are not detected automatically. +If you need to refresh the CIDRs, recreate the peering connection. The request acceptance process is an important safety mechanism. Do not accept a peering request from an unknown account. -1. In [AWS > VPC Dashboard > Peering connections][aws-dashboard], select the peering connection - request from $CLOUD_LONG. +1. **In [AWS > VPC Dashboard > Peering connections][aws-dashboard], select the peering connection + request from $CLOUD_LONG** Copy the peering connection ID to the clipboard. The connection request starts with `pcx-`. -1. In the peering connection, click `Route Tables`, then select the `Route Table ID` - that corresponds to your VPC. -1. In `Routes`, click `Edit routes`. You see the list of existing Destinations. +1. **In the peering connection, click `Route Tables`, then select the `Route Table ID` + that corresponds to your $VPC** + +1. **In `Routes`, click `Edit routes`** + + You see the list of existing destinations. ![Create a new VPC route](https://assets.timescale.com/docs/images/tsc-vpc-add-route.png). If you do not already have a destination that corresponds to the `IP range / CIDR block` of - your Timescale VPC: + your Peering $VPC: 1. Click `Add route`, and set: - * `Destination`: the CIDR block of your Timescale VPC. For example: `10.0.0.7/17`. + * `Destination`: the CIDR block of your Peering $VPC. For example: `10.0.0.7/17`. * `Target`: the peering connection ID you copied to your clipboard. 2. Click `Save changes`. Network traffic is secured between your AWS account and $CLOUD_LONG for this project. + ### Set up security groups in AWS Security groups allow specific inbound and outbound traffic at the resource level. -You can associate a VPC with one or more security groups and each instance in your -VPC may belong to a different set of security groups. The security group choices -for your VPC are: +You can associate a $VPC with one or more security groups, and each instance in your +$VPC may belong to a different set of security groups. The security group choices +for your $VPC are: -* Create a security group to use for your $CLOUD_LONG VPC only. -* Associate your VPC with an existing security group. -* Do nothing, your VPC is automatically associated with the default one. +* Create a security group to use for your $CLOUD_LONG $VPC only. +* Associate your $VPC with an existing security group. +* Do nothing, your $VPC is automatically associated with the default one. -To create a security group specific to your $CLOUD_LONG VPC: +To create a security group specific to your $CLOUD_LONG Peering VPC: -1. [AWS > VPC Dashboard > Security Groups][aws-security-groups], click `Create security group`. +1. **[AWS > VPC Dashboard > Security Groups][aws-security-groups], click `Create security group`** -1. Enter the rules for this security group: +1. **Enter the rules for this security group**: The AWS Security Groups dashboard - * `VPC`: select the VPC that is peered with Timescale. + * `VPC`: select the $VPC that is peered with $CLOUD_LONG. * `Inbound rules`: leave empty. * `Outbound rules`: * `Type`: `Custom TCP` * `Protocol`: `TCP` * `Port range`: `5432` * `Destination`: `Custom` - * `Info`: the CIDR block of your $CLOUD_LONG VPC. -1. Click `Add rule`, then click `Create security group`. + * `Info`: the CIDR block of your $CLOUD_LONG Peering $VPC. +1. **Click `Add rule`, then click `Create security group`** -### Attach a $CLOUD_LONG service to the Peering VPC +### Attach a $SERVICE_LONG to the Peering $VPC Now that $CLOUD_LONG is communicating securely with your AWS infrastructure, you can attach -one or more services to the VPC. - -After you attach a $SERVICE_SHORT to a VPC, you can only access it through the peered -AWS VPC. It is no longer accessible using the public internet. +one or more $SERVICE_SHORTs to the Peering $VPC. +After you attach a $SERVICE_SHORT to a Peering $VPC, you can only access it through the peered +AWS $VPC. It is no longer accessible using the public internet. -1. In [$CONSOLE > Services][console-services] select the $SERVICE_SHORT you want to - connect to the VPC. -1. Click `Security` > `VPC`. -1. Select the VPC, then click `Attach VPC`. +1. **In [$CONSOLE > Services][console-services] select the $SERVICE_SHORT you want to + connect to the Peering $VPC** +1. **Click `Operations` > `Security` > `VPC`** +1. **Select the $VPC, then click `Attach VPC`** And that is it, your $SERVICE_SHORT is now securely communicating with your AWS -account inside a VPC. +account inside a $VPC. ## Migrate a $CLOUD_LONG $SERVICE_SHORT between VPCs To ensure that your applications continue to run without interruption, you keep -$SERVICE_SHORT attached to the VPC. However, you can change the VPC your -$SERVICE_SHORT is attached to, or disconnect from a VPC and enable access to the +$SERVICE_SHORT attached to the Peering $VPC. However, you can change the Peering $VPC your +$SERVICE_SHORT is attached to, or disconnect from the Peering $VPC and enable access to the $SERVICE_SHORT from the public internet. -$CLOUD_LONG uses a different DNS for services that are attached to a VPC. -When you migrate a $SERVICE_SHORT between public access and a VPC, you need +$CLOUD_LONG uses a different DNS for $SERVICE_SHORTs that are attached to a Peering $VPC. +When you migrate a $SERVICE_SHORT between public access and a Peering $VPC, you need to update your connection string. -1. In [$CONSOLE > Services][console-services] select the $SERVICE_SHORT to migrate. +1. **In [$CONSOLE > Services][console-services] select the $SERVICE_SHORT to migrate** If you don't have a $SERVICE_SHORT, [create a new one][create-service]. -1. Click `Security` > `VPC`. -1. Select the VPC, then click `Attach VPC`. +1. **Click `Operations` > `Security` > `VPC`** +1. **Select the $VPC, then click `Attach VPC`** Migration takes a few minutes to complete and requires a change to DNS settings for the -Service. The $SERVICE_SHORT is not accessible during this time. If you receive a DNS error, allow +$SERVICE_SHORT. The $SERVICE_SHORT is not accessible during this time. If you receive a DNS error, allow some time for DNS propagation. [aws-dashboard]: https://console.aws.amazon.com/vpc/home#PeeringConnections: @@ -238,14 +222,10 @@ some time for DNS propagation. [console-services]: https://console.cloud.timescale.com/dashboard/services [timescale-support]: https://www.timescale.com/contact/ [tsc-regions]: /use-timescale/:currentVersion:/regions/ - - [aws-vpc-setup-vpc]: /use-timescale/:currentVersion:/security/vpc/#create-a-peering-vpc-in-timescale-console [aws-vpc-complete]: /use-timescale/:currentVersion:/security/vpc/#complete-the-vpc-connection-in-aws [aws-vpc-security-groups]: /use-timescale/:currentVersion:/security/vpc/#set-up-security-groups-in-aws [aws-vpc-connect-vpcs]: /use-timescale/:currentVersion:/security/vpc/#attach-a-timescale-service-to-the-peering-vpc - - [create-service]: /getting-started/:currentVersion:/services/#create-a-timescale-cloud-service [pricing-plans]: /about/:currentVersion:/pricing-and-account-management/ [project-members]: /use-timescale/:currentVersion:/members/ diff --git a/use-timescale/services/OLD_create-a-pg-service.md b/use-timescale/services/OLD_create-a-pg-service.md index 7448a2c3d0..0578c9679d 100644 --- a/use-timescale/services/OLD_create-a-pg-service.md +++ b/use-timescale/services/OLD_create-a-pg-service.md @@ -5,7 +5,7 @@ products: [cloud] keywords: [services, PostgreSQL, create, install] --- -import EarlyAccess from "versionContent/_partials/_early_access.mdx"; +import EarlyAccess from "versionContent/_partials/_early_access_2_18_0.mdx"; # Create a PostgreSQL service diff --git a/use-timescale/time-buckets/about-time-buckets.md b/use-timescale/time-buckets/about-time-buckets.md index c2ff30d373..0dab19f29e 100644 --- a/use-timescale/time-buckets/about-time-buckets.md +++ b/use-timescale/time-buckets/about-time-buckets.md @@ -7,19 +7,17 @@ keywords: [time buckets] # About time buckets -The [`time_bucket`][time_bucket] function allows you to aggregate data in a -[hypertable][create-hypertable] into buckets of time. For example: 5 minutes, 1 hour, or 3 days. +Time bucketing is essential for real-time analytics. The [`time_bucket`][time_bucket] function enables you to aggregate data in a [hypertable][create-hypertable] into buckets of time. For example, 5 minutes, 1 hour, or 3 days. It's similar to PostgreSQL's [`date_bin`][date_bin] function, but it gives you more -flexibility in bucket size and start time. +flexibility in the bucket size and start time. -Time bucketing is essential to working with time-series data. You can use it to -roll up data for analysis or downsampling. For example, you can calculate +You can use it to roll up data for analysis or downsampling. For example, you can calculate 5-minute averages for a sensor reading over the last day. You can perform these rollups as needed, or pre-calculate them in [continuous aggregates][caggs]. This section explains how time bucketing works. For examples of the `time_bucket` function, see the section on -[using time buckets][use-time-buckets]. +[Aggregate time-series data with `time_bucket`][use-time-buckets]. ## How time bucketing works diff --git a/use-timescale/time-buckets/index.md b/use-timescale/time-buckets/index.md index d766900317..d728b6d522 100644 --- a/use-timescale/time-buckets/index.md +++ b/use-timescale/time-buckets/index.md @@ -10,7 +10,7 @@ keywords: [time buckets] Time buckets enable you to aggregate data in [hypertables][create-hypertable] by time interval. For example, you can group data into 5-minute, 1-hour, and 3-day buckets to calculate summary values. -* [Learn how time buckets work][about-time-buckets] in Timescale +* [Learn how time buckets work][about-time-buckets] in $CLOUD_LONG * [Use time buckets][use-time-buckets] to aggregate data [about-time-buckets]: /use-timescale/:currentVersion:/time-buckets/about-time-buckets/ diff --git a/use-timescale/troubleshoot-timescaledb.md b/use-timescale/troubleshoot-timescaledb.md index 13f3ff7e3e..395dd666bd 100644 --- a/use-timescale/troubleshoot-timescaledb.md +++ b/use-timescale/troubleshoot-timescaledb.md @@ -132,7 +132,7 @@ tsdb=> \dp _timescaledb_internal._compressed_hypertable_2 This means that the `relacl` column of `pg_class` needs to be updated and the offending user removed, but it is not possible to drop a user by numerical value. Instead, you can use the internal function -`repair_relation_acls` in `_timescaledb_function` schema: +`repair_relation_acls` in the `_timescaledb_function` schema: ```sql tsdb=> CALL _timescaledb_functions.repair_relation_acls(); diff --git a/use-timescale/upgrades.md b/use-timescale/upgrades.md index 672e70f4ca..effed6dd80 100644 --- a/use-timescale/upgrades.md +++ b/use-timescale/upgrades.md @@ -8,6 +8,8 @@ cloud_ui: - [services, :serviceId, operations, maintenance] --- +import SupportMatrix from "versionContent/_partials/_migrate_self_postgres_timescaledb_compatibility.mdx"; + # Maintenance and upgrades On Timescale, minor software updates are handled automatically, and @@ -19,8 +21,10 @@ is a critical security vulnerability that affects you, maintenance might need to occur outside of the scheduled maintenance window. + After a maintenance update, the DNS name remains the same, but the IP address it points to often changes. + In most cases, the updates that occur during your maintenance windows do not @@ -71,14 +75,15 @@ For more information about replicas, see the ## Non-critical maintenance updates -Non-critical upgrades are made available before the upgrade is performed -automatically. During this time you can click `Apply upgrades` to start the -upgrade at any time. However, after the time expires, usually around a week, -the upgrade is triggered automatically in the next available maintenance window -for your service. You can configure the maintenance window so that these -upgrades are started only at a particular time, on a set day of the week. If -there are no pending upgrades available during a regular maintenance window, no -changes are performed. +Non-critical upgrades are available before the upgrade is performed +automatically by $CLOUD_LONG. To upgrade $TIMESCALE_DB manually, run `ALTER +EXTENSION timescaledb UPDATE` in your $SERVICE_LONG, or `Pause` and `Resume`. +If no action is taken by the user, the upgrade is triggered in the next +available maintenance window. You can configure the maintenance window so that +these upgrades are started at a particular time, on a set day of the week. + +If there are no pending upgrades available during a regular maintenance window, +no changes are performed. When you are considering your maintenance window schedule, you might prefer to choose a day and time that usually has very low activity, such as during the @@ -119,33 +124,30 @@ can plan accordingly. However, in some cases, we might not be able to do so. ## Upgrade to a new PostgreSQL version -Timescale currently supports PostgreSQL 14, 15, 16, and 17. You can see -your PostgreSQL and Timescale versions from the Timescale service +You can also manually upgrade to the newest supported PostgreSQL version from the service overview page. - - -You can also manually upgrade to the newest supported PostgreSQL version -(PostgreSQL 16) from the service overview page. - Upgrading to a newer version of PostgreSQL allows you to take advantage of new features, enhancements, and security fixes. It also ensures that you are using a version of PostgreSQL that's compatible with the newest version of Timescale, -allowing you to take advantage of everything Timescale has to offer. For more -information about feature changes between versions, see the +allowing you to take advantage of everything Timescale has to offer. + +The following table shows you the compatible versions of $PG and $TIMESCALE_DB. + + + +For more information about feature changes between versions, see the [PostgreSQL release notes][postgres-relnotes] and [Timescale release notes][timescale-relnotes]. + + + Your Timescale service is unavailable until the upgrade is complete. This can take up to 20 minutes. It is recommended to test on a fork first for a better estimate. + ### Recommended practices for upgrading @@ -164,13 +166,14 @@ For a smooth upgrade experience, make sure you: this fork to only pay for storage until you are comfortable deleting it. + Timescale services with replicas cannot be upgraded. To upgrade a service with a replica, you must first delete the replica and then upgrade the service. -### Upgrading to a new PostgreSQL version +### Upgrade to a new PostgreSQL version 1. In the Timescale console, navigate to `Services` and click the service you want to upgrade. @@ -185,6 +188,7 @@ with a replica, you must first delete the replica and then upgrade the service. + [cloud-login]: https://cloud.timescale.com diff --git a/use-timescale/user-defined-actions/about-user-defined-actions.md b/use-timescale/user-defined-actions/about-user-defined-actions.md deleted file mode 100644 index 8361381058..0000000000 --- a/use-timescale/user-defined-actions/about-user-defined-actions.md +++ /dev/null @@ -1,121 +0,0 @@ ---- -title: About user-defined actions -excerpt: User-defined actions are custom PostgreSQL functions and procedures that you set up to run on a schedule. This provides a set-it-and-forget-it approach to maintaining your service and data -products: [cloud, mst, self_hosted] -keywords: [actions] -tags: [user-defined actions, background jobs, scheduled jobs, automation framework] ---- - -# About user-defined actions - -You can create user-defined actions that periodically run custom functions or -procedures on your database. - -## PostgreSQL functions - -You can use PostgreSQL functions to create database operations that would -typically take several queries or steps. Functions can also be used by other -applications that interact with your database to perform tasks without -additional code. - -Your function needs to be wrapped in a `CREATE` statement. You can also use -`CREATE OR REPLACE`, although this is not recommended. This statement also -allows you to define the language of your commands in this statement. -User-defined actions can be written in any language of your choice. This guide uses -the SQL procedural language [PL/pgSQL][plpgsql]. - -The basic syntax of a function suitable for a user-defined action is: - -```sql -CREATE FUNCTION (required arguments) -RETURNS AS $$ - DECLARE - ; - BEGIN - ; - RETURN { | value } - END; LANGUAGE ; -``` - -This is an example of a function that returns the total row count of a table -within a database. - -Use this code to create the function. Note that `job_id` and `config` are required arguments. - -```sql -CREATE FUNCTION totalRecords (job_id INT DEFAULT NULL, config JSONB DEFAULT NULL) -RETURNS integer AS $total$ -declare - total integer; -BEGIN - SELECT count(*) into total FROM fill_measurements; - RETURN total; -END; -$total$ LANGUAGE plpgsql; -``` - -When you execute this code in psql, it returns `CREATE FUNCTION` to indicate -that it has successfully been created. You can then execute a call to the -function, like this: - -```sql -select totalRecords(); -``` - -The result looks like this: - -```sql - totalrecords --------------- - 48600500 -(1 row) -``` - -## The job scheduler - -When you have created your function, you need to register it with the job -scheduler to make the function run regularly. You can do with the `add_job` function. -This example adds the `totalRecords` function, and tells it to run every hour: - -```sql -SELECT add_job('totalRecords', '1h', config => '{"hypertable":"metr"}'); -``` - -For the `config` value, if you don't need any special configuration parameters, -you can use `NULL`. - -You can see a list of all your currently registered jobs by querying the job -scheduler, like this: - -```sql -SELECT * FROM timescaledb_information.jobs; -``` - -The result looks like this: - -```sql - job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | config | next_start | hypertable_schema | hypertable_name ---------+----------------------------+-------------------+-------------+-------------+--------------+-----------------------+------------------+-----------+-----------+------------------------+-------------------------------+-------------------+----------------- - 1 | Telemetry Reporter [1] | 24:00:00 | 00:01:40 | -1 | 01:00:00 | _timescaledb_internal | policy_telemetry | postgres | t | | 2022-08-18 06:26:39.524065+00 | | - 1000 | User-Defined Action [1000] | 01:00:00 | 00:00:00 | -1 | 00:05:00 | public | totalrecords | tsdbadmin | t | {"hypertable": "metr"} | 2022-08-17 07:17:24.831698+00 | | -(2 rows) -``` - -## Native job-scheduling policies - -TimescaleDB natively includes some job-scheduling policies, such as: - -* [Continuous aggregate policies][caggs] to automatically refresh continuous - aggregates -* [Compression policies][compressing] to compress historical data -* [Retention policies][retention] to drop historical data -* [Reordering policies][reordering] to reorder data within chunks - -If these don't cover your use case, or if you want to expand upon the native -policy features, you can write a user-defined action. - -[caggs]: /use-timescale/:currentVersion:/continuous-aggregates/refresh-policies/ -[compressing]: /use-timescale/:currentVersion:/compression/ -[reordering]: /api/:currentVersion:/hypertable/add_reorder_policy/ -[retention]: /use-timescale/:currentVersion:/data-retention/create-a-retention-policy/ -[plpgsql]: https://www.postgresql.org/docs/current/plpgsql-overview.html diff --git a/use-timescale/user-defined-actions/alter-and-delete.md b/use-timescale/user-defined-actions/alter-and-delete.md deleted file mode 100644 index ee0726d34b..0000000000 --- a/use-timescale/user-defined-actions/alter-and-delete.md +++ /dev/null @@ -1,63 +0,0 @@ ---- -title: Alter and delete user-defined actions -excerpt: When a user-defined action has run its course, you can update, reschedule, or delete it. See how to manage user-defined actions in Timescale Cloud -products: [cloud, mst, self_hosted] -keywords: [actions] -tags: [user-defined actions, scheduled jobs, background jobs, automation framework] ---- - -# Alter and delete user-defined actions - -Alter an existing job by using [`alter_job`][api-alter_job]. You can change both -the config and the schedule on which the job runs. Delete a job by using -[`delete_job`][api-delete_job]. - - -To alter or delete a job, you need to know its `job_id`. Find the `job_id` by -querying the `timescaledb_information.jobs` table. - -```sql -SELECT * FROM timescaledb_information.jobs; -``` - - - -## Change a job's config - -To replace the entire JSON config for a job, call `alter_job` with a new -`config` object. For example, replace the JSON config for a job with id `1000`: - -```sql -SELECT alter_job(1000, config => '{"hypertable":"metrics"}'); -``` - -## Turn off scheduling of a job - -To turn off automatic scheduling of a job, call `alter_job` and set `scheduled` -to `false`. You can still run the job manually with `run_job`. For example, -turn off the scheduling for a job with id `1000`: - -```sql -SELECT alter_job(1000, scheduled => false); -``` - -## Re-enable automatic scheduling of a job - -To re-enable automatic scheduling of a job, call `alter_job` and set `scheduled` -to `true`. For example, re-enable scheduling for a job with id `1000`: - -```sql -SELECT alter_job(1000, scheduled => true); -``` - -## Delete a job - -Delete a job from the automation framework with [`delete_job`][api-delete_job]. -For example, to delete a job with id `1000`: - -```sql -SELECT delete_job(1000); -``` - -[api-alter_job]: /api/:currentVersion:/actions/alter_job -[api-delete_job]: /api/:currentVersion:/actions/delete_job diff --git a/use-timescale/user-defined-actions/create-and-register.md b/use-timescale/user-defined-actions/create-and-register.md deleted file mode 100644 index b8f11d8608..0000000000 --- a/use-timescale/user-defined-actions/create-and-register.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -title: Create and register user-defined actions -excerpt: Creating a user-defined action in your Timescale Cloud service is a two-step process. First, define a function or procedure, then register it with add_job. See usage examples -products: [cloud, mst, self_hosted] -keywords: [actions] -tags: [user-defined actions, scheduled jobs, background jobs, automation framework] ---- - -# Create and register user-defined actions - -Adding a user-defined action to your database is a 2-step process: - -1. [Define a function or procedure](#define-a-function-or-procedure) -1. [Register the action with the job scheduler](#register-an-action) - -## Define a function or procedure - -To create an action, begin by defining the -[function][postgres-createfunction] or -[procedure][postgres-createprocedure] that you want your database to execute. - -Your function needs to be wrapped in a `CREATE` statement. You can also use -`CREATE OR REPLACE`, although this is not recommended. This statement also -allows you to define the language of your commands in this statement. -User-defined actions can be written in any language you choose. This guide uses -the SQL procedural language [PL/pgSQL][plpgsql]. - -This example defines a simple procedure that raises a notice: - -```sql -CREATE PROCEDURE user_defined_action(job_id INT, config JSONB) - LANGUAGE PLPGSQL AS - $$ - BEGIN - RAISE NOTICE 'Executing job % with config %', job_id, config; - END - $$; -``` - -## Register an action - -To make the job scheduler run your action, you need to register it. Use the -[`add_job`][api-add_job] function. Supply the name of your action, the schedule -you want it to run on, and the content of your config. If your job needs no -parameters, use a NULL config. - -For example: - -```sql -SELECT add_job('user_defined_action', '1h', config => '{"hypertable":"metrics"}'); -``` - -The `add_job` call returns a `job_id`. It stores the `job_id` and `config` in -the TimescaleDB catalog. - -The action runs on the schedule you set. It also runs when you manually start it -by calling [`run_job`][api-run_job] with the `job_id`. When it runs, the -`job_id` and `config` are passed as arguments to the procedure. - -To list all currently registered jobs, query -[`timescaledb_information.jobs`][api-timescaledb_information-jobs]: - -```sql -SELECT * FROM timescaledb_information.jobs; -``` - -[api-add_job]: /api/:currentVersion:/actions/add_job -[api-run_job]: /api/:currentVersion:/actions/run_job -[api-timescaledb_information-jobs]: /api/:currentVersion:/informational-views/jobs/ -[postgres-createfunction]: https://www.postgresql.org/docs/current/xfunc.html -[postgres-createprocedure]: https://www.postgresql.org/docs/current/xproc.html -[plpgsql]: https://www.postgresql.org/docs/current/plpgsql-overview.html diff --git a/use-timescale/user-defined-actions/index.md b/use-timescale/user-defined-actions/index.md deleted file mode 100644 index 6cefc225f4..0000000000 --- a/use-timescale/user-defined-actions/index.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: User-defined actions -excerpt: Increase efficiency and reduce overhead by automating routine tasks. User-defined actions help you set up custom jobs on top of Timescale Cloud policies. Learn how to create them in your service -products: [cloud, mst, self_hosted] -keywords: [actions] -tags: [user-defined actions, scheduled jobs, background jobs, automation framework] ---- - -# User-defined actions - -User-defined actions let you schedule custom-defined procedures to run within -Timescale. They help you automate periodic tasks that aren't covered by -Timescale's policy features. - -In this section: - -* [Learn about user-defined actions][about-user-defined-actions] before you - start using them -* [Create and register][create-and-register] a user-defined action -* [Test and debug][test-and-debug] a user-defined action -* [Alter and delete][alter-and-delete] a user-defined action -* Examples of user-defined actions: - * Use a user-defined action to set up a - [generic data retention][generic-retention] policy that applies across - all hypertables - * Use a user-defined action to implement - [automatic moving of chunks between tablespaces][manage-storage] - * Use a user-defined action to automatically - [downsample and compress][downsample-compress] older chunks - -[about-user-defined-actions]: /use-timescale/:currentVersion:/user-defined-actions/about-user-defined-actions/ -[alter-and-delete]: /use-timescale/:currentVersion:/user-defined-actions/alter-and-delete/ -[create-and-register]: /use-timescale/:currentVersion:/user-defined-actions/create-and-register/ -[downsample-compress]: /use-timescale/:currentVersion:/user-defined-actions/example-downsample-and-compress -[generic-retention]: /use-timescale/:currentVersion:/user-defined-actions/example-generic-retention -[manage-storage]: /use-timescale/:currentVersion:/user-defined-actions/example-tiered-storage/ -[test-and-debug]: /use-timescale/:currentVersion:/user-defined-actions/test-and-debug/ diff --git a/use-timescale/user-defined-actions/test-and-debug.md b/use-timescale/user-defined-actions/test-and-debug.md deleted file mode 100644 index d21f5a664a..0000000000 --- a/use-timescale/user-defined-actions/test-and-debug.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -title: Test and debug a user-defined action -excerpt: Having issues while setting up user-defined actions in Timescale Cloud? Find solutions to the most common ones -products: [cloud, mst, self_hosted] -keywords: [actions, debug] ---- - -# Test and debug a user-defined action - -To debug an action, you can increase the log level and run the job manually in -the foreground. - -Use the [`run_job`][api-run_job] procedure, which takes a `job_id` argument. -Because `run_job` is a stored procedure and not a function, it needs to be -executed with [`CALL`][postgres-call] instead of `SELECT`. - - - -## Testing and debugging a job - -1. Change the minimum log level shown to the client. Set it to `DEBUG1`. - - ```sql - SET client_min_messages TO DEBUG1; - ``` - -1. Run the job. Replace `1000` with your actual `job_id`. - - ```sql - CALL run_job(1000); - ``` - - -To find the `job_id` for your job, query the `timescaledb_information.jobs` -table. - -```sql -SELECT * FROM timescaledb_information.jobs; -``` - - - - - -[api-run_job]: /api/:currentVersion:/actions/run_job -[postgres-call]: https://www.postgresql.org/docs/current/sql-call.html diff --git a/use-timescale/write-data/about-writing-data.md b/use-timescale/write-data/about-writing-data.md index 1e40441c0b..1d28585a5d 100644 --- a/use-timescale/write-data/about-writing-data.md +++ b/use-timescale/write-data/about-writing-data.md @@ -12,7 +12,7 @@ Timescale supports writing data in the same way as PostgreSQL, using `INSERT`, `UPDATE`, `INSERT ... ON CONFLICT`, and `DELETE`. -Because Timescale is a time-series database, hypertables are optimized for +$CLOUD_LONG is optimized for running real-time analytics workloads on time-series data. For this reason, hypertables are optimized for inserts to the most recent time intervals. Inserting data with recent time values gives [excellent performance](https://www.timescale.com/blog/timescaledb-vs-6a696248104e/).