From ec3feb5d8dc233bfad514e8a8f4cb2d8677fe789 Mon Sep 17 00:00:00 2001 From: atovpeko Date: Thu, 6 Feb 2025 16:14:34 +0200 Subject: [PATCH 01/90] draft --- use-timescale/integrations/airbyte.md | 76 +++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 use-timescale/integrations/airbyte.md diff --git a/use-timescale/integrations/airbyte.md b/use-timescale/integrations/airbyte.md new file mode 100644 index 0000000000..e88f5553d9 --- /dev/null +++ b/use-timescale/integrations/airbyte.md @@ -0,0 +1,76 @@ +--- +title: Integrate Airbyte with Timescale Cloud +excerpt: Integrate Airbyte with Timescale Cloud to enable seamless data movement between different sources and your service +products: [cloud, mst, self_hosted] +keywords: [Airbyte, integrate] + +--- + +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; + +# Integrate Airbyte with $CLOUD_LONG + +[Airbyte][airbyte] is an open-source data integration platform that enables you to sync data from various sources to different destinations, including $CLOUD_LONG. + +This page explains how to integrate Airbyte with your $SERVICE_LONG and move data efficiently for further analysis and storage. + +## Prerequisites + + + +- Install [Airbyte](https://docs.airbyte.com/deploying-airbyte) // Mention both cloud and self-hosted versions, if present. Link to installation pages. +- Ensure you have a running Timescale Cloud instance. +- Retrieve your Timescale Cloud connection details ([Find your connection details][connection-info]). + +## Connect Airbyte to Timescale Cloud + +To connect Airbyte to $CLOUD_LONG: + + + +1. **Log in to Airbyte** + + - Open the [Airbyte dashboard](https://cloud.airbyte.com/) or your self-hosted Airbyte instance. + - Log in to your Airbyte account. + +2. **Set up Timescale Cloud as a Destination** + + - Navigate to the `Destinations` tab in Airbyte. + - Click on **+ New Destination**. + - Select `PostgreSQL` as the destination type (Timescale Cloud is PostgreSQL-compatible). + +3. **Configure the Connection** + + - Enter the following details from your Timescale Cloud instance: + + - **Host**: `` + - **Port**: `5432` + - **Database Name**: `` + - **User**: `` + - **Password**: `` + - **Schema**: `public` (or your target schema) + + - Click **Test Connection** to verify the settings. + +4. **Set Up a Data Source** + + - Navigate to the `Sources` tab in Airbyte. + - Click on **+ New Source**. + - Choose the data source you want to sync (e.g., PostgreSQL, MySQL, API, CSV, etc.). + - Enter the connection details for your data source. + - Click **Test Connection**. + +5. **Create a Sync Job** + + - Click on **Connections** > **New Connection**. + - Select your configured source and Timescale Cloud as the destination. + - Choose the replication mode (`Full Refresh`, `Incremental`, etc.). + - Configure transformation settings if needed. + - Click **Save & Run** to start data synchronization. + + + +You have successfully integrated Airbyte with $CLOUD_LONG. + +[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[airbyte]: https://airbyte.com/ From 95abe772191ffb4baeb59222e2b849f15d0082b9 Mon Sep 17 00:00:00 2001 From: atovpeko Date: Fri, 7 Feb 2025 16:33:31 +0200 Subject: [PATCH 02/90] draft --- use-timescale/integrations/airbyte.md | 66 ++++++++------------------- 1 file changed, 20 insertions(+), 46 deletions(-) diff --git a/use-timescale/integrations/airbyte.md b/use-timescale/integrations/airbyte.md index e88f5553d9..6b4d1ccce8 100644 --- a/use-timescale/integrations/airbyte.md +++ b/use-timescale/integrations/airbyte.md @@ -1,28 +1,25 @@ --- -title: Integrate Airbyte with Timescale Cloud -excerpt: Integrate Airbyte with Timescale Cloud to enable seamless data movement between different sources and your service -products: [cloud, mst, self_hosted] -keywords: [Airbyte, integrate] - +title: Integrate Airbyte with Timescale Cloud +excerpt: Airbyte is an open-source data integration platform. Integrate Airbyte with Timescale Cloud to enable seamless data movement and analytics. +products: [cloud, mst, self_hosted] +keywords: [Airbyte, integration] --- import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; # Integrate Airbyte with $CLOUD_LONG -[Airbyte][airbyte] is an open-source data integration platform that enables you to sync data from various sources to different destinations, including $CLOUD_LONG. +[Airbyte](https://airbyte.com/) is an open-source data integration platform that enables you to move and consolidate data from various sources to a destination of your choice. Airbyte provides pre-built connectors, simplifying ETL (Extract, Transform, Load) processes. -This page explains how to integrate Airbyte with your $SERVICE_LONG and move data efficiently for further analysis and storage. +This pages explains how to integrate Airbyte with $CLOUD_LONG to facilitate efficient data ingestion and analysis in a $SERVICE_LONG. ## Prerequisites -- Install [Airbyte](https://docs.airbyte.com/deploying-airbyte) // Mention both cloud and self-hosted versions, if present. Link to installation pages. -- Ensure you have a running Timescale Cloud instance. -- Retrieve your Timescale Cloud connection details ([Find your connection details][connection-info]). +- Install [Airbyte Self-Managed Community][airbyte-server] or sign up for [Airbyte Cloud][airbyte-cloud]. -## Connect Airbyte to Timescale Cloud +## Connect Airbyte to $CLOUD_LONG To connect Airbyte to $CLOUD_LONG: @@ -30,47 +27,24 @@ To connect Airbyte to $CLOUD_LONG: 1. **Log in to Airbyte** - - Open the [Airbyte dashboard](https://cloud.airbyte.com/) or your self-hosted Airbyte instance. - - Log in to your Airbyte account. - -2. **Set up Timescale Cloud as a Destination** - - - Navigate to the `Destinations` tab in Airbyte. - - Click on **+ New Destination**. - - Select `PostgreSQL` as the destination type (Timescale Cloud is PostgreSQL-compatible). - -3. **Configure the Connection** - - - Enter the following details from your Timescale Cloud instance: - - - **Host**: `` - - **Port**: `5432` - - **Database Name**: `` - - **User**: `` - - **Password**: `` - - **Schema**: `public` (or your target schema) - - - Click **Test Connection** to verify the settings. +1. **Configure the $CLOUD_LONG destination** -4. **Set Up a Data Source** + 1. Open the `Destinations` tab. + 1. Click `New Destination` and select `Postgres` as the destination connector. + 1. Configure the destination using your [connection details][connection-info]. + 1. Click `Set up destination`. - - Navigate to the `Sources` tab in Airbyte. - - Click on **+ New Source**. - - Choose the data source you want to sync (e.g., PostgreSQL, MySQL, API, CSV, etc.). - - Enter the connection details for your data source. - - Click **Test Connection**. + Airbyte tests the connection. -5. **Create a Sync Job** +1. **Test the connection** - - Click on **Connections** > **New Connection**. - - Select your configured source and Timescale Cloud as the destination. - - Choose the replication mode (`Full Refresh`, `Incremental`, etc.). - - Configure transformation settings if needed. - - Click **Save & Run** to start data synchronization. + 1. Click `Test Connection` to verify connectivity. + 1. If successful, click `Save & Continue`. -You have successfully integrated Airbyte with $CLOUD_LONG. +You have successfully added $CLOUD_LONG as a destination in Airbyte. You can now add a source and link the source to the destination to move your data. [connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ -[airbyte]: https://airbyte.com/ +[airbyte-server]: +[airbyte-cloud]: \ No newline at end of file From d17d6da4eb86e39ff6f9affc3c07dd7ecbf89437 Mon Sep 17 00:00:00 2001 From: atovpeko Date: Mon, 10 Feb 2025 11:33:25 +0200 Subject: [PATCH 03/90] draft --- use-timescale/integrations/airbyte.md | 122 ++++++++++++++++++++++--- use-timescale/integrations/index.md | 8 +- use-timescale/page-index/page-index.js | 37 ++++---- 3 files changed, 134 insertions(+), 33 deletions(-) diff --git a/use-timescale/integrations/airbyte.md b/use-timescale/integrations/airbyte.md index 6b4d1ccce8..5c1db261be 100644 --- a/use-timescale/integrations/airbyte.md +++ b/use-timescale/integrations/airbyte.md @@ -9,42 +9,136 @@ import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.md # Integrate Airbyte with $CLOUD_LONG -[Airbyte](https://airbyte.com/) is an open-source data integration platform that enables you to move and consolidate data from various sources to a destination of your choice. Airbyte provides pre-built connectors, simplifying ETL (Extract, Transform, Load) processes. +[Airbyte](https://airbyte.com/) is an open-source data integration platform that enables you to move and consolidate data from various sources to a destination of your choice. Airbyte provides pre-built connectors, simplifying ETL (Extract, Transform, Load) processes. -This pages explains how to integrate Airbyte with $CLOUD_LONG to facilitate efficient data ingestion and analysis in a $SERVICE_LONG. +[Ubidots][ubidots] is an IoT platform that enables real-time IoT data collection and management. + +This page explains how to ingest IoT sensor data from Ubidots into $CLOUD_LONG with Airbyte. ## Prerequisites -- Install [Airbyte Self-Managed Community][airbyte-server] or sign up for [Airbyte Cloud][airbyte-cloud]. +- Install [Airbyte self-managed][airbyte-server] or sign up for [Airbyte Cloud][airbyte-cloud]. +- Sign up for [Ubidots][ubidots-signup]. + +## Simulate IoT sensor data in Ubidots + +To generate simulated IoT data in Ubidots: + + + +1. **Log in to Ubidots** + +1. **Create a new device** + + 1. Navigate to `Devices` > `Create Device`. + 1. Select `Blank Device` and name it `airbyte-iot-sensor`. + +1. **Add variables to the device** + + 1. Click `airbyte-iot-sensor` and select `Add Variable`. + 1. Create the following variables of type `float`: + - `temperature` + - `humidity` + - `pressure` + +1. **Simulate data for the device** + + 1. Click `temperature`, then select `Edit Settings` > `Synthetic Data`. + 1. Enable synthetic data and set an update interval. For example, every 5 seconds. + 1. Enable synthetic data for `humidity` and `pressure` variables. + + + +## Create a table to store IoT sensor data in $CLOUD_LONG + +To store IoT data in $CLOUD_LONG, create a table: + + + +1. **Connect to your $SERVICE_LONG** -## Connect Airbyte to $CLOUD_LONG + For $CLOUD_LONG, open an [SQL editor][run-queries] in [$CONSOLE][console]. For self-hosted, use `psql`. -To connect Airbyte to $CLOUD_LONG: +1. **Create a table** + + Run the following query: + + ```sql + CREATE TABLE iot_sensor_data ( + time TIMESTAMPTZ DEFAULT now(), + device_id TEXT, + temperature DOUBLE PRECISION, + humidity DOUBLE PRECISION, + pressure DOUBLE PRECISION + ); + SELECT create_hypertable('iot_sensor_data', 'time'); + ``` + + + +## Connect Ubidots to $CLOUD_LONG using Airbyte + +To connect Ubidots to $CLOUD_LONG: 1. **Log in to Airbyte** +1. **Configure the Ubidots source** + + 1. Click `Sources` > `New source` and select `Ubidots`. + 1. Enter the required connection details: + - `API Key`: your Ubidots [API key][ubidots-api-key] + - `Device ID`: `airbyte-iot-sensor` + 1. Click `Test & Save`. + 1. **Configure the $CLOUD_LONG destination** - 1. Open the `Destinations` tab. - 1. Click `New Destination` and select `Postgres` as the destination connector. - 1. Configure the destination using your [connection details][connection-info]. + 1. Click `Destinations` > `New Destination` and select `Postgres`. + 1. Enter the required [connection details][connection-info]. 1. Click `Set up destination`. Airbyte tests the connection. -1. **Test the connection** + 1. If the connection is successful, click `Save & Continue`. + +1. **Create a connection between Ubidots and $CLOUD_LONG** + + 1. Click `Connections` > `New Connection`. + 1. Select Ubidots as the source and $CLOUD_LONG as the destination. + 1. Configure the sync mode. For example, `Incremental Append`. + 1. Map Ubidots variables `temperature`, `humidity`, `pressure` to the corresponding columns in `iot_sensor_data`. + 1. Click `Save & Activate`. + + + +## Check the connection + +To check if the data is successfully ingested: + + + +1. **Connect to your $SERVICE_LONG** +1. **Query data from the table** + + Run the following SQL query: - 1. Click `Test Connection` to verify connectivity. - 1. If successful, click `Save & Continue`. + ```sql + SELECT * FROM iot_sensor_data ORDER BY time DESC LIMIT 10; + ``` + You should now see simulated IoT data from Ubidots appear in the table. -You have successfully added $CLOUD_LONG as a destination in Airbyte. You can now add a source and link the source to the destination to move your data. +You have successfully ingested data from Ubidots into $CLOUD_LONG using Airbyte! [connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ -[airbyte-server]: -[airbyte-cloud]: \ No newline at end of file +[airbyte-server]: https://docs.airbyte.com/deploying-airbyte/ +[airbyte-cloud]: https://cloud.airbyte.com/signup +[ubidots]: https://ubidots.com/ +[ubidots-signup]: https://ubidots.com/#signup-modal +[run-queries]: /getting-started/:currentVersion:/run-queries-from-console/ +[console]: https://console.cloud.timescale.com/dashboard/services +[ubidots-api-key]: https://help.ubidots.com/en/articles/570026-api-authentication \ No newline at end of file diff --git a/use-timescale/integrations/index.md b/use-timescale/integrations/index.md index 8d1ca0fda7..fb58a1846c 100644 --- a/use-timescale/integrations/index.md +++ b/use-timescale/integrations/index.md @@ -47,9 +47,10 @@ Some of the most in-demand integrations for $CLOUD_LONG are listed below, with l ## Data engineering and extract, transform, load -| Name | Description | -|:--------------------------------:|----------------------------------------------------------| -| [Apache Airflow][apache-airflow] | Programmatically author, schedule, and monitor workflows. | +| Name | Description | +|:--------------------------------:|----------------------------------------------------------------------------------------------------------------------------------------------| +| [Apache Airflow][apache-airflow] | A community platform designed to programmatically author, schedule, and monitor workflows. | +| [Airbyte][airbyte] | An open-source data integration platform that enables you to move and consolidate data from various sources to a destination of your choice. | @@ -66,3 +67,4 @@ Some of the most in-demand integrations for $CLOUD_LONG are listed below, with l [apache-airflow]: /use-timescale/:currentVersion:/integrations/apache-airflow [postgresql-integrations]: https://slashdot.org/software/p/PostgreSQL/integrations/ [prometheus]: /use-timescale/:currentVersion:/integrations/prometheus +[airbyte]: /use-timescale/:currentVersion:/integrations/airbyte diff --git a/use-timescale/page-index/page-index.js b/use-timescale/page-index/page-index.js index 83af6784b0..db85bde43c 100644 --- a/use-timescale/page-index/page-index.js +++ b/use-timescale/page-index/page-index.js @@ -776,22 +776,27 @@ module.exports = [ { title: "Find your connection details", href: "find-connection-details", - excerpt: "Learn about connecting to your Timescale database", + excerpt: "Find connection details for your Timescale Cloud service", }, { - title: "Apache Airflow", - href: "apache-airflow", - excerpt: "Integrate Apache Airflow with Timescale products", + title: "Airbyte", + href: "airbyte", + excerpt: "Integrate Airbyte with Timescale Cloud", }, { title: "Amazon CloudWatch", href: "cloudwatch", excerpt: "Integrate Amazon Cloudwatch with Timescale Cloud", }, + { + title: "Apache Airflow", + href: "apache-airflow", + excerpt: "Integrate Apache Airflow with Timescale Cloud", + }, { title: "Azure Data Studio", href: "azure-data-studio", - excerpt: "Integrate Azure Data Studio with Timescale products", + excerpt: "Integrate Azure Data Studio with Timescale Cloud", }, { title: "Datadog", @@ -801,12 +806,17 @@ module.exports = [ { title: "DBeaver", href: "dbeaver", - excerpt: "Integrate DBeaver with Timescale products", + excerpt: "Integrate DBeaver with Timescale Cloud", + }, + { + title: "Grafana", + href: "grafana", + excerpt: "Integrate Grafana with Timescale Cloud", }, { title: "pgAdmin", href: "pgadmin", - excerpt: "Integrate pgAdmin with Timescale products", + excerpt: "Integrate pgAdmin with Timescale Cloud", }, { title: "Prometheus", @@ -816,27 +826,22 @@ module.exports = [ { title: "psql", href: "psql", - excerpt: "Connect to Timescale products with psql", + excerpt: "Connect to Timescale Cloud with psql", }, { title: "qStudio", href: "qstudio", - excerpt: "Integrate qstudio with Timescale products", - }, - { - title: "Grafana", - href: "grafana", - excerpt: "Integrate Grafana with Timescale products", + excerpt: "Integrate qstudio with Timescale Cloud", }, { title: "Tableau", href: "tableau", - excerpt: "Integrate Tableau with Timescale products", + excerpt: "Integrate Tableau with Timescale Cloud", }, { title: "Terraform", href: "terraform", - excerpt: "Manage your Timescale services with Terraform", + excerpt: "Manage your Timescale Cloud services with Terraform", }, { title: "Troubleshooting Timescale integrations", From a50d152d17e292acebbdbc39e7e4bd4dcc0e91c8 Mon Sep 17 00:00:00 2001 From: Anagha Mittal Date: Fri, 28 Feb 2025 23:26:36 +0530 Subject: [PATCH 04/90] established the connection, unable to transfer data --- use-timescale/integrations/airbyte.md | 90 ++++++++++++++++----------- 1 file changed, 53 insertions(+), 37 deletions(-) diff --git a/use-timescale/integrations/airbyte.md b/use-timescale/integrations/airbyte.md index 5c1db261be..529a9a4869 100644 --- a/use-timescale/integrations/airbyte.md +++ b/use-timescale/integrations/airbyte.md @@ -9,7 +9,9 @@ import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.md # Integrate Airbyte with $CLOUD_LONG -[Airbyte](https://airbyte.com/) is an open-source data integration platform that enables you to move and consolidate data from various sources to a destination of your choice. Airbyte provides pre-built connectors, simplifying ETL (Extract, Transform, Load) processes. +[Airbyte][airbyte] is an open-source data integration platform that enables you to move and consolidate +data from various sources to a destination of your choice. Airbyte provides pre-built connectors, +simplifying ETL (Extract, Transform, Load) processes. [Ubidots][ubidots] is an IoT platform that enables real-time IoT data collection and management. @@ -28,26 +30,38 @@ To generate simulated IoT data in Ubidots: -1. **Log in to Ubidots** +1. **Log in to [Ubidots][ubidots-signup]** 1. **Create a new device** - 1. Navigate to `Devices` > `Create Device`. + 1. Navigate to `Devices` > `Create a device`. 1. Select `Blank Device` and name it `airbyte-iot-sensor`. 1. **Add variables to the device** 1. Click `airbyte-iot-sensor` and select `Add Variable`. - 1. Create the following variables of type `float`: - - `temperature` + 1. Create the following variables: + - `temperature` - `humidity` - - `pressure` + - `pressure` + 1. Note the `variable-id` for each 1. **Simulate data for the device** - - 1. Click `temperature`, then select `Edit Settings` > `Synthetic Data`. - 1. Enable synthetic data and set an update interval. For example, every 5 seconds. - 1. Enable synthetic data for `humidity` and `pressure` variables. + + 1. In a terminal window, run the following command separately for each variable using the `variable-id` and [Default Token][ubidots-token]: + + ```bash + curl -X POST 'https://industrial.api.ubidots.com/api/v1.6/variables//values' \ + -H 'Content-Type: application/json' \ + -H 'X-Auth-Token: ' \ + -d '[ + {"value": 25.4}, + {"value": 26.1}, + {"value": 24.9} + ]' + ``` + + You may change the values for `pressure` and `humidity`. @@ -65,16 +79,16 @@ To store IoT data in $CLOUD_LONG, create a table: Run the following query: - ```sql - CREATE TABLE iot_sensor_data ( - time TIMESTAMPTZ DEFAULT now(), - device_id TEXT, - temperature DOUBLE PRECISION, - humidity DOUBLE PRECISION, - pressure DOUBLE PRECISION - ); - SELECT create_hypertable('iot_sensor_data', 'time'); - ``` + ```sql + CREATE TABLE iot_sensor_data ( + time TIMESTAMPTZ DEFAULT now(), + device_id TEXT, + temperature DOUBLE PRECISION, + humidity DOUBLE PRECISION, + pressure DOUBLE PRECISION + ); + SELECT create_hypertable('iot_sensor_data', 'time'); + ``` @@ -84,33 +98,31 @@ To connect Ubidots to $CLOUD_LONG: -1. **Log in to Airbyte** +1. **Log in to [Airbyte][airbyte-cloud]** 1. **Configure the Ubidots source** 1. Click `Sources` > `New source` and select `Ubidots`. - 1. Enter the required connection details: - - `API Key`: your Ubidots [API key][ubidots-api-key] - - `Device ID`: `airbyte-iot-sensor` - 1. Click `Test & Save`. + 2. Enter the `API Key` which is your Ubidots `airbyte-iot-sensor` device [Token][ubidots-device-token]. + 3. Click `Set up source`. 1. **Configure the $CLOUD_LONG destination** - 1. Click `Destinations` > `New Destination` and select `Postgres`. - 1. Enter the required [connection details][connection-info]. - 1. Click `Set up destination`. - - Airbyte tests the connection. - - 1. If the connection is successful, click `Save & Continue`. + 1. Click `Destinations` > `New destination` and select `Postgres`. + 2. Enter the Destination name as $CLOUD_LONG and populate the required [connection details][connection-info]. + 3. Click `Set up destination`. 1. **Create a connection between Ubidots and $CLOUD_LONG** - 1. Click `Connections` > `New Connection`. - 1. Select Ubidots as the source and $CLOUD_LONG as the destination. - 1. Configure the sync mode. For example, `Incremental Append`. - 1. Map Ubidots variables `temperature`, `humidity`, `pressure` to the corresponding columns in `iot_sensor_data`. - 1. Click `Save & Activate`. + 1. Click `Connections` > `New connection`. + 2. Select Ubidots as the source and $CLOUD_LONG as the destination. + 3. Select the sync mode and select the schema to sync with $CLOUD_LONG and click on `Next`. + 4. Configure the connection for with desired settings. + 5. Click on `Finish & Sync`. + + **// PLEASE TEST FROM HERE.** + + 6. Map Ubidots variables `temperature`, `humidity`, `pressure` to the corresponding columns in `iot_sensor_data`. @@ -128,6 +140,7 @@ To check if the data is successfully ingested: ```sql SELECT * FROM iot_sensor_data ORDER BY time DESC LIMIT 10; ``` + You should now see simulated IoT data from Ubidots appear in the table. @@ -135,10 +148,13 @@ To check if the data is successfully ingested: You have successfully ingested data from Ubidots into $CLOUD_LONG using Airbyte! [connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[airbyte]: https://airbyte.com/ [airbyte-server]: https://docs.airbyte.com/deploying-airbyte/ [airbyte-cloud]: https://cloud.airbyte.com/signup [ubidots]: https://ubidots.com/ [ubidots-signup]: https://ubidots.com/#signup-modal +[ubidots-token]: https://docs.ubidots.com/v1.6/reference/authentication?utm_source=api_docs&utm_medium=internal_referral&utm_campaign=Send%20data%20to%20a%20Variable [run-queries]: /getting-started/:currentVersion:/run-queries-from-console/ +[ubidots-device-token]: https://help.ubidots.com/en/articles/3832305-security-managing-device-tokens [console]: https://console.cloud.timescale.com/dashboard/services [ubidots-api-key]: https://help.ubidots.com/en/articles/570026-api-authentication \ No newline at end of file From cb50189d7f99d9e5a121d508cbb68d29ec990420 Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Mon, 3 Mar 2025 11:19:29 +0200 Subject: [PATCH 05/90] Fixed numbering on the VPC page Co-authored-by: Iain Cox --- use-timescale/security/vpc.md | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/use-timescale/security/vpc.md b/use-timescale/security/vpc.md index e84d1ac780..770fd70d22 100644 --- a/use-timescale/security/vpc.md +++ b/use-timescale/security/vpc.md @@ -82,16 +82,12 @@ between $CLOUD_LONG and your own VPC in a logically isolated virtual network. * You can attach: * Up to 50 Customer $VPCs to a $CLOUD_LONG $VPC. * A $SERVICE_LONG to a single $CLOUD_LONG $VPC at a time. - - The $SERVICE_SHORT and $VPC must be in the same AWS region. However, you can peer a Customer $VPC and a - $CLOUD_LONG $VPC that are in different regions. + The $SERVICE_SHORT and $VPC must be in the same AWS region. However, you can peer a Customer $VPC and a $CLOUD_LONG $VPC that are in different regions. * Multiple $SERVICE_LONGs to the same $CLOUD_LONG $VPC. - * You cannot attach a $SERVICE_LONG to multiple $CLOUD_LONG $VPCs at the same time. - - The number of $CLOUD_LONG $VPCs you can create in your project depends on your [pricing plan][pricing-plans]. - If you need another $CLOUD_LONG $VPC, either contact [support@timescale.com](mailto:support@timescale.com) or change your - pricing plan in [$CONSOLE][console-login]. + * You cannot attach a $SERVICE_LONG to multiple $CLOUD_LONG $VPCs at the same time. + The number of $CLOUD_LONG $VPCs you can create in your project depends on your [pricing plan][pricing-plans]. + If you need another $CLOUD_LONG $VPC, either contact [support@timescale.com](mailto:support@timescale.com) or change your pricing plan in [$CONSOLE][console-login]. 1. Choose your region and IP range, name your VPC, then click `Create VPC`. From 38e3235fa2e1fdf2d140e9f0cb9506e1a6add6a7 Mon Sep 17 00:00:00 2001 From: Jeff Lambert Date: Mon, 3 Mar 2025 09:43:57 -0500 Subject: [PATCH 06/90] Update compress_chunk.md (#3872) Signed-off-by: Jeff Lambert Co-authored-by: Iain Cox --- api/compress_chunk.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/compress_chunk.md b/api/compress_chunk.md index d8c4962439..75d5e65c89 100644 --- a/api/compress_chunk.md +++ b/api/compress_chunk.md @@ -41,9 +41,9 @@ You can get a list of chunks belonging to a hypertable using the ### Returns -|Column|Description| -|---|---| -| `compress_chunk` | (REGCLASS) Name of the chunk that was compressed| +|Column|Type|Description| +|---|---|---| +| `compress_chunk` | REGCLASS | Name of the chunk that was compressed| ### Sample usage From afb71bad07ef8cbf505e2fc077cf8f00401bed45 Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Mon, 3 Mar 2025 17:10:17 +0200 Subject: [PATCH 07/90] Update Transit gateway docs to cover more clouds (#3887) Co-authored-by: Iain Cox --- use-timescale/page-index/page-index.js | 10 +++++----- use-timescale/security/index.md | 7 +++---- use-timescale/security/transit-gateway.md | 8 +++++--- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/use-timescale/page-index/page-index.js b/use-timescale/page-index/page-index.js index 66500c3a2e..9891239c36 100644 --- a/use-timescale/page-index/page-index.js +++ b/use-timescale/page-index/page-index.js @@ -919,16 +919,16 @@ module.exports = [ excerpt: "Connect to Timescale with a stricter SSL mode of verify-ca or verify-full", }, + { + title: "Connect securely from any cloud", + href: "transit-gateway", + excerpt: "Peer your Timescale Cloud service with AWS Transit Gateway", + }, { title: "VPC peering and AWS PrivateLink", href: "vpc", excerpt: "Secure your Timescale Service with VPC peering and AWS PrivateLink", }, - { - title: "AWS Transit Gateway", - href: "transit-gateway", - excerpt: "Peer your Timescale Cloud service with AWS Transit Gateway", - }, { title: "IP allow list", href: "ip-allow-list", diff --git a/use-timescale/security/index.md b/use-timescale/security/index.md index ad4c204028..1471ba1a00 100644 --- a/use-timescale/security/index.md +++ b/use-timescale/security/index.md @@ -14,12 +14,11 @@ privacy. * Set up [multi-factor][mfa] and [SAML][saml] authentication * Generate multiple [client credentials][client-credentials] instead of using your username and password * Grant [read-only access][read-only] to your $SERVICE_LONGs -* Learn how to [connect with a stricter SSL mode][ssl] -* Secure your Timescale Cloud services with [VPC peering][vpc-peering] -* Peer your $SERVICE_LONGs with [AWS Transit Gateway][transit-gateway] +* Connect with a [stricter SSL mode][ssl] +* Secure your $SERVICE_LONGs with [VPC peering][vpc-peering] +* Connect to your $SERVICE_SHORTs from any cloud with [AWS Transit Gateway][transit-gateway] * Restrict access with an [IP address allow list][ip-allowlist] - [overview]: /use-timescale/:currentVersion:/security/overview/ [ssl]: /use-timescale/:currentVersion:/security/strict-ssl/ [mfa]: /use-timescale/:currentVersion:/security/multi-factor-authentication/ diff --git a/use-timescale/security/transit-gateway.md b/use-timescale/security/transit-gateway.md index a5ebc5a07c..2a58786037 100644 --- a/use-timescale/security/transit-gateway.md +++ b/use-timescale/security/transit-gateway.md @@ -9,11 +9,13 @@ cloud_ui: - [services, :serviceId, operations, vpc] --- -# Peer your $SERVICE_LONGs with AWS Transit Gateway +# Securely connect to $CLOUD_LONG from any cloud or on-premise environment -You use [AWS Transit Gateway][aws-transit-gateway] as a traffic controller for your network. Instead of setting up lots of direct connections to virtual private clouds, on-premise data centers, and other AWS services, you connect everything to Transit Gateway. This simplifies your network and makes it easier to manage and scale. +[AWS Transit Gateway][aws-transit-gateway] enables you to securely connect to your $CLOUD_LONG from AWS, GCP, Azure, or any other cloud or on-premise environment. -You can create a peering connection between your $SERVICE_LONGs and AWS Transit Gateway in $CLOUD_LONG. This means that, no matter how big or complex your infrastructure is, you can connect securely to your $SERVICE_LONGs. +You use AWS Transit Gateway as a traffic controller for your network. Instead of setting up multiple direct connections to different clouds, on-premise data centers, and other AWS services, you connect everything to Transit Gateway. This simplifies your network and makes it easier to manage and scale. + +You can then create a peering connection between your $SERVICE_LONGs and AWS Transit Gateway in $CLOUD_LONG. This means that, no matter how big or complex your infrastructure is, you can connect securely to your $SERVICE_LONGs. To configure this secure connection, you: From aa9b7d4dbf25d02128e137a45a2b205c8cf26bd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B4natas=20Davi=20Paganini?= Date: Tue, 4 Mar 2025 12:56:27 +0100 Subject: [PATCH 08/90] First draft of Supabase + Timescale integration (#3811) * First draft of Supabase + Timescale integration * chore: update in review. Co-authored-by: Iain Cox --- use-timescale/integrations/supabase.md | 272 +++++++++++++++++++++++++ use-timescale/page-index/page-index.js | 5 + 2 files changed, 277 insertions(+) create mode 100644 use-timescale/integrations/supabase.md diff --git a/use-timescale/integrations/supabase.md b/use-timescale/integrations/supabase.md new file mode 100644 index 0000000000..55b0ae8b25 --- /dev/null +++ b/use-timescale/integrations/supabase.md @@ -0,0 +1,272 @@ +--- +title: Integrate Supabase with Timescale Cloud +excerpt: Supabase is an open source Firebase alternative. Integrate Supabase with Timescale Cloud +products: [cloud, mst, self_hosted] +keywords: [integrate] +--- + +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; + +# Integrate Supabase with Timescale Cloud + +[Supabase][supabase] is an open source Firebase alternative. This page shows how to run real-time analytical queries +against a $SERVICE_LONG through Supabase using a foreign data wrapper (fdw) to bring aggregated data from your +$SERVICE_LONG. + +## Prerequisites + + + +- Create a [Supabase project][supabase-new-project] + +## Setup your $SERVICE_LONG + +To setup a $SERVICE_LONG optimized for analytics to receive data from Supabase: + + + +1. **Optimize time-series data in hypertables** + + Time-series data represents how a system, process, or behavior changes over time. [Hypertables][hypertables-section] + are PostgreSQL tables that help you improve insert and query performance by automatically partitioning your data by + time. + + 1. [Connect to your $SERVICE_LONG][connect] and create a table that will point to a Supabase database: + + ```sql + CREATE TABLE signs ( + time timestamptz NOT NULL DEFAULT now(), + origin_time timestamptz NOT NULL, + name TEXT + ); + ``` + + 1. Turn the table to a hypertable: + + ```sql + SELECT create_hypertable('signs', by_range('time')); + ``` +1. **Optimize cooling data for analytics** + + Hypercore is the Timescale hybrid row-columnar storage engine, designed specifically for real-time analytics + and powered by time-series data. The advantage of Hypercore is its ability to seamlessly switch between row-oriented + and column-oriented storage. This flexibility enables $CLOUD_LONG to deliver the best of both worlds, solving the + key challenges in real-time analytics. + + ```sql + ALTER TABLE signs SET ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'name'); + ``` + +1. **Create optimized analytical queries** + + Continuous aggregates are designed to make queries on very large datasets run + faster. Continuous aggregates in $CLOUD_LONG use PostgreSQL [materialized views][postgres-materialized-views] to + continuously, and incrementally refresh a query in the background, so that when you run the query, + only the data that has changed needs to be computed, not the entire dataset. + + 1. Create a continuous aggregate pointing to the Supabase database. + + ```sql + CREATE MATERIALIZED VIEW IF NOT EXISTS signs_per_minute + WITH (timescaledb.continuous) + AS + SELECT time_bucket('1 minute', time) as ts, + name, + count(*) as total + FROM signs + GROUP BY 1, 2 + WITH NO DATA; + ``` + + 1. Setup a delay stats comparing `origin_time` to `time`. + + ```sql + CREATE MATERIALIZED VIEW IF NOT EXISTS _signs_per_minute_delay + WITH (timescaledb.continuous) + AS + SELECT time_bucket('1 minute', time) as ts, + stats_agg(extract(epoch from origin_time - time)::float8) as delay_agg, + candlestick_agg(time, extract(epoch from origin_time - time)::float8, 1) as delay_candlestick + FROM signs GROUP BY 1 + WITH NO DATA; + ``` + + 1. Setup a view to recieve the data from Supabase. + + ```sql + CREATE VIEW signs_per_minute_delay + AS + SELECT ts, + average(delay_agg) as avg_delay, + stddev(delay_agg) as stddev_delay, + open(delay_candlestick) as open, + high(delay_candlestick) as high, + low(delay_candlestick) as low, + close(delay_candlestick) as close + FROM _signs_per_minute_delay + ``` + +1. **Add refresh policies for your analytical queries** + + You use `start_offset` and `end_offset` to define the time range that the continuous aggregate will cover. Assuming + that the data is being inserted without any delay, set the `start_offset` to `5 minutes` and the `end_offset` to + `1 minute`. This means that the continuous aggregate is refreshed every minute, and the refresh covers the last 5 + minutes. + You set `schedule_interval` to `INTERVAL '1 minute'` so the continuous aggregate refreshes on your $SERVICE_LONG + every minute. The data is accessed from Supabase, and the continuous aggregate is refreshed every minute in + the other side. + + ```sql + SELECT add_continuous_aggregate_policy('signs_per_minute', + start_offset => INTERVAL '5 minutes', + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 minute'); + ``` + Do the same thing for data inserted with a delay: + ```sql + SELECT add_continuous_aggregate_policy('_signs_per_minute_delay', + start_offset => INTERVAL '5 minutes', + end_offset => INTERVAL '1 minute', + schedule_interval => INTERVAL '1 minute'); + ``` + + + + +## Setup a Supabase database + +To setup a Supabase database that injects data into your $SERVICE_LONG: + + + +1. **Connect a foreign server in Supabase to your $SERVICE_LONG** + + 1. Connect to your Supabase project using Supabase dashboard or psql. + 1. Enable the `postgres_fdw` extension. + + ```sql + CREATE EXTENSION postgres_fdw; + ``` + 1. Create a foreign server that points to your $SERVICE_LONG. + + Update the following command with your [connection details][connection-info], then run it + in the Supabase database: + + ```sql + CREATE SERVER timescale + FOREIGN DATA WRAPPER postgres_fdw + OPTIONS ( + host '', + port '', + dbname '', + sslmode 'require', + extensions 'timescaledb' + ); + ``` + +1. **Create the user mapping for the foreign server** + + Update the following command with your [connection details][connection-info], the run it + in the Supabase database: + + ```sql + CREATE USER MAPPING FOR CURRENT_USER + SERVER timescale + OPTIONS ( + user '', + password '' + ); + ``` + +1. **Create a foreign table that points to a table in your $SERVICE_LONG.** + + This query introduced the following columns: + - `time`: with a default value of `now()`. This is because the `time` column is used by $CLOUD_LONG to optimize data + in the columnstore. + - `origin_time`: store the original timestamp of the data. + + Using both columns, you understand the delay between Supabase (`origin_time`) and the time the data is + inserted into your $SERVICE_LONG (`time`). + + ```sql + CREATE FOREIGN TABLE signs ( + TIME timestamptz NOT NULL DEFAULT now(), + origin_time timestamptz NOT NULL, + NAME TEXT) + SERVER timescale OPTIONS ( + schema_name 'public', + table_name 'signs' + ); + ``` + +1. **Create a foreign table in Supabase** + + 1. Create a foreign table that matches the `signs_per_minute` view in your $SERVICE_LONG. It represents a top level + view of the data. + + ```sql + CREATE FOREIGN TABLE signs_per_minute ( + ts timestamptz, + name text, + total int + ) + SERVER timescale OPTIONS (schema_name 'public', table_name 'signs_per_minute'); + ``` + + 1. Create a foreign table that matches the `signs_per_minute_delay` view in your $SERVICE_LONG. + + ```sql + CREATE FOREIGN TABLE signs_per_minute_delay ( + ts timestamptz, + avg_delay float8, + stddev_delay float8, + open float8, + high float8, + low float8, + close float8 + ) SERVER timescale OPTIONS (schema_name 'public', table_name 'signs_per_minute_delay'); + ``` + + + +## Test the integration + +To inject data into your $SERVICE_LONG from a Supabase database using a foreign table: + + + +1. **Insert data into your Supabase database** + + Connect to Supabase and run the following query: + + ```sql + INSERT INTO signs (origin_time, name) VALUES (now(), 'test') + ``` + +1. **Check the data in your $SERVICE_LONG** + + [Connect to your $SERVICE_LONG][connect] and run the following query: + + ```sql + SELECT * from signs; + ``` + You see something like: + + | origin_time | time | name | + |-------------|------|------| + | 2025-02-27 16:30:04.682391+00 | 2025-02-27 16:30:04.682391+00 | test | + + + +You have successfully integrated Supabase with your $SERVICE_LONG. + +[supabase]: https://supabase.com/ +[supabase-new-project]: https://supabase.com/dashboard/new + +[hypertables-section]: /use-timescale/:currentVersion:/hypertables/ +[connect]: /getting-started/:currentVersion:/run-queries-from-console/ +[hypercore]: /use-timescale/:currentVersion:/hypercore/ +[postgres-materialized-views]: https://www.postgresql.org/docs/current/rules-materializedviews.html +[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ diff --git a/use-timescale/page-index/page-index.js b/use-timescale/page-index/page-index.js index 9891239c36..9108cae496 100644 --- a/use-timescale/page-index/page-index.js +++ b/use-timescale/page-index/page-index.js @@ -864,6 +864,11 @@ module.exports = [ href: "qstudio", excerpt: "Integrate qstudio with Timescale Cloud", }, + { + title: "Supabase", + href: "supabase", + excerpt: "Integrate Supabase with Timescale products", + }, { title: "Tableau", href: "tableau", From 65cc02591ad4100aa90bc15337f7e58cb9c86ca4 Mon Sep 17 00:00:00 2001 From: Iain Cox Date: Tue, 4 Mar 2025 13:10:42 +0100 Subject: [PATCH 09/90] 3726 docs rfc create a doc showing how to integrate decodable with timescale cloud (#3892) * draft * Apply suggestions from code review Co-authored-by: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Signed-off-by: Iain Cox --- .../integrations/amazon-sagemaker.md | 4 +- use-timescale/integrations/decodable.md | 75 +++++++++++++++++++ use-timescale/integrations/index.md | 22 +++--- use-timescale/page-index/page-index.js | 5 ++ 4 files changed, 92 insertions(+), 14 deletions(-) create mode 100644 use-timescale/integrations/decodable.md diff --git a/use-timescale/integrations/amazon-sagemaker.md b/use-timescale/integrations/amazon-sagemaker.md index 4e6ddb870d..6021043014 100644 --- a/use-timescale/integrations/amazon-sagemaker.md +++ b/use-timescale/integrations/amazon-sagemaker.md @@ -31,6 +31,8 @@ Create a table in $SERVICE_LONG to store model predictions generated by SageMake For $CLOUD_LONG, open an [SQL editor][run-queries] in [$CONSOLE][open-console]. For self-hosted, use [`psql`][psql]. +1. **Create a table to store model predictions** + ```sql CREATE TABLE model_predictions ( time TIMESTAMPTZ NOT NULL, @@ -38,7 +40,7 @@ Create a table in $SERVICE_LONG to store model predictions generated by SageMake prediction DOUBLE PRECISION NOT NULL ); ``` - + 1. **For better performance and easier real-time analytics, convert the table to a hypertable** [Hypertables][about-hypertables] are PostgreSQL tables that automatically partition your data by time. You interact diff --git a/use-timescale/integrations/decodable.md b/use-timescale/integrations/decodable.md new file mode 100644 index 0000000000..c6d1724daa --- /dev/null +++ b/use-timescale/integrations/decodable.md @@ -0,0 +1,75 @@ +--- +title: Integrate Decodable with Timescale Cloud +excerpt: Decodable enables you to build, run, and manage data pipelines effortlessly. Seamlessly integrate Decodable with Timescale Cloud to unlock real-time data processing capabilities +products: [cloud, mst, self_hosted] +keywords: [Decodable, Timescale Cloud] +--- + +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; + +# Integrate Decodable with $CLOUD_LONG + +[Decodable][decodable] is a real-time data platform that allows you to build, run, and manage data pipelines effortlessly. + +![Decodable workflow](https://assets.timescale.com/docs/images/integrations-decodable-configuration.png) + +This page explains how to integrate Decodable with your $SERVICE_LONG to enable efficient real-time streaming and analytics. + +## Prerequisites + + + +- Sign up for [Decodable][sign-up-decodable]. + + This page uses the pipeline you create using the [Decodable Quickstart Guide][decodable-quickstart]. + +## Connect Decodable to your $SERVICE_LONG + +To stream data gathered in Decodable to a $SERVICE_LONG: + + + +1. **Create the sync to pipe a Decodable data stream into your $SERVICE_LONG** + + 1. Log in to your [Decodable account][decodable-app]. + 1. Click `Connections`, then click `New Connection`. + 1. Select a `PostgreSQL sink` connection type, then click `Connect`. + 1. Using your [connection details][connection-info], fill in the connection information. + + Leave `schema` and `JDBC options` empty. + 1. Select the `http_events` source stream, then click `Next`. + + Decodable creates the table in your $SERVICE_LONG and starts streaming data. + + + +1. **Test the connection** + + 1. Connect to your $SERVICE_LONG. + + For $CLOUD_LONG, open an [SQL editor][run-queries] in [$CONSOLE][open-console]. For self-hosted, use [`psql`][psql]. + + 1. Check the data from Decodable is streaming into your $SERVICE_LONG. + + ```sql + SELECT * FROM http_events; + ``` + You see something like: + + ![Decodable workflow](https://assets.timescale.com/docs/images/integrations-decodable-data-in-service.png) + + + + +You have successfully integrated Decodable with $CLOUD_LONG. + + +[decodable]: https://www.decodable.co/ +[decodable-app]:https://app.decodable.co/-/accounts +[sign-up-decodable]: https://auth.decodable.co/u/signup/ +[decodable-quickstart]: https://docs.decodable.co/get-started/quickstart.html +[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[run-queries]: /getting-started/:currentVersion:/run-queries-from-console/ +[open-console]: https://console.cloud.timescale.com/dashboard/services +[psql]: /use-timescale/:currentVersion:/integrations/psql/ +[about-hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/ diff --git a/use-timescale/integrations/index.md b/use-timescale/integrations/index.md index 53bc769430..0d3cb8e42b 100644 --- a/use-timescale/integrations/index.md +++ b/use-timescale/integrations/index.md @@ -47,18 +47,13 @@ Some of the most in-demand integrations for $CLOUD_LONG are listed below, with l ## Data engineering and extract, transform, load -| Name | Description | -|:--------------------------------:|----------------------------------------------------------------------------------------------------------------------------------------------| -| [Apache Airflow][apache-airflow] | A community platform designed to programmatically author, schedule, and monitor workflows. | -| [Airbyte][airbyte] | An open-source data integration platform that enables you to move and consolidate data from various sources to a destination of your choice. | -======= -| Name | Description | -|:--------------------------------:|--------------------------------------------------------------------------------------------------------| -| [Amazon SageMaker][amazon-sagemaker]| Build, train, and deploy ML models into a production-ready hosted environment. | -| [Apache Airflow][apache-airflow] | Programmatically author, schedule, and monitor workflows. | -| [Apache Kafka][kafka] | Stream high-performance data pipelines, analytics, and data integration. | -| [AWS Lambda][aws-lambda]| Run code without provisioning or managing servers, scaling automatically as needed. | - +| Name | Description | +|:--------------------------------:|-------------------------------------------------------------------------------------| +| [Amazon SageMaker][amazon-sagemaker]| Build, train, and deploy ML models into a production-ready hosted environment. | +| [Apache Airflow][apache-airflow] | Programmatically author, schedule, and monitor workflows. | +| [Apache Kafka][kafka] | Stream high-performance data pipelines, analytics, and data integration. | +| [AWS Lambda][aws-lambda]| Run code without provisioning or managing servers, scaling automatically as needed. | +| [Decodable][decodable] | Build, run, and manage data pipelines effortlessly. | ## Data ingestion and streaming @@ -86,10 +81,11 @@ Some of the most in-demand integrations for $CLOUD_LONG are listed below, with l [apache-airflow]: /use-timescale/:currentVersion:/integrations/apache-airflow [aws-lambda]: /use-timescale/:currentVersion:/integrations/aws-lambda [postgresql-integrations]: https://slashdot.org/software/p/PostgreSQL/integrations/ +[decodable]: /use-timescale/:currentVersion:/integrations/decodable [power-bi]: /use-timescale/:currentVersion:/integrations/power-bi [fivetran]: /use-timescale/:currentVersion:/integrations/fivetran [prometheus]: /use-timescale/:currentVersion:/integrations/prometheus [airbyte]: /use-timescale/:currentVersion:/integrations/airbyte [amazon-sagemaker]: /use-timescale/:currentVersion:/integrations/amazon-sagemaker [postgresql]: /use-timescale/:currentVersion:/integrations/postgresql -[kafka]: /use-timescale/:currentVersion:/integrations/apache-kafka \ No newline at end of file +[kafka]: /use-timescale/:currentVersion:/integrations/apache-kafka diff --git a/use-timescale/page-index/page-index.js b/use-timescale/page-index/page-index.js index 9108cae496..aa79be2e74 100644 --- a/use-timescale/page-index/page-index.js +++ b/use-timescale/page-index/page-index.js @@ -824,6 +824,11 @@ module.exports = [ href: "dbeaver", excerpt: "Integrate DBeaver with Timescale Cloud", }, + { + title: "Decodable", + href: "decodable", + excerpt: "Integrate Decodable with Timescale Cloud", + }, { title: "Fivetran", href: "fivetran", From 44db4fff1268f72a861aee849abf9010b9e5f5aa Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Tue, 4 Mar 2025 18:39:15 +0200 Subject: [PATCH 10/90] AWS account signup (#3891) * AWS account signup * trigger build * Update _partials/_cloud-installation.md Co-authored-by: Iain Cox Signed-off-by: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> * Update _partials/_cloud-installation.md Co-authored-by: Iain Cox Signed-off-by: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> * Update _partials/_cloud-installation.md Co-authored-by: Iain Cox Signed-off-by: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> * review * review --------- Signed-off-by: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Co-authored-by: Iain Cox --- _partials/_cloud-installation.md | 67 ++++++++++++++++++++++++++++---- 1 file changed, 59 insertions(+), 8 deletions(-) diff --git a/_partials/_cloud-installation.md b/_partials/_cloud-installation.md index 01b7f71ca0..0cd2a4e7d8 100644 --- a/_partials/_cloud-installation.md +++ b/_partials/_cloud-installation.md @@ -1,21 +1,72 @@ -## Create a Timescale Cloud account +## Create a $CLOUD_LONG account -You create a $CLOUD_LONG account to manage your services and data in a centralized and efficient manner. From $CONSOLE you can create and delete services, run queries, manage access and billing, integrate other services, contact support, and more. +You create a $CLOUD_LONG account to manage your $SERVICE_SHORTs and data in a centralized and efficient manner in $CONSOLE. From there, you can create and delete $SERVICE_SHORTs, run queries, manage access and billing, integrate other services, contact support, and more. + + + + + +You create a standalone account to manage $CLOUD_LONG as a separate unit in your infrastructure, which includes separate billing and invoicing. + +To create an account: -To set up Timescale: +1. **Sign up for a 30-day free trial** -1. Sign up for a [30 day free trial][sign-up]. + Open [Sign up for Timescale][timescale-signup] and add your details, then click `Start your free trial`. You receive a confirmation email in your inbox. - You receive a confirmation email in your inbox. -1. In the confirmation email, click the link supplied and sign in to [Timescale Console][tsc-portal]. +1. **In the confirmation email, click the link supplied and sign in to [$CONSOLE][tsc-portal]** - Answer the requirements questions, they help us optimize the Timescale service for your use case. You can now create a Timescale service. +1. **Answer the requirements questions** + + Your answers help us optimize $SERVICE_LONGs for your use cases. + + + + + + + +You create an account through AWS Marketplace to have $CLOUD_LONG as a part of your AWS infrastructure. In this case, $CLOUD_LONG is a line item in your AWS invoice. + +To create an account: + + +1. **Open [AWS Marketplace][aws-marketplace] and search for `Timescale Cloud`** + You see two pricing options, [pay-as-you-go][aws-paygo] and [annual commit][aws-annual-commit]. + +1. **Select the pricing option that suits you and click `View purchase options`** + +1. **Review and configure the purchase details, then click `Subscribe`** + +1. **Click `Set up your account` at the top of the page** + + You are redirected to $CONSOLE. + +1. **Sign up for a 30-day free trial** + + Add your details, then click `Start your free trial`. If you want to link an existing $CLOUD_LONG account to AWS, log in with your existing credentials. + +1. **In `Confirm AWS Marketplace connection`, click `Connect`** + + Your $CLOUD_LONG and AWS accounts are now connected. + +1. **Answer the requirements questions** + + Your answers help us optimize $SERVICE_LONGs for your use cases. -[sign-up]: https://console.cloud.timescale.com/signup + + + + [tsc-portal]: https://console.cloud.timescale.com/ +[timescale-website]: https://www.timescale.com/ +[aws-marketplace]: https://aws.amazon.com/marketplace +[aws-paygo]: https://aws.amazon.com/marketplace/pp/prodview-iestawpo5ihca?applicationId=AWSMPContessa&ref_=beagle&sr=0-1 +[aws-annual-commit]: https://aws.amazon.com/marketplace/pp/prodview-ezxwlmjyr6x4u?applicationId=AWSMPContessa&ref_=beagle&sr=0-2 +[timescale-signup]: https://console.cloud.timescale.com/signup \ No newline at end of file From fd3833f911007c05dd1c6dd0c24fbbecc99e9e04 Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Wed, 5 Mar 2025 17:04:58 +0200 Subject: [PATCH 11/90] Connect from any cloud with AWS TGW (#3893) * draft * trigger build * update with partial * Update use-timescale/security/transit-gateway.md Co-authored-by: Iain Cox Signed-off-by: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> * Update use-timescale/security/transit-gateway.md Co-authored-by: Iain Cox Signed-off-by: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> * Update use-timescale/security/transit-gateway.md Co-authored-by: Iain Cox Signed-off-by: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> * review * review * review * chore: reuse and numbering update. * review --------- Signed-off-by: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Co-authored-by: Iain Cox --- _partials/_transit-gateway-aws.md | 43 ++++++++++ _partials/_transit-gateway-others.md | 43 ++++++++++ .../dual-write-from-other.md | 2 +- use-timescale/integrations/aws.md | 33 ++++++++ .../integrations/corporate-data-center.md | 35 ++++++++ use-timescale/integrations/google-cloud.md | 38 +++++++++ use-timescale/integrations/index.md | 15 ++++ use-timescale/integrations/microsoft-azure.md | 38 +++++++++ use-timescale/page-index/page-index.js | 28 +++++-- use-timescale/security/transit-gateway.md | 79 ++++++++----------- 10 files changed, 302 insertions(+), 52 deletions(-) create mode 100644 _partials/_transit-gateway-aws.md create mode 100644 _partials/_transit-gateway-others.md create mode 100644 use-timescale/integrations/aws.md create mode 100644 use-timescale/integrations/corporate-data-center.md create mode 100644 use-timescale/integrations/google-cloud.md create mode 100644 use-timescale/integrations/microsoft-azure.md diff --git a/_partials/_transit-gateway-aws.md b/_partials/_transit-gateway-aws.md new file mode 100644 index 0000000000..2c4cb51f1e --- /dev/null +++ b/_partials/_transit-gateway-aws.md @@ -0,0 +1,43 @@ +1. **Create a Peering $VPC in [$CONSOLE][console-login]** + + 1. In `Security` > `VPC`, click `Create a VPC`: + + ![$CLOUD_LONG new $VPC](https://assets.timescale.com/docs/images/add-peering-vpc.png) + + 1. Choose your region and IP range, name your VPC, then click `Create VPC`: + + ![Create a new VPC in $CLOUD_LONG](https://assets.timescale.com/docs/images/configure-peering-vpc.png) + + Your $SERVICE_SHORT and Peering $VPC must be in the same AWS region. The number of Peering $VPCs you can create in your project depends on your [price tier][pricing-plans]. If you need another Peering $VPC, either contact [support@timescale.com](mailto:support@timescale.com) or change your tier in [$CONSOLE][console-login]. + + 1. Add a peering connection: + + 1. In the `VPC Peering` column, click `Add`. + 1. Provide your AWS account ID, Transit Gateway ID, CIDR ranges, and AWS region. + 1. Click `Add connection`. + + ![Add peering](https://assets.timescale.com/docs/images/add-peering.png) + +1. **Accept and configure peering connection in your AWS account** + + Once your peering connection appears as `Processing`, you can accept and configure it in AWS: + + 1. Accept the peering request coming from $CLOUD_LONG. The request can take up to 5 min to arrive. Within 5 more minutes after accepting, the peering should appear as `Connected` in $CONSOLE. + + 1. Configure at least the following in yur AWS account networking: + + - Your subnet route table to route traffic to your Transit Gateway for the Peering VPC CIDRs. + - Your Transit Gateway route table to route traffic to the newly created Transit Gateway peering attachment for the Peering VPC CIDRs. + - Security groups to allow outbound TCP 5432. + +1. **Attach a $CLOUD_LONG service to the Peering VPC In [$CONSOLE][console-services]** + + 1. Select the $SERVICE_SHORT you want to connect to the Peering VPC. + 1. Click `Operations` > `Security` > `VPC`. + 1. Select the VPC, then click `Attach VPC`. + + You cannot attach a $SERVICE_LONG to multiple $CLOUD_LONG $VPCs at the same time. + +[console-login]: https://console.cloud.timescale.com/ +[console-services]: https://console.cloud.timescale.com/dashboard/services +[pricing-plans]: /about/:currentVersion:/pricing-and-account-management/ diff --git a/_partials/_transit-gateway-others.md b/_partials/_transit-gateway-others.md new file mode 100644 index 0000000000..b11cea5367 --- /dev/null +++ b/_partials/_transit-gateway-others.md @@ -0,0 +1,43 @@ +2. **Create a Peering $VPC in [$CONSOLE][console-login]** + + 1. In `Security` > `VPC`, click `Create a VPC`: + + ![$CLOUD_LONG new $VPC](https://assets.timescale.com/docs/images/add-peering-vpc.png) + + 1. Choose your region and IP range, name your VPC, then click `Create VPC`: + + ![Create a new VPC in $CLOUD_LONG](https://assets.timescale.com/docs/images/configure-peering-vpc.png) + + Your $SERVICE_SHORT and Peering $VPC must be in the same AWS region. The number of Peering $VPCs you can create in your project depends on your [price tier][pricing-plans]. If you need another Peering $VPC, either contact [support@timescale.com](mailto:support@timescale.com) or change your tier in [$CONSOLE][console-login]. + + 1. Add a peering connection: + + 1. In the `VPC Peering` column, click `Add`. + 1. Provide your AWS account ID, Transit Gateway ID, CIDR ranges, and AWS region. + 1. Click `Add connection`. + + ![Add peering](https://assets.timescale.com/docs/images/add-peering.png) + +1. **Accept and configure peering connection in your AWS account** + + Once your peering connection appears as `Processing`, you can accept and configure it in AWS: + + 1. Accept the peering request coming from $CLOUD_LONG. The request can take up to 5 min to arrive. Within 5 more minutes after accepting, the peering should appear as `Connected` in $CONSOLE. + + 1. Configure at least the following in yur AWS account networking: + + - Your subnet route table to route traffic to your Transit Gateway for the Peering VPC CIDRs. + - Your Transit Gateway route table to route traffic to the newly created Transit Gateway peering attachment for the Peering VPC CIDRs. + - Security groups to allow outbound TCP 5432. + +1. **Attach a $CLOUD_LONG service to the Peering VPC In [$CONSOLE][console-services]** + + 1. Select the $SERVICE_SHORT you want to connect to the Peering VPC. + 1. Click `Operations` > `Security` > `VPC`. + 1. Select the VPC, then click `Attach VPC`. + + You cannot attach a $SERVICE_LONG to multiple $CLOUD_LONG $VPCs at the same time. + +[console-login]: https://console.cloud.timescale.com/ +[console-services]: https://console.cloud.timescale.com/dashboard/services +[pricing-plans]: /about/:currentVersion:/pricing-and-account-management/ diff --git a/migrate/dual-write-and-backfill/dual-write-from-other.md b/migrate/dual-write-and-backfill/dual-write-from-other.md index 9725cef8d4..c32b8e4210 100644 --- a/migrate/dual-write-and-backfill/dual-write-from-other.md +++ b/migrate/dual-write-and-backfill/dual-write-from-other.md @@ -137,4 +137,4 @@ your production workload. -[dual-write-and-backfill]: /migrate/:currentVersion:/dual-write-and-backfill/ \ No newline at end of file +[dual-write-and-backfill]: /migrate/:currentVersion:/dual-write-and-backfill/ diff --git a/use-timescale/integrations/aws.md b/use-timescale/integrations/aws.md new file mode 100644 index 0000000000..c9bcf52a03 --- /dev/null +++ b/use-timescale/integrations/aws.md @@ -0,0 +1,33 @@ +--- +title: Integrate Amazon Web Services with Timescale Cloud +excerpt: AWS enables you to build, run, and manage applications across cloud, hybrid, and edge environments with AI, analytics, security, and scalable infrastructure. Integrate AWS with Timescale Cloud using AWS Transit Gateway +products: [cloud] +keywords: [AWS, integrations] +--- + +import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; +import TransitGatewayAWS from "versionContent/_partials/_transit-gateway-aws.mdx"; + +# Integrate Amazon Web Services with $CLOUD_LONG + +[Amazon Web Services (AWS)][aws] is a comprehensive cloud computing platform that provides on-demand infrastructure, storage, databases, AI, analytics, and security services to help businesses build, deploy, and scale applications in the cloud. + +This page explains how to integrate your AWS infrastructure with $CLOUD_LONG using [AWS Transit Gateway][aws-transit-gateway]. + +## Prerequisites + + + +- Set up [AWS Transit Gateway][gtw-setup]. + +## Connect your AWS infrastructure to your $SERVICE_LONGs + +To connect to $CLOUD_LONG: + + + +You have successfully integrated your AWS infrastructure with $CLOUD_LONG. + +[aws]: https://aws.amazon.com/ +[aws-transit-gateway]: https://aws.amazon.com/transit-gateway/ +[gtw-setup]: https://docs.aws.amazon.com/vpc/latest/tgw/tgw-getting-started.html diff --git a/use-timescale/integrations/corporate-data-center.md b/use-timescale/integrations/corporate-data-center.md new file mode 100644 index 0000000000..ff158d9496 --- /dev/null +++ b/use-timescale/integrations/corporate-data-center.md @@ -0,0 +1,35 @@ +--- +title: Integrate your data center with Timescale Cloud +excerpt: Integrate your on-premise data center with Timescale Cloud using AWS Transit Gateway +products: [cloud] +keywords: [on-premise, integrations] +--- + +import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; +import TransitGatewayOthers from "versionContent/_partials/_transit-gateway-others.mdx"; + +# Integrate your data center with $CLOUD_LONG + +This page explains how to integrate your corporate on-premise infrastructure with $CLOUD_LONG using [AWS Transit Gateway][aws-transit-gateway]. + +## Prerequisites + + + +- Set up [AWS Transit Gateway][gtw-setup]. + +## Connect your on-premise infrastructure to your $SERVICE_LONGs + +To connect to $CLOUD_LONG: + +1. **Connect your infrastructure to AWS Transit Gateway** + + Establish connectivity between your on-premise infrastructure and AWS. See the [Centralize network connectivity using AWS Transit Gateway][aws-onprem]. + + + +You have successfully integrated your Microsoft Azure infrastructure with $CLOUD_LONG. + +[aws-transit-gateway]: https://aws.amazon.com/transit-gateway/ +[gtw-setup]: https://docs.aws.amazon.com/vpc/latest/tgw/tgw-getting-started.html +[aws-onprem]: https://docs.aws.amazon.com/prescriptive-guidance/latest/patterns/centralize-network-connectivity-using-aws-transit-gateway.html diff --git a/use-timescale/integrations/google-cloud.md b/use-timescale/integrations/google-cloud.md new file mode 100644 index 0000000000..e4777dcddb --- /dev/null +++ b/use-timescale/integrations/google-cloud.md @@ -0,0 +1,38 @@ +--- +title: Integrate Google Cloud with Timescale Cloud +excerpt: Google Cloud enables you to deploy, manage, and scale cloud-based applications, databases, and data processing workflows. Integrate Google Cloud with Timescale Cloud using AWS Transit Gateway +products: [cloud] +keywords: [Google Cloud, integrations] +--- + +import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; +import TransitGatewayOthers from "versionContent/_partials/_transit-gateway-others.mdx"; + +# Integrate Google Cloud with $CLOUD_LONG + +[Google Cloud][google-cloud] is a suite of cloud computing services, offering scalable infrastructure, AI, analytics, databases, security, and developer tools to help businesses build, deploy, and manage applications. + +This page explains how to integrate your Google Cloud infrastructure with $CLOUD_LONG using [AWS Transit Gateway][aws-transit-gateway]. + +## Prerequisites + + + +- Set up [AWS Transit Gateway][gtw-setup]. + +## Connect your Google Cloud infrastructure to your $SERVICE_LONGs + +To connect to $CLOUD_LONG: + +1. **Connect your infrastructure to AWS Transit Gateway** + + Establish connectivity between Google Cloud and AWS. See [Connect HA VPN to AWS peer gateways][gcp-aws]. + + + +You have successfully integrated your Google Cloud infrastructure with $CLOUD_LONG. + +[google-cloud]: https://cloud.google.com/?hl=en +[aws-transit-gateway]: https://aws.amazon.com/transit-gateway/ +[gtw-setup]: https://docs.aws.amazon.com/vpc/latest/tgw/tgw-getting-started.html +[gcp-aws]: https://cloud.google.com/network-connectivity/docs/vpn/how-to/connect-ha-vpn-aws-peer-gateway diff --git a/use-timescale/integrations/index.md b/use-timescale/integrations/index.md index 0d3cb8e42b..c73fa372e3 100644 --- a/use-timescale/integrations/index.md +++ b/use-timescale/integrations/index.md @@ -45,6 +45,7 @@ Some of the most in-demand integrations for $CLOUD_LONG are listed below, with l | [Terraform][terraform] | Safely and predictably provision and manage infrastructure in any cloud. | + ## Data engineering and extract, transform, load | Name | Description | @@ -68,6 +69,16 @@ Some of the most in-demand integrations for $CLOUD_LONG are listed below, with l | [Power BI][power-bi] | Visualize data, build interactive dashboards, and share insights. | +## Secure connectivity to $CLOUD_LONG + +| Name | Description | +|:------------------------------------:|-----------------------------------------------------------------------------| +| [Amazon Web Services][aws] | Connect your other services and applications running in AWS to $CLOUD_LONG. | +| [Google Cloud][google-cloud] | Connect your Google Cloud infrastructure to $CLOUD_LONG. | +| [Microsoft Azure][azure] | Connect your Microsoft Azure infrastructure to $CLOUD_LONG. | +| [Corporate data center][data-center] | Connect your on-premise data center to $CLOUD_LONG. | + + [psql]: /use-timescale/:currentVersion:/integrations/psql/ [qstudio]: /use-timescale/:currentVersion:/integrations/qstudio/ [cloudwatch]: /use-timescale/:currentVersion:/integrations/cloudwatch/ @@ -89,3 +100,7 @@ Some of the most in-demand integrations for $CLOUD_LONG are listed below, with l [amazon-sagemaker]: /use-timescale/:currentVersion:/integrations/amazon-sagemaker [postgresql]: /use-timescale/:currentVersion:/integrations/postgresql [kafka]: /use-timescale/:currentVersion:/integrations/apache-kafka +[aws]: /use-timescale/:currentVersion:/integrations/aws +[google-cloud]: /use-timescale/:currentVersion:/integrations/google-cloud +[azure]: /use-timescale/:currentVersion:/integrations/microsoft-azure +[data-center]: /use-timescale/:currentVersion:/integrations/corporate-data-center diff --git a/use-timescale/integrations/microsoft-azure.md b/use-timescale/integrations/microsoft-azure.md new file mode 100644 index 0000000000..6686a802e6 --- /dev/null +++ b/use-timescale/integrations/microsoft-azure.md @@ -0,0 +1,38 @@ +--- +title: Integrate Microsoft Azure with Timescale Cloud +excerpt: Microsoft Azure enables you to build, deploy, and manage applications across cloud, hybrid, and edge environments. Integrate Microsoft Azure with Timescale Cloud using AWS Transit Gateway +products: [cloud] +keywords: [Azure, integrations] +--- + +import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; +import TransitGatewayOthers from "versionContent/_partials/_transit-gateway-others.mdx"; + +# Integrate Microsoft Azure with $CLOUD_LONG + +[Microsoft Azure][azure] is a cloud computing platform and services suite, offering infrastructure, AI, analytics, security, and developer tools to help businesses build, deploy, and manage applications. + +This page explains how to integrate your Microsoft Azure infrastructure with $CLOUD_LONG using [AWS Transit Gateway][aws-transit-gateway]. + +## Prerequisites + + + +- Set up [AWS Transit Gateway][gtw-setup]. + +## Connect your Microsoft Azure infrastructure to your $SERVICE_LONGs + +To connect to $CLOUD_LONG: + +1. **Connect your infrastructure to AWS Transit Gateway** + + Establish connectivity between Azure and AWS. See the [AWS architectural documentation][azure-aws] for details. + + + +You have successfully integrated your Microsoft Azure infrastructure with $CLOUD_LONG. + +[aws-transit-gateway]: https://aws.amazon.com/transit-gateway/ +[gtw-setup]: https://docs.aws.amazon.com/vpc/latest/tgw/tgw-getting-started.html +[azure]: https://azure.microsoft.com/en-gb/ +[azure-aws]: https://aws.amazon.com/blogs/modernizing-with-aws/designing-private-network-connectivity-aws-azure/ diff --git a/use-timescale/page-index/page-index.js b/use-timescale/page-index/page-index.js index aa79be2e74..c0ba3a0ba3 100644 --- a/use-timescale/page-index/page-index.js +++ b/use-timescale/page-index/page-index.js @@ -776,7 +776,6 @@ module.exports = [ { title: "Find your connection details", href: "find-connection-details", - excerpt: "Find connection details for your Timescale Cloud service", excerpt: "Find connection information for your Timescale Cloud service", }, { @@ -790,9 +789,14 @@ module.exports = [ excerpt: "Integrate Amazon SageMaker with Timescale Cloud", }, { - title: "Airbyte", - href: "airbyte", - excerpt: "Integrate Airbyte with Timescale Cloud", + title: "Amazon Web Services", + href: "aws", + excerpt: "Integrate AWS with Timescale Cloud", + }, + { + title: "Apache Airflow", + href: "apache-airflow", + excerpt: "Integrate Apache Airflow with Timescale products", }, { title: "Apache Kafka", @@ -814,6 +818,11 @@ module.exports = [ href: "azure-data-studio", excerpt: "Integrate Azure Data Studio with Timescale Cloud", }, + { + title: "Corporate data center", + href: "corporate-data-center", + excerpt: "Integrate your corporate data center with Timescale Cloud", + }, { title: "Datadog", href: "datadog", @@ -834,11 +843,21 @@ module.exports = [ href: "fivetran", excerpt: "Integrate Fivetran with Timescale Cloud", }, + { + title: "Google Cloud", + href: "google-cloud", + excerpt: "Integrate Google Cloud with Timescale Cloud", + }, { title: "Grafana", href: "grafana", excerpt: "Integrate Grafana with Timescale products", }, + { + title: "Microsoft Azure", + href: "microsoft-azure", + excerpt: "Integrate Microsoft Azure with Timescale Cloud", + }, { title: "pgAdmin", href: "pgadmin", @@ -882,7 +901,6 @@ module.exports = [ { title: "Terraform", href: "terraform", - excerpt: "Manage your Timescale Cloud services with Terraform", excerpt: "Integrate Terraform with Timescale Cloud", }, { diff --git a/use-timescale/security/transit-gateway.md b/use-timescale/security/transit-gateway.md index 2a58786037..623f55da1b 100644 --- a/use-timescale/security/transit-gateway.md +++ b/use-timescale/security/transit-gateway.md @@ -9,85 +9,72 @@ cloud_ui: - [services, :serviceId, operations, vpc] --- +import TransitGatewayAWS from "versionContent/_partials/_transit-gateway-aws.mdx"; +import TransitGatewayOthers from "versionContent/_partials/_transit-gateway-others.mdx"; + # Securely connect to $CLOUD_LONG from any cloud or on-premise environment -[AWS Transit Gateway][aws-transit-gateway] enables you to securely connect to your $CLOUD_LONG from AWS, GCP, Azure, or any other cloud or on-premise environment. +[AWS Transit Gateway][aws-transit-gateway] enables you to securely connect to your $CLOUD_LONG from AWS, Google Cloud, Microsoft Azure, or any other cloud or on-premise environment. -You use AWS Transit Gateway as a traffic controller for your network. Instead of setting up multiple direct connections to different clouds, on-premise data centers, and other AWS services, you connect everything to Transit Gateway. This simplifies your network and makes it easier to manage and scale. +You use AWS Transit Gateway as a traffic controller for your network. Instead of setting up multiple direct connections to different clouds, on-premise data centers, and other AWS services, you connect everything to AWS Transit Gateway. This simplifies your network and makes it easier to manage and scale. You can then create a peering connection between your $SERVICE_LONGs and AWS Transit Gateway in $CLOUD_LONG. This means that, no matter how big or complex your infrastructure is, you can connect securely to your $SERVICE_LONGs. To configure this secure connection, you: -1. Create a $CLOUD_LONG Peering $VPC with a peering connection to your AWS Transit Gateway. +1. Connect your infrastructure to AWS Transit Gateway. +1. Create a $CLOUD_LONG Peering $VPC with a peering connection to AWS Transit Gateway. 1. Accept and configure the peering connection on your side. 1. Attach individual $SERVICE_SHORTs to the Peering $VPC. -Transit Gateway feature is available for Scale and Enterprise [pricing plans][pricing-plans]. - -## Create a Peering $VPC - -To create a Peering $VPC: - - - -1. **In [$CONSOLE][console-login] > `Security` > `VPC`, click `Create a VPC`** - - ![$CLOUD_LONG new $VPC](https://assets.timescale.com/docs/images/add-peering-vpc.png) - -1. **Choose your region and IP range, name your VPC, then click `Create VPC`** +The AWS Transit Gateway feature is available for Scale and Enterprise [pricing plans][pricing-plans]. - ![Create a new VPC in $CLOUD_LONG](https://assets.timescale.com/docs/images/configure-peering-vpc.png) +AWS Transit Gateway enable you to connect from almost any environment, this page provides examples for the most common use cases. - Your $SERVICE_SHORT and Peering $VPC must be in the same AWS region. The number of Peering $VPCs you can create in your project depends on your [pricing plan][pricing-plans]. If you need another Peering $VPC, either contact [support@timescale.com](mailto:support@timescale.com) or change your pricing plan in [$CONSOLE][console-login]. + -1. **Add a peering connection** + - 1. In the `VPC Peering` column, click `Add`. - 1. Provide your AWS account ID, VPC ID or Transit Gateway ID, CIDR ranges, and AWS region. - 1. Click `Add connection`. + - ![Add peering](https://assets.timescale.com/docs/images/add-peering.png) + - + -## Accept and configure peering connection +1. **Connect your infrastructure to AWS Transit Gateway** -Once your peering connection appears as `Processing`, you can accept and configure it in AWS: + Establish connectivity between Azure and AWS. See the [AWS architectural documentation][azure-aws] for details. - + -1. **Accept the peering request** + - In your AWS account, accept the peering request coming from $CLOUD_LONG. The request can take up to 5 min to arrive. Within 5 more minutes after accepting, the peering should appear as `Connected` in $CONSOLE. + -1. **Configure networking in your AWS account** +1. **Connect your infrastructure to AWS Transit Gateway** - Configure at least the following: + Establish connectivity between Google Cloud and AWS. See [Connect HA VPN to AWS peer gateways][gcp-aws]. - 1. Your subnet route table to route traffic to your Transit Gateway for the Peering VPC CIDRs. - 1. Your Transit Gateway route table to route traffic to the newly created Transit Gateway peering attachment for the Peering VPC CIDRs. - 1. Security groups to allow outbound TCP 5432. + - + -## Attach a $CLOUD_LONG service to the Peering VPC + -To attach a $SERVICE_SHORT to the Peering VPC: +1. **Connect your infrastructure to AWS Transit Gateway** - + Establish connectivity between your on-premise infrastructure and AWS. See the [Centralize network connectivity using AWS Transit Gateway][aws-onprem]. -1. **In [$CONSOLE][console-services], select the $SERVICE_SHORT you want to connect to the Peering VPC** -1. **Click `Operations` > `Security` > `VPC`** -1. **Select the VPC, then click `Attach VPC`** + - You cannot attach a $SERVICE_LONG to multiple $CLOUD_LONG $VPCs at the same time. + - + -You can now securely access your $SERVICE_SHORTs from any private cloud or on-premise data center connected to AWS Transit Gateway. +You can now securely access your $SERVICE_SHORTs in $CLOUD_LONG. [aws-transit-gateway]: https://aws.amazon.com/transit-gateway/ [pricing-plans]: /about/:currentVersion:/pricing-and-account-management/ -[console-login]: https://console.cloud.timescale.com/ -[console-services]: https://console.cloud.timescale.com/dashboard/services \ No newline at end of file +[azure-aws]: https://aws.amazon.com/blogs/modernizing-with-aws/designing-private-network-connectivity-aws-azure/ +[gcp-aws]: https://cloud.google.com/network-connectivity/docs/vpn/how-to/connect-ha-vpn-aws-peer-gateway +[aws-onprem]: https://docs.aws.amazon.com/prescriptive-guidance/latest/patterns/centralize-network-connectivity-using-aws-transit-gateway.html From 75a3b4106bb6abaa58d8c33f1d1f79e7a27d154f Mon Sep 17 00:00:00 2001 From: Florian Maurer Date: Thu, 6 Mar 2025 10:39:49 +0100 Subject: [PATCH 12/90] Fix upgrade path for timescaledb extension (#3890) Before, the docs described to upgrade timescaledb extensions to a version, which would be incompatible with the mentioned PG13 version Signed-off-by: Florian Maurer Co-authored-by: Iain Cox --- _partials/_migrate_self_postgres_plan_migration_path.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/_partials/_migrate_self_postgres_plan_migration_path.md b/_partials/_migrate_self_postgres_plan_migration_path.md index 0ecb649776..c00f5ffd42 100644 --- a/_partials/_migrate_self_postgres_plan_migration_path.md +++ b/_partials/_migrate_self_postgres_plan_migration_path.md @@ -7,10 +7,10 @@ Cloud and always get latest update without any hassle. Check the following support matrix against the versions of TimescaleDB and PostgreSQL that you are running currently and the versions you want to update to, then choose your upgrade path. -For example, to upgrade from TimescaleDB 2.13 on PostgreSQL 13 to TimescaleDB 2.17.2 you need to: -1. Upgrade TimescaleDB to 2.16 -1. Upgrade PostgreSQL to 14 or higher -1. Upgrade TimescaleDB to 2.17.2. +For example, to upgrade from TimescaleDB 2.13 on PostgreSQL 13 to TimescaleDB 2.18.2 you need to: +1. Upgrade TimescaleDB to 2.15 +1. Upgrade PostgreSQL to 14, 15 or 16. +1. Upgrade TimescaleDB to 2.18.2. You may need to [upgrade to the latest PostgreSQL version][upgrade-pg] before you upgrade TimescaleDB. Also, if you use [Timescale Toolkit][toolkit-install], ensure the `timescaledb_toolkit` extension is >= From 99dfc638caadb3f52086472af46628ad438b9693 Mon Sep 17 00:00:00 2001 From: Iain Cox Date: Thu, 6 Mar 2025 15:39:38 +0100 Subject: [PATCH 13/90] chore: update (#3901) --- use-timescale/security/transit-gateway.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/use-timescale/security/transit-gateway.md b/use-timescale/security/transit-gateway.md index 623f55da1b..2cd90e7217 100644 --- a/use-timescale/security/transit-gateway.md +++ b/use-timescale/security/transit-gateway.md @@ -12,7 +12,7 @@ cloud_ui: import TransitGatewayAWS from "versionContent/_partials/_transit-gateway-aws.mdx"; import TransitGatewayOthers from "versionContent/_partials/_transit-gateway-others.mdx"; -# Securely connect to $CLOUD_LONG from any cloud or on-premise environment +# Securely connect to $CLOUD_LONG using AWS Transit Gateway [AWS Transit Gateway][aws-transit-gateway] enables you to securely connect to your $CLOUD_LONG from AWS, Google Cloud, Microsoft Azure, or any other cloud or on-premise environment. From e9a6b26e774881b9c5f309f6a3cd4476688e11fc Mon Sep 17 00:00:00 2001 From: Grant Godeke Date: Fri, 7 Mar 2025 08:08:50 -0500 Subject: [PATCH 14/90] March7changelog.md (#3905) Signed-off-by: Grant Godeke --- about/changelog.md | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/about/changelog.md b/about/changelog.md index 3443dc96df..06df29e191 100644 --- a/about/changelog.md +++ b/about/changelog.md @@ -8,6 +8,33 @@ keywords: [changelog, upgrades, updates, releases] All the latest features and updates to Timescale products. +## Agent Mode for PopSQL and more + + +### 🤖 Agent Mode for PopSQL + +Introducing Agent Mode, a new feature in $CONSOLE SQL Assistant. SQL Assistant lets you query your database using natural language. However, if you ran into errors, you have to approve the implementation of the Assistant's suggestions. + +With Agent Mode on, SQL Assistant automatically adjusts and executes your query without intervention. It runs, diagnoses, and fixes any errors that it runs into until you get your desired results. + +Below you can see SQL Assistant run into an error, identify the resolution, execute the fixed query, display results, and even change the title of the query: + +![Timescale SQL Assistant Agent Mode](https://assets.timescale.com/docs/images/timescale-sql-assistant-agent-mode.gif) + +To use Agent Mode, make sure you have SQL Assistant enabled, then click on the model selector dropdown, and tick the `Agent Mode` checkbox. + +### Improved AWS Marketplace integration for a smoother experience + +We've enhanced the AWS Marketplace workflow to make your experience even better! Now, everything is fully automated, ensuring a seamless process from setup to billing. If you're using the AWS Marketplace integration, you'll notice a smoother transition and clearer billing visibility—your $CLOUD_LONG subscription will be reflected directly in AWS Marketplace! + +### $CONSOLE recommendations + +Sometimes it can be hard to know if you are getting the best use out of your service. To help with this, $CLOUD_LONG now provides recommendations based on your service's context, assisting with onboarding or notifying if there is a configuration concern with your service, such as consistently failing jobs. + +To start, recommendations are focused primarily on onboarding or service health, though we will regularly add new ones. You can see if you have any existing recommendations for your service by going to the `Actions` tab in $CONSOLE. + +![Timescale Console recommendations](https://assets.timescale.com/docs/images/timescale-console-recommendations.png) + ## 🛣️ Configuration Options for Secure Connections and More From ec23c6f1f457cf47794f236dc3da6bee6247bb7f Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Fri, 7 Mar 2025 17:03:37 +0200 Subject: [PATCH 15/90] AWS pricing added to the pricing page (#3898) --- about/pricing-and-account-management.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/about/pricing-and-account-management.md b/about/pricing-and-account-management.md index 9f9ea8978e..b7cc42a55c 100644 --- a/about/pricing-and-account-management.md +++ b/about/pricing-and-account-management.md @@ -34,7 +34,9 @@ Pricing plans give you: * **Cost efficiency**: by aligning our pricing with the value delivered, we ensure that you get the most out of every dollar spent. Our goal is to help you achieve more with less. -Using self-hosted TimescaleDB and our open-source products is still free. +Using self-hosted TimescaleDB and our open-source products is still free. + +If you create a $COMPANY account from AWS Marketplace, the pricing options are pay-as-you-go and annual commit. See [AWS pricing][aws-pricing] for details. This page explains pricing plans for Timescale Cloud, and how to easily manage your Timescale account. @@ -231,7 +233,14 @@ alt="Adding a payment method in Timescale"/> - **Plans**: choose the plan supplying the [features][plan-features] that suit your business and engineering needs. -- **Add-ons**: add `Production support` and improved database performance for mission critical workloads. +- **Add-ons**: add `Production support` and improved database performance for mission-critical workloads. + +## AWS Marketplace pricing + +When you get $CLOUD_LONG at AWS Marketplace, the following pricing options are available: + +- **Pay-as-you-go**: your consumption is calculated at the end of the month and included in your AWS invoice. No upfront costs, standard $CLOUD_LONG rates apply. +- **Annual commit**: your consumption is calculated at the end of the month ensuring predictable pricing and seamless billing through your AWS account. We confirm the contract terms with you before finalizing the commitment. [cloud-login]: https://console.cloud.timescale.com/ [compression]: /use-timescale/:currentVersion:/compression/ @@ -243,3 +252,4 @@ alt="Adding a payment method in Timescale"/> [production-support]: https://www.timescale.com/support [get-in-touch]: https://www.timescale.com/contact [hipaa-compliance]: https://www.hhs.gov/hipaa/for-professionals/index.html +[aws-pricing]: #aws-pricing \ No newline at end of file From 0c9550fc3b97d064eed3209a2f336196125d6197 Mon Sep 17 00:00:00 2001 From: Victor Meireles <129079167+victormeirelests@users.noreply.github.com> Date: Mon, 10 Mar 2025 08:15:55 -0300 Subject: [PATCH 16/90] chore: testing preview link three (#3902) Co-authored-by: Robson Coelho --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 83f4163786..52b1be28ee 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -27,7 +27,7 @@ jobs: token: ${{ secrets.ORG_AUTOMATION_TOKEN }} repository: timescale/web-documentation event-type: build-docs-content - client-payload: '{"branch": "${{ steps.timescale.outputs.DEV_FOLDER }}", "pr_number": "${{ env.PR_NUMBER }}"}' + client-payload: '{"branch": "${{ env.HYPHENATED_BRANCH_NAME }}", "pr_number": "${{ env.PR_NUMBER }}"}' - name: Write comment uses: marocchino/sticky-pull-request-comment@efaaab3fd41a9c3de579aba759d2552635e590fd From f9d6f61e8ab140ccda61518468f11f5f7edd554a Mon Sep 17 00:00:00 2001 From: Victor Meireles <129079167+victormeirelests@users.noreply.github.com> Date: Wed, 12 Mar 2025 10:23:19 -0300 Subject: [PATCH 17/90] fix: reverting changes to preview link (#3920) --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 52b1be28ee..83f4163786 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -27,7 +27,7 @@ jobs: token: ${{ secrets.ORG_AUTOMATION_TOKEN }} repository: timescale/web-documentation event-type: build-docs-content - client-payload: '{"branch": "${{ env.HYPHENATED_BRANCH_NAME }}", "pr_number": "${{ env.PR_NUMBER }}"}' + client-payload: '{"branch": "${{ steps.timescale.outputs.DEV_FOLDER }}", "pr_number": "${{ env.PR_NUMBER }}"}' - name: Write comment uses: marocchino/sticky-pull-request-comment@efaaab3fd41a9c3de579aba759d2552635e590fd From e7a5da438ac62a78852f19fa944411a08bbfb37a Mon Sep 17 00:00:00 2001 From: Iain Cox Date: Thu, 13 Mar 2025 10:38:14 +0100 Subject: [PATCH 18/90] chore: release branch for TimescaleDB v2.18.0. (#3764) * chore: milestone branch for TimescaleDB v2.18.0. Co-authored-by: atovpeko --- .github/pull_request_template.md | 2 +- .github/styles/Vocab/Timescale/accept.txt | 2 + .../config/vocabularies/Timescale/accept.txt | 160 ++++++ CONTRIBUTING.md | 314 +++++------ README.md | 26 +- _multi-code-block.md | 2 +- _partials/_add-data-blockchain.md | 11 +- _partials/_add-data-energy.md | 2 - _partials/_add-data-nyctaxis.md | 2 - _partials/_add-data-twelvedata-crypto.md | 35 +- _partials/_add-data-twelvedata-stocks.md | 4 - _partials/_caggs-intro.md | 18 +- _partials/_cloud-connect.md | 24 +- _partials/_cloud-create-connect-tutorials.md | 4 +- _partials/_cloud-installation.md | 14 +- _partials/_cloud-intro.md | 16 +- _partials/_cloud_self_configuration.md | 81 +++ _partials/_create-hypertable-blockchain.md | 4 +- _partials/_create-hypertable-energy.md | 4 +- _partials/_create-hypertable-nyctaxis.md | 27 +- .../_create-hypertable-twelvedata-crypto.md | 47 +- .../_create-hypertable-twelvedata-stocks.md | 4 +- _partials/_deprecated.md | 2 + _partials/_deprecated_2_18_0.md | 1 + _partials/_early_access.md | 6 +- .../_financial-industry-data-analysis.md | 1 + _partials/_formatting_examples.md | 167 ++++++ _partials/_grafana-connect.md | 29 +- _partials/_graphing-ohlcv-data.md | 4 +- _partials/_high-availability-setup.md | 13 + _partials/_hypercore-conversion-overview.md | 21 + _partials/_hypercore_manual_workflow.md | 44 ++ _partials/_hypercore_policy_workflow.md | 118 ++++ _partials/_hypertables-intro.md | 27 +- _partials/_import-data-iot.md | 86 +++ _partials/_import-data-nyc-taxis.md | 168 ++++++ _partials/_integration-prereqs-cloud-only.md | 9 +- _partials/_integration-prereqs.md | 7 +- .../_migrate_awsrds_connect_intermediary.md | 4 +- .../_migrate_install_psql_ec2_instance.md | 1 - _partials/_migrate_post_schema_caggs_etal.md | 5 +- _partials/_multi-node-deprecation.md | 3 +- _partials/_prereqs-cloud-and-self.md | 8 + _partials/_prereqs-cloud-only.md | 5 + _partials/_since_2_18_0.md | 1 + _partials/_usage-based-storage-intro.md | 6 +- _partials/_use-case-iot-create-cagg.md | 87 +++ .../_use-case-setup-blockchain-dataset.md | 18 + _partials/_use-case-transport-geolocation.md | 88 +++ _partials/_where-to-next.md | 14 +- _procedure-block.md | 8 +- _queries/getting-started-crypto-cagg.md | 19 + .../getting-started-crypto-srt-orderby.md | 20 + _queries/getting-started-srt-orderby.md | 26 +- _troubleshooting/mst/low-disk-memory-cpu.md | 4 +- about/changelog.md | 1 + about/contribute-to-docs.md | 524 ------------------ about/contribute-to-timescale.md | 64 +-- about/page-index/page-index.js | 7 +- about/pricing-and-account-management.md | 3 +- about/release-notes.md | 2 +- about/timescaledb-editions.md | 72 ++- .../candlestick_agg/examples.md | 6 +- .../count_min_sketch/approx_count.md | 2 +- api/add_continuous_aggregate_policy.md | 2 +- api/add_policies.md | 6 +- api/add_reorder_policy.md | 4 +- api/alter_table_compression.md | 77 --- .../add_compression_policy.md | 8 + api/compression/alter_table_compression.md | 83 +++ .../chunk_compression_stats.md | 4 + api/{ => compression}/compress_chunk.md | 5 + api/{ => compression}/decompress_chunk.md | 6 + .../hypertable_compression_stats.md | 6 + api/{compression.md => compression/index.md} | 9 +- api/{ => compression}/recompress_chunk.md | 4 + .../remove_compression_policy.md | 5 + api/distributed-hypertables.md | 4 +- api/enable_chunk_skipping.md | 6 +- api/hypercore/add_columnstore_policy.md | 122 ++++ api/hypercore/alter_materialized_view.md | 104 ++++ api/hypercore/alter_table.md | 106 ++++ api/hypercore/chunk_columnstore_settings.md | 57 ++ api/hypercore/chunk_columnstore_stats.md | 109 ++++ api/hypercore/convert_to_columnstore.md | 54 ++ api/hypercore/convert_to_rowstore.md | 46 ++ .../hypertable_columnstore_settings.md | 62 +++ api/hypercore/hypertable_columnstore_stats.md | 82 +++ api/hypercore/index.md | 99 ++++ api/hypercore/page-index/page-index.js | 55 ++ api/hypercore/remove_columnstore_policy.md | 47 ++ api/merge_chunks.md | 56 ++ api/page-index/page-index.js | 212 ++++--- api/remove_continuous_aggregate_policy.md | 5 +- api/rollup.md | 2 +- getting-started/aggregation.md | 214 ------- getting-started/index.md | 25 +- getting-started/next-steps.md | 25 - getting-started/page-index/page-index.js | 30 +- getting-started/queries.md | 217 -------- getting-started/run-queries-from-console.md | 7 +- getting-started/services.md | 27 +- getting-started/tables-hypertables.md | 119 ---- getting-started/time-series-data.md | 62 --- .../try-key-features-timescale-products.md | 437 +++++++++++++++ lambda/redirects.js | 66 ++- .../dual-write-from-other.md | 2 +- .../dual-write-from-postgres.md | 2 +- quick-start/golang.md | 14 +- self-hosted/configuration/docker-config.md | 2 +- .../configuration/timescaledb-config.md | 26 +- self-hosted/migration/migrate-influxdb.md | 4 +- self-hosted/migration/same-db.md | 2 +- self-hosted/multinode-timescaledb/index.md | 2 +- self-hosted/page-index/page-index.js | 168 +++--- self-hosted/tooling/install-toolkit.md | 6 +- .../OLD_grafana/visualizations/pie-chart.md | 2 +- .../blockchain-analyze/blockchain-dataset.md | 35 +- tutorials/blockchain-analyze/index.md | 17 +- .../blockchain-query/blockchain-dataset.md | 29 +- tutorials/blockchain-query/index.md | 11 +- tutorials/energy-data/dataset-energy.md | 94 +--- tutorials/energy-data/index.md | 12 +- .../financial-ingest-dataset.md | 58 +- .../financial-ingest-query.md | 2 +- tutorials/financial-ingest-real-time/index.md | 7 +- .../financial-tick-compress.md | 10 +- .../financial-tick-dataset.md | 42 +- .../financial-tick-query.md | 2 +- tutorials/financial-tick-data/index.md | 43 +- tutorials/index.md | 23 +- tutorials/ingest-real-time-websocket-data.md | 4 +- tutorials/nyc-taxi-cab/dataset-nyc.md | 23 +- tutorials/nyc-taxi-geospatial/dataset-nyc.md | 28 +- tutorials/page-index/page-index.js | 117 ++-- .../real-time-analytics-energy-consumption.md | 226 ++++++++ tutorials/real-time-analytics-transport.md | 189 +++++++ use-timescale/alerting.md | 6 +- .../compression/about-compression.md | 4 +- .../compression/compression-design.md | 5 + .../compression/compression-methods.md | 5 + .../compression/compression-policy.md | 5 +- .../compression/decompress-chunks.md | 6 + use-timescale/compression/index.md | 9 +- .../compression/manual-compression.md | 3 + use-timescale/compression/modify-a-schema.md | 4 + .../compression/modify-compressed-data.md | 3 + .../configuration/advanced-parameters.md | 13 +- .../hierarchical-continuous-aggregates.md | 6 +- use-timescale/continuous-aggregates/index.md | 12 +- .../real-time-aggregates.md | 29 +- .../data-retention/about-data-retention.md | 15 +- .../data-tiering/enabling-data-tiering.md | 12 +- .../ha-replicas/high-availability.md | 18 +- ...best-practice-for-large-amounts-of-data.md | 188 +++++++ .../efficient-data-access-in-hypercore.md | 75 +++ use-timescale/hypercore/index.md | 60 ++ .../hypercore/modify-data-in-hypercore.md | 110 ++++ .../real-time-analytics-in-hypercore.md | 83 +++ .../hyperfunctions/about-hyperfunctions.md | 23 +- use-timescale/hyperfunctions/heartbeat-agg.md | 28 +- use-timescale/hyperfunctions/index.md | 10 +- .../hypertables/about-hypertables.md | 11 +- .../hypertables/improve-query-performance.md | 1 + use-timescale/hypertables/index.md | 6 +- use-timescale/integrations/apache-airflow.md | 10 +- use-timescale/integrations/datadog.md | 2 +- use-timescale/integrations/grafana.md | 11 +- use-timescale/integrations/index.md | 1 + use-timescale/integrations/pgadmin.md | 2 +- use-timescale/integrations/supabase.md | 2 +- use-timescale/page-index/page-index.js | 120 ++-- .../time-buckets/about-time-buckets.md | 10 +- use-timescale/time-buckets/index.md | 2 +- use-timescale/troubleshoot-timescaledb.md | 2 +- .../write-data/about-writing-data.md | 2 +- 176 files changed, 4797 insertions(+), 2499 deletions(-) create mode 100644 .github/styles/config/vocabularies/Timescale/accept.txt create mode 100644 _partials/_cloud_self_configuration.md create mode 100644 _partials/_deprecated_2_18_0.md create mode 100644 _partials/_financial-industry-data-analysis.md create mode 100644 _partials/_formatting_examples.md create mode 100644 _partials/_high-availability-setup.md create mode 100644 _partials/_hypercore-conversion-overview.md create mode 100644 _partials/_hypercore_manual_workflow.md create mode 100644 _partials/_hypercore_policy_workflow.md create mode 100644 _partials/_import-data-iot.md create mode 100644 _partials/_import-data-nyc-taxis.md create mode 100644 _partials/_prereqs-cloud-and-self.md create mode 100644 _partials/_prereqs-cloud-only.md create mode 100644 _partials/_since_2_18_0.md create mode 100644 _partials/_use-case-iot-create-cagg.md create mode 100644 _partials/_use-case-setup-blockchain-dataset.md create mode 100644 _partials/_use-case-transport-geolocation.md create mode 100644 _queries/getting-started-crypto-cagg.md create mode 100644 _queries/getting-started-crypto-srt-orderby.md delete mode 100644 about/contribute-to-docs.md delete mode 100644 api/alter_table_compression.md rename api/{ => compression}/add_compression_policy.md (91%) create mode 100644 api/compression/alter_table_compression.md rename api/{ => compression}/chunk_compression_stats.md (94%) rename api/{ => compression}/compress_chunk.md (79%) rename api/{ => compression}/decompress_chunk.md (86%) rename api/{ => compression}/hypertable_compression_stats.md (92%) rename api/{compression.md => compression/index.md} (88%) rename api/{ => compression}/recompress_chunk.md (92%) rename api/{ => compression}/remove_compression_policy.md (84%) create mode 100644 api/hypercore/add_columnstore_policy.md create mode 100644 api/hypercore/alter_materialized_view.md create mode 100644 api/hypercore/alter_table.md create mode 100644 api/hypercore/chunk_columnstore_settings.md create mode 100644 api/hypercore/chunk_columnstore_stats.md create mode 100644 api/hypercore/convert_to_columnstore.md create mode 100644 api/hypercore/convert_to_rowstore.md create mode 100644 api/hypercore/hypertable_columnstore_settings.md create mode 100644 api/hypercore/hypertable_columnstore_stats.md create mode 100644 api/hypercore/index.md create mode 100644 api/hypercore/page-index/page-index.js create mode 100644 api/hypercore/remove_columnstore_policy.md create mode 100644 api/merge_chunks.md delete mode 100644 getting-started/aggregation.md delete mode 100644 getting-started/next-steps.md delete mode 100644 getting-started/queries.md delete mode 100644 getting-started/tables-hypertables.md delete mode 100644 getting-started/time-series-data.md create mode 100644 getting-started/try-key-features-timescale-products.md create mode 100644 tutorials/real-time-analytics-energy-consumption.md create mode 100644 tutorials/real-time-analytics-transport.md create mode 100644 use-timescale/hypercore/best-practice-for-large-amounts-of-data.md create mode 100644 use-timescale/hypercore/efficient-data-access-in-hypercore.md create mode 100644 use-timescale/hypercore/index.md create mode 100644 use-timescale/hypercore/modify-data-in-hypercore.md create mode 100644 use-timescale/hypercore/real-time-analytics-in-hypercore.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 567d11e083..aa46627a82 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -8,7 +8,7 @@ Fixes #[insert issue link, if any] # Writing help -For information about style and word usage, see the [style guide](https://docs.timescale.com/about/latest/contribute-to-docs/) +For information about style and word usage, see the [Contribution guide](https://github.com/timescale/docs/blob/latest/CONTRIBUTING.md) # Review checklists diff --git a/.github/styles/Vocab/Timescale/accept.txt b/.github/styles/Vocab/Timescale/accept.txt index 0f12e77ca5..cd7dfb7f28 100644 --- a/.github/styles/Vocab/Timescale/accept.txt +++ b/.github/styles/Vocab/Timescale/accept.txt @@ -2,6 +2,7 @@ Aiven Alertmanager API +api Anthropic async [Aa]utoscal(?:e|ing) @@ -16,6 +17,7 @@ https? BSD [Cc]allouts? COPY +[Cc]opy [Cc]lickstreams? Cloudformation Cloudwatch diff --git a/.github/styles/config/vocabularies/Timescale/accept.txt b/.github/styles/config/vocabularies/Timescale/accept.txt new file mode 100644 index 0000000000..32ad4ac714 --- /dev/null +++ b/.github/styles/config/vocabularies/Timescale/accept.txt @@ -0,0 +1,160 @@ +[Aa]ccessor +Aiven +Alertmanager +API +api +Anthropic +async +[Aa]utoscal(?:e|ing) +[Bb]ackfill(?:ing)? +cron +csv +https? +(?i)bigint +[Bb]itemporal +[Bb]lockchains? +[Bb]oolean +BSD +[Cc]allouts? +COPY +[Cc]opy +[Cc]lickstreams? +Cloudformation +Cloudwatch +config +[Cc]onstif(?:y|ied) +[Cc]rypto +[Cc]ryptocurrenc(?:y|ies) +Datadog +[Dd]efragment(:?s|ed)? +Django +distincts +Docker +[Dd]ownsampl(?:e|ing) +erroring +Ethereum +[Ff]ailover +[Ff]inalizers? +[Ff]orex +[Gg]apfill(?:ing)? +[Gg]eospatial +GitHub +GNU +Grafana +GUC +gsed +gzip(?:ped)? +Hasura +HipChat +[Hh]ostname +href +[Hh]ypercore +[Hh]yperloglog +[Hh]yperfunctions? +[Hh]ypertables? +[Hh]ypershift +img +Indri +[Ii]nserter +[Ii]ntraday +[Iin]validation +ivfflat +jpg +JDBC +JDK +JSON +Kafka +Kaggle +Kinesis +[Ll]ambda +LangChain +LlamaIndex +LLMs +[Ll]ookups? +loopback +[Mm]atchers? +[Mm]aterializer +(?i)MST +matplotlib +[Mm]utators? +Nagios +[Nn]amespaces? +[Nn]ullable +Outflux +[Pp]ageviews? +[Pp]aralleliza(?:ble|tion) +[Pp]athname +Patroni +Paulo +[Pp]erformant +pg_dump +pg_restore +[Pp]gvector +[Pp]laintext +Plotly +pre +POSIX +PostgreSQL +[Pp]ooler? +Prometheus +PromLens +PromQL +Promscale +Protobuf +psql +[Qq]uantiles? +qStudio +RDS +[Rr]edistribut(?:e|able) +[Rr]eindex(?:ed)? +reltuples +[Rr]eusability +[Rr]unbooks? +[Ss]crollable +Sequelize +[Ss]ignups? +[Ss]iloed +(?i)smallint +sed +src +[Ss]ubquer(?:y|ies) +[Ss]ubsets? +[Ss]upersets? +[Tt]ablespaces? +Telegraf +Thanos +Threema +[Tt]iering +[Tt]imevectors? +Timescale(?:DB)? +tobs +[Tt]ransactionally +tsdbadmin +Uber +[Uu]nary +[Uu]ncomment +[Uu]nencrypted +Unix +[Uu]nmaterialized +[Uu]nregister +[Uu]nthrottled? +[Uu]ntier +[Uu]pserts? +[Rr]ebalanc(?:e|ing) +[Rr]epos? +[Rr]ollups? +[Ss]erverless +[Ss]hard(?:s|ing)? +SkipScan +(?i)timestamptz +URLs? +URIs? +UUID +versionContent +[Vv]irtualenv +WAL +[Ww]ebsockets? +Worldmap +www +Zabbix +Zipkin diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5d2be4e9bc..f65a02313e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,202 +1,190 @@ -# Introduction +# Contribute to Timescale documentation -There are multiple ways to help make Timescale better, including contributing -to the documentation. All of our documentation is available to use and review -with GitHub. +Timescale documentation is open for contribution from all community members. The current source is in this repository. -## First contribution +This page explains the process and guidelines to follow when contributing to Timescale documentation. -You can make contributions to the documentation by creating a fork of the -repository. However, if you have write access to the repository, use a branch -instead. Some of our automation does not work correctly on forks. +## Contribution process - +You can contribute to Timescale documentation in the following ways: -### Contributing using a fork +- [Create an issue][docs-issues] in this repository and describe the proposed change. Our doc team will take care of it. +- Update the docs yourself and have your change reviewed and published by our doc team. -1. Make sure you have a [GitHub](https://github.com) account, and that - you're signed in. -1. Navigate to the - [Timescale documentation repo](https://github.com/timescale/docs), - click the `Fork` button in the top-right corner, and select the account you - want to use. -1. Wait for GitHub to create your fork and redirect you. -1. Clone the repository to your local machine. To find this URL, click the green - `Code` button and copy the HTTPS URL: +To make the contribution yourself: - ```bash - git clone https://github.com//docs.git - ``` +1. Get the documentation source: + + - No write access? [Fork this repository][github-fork]. + - Already have a write access? [Clone this repository][github-clone]. -1. List the current remote branches: +2. Create a branch from `latest`, make your changes, and raise a pull request back to `latest`. - ```bash - git remote -v - ``` +3. Sign a Contributor License Agreement (CLA). - This command should list two remotes, both marked `origin`, like this: + You have to sign the CLA only the first time you raise a PR. This helps to ensure that the community is free to use your contributions. + +4. Review your changes. - ```bash - origin https://github.com//docs.git (fetch) - origin https://github.com//docs.git (push) - ``` + The documentation site is generated in a separate private repository using [Gatsby][gatsby]. Once you raise a PR for any branch, GitHub will **automatically** generate a preview for your changes and attach the link in the comments. Any new commits will be visible at the same URL. If you don't see the latest changes, try an incognito browser window. Automated builds are not available for PRs from forked repositories. - The `origin` remotes are your own fork, and you can do whatever you want - here without changing the upstream repository. -1. Add the docs repo as an upstream: +## Language - ```bash - git remote add upstream https://github.com/timescale/docs.git - ``` +Write in a clear, concise, and actionable manner. Timescale documentation uses the [Google Developer Documentation Style Guide][google-style] with the following exceptions: -1. Check: +- Do not capitalize the first word after a colon. +- Use code font (back ticks) for UI elements instead of semi-bold. - ```bash - git remote -v - ``` +## Edit individual pages - This command should now have the same two `origin` remotes as before, plus - two more labelled `upstream`, like this: +Each major doc section has a dedicated directory with `.md` files inside, representing its child pages. This includes an `index.md` file that serves as a landing page for that doc section by default, unless specifically changed in the navigation tree. To edit a page, modify the corresponding `.md` file following these recommendations: - ```bash - origin https://github.com//docs.git (fetch) - origin https://github.com//docs.git (push) - upstream https://github.com/timescale/docs.git (fetch) - upstream https://github.com/timescale/docs.git (push) - ``` +- **Regular pages** should include: -1. Fetch the branches in the upstream repository: + - A short intro describing the main subject of the page. + - A visual illustrating the main concept, if relevant. + - Paragraphs with descriptive headers, organizing the content into logical sections. + - Procedures to describe the sequence of steps to reach a certain goal. For example, create a Timescale service. + - Other visual aids, if necessary. + - Links to other relevant resources. - ```bash - git fetch upstream - ``` +- **API pages** should include: -1. Merge the changes from the upstream `latest` branch, into your fork's - `latest` branch: + - The function name, with empty parentheses if it takes arguments. + - A brief, specific description of the function, including any possible warnings. + - One or two samples of the function being used to demonstrate argument syntax. + - An argument table with `Name`, `Type`, `Default`, `Required`, `Description` columns. + - A return table with `Column`, `Type`, and `Description` columns. - ```bash - git merge upstream/latest - ``` +- **Troubleshooting pages** are not written as whole Markdown files, but are programmatically assembled from individual files in the`_troubleshooting` folder. Each entry describes a single troubleshooting case and its solution, and contains the following front matter: + + |Key| Type |Required| Description | + |-|-------|-|--------------------------------------------------------------------------| + |`title`| string |✅| The title of the troubleshooting entry, displayed as a heading above it | + |`section`| The literal string `troubleshooting` |✅| Must be `troubleshooting`, used to identify troubleshooting entries during site build | + |`products` or `topics`| array of strings |✅ (can have either or both, but must have at least one)| The products or topics related to the entry. The entry will show up on the troubleshooting pages for the listed products and topics. | + |`errors`| object of form `{language: string, message: string}` |❌| The error, if any, related to the troubleshooting entry. Displayed as a code block right underneath the title. `language` is the programming language to use for syntax highlighting. | + |`keywords`| array of strings |❌| These are displayed at the bottom of every troubleshooting page. Each keyword links to a collection of all pages associated with that keyword. | + |`tags`| array of strings |❌| Concepts, actions, or things associated with the troubleshooting entry. These are not displayed in the UI, but they affect the calculation of related pages. | + + Beneath the front matter, describe the error and its solution in regular Markdown. You can also use any other components allowed within the docs site. + + The entry shows up on the troubleshooting pages for its associated products and topics. If the page doesn't already exist, add an entry for it in the page + index, setting `type` to `placeholder`. See [Navigation tree](#navigation-tree). -1. Create a new branch for the work you want to do. Make sure you give it an - appropriate name, and include your username: +## Edit the navigation hierarchy - ```bash - git checkout -b update-readme-username - ``` +The navigation hierarchy of a doc section is governed by `page-index/page-index.js` within the corresponding directory. For example: - +```js + { + title: "Timescale Cloud services", + href: "services", + excerpt: "About Timescale Cloud services", + children: [ + { + title: "Services overview", + href: "service-overview", + excerpt: "Timescale services overview", + }, + { + title: "Service explorer", + href: "service-explorer", + excerpt: "Timescale services explorer", + }, + { + title: "Troubleshooting Timescale services", + href: "troubleshooting", + type: "placeholder", + }, + ], + }, +``` - +See [Use Timescale section navigation][use-navigation] for reference. -### Committing changes and creating a pull request +To change the structure, add or delete pages in a section, modify the corresponding `page-index.js`. An entry in a `page-index.js` includes the following fields: -1. Make your changes. -1. Add the updated files to your commit: +| Key | Type | Required | Description | +|--------------------|-----------------------------------------------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `href` | string | ✅ | The URL segment to use for the page. If there is a corresponding Markdown file, `href` must match the name of the Markdown file, minus the file extension. | +| `title` | string | ✅ | The title of the page, used as the page name within the TOC on the left. Must be the same as the first header in the corresponding Markdown file. | +| `excerpt` | string | ✅ | The short description of the page, used for the page card if `pageComponents` is set to `featured-cards`. Should be up to 100 characters. See `pageComponents` for details. | +| `type` | One of `[directory, placeholder, redirect-to-child-page]` | ❌ | If no type is specified, the page is built as a regular webpage. The structure of its children, if present, is defined by `children` entries and the corresponding structure of subfolders. If the type is `directory`, the corresponding file becomes a directory. The difference of the directory page is that its child pages sit at the same level as the `directory` page. They only become children during the site build. If the type is `placeholder`, the corresponding page is produced programmatically upon site build. If not produced, the link in the navigation tree returns a 404. In particular, this is used for troubleshooting pages. If the type is `redirect-to-child-page`, no page is built and the link in the navigation tree goes directly to the first child. | +| `children` | Array of page entries | ❌ | Child pages of the current page. For regular pages, the children should be located in a directory with the same name as the parent. The parent is the `index.md` file in that directory. For`directory` pages, the children should be located in the same directory as the parent. | +| `pageComponents` | One of `[['featured-cards'], ['content-list']]` | ❌ | Any page that has child pages can list its children in either card or list style at the bottom of the page. Specify the desired style with this key. | +| `featuredChildren` | Array of URLs | ❌ | Similar to `pageComponents`, this displays the children of the current page, but only the selected ones. | +| `index` | string | ❌ | If a section landing page needs to be different from the `index.md` file in that directory, this field specifies the corresponding Markdown file name. | - ```bash - git add . - ``` +## Reuse text in multiple pages -1. Commit your changes: +Partials allow you to reuse snippets of content in multiple places. All partials +live in the `_partials` top-level directory. To make a new partial, create a new +`.md` file in this directory. The filename must start with an underscore. Then import it into the target page and reference in the relevant place. See [Formatting examples][formatting]. - ```bash - git commit -m "Commit message here" - ``` +## Formatting -1. Push your changes: +In addition to all the [regular Markdown formatting][markdown-syntax], the following elements are available for Timescale docs: - ```bash - git push - ``` +- Procedure blocks +- Highlight blocks +- Tabs +- Code blocks without line numbers and the copy button +- Multi-tab code blocks +- Tags + +See [Formatting examples][formatting] for how to use them. - If git prompts you to set an upstream in order to push, use this command: +## Variables + +Timescale documentation uses variables for its product names, features, and UI elements in Timescale Console with the following syntax: `$VARIABLE_NAME`. Variables do not work inside the following: + +- Front matter on each page +- HTML tables and tabs + +See the [full list of available variables][variables]. + +## Links + +- Internal page links: internal links do not need to include the domain name `https://docs.timescale.com`. Use the `:currentVersion:` variable instead of `latest` in the URL. +- External links: input external links as is. + +See [Formatting examples][formatting] for details. + +## Visuals + +When adding screenshots to the docs, aim for a full-screen view to provide better context. Reduce the size of your browser so there is as little wasted space as possible. + +Attach the image to your issue or PR, and the doc team will upload and insert it for you. + +## SEO optimization + +To make a documentation page more visible and clear for Google: + +- Include the `title` and `excerpt` meta tags at the top of the page. These represent meta title and description required for SEO optimization. - ```bash - git push --set-upstream origin - ``` + - `title`: up to 60 characters, a short description of the page contents. In most cases a variation of the page title. + - `excerpt`: under 200 characters, a longer description of the page contents. In most cases a variation of the page intro. -1. Create a pull request (PR) by navigating to - and clicking - `Compare and Create Pull Request`. Write an informative commit message - detailing your changes, choose reviewers, and save your PR. If you haven't - yet finished the work you want to do, make sure you create a draft PR by - selecting it from the drop down box in the GitHub web UI. This lets your - reviewers know that you haven't finished work yet, while still being - transparent about what you are working on, and making sure we all understand - current progress. +- Summarize the contents of each paragraph in the first sentence of that paragraph. +- Include main page keywords into the meta tags, page title, first header, and intro. These are usually the names of features described in the page. For example, for a page dedicated to creating hypertables, you can use the keyword **hypertable** in the following way: - + - Title: Create a hypertable in Timescale Cloud + - Description: Turn a regular PostgreSQL table into a hypertable in a few steps, using Timescale Console. + - First header: Create a hypertable - -Choose your reviewers carefully! If you have made changes to the technical -detail of the documentation, choose an appropriate subject matter expert (SME) -to review those changes. Additionally, every change requires at least one -documentation team member to approve. Ask the documentation team for a review by -adding the `timescale/documentation` group as a reviewer. - +## Docs for deprecated products -## Second contribution - -When you have checked out the repo, if you want to keep working on things, you -need to make sure that your local copy of the repo stays up to date. If you -don't do this, you *will* end up with merge conflicts. - - - -### Second contribution - -1. Check out your fork's `latest` branch: - - ```bash - git checkout latest - ``` - - You get a message like this: - - ```bash - Switched to branch 'latest' - Your branch is up to date with 'origin/latest'. - ``` - - BEWARE! This is usually a lie! -1. Fetch the branches in the upstream repository: - - ```bash - git fetch upstream - ``` - -1. Merge the changes from the upstream `latest` branch, into your fork's - `latest` branch: - - ```bash - git merge upstream/latest - ``` - -1. If you are continuing work you began earlier, check out the branch that - contains your work. For new work, create a new branch. Doing this regularly - as you are working means you keep your local copies up to date and avoid - conflicts. You should do it at least every day before you begin work, and - again whenever you switch branches. - - - - -Never leave branches lying around on your local system. Create your PR as soon -as possible, and make good use of the Draft feature. Commit to your feature -branch early and often! Update your local copy from latest whenever you switch -branches. - - -## Reviewing standards - -When you create a pull request, a member of the documentation team will review -it for accuracy and adherance to our standards. You can see a list of the things -that reviewers check for in the pull request template. - -## Writing standards - -Timescale has comprehensive writing and style standards, that are constantly -being updated and improved. For the current guidelines, see -[contributing to documentation](https://docs.timescale.com/about/latest/contribute-to-docs/). +The previous documentation source is in the deprecated repository called [docs.timescale.com-content][legacy-source]. +[legacy-source]: https://github.com/timescale/docs.timescale.com-content +[docs-issues]: https://github.com/timescale/docs/issues +[github-fork]: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo +[github-clone]: https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository +[gatsby]: https://www.gatsbyjs.com/ +[google-style]: https://developers.google.com/style +[markdown-syntax]: https://www.markdownguide.org/extended-syntax/ +[github-docs]: https://github.com/timescale/docs +[use-navigation]: use-timescale/page-index/page-index.js +[formatting]: _partials/_formatting_examples.md +[variables]: https://docs.timescale.com/variables-for-contributors/ \ No newline at end of file diff --git a/README.md b/README.md index 6148df9aca..3fc140ce91 100644 --- a/README.md +++ b/README.md @@ -196,23 +196,23 @@ Where you need the partial to display, insert it as a self-closing tag: To maintain consistency, please follow these general rules. * Maintain text editor width for paragraphs at 80 characters. We ask you to do -this to assist in reviewing documentation changes. When text is very wide, it -is difficult to visually see where text has changed within a paragraph and keeping -a narrow width on text assists in making PRs easier to review. **Most editors such -as Visual Studio Code have settings to do this visually.** + this to assist in reviewing documentation changes. When text is very wide, it + is difficult to visually see where text has changed within a paragraph and keeping + a narrow width on text assists in making PRs easier to review. **Most editors such + as Visual Studio Code have settings to do this visually.** * Most links should be reference-style links where the link address is at the -bottom of the page. The two exceptions are: + bottom of the page. The two exceptions are: * Links within highlight blocks (Note, Important, or Warning). These must be inline links for now * Links to anchors on the same page as the link itself. * All functions, commands and standalone function arguments (ex. `SELECT`, -`time_bucket`) should be set as inline code within backticks ("\`command\`"). + `time_bucket`) should be set as inline code within backticks ("\`command\`"). * Functions should not be written with parentheses unless the function is -being written with arguments within the parentheses. + being written with arguments within the parentheses. * "PostgreSQL" is the way to write the elephant database name, rather than -"Postgres." "TimescaleDB" refers to the database, "Timescale" refers to the -company. + "Postgres." "TimescaleDB" refers to the database, "Timescale" refers to the + company. * Use backticks when referring to the object of a user interface action. -For example: Click `Get started` to proceed with the tutorial. + For example: Click `Get started` to proceed with the tutorial. ### Callout and highlight blocks @@ -264,7 +264,7 @@ Used to indicate an optional step within a procedure. Syntax: `` Multi-code blocks are code blocks with a language or OS selector. For syntax, see [the multi-code-block example](./_multi-code-block.md). -### Tabs +### Tabs Tabs can be used to display content that differs based on a user selection. The syntax is: @@ -329,3 +329,7 @@ There is a specific format for the API section which consists of: * One or two literal examples of the function being used to demonstrate argument syntax. See the API file to get an idea. + + + + diff --git a/_multi-code-block.md b/_multi-code-block.md index fe2c36d061..d638a5123e 100644 --- a/_multi-code-block.md +++ b/_multi-code-block.md @@ -34,4 +34,4 @@ def different_python: - + \ No newline at end of file diff --git a/_partials/_add-data-blockchain.md b/_partials/_add-data-blockchain.md index 924f31d044..5e359ee738 100644 --- a/_partials/_add-data-blockchain.md +++ b/_partials/_add-data-blockchain.md @@ -1,16 +1,14 @@ ## Load financial data -This tutorial uses Bitcoin transactions from the past five days. - -## Ingest the dataset +The dataset contains around 1.5 million Bitcoin transactions, the trades for five days. It includes +information about each transaction, along with the value in [satoshi][satoshi-def]. It also states if a +trade is a [coinbase][coinbase-def] transaction, and the reward a coin miner receives for mining the coin. To ingest data into the tables that you created, you need to download the dataset and copy the data to your database. -### Ingesting the dataset - 1. Download the `bitcoin_sample.zip` file. The file contains a `.csv` file that contains Bitcoin transactions for the past five days. Download: @@ -37,3 +35,6 @@ dataset and copy the data to your database. resources. + +[satoshi-def]: https://www.pcmag.com/encyclopedia/term/satoshi +[coinbase-def]: https://www.pcmag.com/encyclopedia/term/coinbase-transaction diff --git a/_partials/_add-data-energy.md b/_partials/_add-data-energy.md index a10dc6b3e6..c42420ae13 100644 --- a/_partials/_add-data-energy.md +++ b/_partials/_add-data-energy.md @@ -5,8 +5,6 @@ into the `metrics` hypertable. -### Loading energy consumption data - This is a large dataset, so it might take a long time, depending on your network connection. diff --git a/_partials/_add-data-nyctaxis.md b/_partials/_add-data-nyctaxis.md index 60b9f5231a..8f5d1313e6 100644 --- a/_partials/_add-data-nyctaxis.md +++ b/_partials/_add-data-nyctaxis.md @@ -5,8 +5,6 @@ When you have your database set up, you can load the taxi trip data into the -### Loading trip data - This is a large dataset, so it might take a long time, depending on your network connection. diff --git a/_partials/_add-data-twelvedata-crypto.md b/_partials/_add-data-twelvedata-crypto.md index 5e35ecb48a..3021b352d4 100644 --- a/_partials/_add-data-twelvedata-crypto.md +++ b/_partials/_add-data-twelvedata-crypto.md @@ -1,29 +1,29 @@ ## Load financial data This tutorial uses real-time cryptocurrency data, also known as tick data, from -[Twelve Data][twelve-data]. A direct download link is provided below. +[Twelve Data][twelve-data]. To ingest data into the tables that you created, you need to +download the dataset, then upload the data to your $SERVICE_LONG. -### Ingest the dataset + -To ingest data into the tables that you created, you need to download the -dataset and copy the data to your database. - +1. Unzip [crypto_sample.zip](https://assets.timescale.com/docs/downloads/candlestick/crypto_sample.zip) to a ``. -#### Ingesting the dataset + This test dataset contains second-by-second trade data for the most-traded crypto-assets + and a regular table of asset symbols and company names. -1. Download the `crypto_sample.zip` file. The file contains two `.csv` - files; one with company information, and one with real-time stock trades for - the past month. Download: - [crypto_sample.zip](https://assets.timescale.com/docs/downloads/candlestick/crypto_sample.zip) - + To import up to 100GB of data directly from your current PostgreSQL based database, + [migrate with downtime][migrate-with-downtime] using native PostgreSQL tooling. To seamlessly import 100GB-10TB+ + of data, use the [live migration][migrate-live] tooling supplied by $COMPANY. To add data from non-PostgreSQL + data sources, see [Import and ingest data][data-ingest]. -1. In a new terminal window, run this command to unzip the `.csv` files: - ```bash - unzip crypto_sample.zip - ``` + +1. In Terminal, navigate to `` and connect to your $SERVICE_SHORT. + ```bash + psql -d "postgres://:@:/" + ``` + The connection information for a $SERVICE_SHORT is available in the file you downloaded when you created it. 1. At the `psql` prompt, use the `COPY` command to transfer data into your Timescale instance. If the `.csv` files aren't in your current directory, @@ -44,3 +44,6 @@ dataset and copy the data to your database. [twelve-data]: https://twelvedata.com/ +[migrate-with-downtime]: /migrate/:currentVersion:/pg-dump-and-restore/ +[migrate-live]: /migrate/:currentVersion:/live-migration/ +[data-ingest]: /use-timescale/:currentVersion:/ingest-data/ diff --git a/_partials/_add-data-twelvedata-stocks.md b/_partials/_add-data-twelvedata-stocks.md index 7be487e717..fb3a13b13d 100644 --- a/_partials/_add-data-twelvedata-stocks.md +++ b/_partials/_add-data-twelvedata-stocks.md @@ -3,15 +3,11 @@ This tutorial uses real-time stock trade data, also known as tick data, from [Twelve Data][twelve-data]. A direct download link is provided below. -## Ingest the dataset - To ingest data into the tables that you created, you need to download the dataset and copy the data to your database. -#### Ingesting the dataset - 1. Download the `real_time_stock_data.zip` file. The file contains two `.csv` files; one with company information, and one with real-time stock trades for the past month. Download: diff --git a/_partials/_caggs-intro.md b/_partials/_caggs-intro.md index aa8f0d6bcc..1215a8f69f 100644 --- a/_partials/_caggs-intro.md +++ b/_partials/_caggs-intro.md @@ -1,12 +1,12 @@ -Time-series data usually grows very quickly. And that means that aggregating the -data into useful summaries can become very slow. Continuous aggregates makes -aggregating data lightning fast. +In modern applications, data usually grows very quickly. This means that aggregating +it into useful summaries can become very slow. $CLOUD_LONG continuous aggregates make +aggregating data lightning fast, accurate, and easy. If you are collecting data very frequently, you might want to aggregate your -data into minutes or hours instead. For example, if you have a table of -temperature readings taken every second, you can find the average temperature +data into minutes or hours instead. For example, if an IoT device takes +temperature readings every second, you might want to find the average temperature for each hour. Every time you run this query, the database needs to scan the -entire table and recalculate the average every time. +entire table and recalculate the average. Continuous aggregates are a kind of hypertable that is refreshed automatically in the background as new data is added, or old data is modified. Changes to your @@ -22,14 +22,14 @@ database. Because continuous aggregates are based on hypertables, you can query them in exactly the same way as your other tables, and enable [compression][compression] -or [tiered storage][data-tiering] on your continuous aggregates. You can even +or [tiered storage][data-tiering] on them. You can even create -[continuous aggregates on top of your continuous aggregates][hierarchical-caggs]. +[continuous aggregates on top of your continuous aggregates][hierarchical-caggs] - for an even more fine-tuned aggregation. By default, querying continuous aggregates provides you with real-time data. Pre-aggregated data from the materialized view is combined with recent data that hasn't been aggregated yet. This gives you up-to-date results on every query. [data-tiering]: /use-timescale/:currentVersion:/data-tiering/ -[compression]: /use-timescale/:currentVersion:/compression/ +[hypercore]: /use-timescale/:currentVersion:/hypercore/ [hierarchical-caggs]: /use-timescale/:currentVersion:/continuous-aggregates/hierarchical-continuous-aggregates/ diff --git a/_partials/_cloud-connect.md b/_partials/_cloud-connect.md index 4f057c6123..97180eecca 100644 --- a/_partials/_cloud-connect.md +++ b/_partials/_cloud-connect.md @@ -1,45 +1,45 @@ -To connect to a service: +To connect to a $SERVICE_SHORT: -1. **Check your service is running correctly** +1. **Check your $SERVICE_SHORT is running correctly** - In [Timescale Console][services-portal], check that your service is marked as `Running`. + In [$CONSOLE][services-portal], check that your $SERVICE_SHORT is marked as `Running`. ![Check service is running](https://assets.timescale.com/docs/images/console-services-view.png) -1. **Connect to your service** +1. **Connect to your $SERVICE_SHORT** Use either: - - [Data mode][popsql] in Timescale Console: - 1. In the [data mode][portal-data-mode] in Timescale Console, select a service and enter your password, then click **Connect**. + - [Data mode][popsql] in $CONSOLE: + 1. In the [data mode][portal-data-mode] in $CONSOLE, select a $SERVICE_SHORT and enter your password, then click **Connect**. You find your password in the config file you just downloaded. 2. Select a query to edit: ![Select a query to edit](https://assets.timescale.com/docs/images/data-mode-query-window.png) - - [SQL editor][run-sqleditor] in Timescale Console: + - [SQL editor][run-sqleditor] in $CONSOLE: - In the [ops mode][portal-ops-mode] in Timescale Console, select a service, then click **SQL editor**. + In the [ops mode][portal-ops-mode] in $CONSOLE, select a $SERVICE_SHORT, then click **SQL editor**. ![Check service is running](https://assets.timescale.com/docs/images/ops-view-sql-editor.png) - [psql][install-psql] on the command line: - Connect to your service with the value of `Service URL` from the config file you + Connect to your $SERVICE_SHORT with the value of `Service URL` from the config file you just saved: @:/tsdb?sslmode=require" `} /> - You can now run queries for this service. + You can now run queries for this $SERVICE_SHORT. Quick recap. You: -- Manage your services in the [ops mode][portal-ops-mode] in Timescale Console: add read replicas and enable +- Manage your $SERVICE_SHORTs in the [ops mode][portal-ops-mode] in $CONSOLE: add read replicas and enable high-availability, configure compression, change parameters, and so on. -- Analyze your data in the [data mode][portal-data-mode] in Timescale Console: write queries with +- Analyze your data in the [data mode][portal-data-mode] in $CONSOLE: write queries with autocomplete, save them in folders, share them, create charts/dashboards, and much more. - Store configuration and security information in your config file. diff --git a/_partials/_cloud-create-connect-tutorials.md b/_partials/_cloud-create-connect-tutorials.md index 55bae061e2..a96cbf3892 100644 --- a/_partials/_cloud-create-connect-tutorials.md +++ b/_partials/_cloud-create-connect-tutorials.md @@ -6,9 +6,7 @@ command-line utility. If you've used PostgreSQL before, you might already have -### Create a Timescale service and connect to the service - -1. In the [Timescale portal][timescale-portal], click `Create service`. +1. In the [$CONSOLE][timescale-portal], click `Create service`. 1. Click `Download the cheatsheet` to download an SQL file that contains the login details for your new service. You can also copy the details directly from this page. When you have copied your password, diff --git a/_partials/_cloud-installation.md b/_partials/_cloud-installation.md index 0cd2a4e7d8..805caff377 100644 --- a/_partials/_cloud-installation.md +++ b/_partials/_cloud-installation.md @@ -6,17 +6,17 @@ You create a $CLOUD_LONG account to manage your $SERVICE_SHORTs and data in a ce -You create a standalone account to manage $CLOUD_LONG as a separate unit in your infrastructure, which includes separate billing and invoicing. - -To create an account: +You create a standalone account to manage $CLOUD_LONG as a separate unit in your infrastructure, which includes separate billing and invoicing. +To set up $CLOUD_LONG: + 1. **Sign up for a 30-day free trial** Open [Sign up for Timescale][timescale-signup] and add your details, then click `Start your free trial`. You receive a confirmation email in your inbox. -1. **In the confirmation email, click the link supplied and sign in to [$CONSOLE][tsc-portal]** +1. **In the confirmation email, click the link supplied and sign in to [$CONSOLE][tsc-portal]** 1. **Answer the requirements questions** @@ -28,12 +28,12 @@ To create an account: -You create an account through AWS Marketplace to have $CLOUD_LONG as a part of your AWS infrastructure. In this case, $CLOUD_LONG is a line item in your AWS invoice. - -To create an account: +You create an account through AWS Marketplace to have $CLOUD_LONG as a part of your AWS infrastructure. In this case, $CLOUD_LONG is a line item in your AWS invoice. +To set up $CLOUD_LONG via AWS: + 1. **Open [AWS Marketplace][aws-marketplace] and search for `Timescale Cloud`** You see two pricing options, [pay-as-you-go][aws-paygo] and [annual commit][aws-annual-commit]. diff --git a/_partials/_cloud-intro.md b/_partials/_cloud-intro.md index ee0298abe4..1303373253 100644 --- a/_partials/_cloud-intro.md +++ b/_partials/_cloud-intro.md @@ -1,12 +1,14 @@ -$CLOUD_LONG is a cloud-based PostgreSQL platform for resource-intensive workloads. We help you build faster, -scale further, and stay under budget. A $SERVICE_LONG is a single optimized 100% PostgreSQL database instance that you -use as is, or extend with capabilities specific to your business needs. The available capabilities are: +$CLOUD_LONG is the modern PostgreSQL data platform for all your applications. It enhances PostgreSQL to handle time series, events, +real-time analytics, and vector search—all in a single database alongside transactional workloads. + +You get one system that handles live data ingestion, late and out-of-order updates, and low latency queries, with the performance, reliability, and scalability your app needs. Ideal for IoT, crypto, finance, SaaS, and a myriad other domains, $CLOUD_LONG allows you to build data-heavy, mission-critical apps while retaining the familiarity and reliability of PostgreSQL. + +A $SERVICE_LONG is a single optimized 100% PostgreSQL database instance that you +use as is, or extend with the capabilities specific to your business needs: - **Time-series and analytics**: PostgreSQL with TimescaleDB. The PostgreSQL you know and love, supercharged with functionality for storing and querying [time-series data][what-is-time-series] at scale for - analytics and other use cases. - Get faster time-based queries with hypertables, continuous aggregates, and columnar storage. Save on storage with - native compression, data retention policies, and bottomless data tiering to Amazon S3. + real-time analytics and other use cases. Get faster time-based queries with hypertables, continuous aggregates, and columnar storage. Save on storage with native compression, data retention policies, and bottomless data tiering to Amazon S3. - **AI and vector**: PostgreSQL with vector extensions. Use PostgreSQL as a vector database with purpose built extensions for building AI applications from start to scale. Get fast and accurate similarity search with the pgvector and pgvectorscale extensions. Create vector embeddings and perform LLM reasoning on your data with @@ -14,7 +16,7 @@ use as is, or extend with capabilities specific to your business needs. The avai - **PostgreSQL**: the trusted industry-standard RDBMS. Ideal for applications requiring strong data consistency, complex relationships, and advanced querying capabilities. Get ACID compliance, extensive SQL support, JSON handling, and extensibility through custom functions, data types, and extensions. $CLOUD_LONG continuously - monitors your services and prevents common PostgreSQL out-of-memory crashes. + monitors your $SERVICE_SHORTs and prevents common PostgreSQL out-of-memory crashes. All $SERVICE_LONGs include the tooling you expect for production and developer environments: [live migration][live-migration], [automatic backups and PITR][automatic-backups], [high availability][high-availability], [read replicas][readreplica], [data forking][operations-forking], [connection pooling][connection-pooling], [tiered storage][data-tiering], diff --git a/_partials/_cloud_self_configuration.md b/_partials/_cloud_self_configuration.md new file mode 100644 index 0000000000..4a5ab0b425 --- /dev/null +++ b/_partials/_cloud_self_configuration.md @@ -0,0 +1,81 @@ +import EarlyAccess from "versionContent/_partials/_early_access.mdx"; + +## Policies + +### `timescaledb.max_background_workers (int)` + +Max background worker processes allocated to TimescaleDB. Set to at least 1 + +the number of databases loaded with the TimescaleDB extension in a PostgreSQL +instance. Default value is 16. + +## Hypercore features + +### `timescaledb.default_hypercore_use_access_method (bool)` + +The default value for `hypercore_use_access_method` for functions that have this parameter. This function is in `user` context, meaning that any user can set it for the session. The default value is `false`. + + + +## $SERVICE_LONG tuning + +### `timescaledb.disable_load (bool)` + +Disable the loading of the actual extension + +### `timescaledb.enable_cagg_reorder_groupby (bool)` +Enable group by reordering + +### `timescaledb.enable_chunk_append (bool)` +Enable chunk append node + +### `timescaledb.enable_constraint_aware_append (bool)` +Enable constraint-aware append scans + +### `timescaledb.enable_constraint_exclusion (bool)` +Enable constraint exclusion + +### `timescaledb.enable_job_execution_logging (bool)` +Enable job execution logging + +### `timescaledb.enable_optimizations (bool)` +Enable TimescaleDB query optimizations + +### `timescaledb.enable_ordered_append (bool)` +Enable ordered append scans + +### `timescaledb.enable_parallel_chunk_append (bool)` +Enable parallel chunk append node + +### `timescaledb.enable_runtime_exclusion (bool)` +Enable runtime chunk exclusion + +### `timescaledb.enable_tiered_reads (bool)` + +Enable [tiered reads][enabling-data-tiering] to query your data normally when it's distributed across different storage tiers. +Your hypertable is spread across the tiers, so queries and `JOIN`s work and fetch the same data as usual. + +By default, tiered data is not accessed by queries. Querying tiered data may slow down query performance +as the data is not stored locally on Timescale's high-performance storage tier. + + +### `timescaledb.enable_transparent_decompression (bool)` +Enable transparent decompression + + +### `timescaledb.restoring (bool)` +Stop any background workers which could have been performing tasks. This is especially useful if you +migrate data to your [$SERVICE_LONG][pg-dump-and-restore] or [self-hosted database][migrate-entire]. + +### `timescaledb.max_cached_chunks_per_hypertable (int)` +Maximum cached chunks + +### `timescaledb.max_open_chunks_per_insert (int)` +Maximum open chunks per insert + +### `timescaledb.max_tuples_decompressed_per_dml_transaction (int)` + +The max number of tuples that can be decompressed during an INSERT, UPDATE, or DELETE. + +[enabling-data-tiering]: /use-timescale/:currentVersion:/data-tiering/enabling-data-tiering/ +[pg-dump-and-restore]: /migrate/:currentVersion:/pg-dump-and-restore/ +[migrate-entire]: /self-hosted/:currentVersion:/migration/entire-database/ diff --git a/_partials/_create-hypertable-blockchain.md b/_partials/_create-hypertable-blockchain.md index 84761f1e02..9d277e6130 100644 --- a/_partials/_create-hypertable-blockchain.md +++ b/_partials/_create-hypertable-blockchain.md @@ -1,5 +1,5 @@ -## Create a hypertable +## Optimize time-series data in hypertables Hypertables are the core of Timescale. Hypertables enable Timescale to work efficiently with time-series data. Because Timescale is PostgreSQL, all the @@ -9,8 +9,6 @@ with Timescale tables similar to standard PostgreSQL. -### Creating a hypertable - 1. Create a standard PostgreSQL table to store the Bitcoin blockchain data using `CREATE TABLE`: diff --git a/_partials/_create-hypertable-energy.md b/_partials/_create-hypertable-energy.md index 235a90d788..ff75bf552e 100644 --- a/_partials/_create-hypertable-energy.md +++ b/_partials/_create-hypertable-energy.md @@ -1,4 +1,4 @@ -## Create a hypertable +## Optimize time-series data in hypertables Hypertables are the core of Timescale. Hypertables enable Timescale to work efficiently with time-series data. Because Timescale is PostgreSQL, all the @@ -8,8 +8,6 @@ with Timescale tables similar to standard PostgreSQL. -### Creating a hypertable - 1. Create a standard PostgreSQL table to store the energy consumption data using `CREATE TABLE`: diff --git a/_partials/_create-hypertable-nyctaxis.md b/_partials/_create-hypertable-nyctaxis.md index b41a579de4..3400e849b6 100644 --- a/_partials/_create-hypertable-nyctaxis.md +++ b/_partials/_create-hypertable-nyctaxis.md @@ -1,17 +1,16 @@ -## Create a hypertable +## Optimize time-series data in hypertables -Hypertables are the core of Timescale. Hypertables enable Timescale to work -efficiently with time-series data. Because Timescale is PostgreSQL, all the -standard PostgreSQL tables, indexes, stored procedures and other objects can be -created alongside your Timescale hypertables. This makes creating and working -with Timescale tables similar to standard PostgreSQL. +Time-series data represents how a system, process, or behavior changes over time. [Hypertables][hypertables-section] +are PostgreSQL tables that help you improve insert and query performance by automatically partitioning your data by +time. Each hypertable is made up of child tables called chunks. Each chunk is assigned a range of time, and only +contains data from that range. - +Hypertables exist alongside regular PostgreSQL tables. You use regular PostgreSQL tables for relational data, and +interact with hypertables and regular PostgreSQL tables in the same way. -### Creating a hypertable + -1. Create a standard PostgreSQL table to store the taxi trip data - using `CREATE TABLE`: +1. **Create a standard PostgreSQL table to store the taxi trip data** ```sql CREATE TABLE "rides"( @@ -36,7 +35,8 @@ with Timescale tables similar to standard PostgreSQL. ); ``` -1. Convert the standard table into a hypertable partitioned on the `time` +1. **Convert the standard table into a hypertable** + Partitioned on the `time` column using the `create_hypertable()` function provided by Timescale. You must provide the name of the table and the column in that table that holds the timestamp data to use for partitioning: @@ -69,8 +69,6 @@ there are two other tables of data, called `payment_types` and `rates`. -### Creating standard PostgreSQL tables - 1. Add a table to store the payment types data: ```sql @@ -117,3 +115,6 @@ the `psql` command line. You should see this: public | rides | table | tsdbadmin (3 rows) ``` + + +[hypertables-section]: /use-timescale/:currentVersion:/hypertables/ diff --git a/_partials/_create-hypertable-twelvedata-crypto.md b/_partials/_create-hypertable-twelvedata-crypto.md index 4412295ea7..737169cdd7 100644 --- a/_partials/_create-hypertable-twelvedata-crypto.md +++ b/_partials/_create-hypertable-twelvedata-crypto.md @@ -1,18 +1,22 @@ -## Create a hypertable +## Optimize time-series data in a hypertable -Hypertables are the core of Timescale. Hypertables enable Timescale to work -efficiently with time-series data. Because Timescale is PostgreSQL, all the -standard PostgreSQL tables, indexes, stored procedures, and other objects can be -created alongside your Timescale hypertables. This makes creating and working -with Timescale tables similar to standard PostgreSQL. +Hypertables are the core of $TIMESCALE_DB, they enable $CLOUD_LONG to work +efficiently with time-series data. Hypertables are PostgreSQL tables that automatically +partition your time-series data by time. When you run a query, $CLOUD_LONG identifies the +correct partition and runs the query on it, instead of going through the entire table. + +Because $TIMESCALE_DB is 100% PostgreSQL, you can create standard PostgreSQL tables, indexes, stored +procedures, and other objects alongside your Timescale hypertables. This makes creating and working +with hypertables similar to standard PostgreSQL. -### Creating a hypertable +1. Connect to your $SERVICE_LONG. + + In [$CONSOLE][services-portal] open an [SQL editor][in-console-editors]. You can also connect to your service using [psql][connect-using-psql]. -1. Create a standard PostgreSQL table to store the real-time cryptocurrency data - using `CREATE TABLE`: +1. Create a standard PostgreSQL table to store the real-time cryptocurrency data: ```sql CREATE TABLE crypto_ticks ( @@ -32,23 +36,16 @@ with Timescale tables similar to standard PostgreSQL. SELECT create_hypertable('crypto_ticks', by_range('time')); ``` - - The `by_range` dimension builder is an addition to TimescaleDB 2.13. - - -## Create standard PostgreSQL tables for relational data +## Create a standard PostgreSQL table for relational data -When you have other relational data that enhances your time-series data, you can -create standard PostgreSQL tables just as you would normally. For this dataset, -there is one other table of data called `crypto_assets`. +When you have relational data that enhances your time-series data, store that data in +standard PostgreSQL relational tables. -### Creating standard PostgreSQL tables - -1. Add a table to store the company name and symbol for the stock trade data: +1. Add a table to store the asset symbol and name in a relational table: ```sql CREATE TABLE crypto_assets ( @@ -57,7 +54,11 @@ there is one other table of data called `crypto_assets`. ); ``` -1. You now have two tables within your Timescale database. One hypertable - named `crypto_ticks`, and one normal PostgreSQL table named `crypto_assets`. - + +You now have two tables within your $SERVICE_LONG. A hypertable named `crypto_ticks`, and a normal +PostgreSQL table named `crypto_assets`. + +[in-console-editors]: /getting-started/:currentVersion:/run-queries-from-console/ +[services-portal]: https://console.cloud.timescale.com/dashboard/services +[connect-using-psql]: /use-timescale/:currentVersion:/integrations/psql diff --git a/_partials/_create-hypertable-twelvedata-stocks.md b/_partials/_create-hypertable-twelvedata-stocks.md index 85727a9fdc..6e06273e9b 100644 --- a/_partials/_create-hypertable-twelvedata-stocks.md +++ b/_partials/_create-hypertable-twelvedata-stocks.md @@ -1,5 +1,5 @@ -## Create a hypertable +## Optimize time-series data in hypertables Hypertables are the core of Timescale. Hypertables enable Timescale to work efficiently with time-series data. Because Timescale is PostgreSQL, all the @@ -9,8 +9,6 @@ with Timescale tables similar to standard PostgreSQL. -### Creating a hypertable - 1. Create a standard PostgreSQL table to store the real-time stock trade data using `CREATE TABLE`: diff --git a/_partials/_deprecated.md b/_partials/_deprecated.md index c07cc2db29..53763784a1 100644 --- a/_partials/_deprecated.md +++ b/_partials/_deprecated.md @@ -1,5 +1,7 @@ + This section describes a feature that is deprecated on Timescale. We strongly recommend that you do not use this feature in a production environment. If you need more information, [contact us](https://www.timescale.com/contact/). + diff --git a/_partials/_deprecated_2_18_0.md b/_partials/_deprecated_2_18_0.md new file mode 100644 index 0000000000..77bf9772d6 --- /dev/null +++ b/_partials/_deprecated_2_18_0.md @@ -0,0 +1 @@ +Old API since [TimescaleDB v2.18.0](https://github.com/timescale/timescaledb/releases/tag/2.18.0) diff --git a/_partials/_early_access.md b/_partials/_early_access.md index b27534d7f6..89ad944fba 100644 --- a/_partials/_early_access.md +++ b/_partials/_early_access.md @@ -1,5 +1 @@ - -This feature is early access. Early access features might be subject to billing -changes in the future. If you have feedback, reach out to your customer success -manager, or [contact us](https://www.timescale.com/contact/). - +Early access: TimescaleDB v2.18.0 diff --git a/_partials/_financial-industry-data-analysis.md b/_partials/_financial-industry-data-analysis.md new file mode 100644 index 0000000000..bfd6293928 --- /dev/null +++ b/_partials/_financial-industry-data-analysis.md @@ -0,0 +1 @@ +The financial industry is extremely data-heavy and relies on real-time and historical data for decision-making, risk assessment, fraud detection, and market analysis. Timescale simplifies management of these large volumes of data, while also providing you with meaningful analytical insights and optimizing storage costs. \ No newline at end of file diff --git a/_partials/_formatting_examples.md b/_partials/_formatting_examples.md new file mode 100644 index 0000000000..486b3087af --- /dev/null +++ b/_partials/_formatting_examples.md @@ -0,0 +1,167 @@ +# Formatting examples + +This page illustrates and provides examples of the formatting available for Timescale documentation. Note that for most elements, spacing is important. + +## Procedure + +Use for a logical sequence of steps to achieve a goal. For example, create a hypertable. + +![Procedure example](https://assets.timescale.com/docs/images/procedure-syntax.png) + +See a [use example][data-tiering] in the docs. + +## Highlight blocks + +Use sparingly and only if it's essential to attract the reader's attention. + +- Note + + ![Note highlight](https://assets.timescale.com/docs/images/highlight-note.png) + + See a [use example][disable-chunk-skipping] in the docs. + +- Important + + ![Important highlight](https://assets.timescale.com/docs/images/highlight-important.png) + + See a [use example][decompress-chunks] in the docs. + +- Warning + + ![Caution highlight](https://assets.timescale.com/docs/images/highlight-warning.png) + + See a [use example][alerting] in the docs. + +- Deprecation + + ![Deprecated highlight](https://assets.timescale.com/docs/images/highlight-deprecation.png) + + See a [use example][deprecation] in the docs. + +- Cloud + + ![Cloud highlight](https://assets.timescale.com/docs/images/highlight-cloud.png) + + Syntax example: + + ```text + + + A note dealing specifically with Timescale Cloud. + + + ``` + +## Tabs + +![Tabs](https://assets.timescale.com/docs/images/tabs-example.png) + +See a [use example][live-migration] in the docs. + +## Code blocks + +As a default, use [fenced Markdown code blocks][fenced-code-blocks]: + +![Regular code block](https://assets.timescale.com/docs/images/markdown-code-block.png) + +To remove line numbers and the copy button, use the `CodeBlock` component with `canCopy` and `showLineNumbers` set to `false`: + +![Custom code block](https://assets.timescale.com/docs/images/custom-code-block.png) + +See a [use example][aggregation] in the docs. + +## Multi-tab code blocks + +![Multi-tab code block](https://assets.timescale.com/docs/images/multi-tab-code.png) + +Syntax example: + + + + + + ```ruby + ruby code + ``` + + + + + + ```python + pyhon code + ``` + + + + + + ```go + different python code + ``` + + + + + +## Tags + +- Download + + ![Download tag](https://assets.timescale.com/docs/images/tag-download.png) + + See a [use example][installation-windows] in the docs. + +- Experimental + + ![Experimental tag](https://assets.timescale.com/docs/images/tag-experimental.png) + + See a [use example][time-bucket] in the docs. + +- Toolkit + + ![Tooklit tag](https://assets.timescale.com/docs/images/tag-toolkit.png) + + See a [use example][time-weighted-average] in the docs. + +- Community + + ![Community tag](https://assets.timescale.com/docs/images/tag-community.png) + + See a [use example][remove-reorder-policy] in the docs. + +- Hollow + + ![Hollow tag](https://assets.timescale.com/docs/images/hollow-tag.png) + + Syntax example: + + ```text + Text to display in a tag + ``` + +## Partials + +Import a partial from the `_partials` directory and then reference it in the relevant part of the page. See a [use example][live-migration] in the docs. + +## Links + +Links should be [reference-style Markdown links][reference-links]. For example: + +[A link to the data tiering section in docs][data-tiering] + +[data-tiering]: ../use-timescale/data-tiering/enabling-data-tiering.md +[disable-chunk-skipping]: ../api/disable_chunk_skipping.md +[decompress-chunks]: ../use-timescale/compression/decompress-chunks.md +[alerting]: ../use-timescale/alerting.md +[deprecation]: ../_partials/_deprecated.md +[live-migration]: ../migrate/live-migration.md +[fenced-code-blocks]: https://www.markdownguide.org/extended-syntax/#fenced-code-blocks +[aggregation]: ../getting-started/test-drive-timescale-features.md +[installation-windows]: ../self-hosted/install/installation-windows.md +[time-bucket]: ../api/time_bucket_ng.md +[time-weighted-average]: ../api/time-weighted-averages.md +[remove-reorder-policy]: ../api/remove_reorder_policy.md +[reference-links]: https://www.markdownguide.org/basic-syntax/#reference-style-links + + diff --git a/_partials/_grafana-connect.md b/_partials/_grafana-connect.md index fdc3619001..0bce6cd58b 100644 --- a/_partials/_grafana-connect.md +++ b/_partials/_grafana-connect.md @@ -1,14 +1,6 @@ -## Prerequisites +## Connect Grafana to $CLOUD_LONG -import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; - - - -* Install [self-managed Grafana][grafana-self-managed] or sign up for [Grafana Cloud][grafana-cloud]. - -## Add your $SERVICE_SHORT as a data source - -To connect the data in your $SERVICE_SHORT to Grafana: +To visualize the results of your queries, enable Grafana to read the data in your $SERVICE_SHORT: @@ -21,15 +13,16 @@ To connect the data in your $SERVICE_SHORT to Grafana: 1. Open `Connections` > `Data sources`, then click `Add new data source`. 1. Select `PostgreSQL` from the list. 1. Configure the connection: - - `Host URL`, `Username`, `Password`, and `Database`: configure using your [connection details][connection-info]. - - `Database name`: provide the name for your dataset. - - `TLS/SSL Mode`: select `require`. - - `PostgreSQL options`: enable `TimescaleDB`. - - Leave the default setting for all other fields. - - 1. **Click `Save & test`** + - `Host URL`, `Database name`, `Username`, and `Password` + + Configure using your [connection details][connection-info]. `Host URL` is in the format `:`. + - `TLS/SSL Mode`: select `require`. + - `PostgreSQL options`: enable `TimescaleDB`. + - Leave the default setting for all other fields. + + 1. Click `Save & test`. - Grafana checks that your details are set correctly. + Grafana checks that your details are set correctly. diff --git a/_partials/_graphing-ohlcv-data.md b/_partials/_graphing-ohlcv-data.md index 2e7a51d53b..272ac3d156 100644 --- a/_partials/_graphing-ohlcv-data.md +++ b/_partials/_graphing-ohlcv-data.md @@ -9,12 +9,12 @@ up to connect to your TimescaleDB database. ### Graphing OHLCV data 1. Ensure you have Grafana installed, and you are using the TimescaleDB - database that contains the Twelve Data stocks dataset set up as a + database that contains the Twelve Data dataset set up as a data source. 1. In Grafana, from the `Dashboards` menu, click `New Dashboard`. In the `New Dashboard` page, click `Add a new panel`. 1. In the `Visualizations` menu in the top right corner, select `Candlestick` - from the list. Ensure you have set the Twelve Data stocks dataset as + from the list. Ensure you have set the Twelve Data dataset as your data source. 1. Click `Edit SQL` and paste in the query you used to get the OHLCV values. 1. In the `Format as` section, select `Table`. diff --git a/_partials/_high-availability-setup.md b/_partials/_high-availability-setup.md new file mode 100644 index 0000000000..5eb9eb15d1 --- /dev/null +++ b/_partials/_high-availability-setup.md @@ -0,0 +1,13 @@ + + +1. In [Timescale Console][cloud-login], select the service to enable replication for. +1. Click `Operations`, then select `High availability`. +1. Choose your replication strategy, then click `Change configuration`. + + ![Creating a service replica in Timescale Cloud](https://assets.timescale.com/docs/images/tsc-replication-add.png) + +1. In `Change high availability configuration`, click `Change config`. + + + +[cloud-login]: https://console.cloud.timescale.com diff --git a/_partials/_hypercore-conversion-overview.md b/_partials/_hypercore-conversion-overview.md new file mode 100644 index 0000000000..7cc2bf5ebc --- /dev/null +++ b/_partials/_hypercore-conversion-overview.md @@ -0,0 +1,21 @@ +When you convert chunks from the rowstore to the columnstore, multiple records are grouped into a single row. +The columns of this row hold an array-like structure that stores all the data. For example, data in the following +rowstore chunk: + +| Timestamp | Device ID | Device Type | CPU |Disk IO| +|---|---|---|---|---| +|12:00:01|A|SSD|70.11|13.4| +|12:00:01|B|HDD|69.70|20.5| +|12:00:02|A|SSD|70.12|13.2| +|12:00:02|B|HDD|69.69|23.4| +|12:00:03|A|SSD|70.14|13.0| +|12:00:03|B|HDD|69.70|25.2| + +Is converted and compressed into arrays in a row in the columnstore: + +|Timestamp|Device ID|Device Type|CPU|Disk IO| +|-|-|-|-|-| +|[12:00:01, 12:00:01, 12:00:02, 12:00:02, 12:00:03, 12:00:03]|[A, B, A, B, A, B]|[SSD, HDD, SSD, HDD, SSD, HDD]|[70.11, 69.70, 70.12, 69.69, 70.14, 69.70]|[13.4, 20.5, 13.2, 23.4, 13.0, 25.2]| + +Because a single row takes up less disk space, you can reduce your chunk size by more than 90%, and can also +speed up your queries. This saves on storage costs, and keeps your queries operating at lightning speed. diff --git a/_partials/_hypercore_manual_workflow.md b/_partials/_hypercore_manual_workflow.md new file mode 100644 index 0000000000..1598c3e6b5 --- /dev/null +++ b/_partials/_hypercore_manual_workflow.md @@ -0,0 +1,44 @@ +import EarlyAccess from "versionContent/_partials/_early_access.mdx"; + +1. **Stop the jobs that are automatically adding chunks to the columnstore** + + Retrieve the list of jobs from the [timescaledb_information.jobs][informational-views] view + to find the job you need to [alter_job][alter_job]. + + ``` sql + SELECT alter_job(JOB_ID, scheduled => false); + ``` + +1. **Convert a chunk to update back to the rowstore** + + ``` sql + CALL convert_to_rowstore('_timescaledb_internal._hyper_2_2_chunk'); + ``` + +1. **Update the data in the chunk you added to the rowstore** + + Best practice is to structure your [INSERT][insert] statement to include appropriate + partition key values, such as the timestamp. TimescaleDB adds the data to the correct chunk: + + ``` sql + INSERT INTO metrics (time, value) + VALUES ('2025-01-01T00:00:00', 42); + ``` + +1. **Convert the updated chunks back to the columnstore** + + ``` sql + CALL convert_to_columnstore('_timescaledb_internal._hyper_1_2_chunk'); + ``` + +1. **Restart the jobs that are automatically converting chunks to the columnstore** + + ``` sql + SELECT alter_job(JOB_ID, scheduled => true); + ``` + +[alter_job]: /api/:currentVersion:/actions/alter_job/ +[informational-views]: /api/:currentVersion:/informational-views/jobs/ +[insert]: /use-timescale/:currentVersion:/write-data/insert/ +[setup-hypercore]: /use-timescale/:currentVersion:/hypercore/real-time-analytics-in-hypercore/ +[compression_alter-table]: /api/:currentVersion:/hypercore/alter_table/ diff --git a/_partials/_hypercore_policy_workflow.md b/_partials/_hypercore_policy_workflow.md new file mode 100644 index 0000000000..26e8cbac22 --- /dev/null +++ b/_partials/_hypercore_policy_workflow.md @@ -0,0 +1,118 @@ + + + +1. **Connect to your $SERVICE_LONG** + + In [$CONSOLE][services-portal] open an [SQL editor][in-console-editors]. You can also connect to your service using [psql][connect-using-psql]. + +1. **Enable columnstore on a hypertable** + + Create a [job][job] that automatically moves chunks in a hypertable to the columnstore at a specific time interval. + By default, your table is `orderedby` the time column. For efficient queries on columnstore data, remember to + `segmentby` the column you will use most often to filter your data: + + * [Use `ALTER TABLE` for a hypertable][alter_table_hypercore] + ```sql + ALTER TABLE crypto_ticks SET ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'symbol'); + ``` + * [Use ALTER MATERIALIZED VIEW for a continuous aggregate][compression_continuous-aggregate] + ```sql + ALTER MATERIALIZED VIEW assets_candlestick_daily set ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'symbol' ); + ``` + Before you say `huh`, a continuous aggregate is a specialized hypertable. + +1. **Add a policy to convert chunks to the columnstore at a specific time interval** + + For example, move yesterday's crypto trading data to the columnstore: + ``` sql + CALL add_columnstore_policy('crypto_ticks', after => INTERVAL '1d'); + ``` + See [add_columnstore_policy][add_columnstore_policy]. + +1. **Check the columstore policy** + + 1. View your data space saving: + + When you convert data to the columnstore, as well as being optimized for analytics, it is compressed by more than + 90%. This saves on storage costs and keeps your queries operating at lightning speed. To see the amount of space + saved: + + ``` sql + SELECT + pg_size_pretty(before_compression_total_bytes) as before, + pg_size_pretty(after_compression_total_bytes) as after + FROM hypertable_compression_stats('crypto_ticks'); + ``` + You see something like: + + | before | after | + |---------|--------| + | 194 MB | 24 MB | + + 1. View the policies that you set or the policies that already exist: + + ``` sql + SELECT * FROM timescaledb_information.jobs + WHERE proc_name='policy_compression'; + ``` + See [timescaledb_information.jobs][informational-views]. + +1. **Pause a columnstore policy** + + If you need to modify or add a lot of data to a chunk in the columnstore, best practice is to stop any jobs moving + chunks to the columnstore, [convert the chunk back to the rowstore][convert_to_rowstore], then modify the data. + After the update, [convert the chunk to the columnstore][convert_to_columnstore] and restart the jobs. + + ``` sql + SELECT * FROM timescaledb_information.jobs where + proc_name = 'policy_compression' AND relname = 'crypto_ticks' + + -- Select the JOB_ID from the results + + SELECT alter_job(JOB_ID, scheduled => false); + ``` + See [alter_job][alter_job]. + +1. **Restart a columnstore policy** + + ``` sql + SELECT alter_job(JOB_ID, scheduled => true); + ``` + See [alter_job][alter_job]. + +1. **Remove a columnstore policy** + + ``` sql + CALL remove_columnstore_policy('crypto_ticks'); + ``` + See [remove_columnstore_policy][remove_columnstore_policy]. + +1. **Disable columnstore** + + If your table has chunks in the columnstore, you have to + [convert the chunks back to the rowstore][convert_to_rowstore] before you disable the columnstore. + ``` sql + ALTER TABLE crypto_ticks SET (timescaledb.enable_columnstore = false); + ``` + See [alter_table_hypercore][alter_table_hypercore]. + + + +[job]: /api/:currentVersion:/actions/add_job/ +[alter_table_hypercore]: /api/:currentVersion:/hypercore/alter_table/ +[compression_continuous-aggregate]: /api/:currentVersion:/hypercore/alter_materialized_view/ +[convert_to_rowstore]: /api/:currentVersion:/hypercore/convert_to_rowstore/ +[convert_to_columnstore]: /api/:currentVersion:/hypercore/convert_to_columnstore/ +[informational-views]: /api/:currentVersion:/informational-views/jobs/ +[add_columnstore_policy]: /api/:currentVersion:/hypercore/add_columnstore_policy/ +[hypercore_workflow]: /api/:currentVersion:/hypercore/#hypercore-workflow +[alter_job]: /api/:currentVersion:/actions/alter_job/ +[remove_columnstore_policy]: /api/:currentVersion:/hypercore/remove_columnstore_policy/ +[in-console-editors]: /getting-started/:currentVersion:/run-queries-from-console/ +[services-portal]: https://console.cloud.timescale.com/dashboard/services +[connect-using-psql]: /use-timescale/:currentVersion:/integrations/query-admin/psql#connect-to-your-service +[insert]: /use-timescale/:currentVersion:/write-data/insert/ diff --git a/_partials/_hypertables-intro.md b/_partials/_hypertables-intro.md index 5b3d450076..fb2d811fc3 100644 --- a/_partials/_hypertables-intro.md +++ b/_partials/_hypertables-intro.md @@ -1,15 +1,12 @@ -Hypertables are PostgreSQL tables that automatically partition your data by -time. You interact with hypertables in the same way as regular PostgreSQL -tables, but with extra features that makes managing your time-series data much -easier. - -In Timescale, hypertables exist alongside regular PostgreSQL tables. Use -hypertables to store time-series data. This gives you improved insert and query -performance, and access to useful time-series features. Use regular PostgreSQL -tables for other relational data. - -With hypertables, Timescale makes it easy to improve insert and query -performance by partitioning time-series data on its time parameter. Behind the -scenes, the database performs the work of setting up and maintaining the -hypertable's partitions. Meanwhile, you insert and query your data as if it all -lives in a single, regular PostgreSQL table. +$CLOUD_LONG supercharges your real-time analytics by letting you run complex queries continuously, with near-zero latency. Under the hood, this is achieved by using hypertables - PostgreSQL tables that automatically partition your time-series data by time and optionally by space. When you run a query, $CLOUD_LONG identifies the correct partition and runs the query on it, instead of going through the entire table. + +Hypertables offer a range of other features, such as [skipping partitions][chunk-skipping] or running [hyperfunctions][hyperfunctions], that boost the performance of your analytical queries even more. + +To top it all, there is no added complexity - you interact with hypertables in the same way as you would with regular PostgreSQL tables. All the optimization magic happens behind the scenes. + +[chunk-skipping]: /use-timescale/:currentVersion:/hypertables/improve-query-performance/ +[hyperfunctions]: /use-timescale/:currentVersion:/hyperfunctions/ + + + + diff --git a/_partials/_import-data-iot.md b/_partials/_import-data-iot.md new file mode 100644 index 0000000000..88f89c3da7 --- /dev/null +++ b/_partials/_import-data-iot.md @@ -0,0 +1,86 @@ +Time-series data represents how a system, process, or behavior changes over time. [Hypertables][hypertables-section] +are PostgreSQL tables that help you improve insert and query performance by automatically partitioning your data by +time, speeding up queries for real-time analytics and other challenging workloads. Each hypertable is made up of child +tables called chunks. Each chunk is assigned a range of time, and only contains data from that range. + + + +1. **Import time-series data into a hypertable** + + 1. Unzip [metrics.csv.gz](https://assets.timescale.com/docs/downloads/metrics.csv.gz) to a ``. + + This test dataset contains energy consumption data. + + To import up to 100GB of data directly from your current PostgreSQL based database, + [migrate with downtime][migrate-with-downtime] using native PostgreSQL tooling. To seamlessly import 100GB-10TB+ + of data, use the [live migration][migrate-live] tooling supplied by $COMPANY. To add data from non-PostgreSQL + data sources, see [Import and ingest data][data-ingest]. + + 1. In Terminal, navigate to `` and update the following string with [your connection details][connection-info] + to connect to your $SERVICE_SHORT. + + ```bash + psql -d "postgres://:@:/?sslmode=require" + ``` + + 1. Create tables to import time-series data: + + 1. In your sql client, create a normal PostgreSQL table: + + ```sql + CREATE TABLE "metrics"( + created timestamp with time zone default now() not null, + type_id integer not null, + value double precision not null + ); + ``` + + 1. Convert `metrics` to a hypertable and partitioned on time: + ```sql + SELECT create_hypertable('metrics', by_range('created')); + ``` + To more fully understand how hypertables work, and how to optimize them for performance by + tuning chunk intervals and enabling chunk skipping, see [the hypertables documentation][hypertables-section]. + + 1. Upload the dataset to your $SERVICE_SHORT + ```sql + \COPY metrics FROM metrics.csv CSV; + ``` + To more fully understand how hypertables work, and how to optimize them for performance by + tuning chunk intervals and enabling chunk skipping, see [the hypertables documentation][hypertables-section]. + +1. **Have a quick look at your data** + + You query hypertables in exactly the same way as you would a relational PostgreSQL table. + Use one of the following SQL editors to run a query and see the data you uploaded: + - **Data mode**: write queries, visualize data, and share your results in [$CONSOLE][portal-data-mode] for all your $SERVICE_LONGs. + - **SQL editor**: write, fix, and organize SQL faster and more accurately in [$CONSOLE][portal-ops-mode] for a $SERVICE_LONG. + - **psql**: easily run queries on your $SERVICE_LONGs or self-hosted TimescaleDB deployment from Terminal. + + ```sql + SELECT time_bucket('1 day', created, 'Europe/Berlin') AS "time", + round((last(value, created) - first(value, created)) * 100.) / 100. AS value + FROM metrics + WHERE type_id = 5 + GROUP BY 1; + ``` + + On this amount of data, this query on data in the rowstore takes about 3.6 seconds. You see something like: + + | Time | value | + |------------------------------|-------| + | 2023-05-29 22:00:00+00 | 23.1 | + | 2023-05-28 22:00:00+00 | 19.5 | + | 2023-05-30 22:00:00+00 | 25 | + | 2023-05-31 22:00:00+00 | 8.1 | + + + +[hypertables-section]: /use-timescale/:currentVersion:/hypertables/ +[portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services +[portal-data-mode]: https://console.cloud.timescale.com/dashboard/services?popsql +[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[migrate-with-downtime]: /migrate/:currentVersion:/pg-dump-and-restore/ +[migrate-live]: /migrate/:currentVersion:/live-migration/ +[data-ingest]: /use-timescale/:currentVersion:/ingest-data/ + diff --git a/_partials/_import-data-nyc-taxis.md b/_partials/_import-data-nyc-taxis.md new file mode 100644 index 0000000000..e3207b1dca --- /dev/null +++ b/_partials/_import-data-nyc-taxis.md @@ -0,0 +1,168 @@ +Time-series data represents how a system, process, or behavior changes over time. [Hypertables][hypertables-section] +are PostgreSQL tables that help you improve insert and query performance by automatically partitioning your data by +time, speeding up queries for real-time analytics and other challenging workloads. Each hypertable is made up of child +tables called chunks. Each chunk is assigned a range of time, and only contains data from that range. + + + +1. **Import time-series data into a hypertable** + + 1. Unzip [nyc_data.tar.gz](https://assets.timescale.com/docs/downloads/nyc_data.tar.gz) to a ``. + + This test dataset contains historical data from New York's yellow taxi network. + + To import up to 100GB of data directly from your current PostgreSQL based database, + [migrate with downtime][migrate-with-downtime] using native PostgreSQL tooling. To seamlessly import 100GB-10TB+ + of data, use the [live migration][migrate-live] tooling supplied by $COMPANY. To add data from non-PostgreSQL + data sources, see [Import and ingest data][data-ingest]. + + 1. In Terminal, navigate to `` and update the following string with [your connection details][connection-info] + to connect to your $SERVICE_SHORT. + + ```bash + psql -d "postgres://:@:/?sslmode=require" + ``` + + 1. Create tables to import time-series data: + + 1. In your sql client, create a normal PostgreSQL table: + + ```sql + CREATE TABLE "rides"( + vendor_id TEXT, + pickup_datetime TIMESTAMP WITHOUT TIME ZONE NOT NULL, + dropoff_datetime TIMESTAMP WITHOUT TIME ZONE NOT NULL, + passenger_count NUMERIC, + trip_distance NUMERIC, + pickup_longitude NUMERIC, + pickup_latitude NUMERIC, + rate_code INTEGER, + dropoff_longitude NUMERIC, + dropoff_latitude NUMERIC, + payment_type INTEGER, + fare_amount NUMERIC, + extra NUMERIC, + mta_tax NUMERIC, + tip_amount NUMERIC, + tolls_amount NUMERIC, + improvement_surcharge NUMERIC, + total_amount NUMERIC + ); + ``` + + 1. Convert `rides` to a hypertable and partitioned on time: + ```sql + SELECT create_hypertable('rides', by_range('pickup_datetime'), create_default_indexes=>FALSE); + SELECT add_dimension('rides', by_hash('payment_type', 2)); + ``` + To more fully understand how hypertables work, and how to optimize them for performance by + tuning chunk intervals and enabling chunk skipping, see [the hypertables documentation][hypertables-section]. + + 1. Create an index to support efficient queries by vendor, rate code, and passenger count: + ```sql + CREATE INDEX ON rides (vendor_id, pickup_datetime DESC); + CREATE INDEX ON rides (rate_code, pickup_datetime DESC); + CREATE INDEX ON rides (passenger_count, pickup_datetime DESC); + ``` + + 1. Create tables for relational data: + + 1. Add a table to store the payment types data: + + ```sql + CREATE TABLE IF NOT EXISTS "payment_types"( + payment_type INTEGER, + description TEXT + ); + INSERT INTO payment_types(payment_type, description) VALUES + (1, 'credit card'), + (2, 'cash'), + (3, 'no charge'), + (4, 'dispute'), + (5, 'unknown'), + (6, 'voided trip'); + ``` + + 1. Add a table to store the rates data: + + ```sql + CREATE TABLE IF NOT EXISTS "rates"( + rate_code INTEGER, + description TEXT + ); + INSERT INTO rates(rate_code, description) VALUES + (1, 'standard rate'), + (2, 'JFK'), + (3, 'Newark'), + (4, 'Nassau or Westchester'), + (5, 'negotiated fare'), + (6, 'group ride'); + ``` + + 1. Upload the dataset to your $SERVICE_SHORT + ```sql + \COPY rides FROM nyc_data_rides.csv CSV; + ``` + To more fully understand how hypertables work, and how to optimize them for performance by + tuning chunk intervals and enabling chunk skipping, see [the hypertables documentation][hypertables-section]. + +1. **Have a quick look at your data** + + You query hypertables in exactly the same way as you would a relational PostgreSQL table. + Use one of the following SQL editors to run a query and see the data you uploaded: + - **Data mode**: write queries, visualize data, and share your results in [$CONSOLE][portal-data-mode] for all your $SERVICE_LONGs. + - **SQL editor**: write, fix, and organize SQL faster and more accurately in [$CONSOLE][portal-ops-mode] for a $SERVICE_LONG. + - **psql**: easily run queries on your $SERVICE_LONGs or self-hosted TimescaleDB deployment from Terminal. + + For example: + - Display the number of rides for each fare type: + ```sql + SELECT rate_code, COUNT(vendor_id) AS num_trips + FROM rides + WHERE pickup_datetime < '2016-01-08' + GROUP BY rate_code + ORDER BY rate_code; + ``` + This simple query runs in 3 seconds. You see something like: + + | rate_code | num_trips | + |-----------------|-----------| + |1 | 2266401| + |2 | 54832| + |3 | 4126| + |4 | 967| + |5 | 7193| + |6 | 17| + |99 | 42| + + - To select all rides taken in the first week of January 2016, and return the total number of trips taken for each rate code: + ```sql + SELECT rates.description, COUNT(vendor_id) AS num_trips + FROM rides + JOIN rates ON rides.rate_code = rates.rate_code + WHERE pickup_datetime < '2016-01-08' + GROUP BY rates.description + ORDER BY LOWER(rates.description); + ``` + On this large amount of data, this analytical query on data in the rowstore takes about 59 seconds. You see something like: + + | description | num_trips | + |-----------------|-----------| + | group ride | 17 | + | JFK | 54832 | + | Nassau or Westchester | 967 | + | negotiated fare | 7193 | + | Newark | 4126 | + | standard rate | 2266401 | + + + + +[hypertables-section]: /use-timescale/:currentVersion:/hypertables/ +[portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services +[portal-data-mode]: https://console.cloud.timescale.com/dashboard/services?popsql +[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[migrate-with-downtime]: /migrate/:currentVersion:/pg-dump-and-restore/ +[migrate-live]: /migrate/:currentVersion:/live-migration/ +[data-ingest]: /use-timescale/:currentVersion:/ingest-data/ + diff --git a/_partials/_integration-prereqs-cloud-only.md b/_partials/_integration-prereqs-cloud-only.md index b46e842697..0b21e814f9 100644 --- a/_partials/_integration-prereqs-cloud-only.md +++ b/_partials/_integration-prereqs-cloud-only.md @@ -1,5 +1,10 @@ -To follow the procedure on this page, you need to: -* Create a target [$SERVICE_LONG][create-service]. +To follow the steps on this page: + +* Create a target [$SERVICE_LONG][create-service] with time-series and analytics enabled. + + You need your [connection details][connection-info]. + [create-service]: /getting-started/:currentVersion:/services/ +[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ diff --git a/_partials/_integration-prereqs.md b/_partials/_integration-prereqs.md index 26a70caf25..26d94388b6 100644 --- a/_partials/_integration-prereqs.md +++ b/_partials/_integration-prereqs.md @@ -1,10 +1,11 @@ -To follow the procedure on this page, you need to: +To follow the steps on this page: -* Create a target [$SERVICE_LONG][create-service] +* Create a target [$SERVICE_LONG][create-service] with time-series and analytics enabled. - You need your [connection details][connection-info] to follow the steps in this page. This procedure also + You need [your connection details][connection-info]. This procedure also works for [self-hosted $TIMESCALE_DB][enable-timescaledb]. + [create-service]: /getting-started/:currentVersion:/services/ [enable-timescaledb]: /self-hosted/:currentVersion:/install/ [connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ diff --git a/_partials/_migrate_awsrds_connect_intermediary.md b/_partials/_migrate_awsrds_connect_intermediary.md index dd056c0c14..86c1abaa2a 100644 --- a/_partials/_migrate_awsrds_connect_intermediary.md +++ b/_partials/_migrate_awsrds_connect_intermediary.md @@ -10,7 +10,7 @@ - **Key pair**: use an existing pair or create a new one that you will use to access the intermediary machine. - **VPC**: by default, this is the same as the database instance. - **Configure Storage**: adjust the volume to at least the size of RDS/Aurora PostgreSQL instance you are migrating from. - You can reduce the space used by your data on Timescale Cloud using [data compression][data-compression]. + You can reduce the space used by your data on Timescale Cloud using [Hypercore][hypercore]. 1. Click `Lauch instance`. AWS creates your EC2 instance, then click `Connect to instance` > `SSH client`. Follow the instructions to create the connection to your intermediary EC2 instance. @@ -89,5 +89,5 @@ [about-hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/ -[data-compression]: /use-timescale/:currentVersion:/compression/about-compression/ +[hypercore]: /use-timescale/:currentVersion:/hypercore/ [databases]: https://console.aws.amazon.com/rds/home#databases: diff --git a/_partials/_migrate_install_psql_ec2_instance.md b/_partials/_migrate_install_psql_ec2_instance.md index 00f0d5326c..a90af1989e 100644 --- a/_partials/_migrate_install_psql_ec2_instance.md +++ b/_partials/_migrate_install_psql_ec2_instance.md @@ -71,4 +71,3 @@ [about-hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/ -[data-compression]: /use-timescale/:currentVersion:/compression/about-compression/ diff --git a/_partials/_migrate_post_schema_caggs_etal.md b/_partials/_migrate_post_schema_caggs_etal.md index b03f4566dd..db130636a7 100644 --- a/_partials/_migrate_post_schema_caggs_etal.md +++ b/_partials/_migrate_post_schema_caggs_etal.md @@ -131,8 +131,7 @@ separately. Recreate them on your Timescale database. 1. Recreate each policy. For more information about recreating policies, see the sections on [continuous-aggregate refresh policies][cagg-policy], - [retention policies][retention-policy], [compression - policies][compression-policy], and [reorder policies][reorder-policy]. + [retention policies][retention-policy], [Hypercore policies][setup-hypercore], and [reorder policies][reorder-policy]. @@ -160,7 +159,7 @@ accessed. Skipping them does not affect statistics on your data. [analyze]: https://www.postgresql.org/docs/10/sql-analyze.html [cagg-policy]: /use-timescale/:currentVersion:/continuous-aggregates/refresh-policies/ -[compression-policy]: /use-timescale/:currentVersion:/compression/ +[setup-hypercore]: /use-timescale/:currentVersion:/hypercore/real-time-analytics-in-hypercore/ [retention-policy]: /use-timescale/:currentVersion:/data-retention/create-a-retention-policy/ [reorder-policy]: /api/:currentVersion:/hypertable/add_reorder_policy/ [timescaledb-parallel-copy]: https://github.com/timescale/timescaledb-parallel-copy diff --git a/_partials/_multi-node-deprecation.md b/_partials/_multi-node-deprecation.md index a5e1eda837..31f68e1bd0 100644 --- a/_partials/_multi-node-deprecation.md +++ b/_partials/_multi-node-deprecation.md @@ -1,9 +1,10 @@ -[Multi-node support is deprecated][multi-node-deprecation]. +[Multi-node support is sunsetted][multi-node-deprecation]. TimescaleDB v2.13 is the last release that includes multi-node support for PostgreSQL versions 13, 14, and 15. + [multi-node-deprecation]: https://github.com/timescale/timescaledb/blob/main/docs/MultiNodeDeprecation.md diff --git a/_partials/_prereqs-cloud-and-self.md b/_partials/_prereqs-cloud-and-self.md new file mode 100644 index 0000000000..1a6d3a6619 --- /dev/null +++ b/_partials/_prereqs-cloud-and-self.md @@ -0,0 +1,8 @@ +To follow the procedure on this page you need to: + +* Create a [target $SERVICE_LONG][create-service] + + This procedure also works for [self-hosted $TIMESCALE_DB][enable-timescaledb]. + +[create-service]: /getting-started/:currentVersion:/services/ +[enable-timescaledb]: /self-hosted/:currentVersion:/install/ diff --git a/_partials/_prereqs-cloud-only.md b/_partials/_prereqs-cloud-only.md new file mode 100644 index 0000000000..3c9a7d93bb --- /dev/null +++ b/_partials/_prereqs-cloud-only.md @@ -0,0 +1,5 @@ +To follow the procedure on this page you need to: + +* Create a [target $SERVICE_LONG][create-service] + +[create-service]: /getting-started/:currentVersion:/services/ diff --git a/_partials/_since_2_18_0.md b/_partials/_since_2_18_0.md new file mode 100644 index 0000000000..19a03870be --- /dev/null +++ b/_partials/_since_2_18_0.md @@ -0,0 +1 @@ +Since [TimescaleDB v2.18.0](https://github.com/timescale/timescaledb/releases/tag/2.18.0) diff --git a/_partials/_usage-based-storage-intro.md b/_partials/_usage-based-storage-intro.md index 6823fdbb60..5bee4f7a55 100644 --- a/_partials/_usage-based-storage-intro.md +++ b/_partials/_usage-based-storage-intro.md @@ -1,9 +1,9 @@ $CLOUD_LONG charges are based on the amount of storage you use. You don't pay for fixed storage size, and you don't need to worry about scaling disk size as your -data grows; We handle it all for you. To reduce your data costs further, -use [compression][compression], a [data retention policy][data-retention], and +data grows - we handle it all for you. To reduce your data costs further, +combine [Hypercore][hypercore], a [data retention policy][data-retention], and [tiered storage][data-tiering]. -[compression]: /use-timescale/:currentVersion:/compression/about-compression +[hypercore]: /api/:currentVersion:/hypercore/ [data-retention]: /use-timescale/:currentVersion:/data-retention/ [data-tiering]: /use-timescale/:currentVersion:/data-tiering/ diff --git a/_partials/_use-case-iot-create-cagg.md b/_partials/_use-case-iot-create-cagg.md new file mode 100644 index 0000000000..7a8f13940f --- /dev/null +++ b/_partials/_use-case-iot-create-cagg.md @@ -0,0 +1,87 @@ +1. **Monitor energy consumption on a day-to-day basis** + + 1. Create a continuous aggregate `kwh_day_by_day` for energy consumption: + + ```sql + CREATE MATERIALIZED VIEW kwh_day_by_day(time, value) + with (timescaledb.continuous) as + SELECT time_bucket('1 day', created, 'Europe/Berlin') AS "time", + round((last(value, created) - first(value, created)) * 100.) / 100. AS value + FROM metrics + WHERE type_id = 5 + GROUP BY 1; + ``` + + 1. Add a refresh policy to keep `kwh_day_by_day` up-to-date: + + ```sql + SELECT add_continuous_aggregate_policy('kwh_day_by_day', + start_offset => NULL, + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); + ``` + +1. **Monitor energy consumption on an hourly basis** + + 1. Create a continuous aggregate `kwh_hour_by_hour` for energy consumption: + + ```sql + CREATE MATERIALIZED VIEW kwh_hour_by_hour(time, value) + with (timescaledb.continuous) as + SELECT time_bucket('01:00:00', metrics.created, 'Europe/Berlin') AS "time", + round((last(value, created) - first(value, created)) * 100.) / 100. AS value + FROM metrics + WHERE type_id = 5 + GROUP BY 1; + ``` + + 1. Add a refresh policy to keep the continuous aggregate up-to-date: + + ```sql + SELECT add_continuous_aggregate_policy('kwh_hour_by_hour', + start_offset => NULL, + end_offset => INTERVAL '1 hour', + schedule_interval => INTERVAL '1 hour'); + ``` + +1. **Analyze your data** + + Now you have made continuous aggregates, it could be a good idea to use them to perform analytics on your data. + For example, to see how average energy consumption changes during weekdays over the last year, run the following query: + ```sql + WITH per_day AS ( + SELECT + time, + value + FROM kwh_day_by_day + WHERE "time" at time zone 'Europe/Berlin' > date_trunc('month', time) - interval '1 year' + ORDER BY 1 + ), daily AS ( + SELECT + to_char(time, 'Dy') as day, + value + FROM per_day + ), percentile AS ( + SELECT + day, + approx_percentile(0.50, percentile_agg(value)) as value + FROM daily + GROUP BY 1 + ORDER BY 1 + ) + SELECT + d.day, + d.ordinal, + pd.value + FROM unnest(array['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat']) WITH ORDINALITY AS d(day, ordinal) + LEFT JOIN percentile pd ON lower(pd.day) = lower(d.day); + ``` + + You see something like: + + | day | ordinal | value | + | --- | ------- | ----- | + | Mon | 2 | 23.08078714975423 | + | Sun | 1 | 19.511430831944395 | + | Tue | 3 | 25.003118897837307 | + | Wed | 4 | 8.09300571759772 | diff --git a/_partials/_use-case-setup-blockchain-dataset.md b/_partials/_use-case-setup-blockchain-dataset.md new file mode 100644 index 0000000000..c1ddd2669f --- /dev/null +++ b/_partials/_use-case-setup-blockchain-dataset.md @@ -0,0 +1,18 @@ + + +import CreateHypertableBlockchain from "versionContent/_partials/_create-hypertable-blockchain.mdx"; +import AddDataBlockchain from "versionContent/_partials/_add-data-blockchain.mdx"; +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; + +# Ingest data into a $SERVICE_LONG + +This tutorial uses a dataset that contains Bitcoin blockchain data for +the past five days, in a hypertable named `transactions`. + +## Prerequisites + + + + + + diff --git a/_partials/_use-case-transport-geolocation.md b/_partials/_use-case-transport-geolocation.md new file mode 100644 index 0000000000..fbdefd6cad --- /dev/null +++ b/_partials/_use-case-transport-geolocation.md @@ -0,0 +1,88 @@ + +### Set up your data for geospatial queries + +To add geospatial analysis to your ride count visualization, you need geospatial data to work out which trips +originated where. As $TIMESCALE_DB is compatible with all PostgreSQL extensions, use [PostGIS][postgis] to slice +data by time and location. + + + +1. Connect to your [$SERVICE_LONG][in-console-editors] and add the PostGIS extension: + + ```sql + CREATE EXTENSION postgis; + ``` + +1. Add geometry columns for pick up and drop off locations: + + ```sql + ALTER TABLE rides ADD COLUMN pickup_geom geometry(POINT,2163); + ALTER TABLE rides ADD COLUMN dropoff_geom geometry(POINT,2163); + ``` + +1. Convert the latitude and longitude points into geometry coordinates that work with PostGIS: + + ```sql + UPDATE rides SET pickup_geom = ST_Transform(ST_SetSRID(ST_MakePoint(pickup_longitude,pickup_latitude),4326),2163), + dropoff_geom = ST_Transform(ST_SetSRID(ST_MakePoint(dropoff_longitude,dropoff_latitude),4326),2163); + ``` + This updates 10,906,860 rows of data on both columns, it takes a while. Coffee is your friend. + + + +### Visualize the area where you can make the most money + +In this section you visualize a query that returns rides longer than 5 miles for +trips taken within 2 km of Times Square. The data includes the distance travelled and +is `GROUP BY` `trip_distance` and location so that Grafana can plot the data properly. + +This enables you to see where a taxi driver is most likely to pick up a passenger who wants a longer ride, +and make more money. + + + +1. **Create a geolocalization dashboard** + + 1. In Grafana, create a new dashboard that is connected to your $SERVICE_LONG data source with a Geomap + visualization. + + 1. In the `Queries` section, select `Code`, then select the Time series `Format`. + + ![Real-time analytics geolocation](https://assets.timescale.com/docs/images/use-case-rta-grafana-timescale-configure-dashboard.png) + + 1. To find rides longer than 5 miles in Manhattan, paste the following query: + + ```sql + SELECT time_bucket('5m', rides.pickup_datetime) AS time, + rides.trip_distance AS value, + rides.pickup_latitude AS latitude, + rides.pickup_longitude AS longitude + FROM rides + WHERE rides.pickup_datetime BETWEEN '2016-01-01T01:41:55.986Z' AND '2016-01-01T07:41:55.986Z' AND + ST_Distance(pickup_geom, + ST_Transform(ST_SetSRID(ST_MakePoint(-73.9851,40.7589),4326),2163) + ) < 2000 + GROUP BY time, + rides.trip_distance, + rides.pickup_latitude, + rides.pickup_longitude + ORDER BY time + LIMIT 500; + ``` + You see a world map with a dot on New York. + 1. Zoom into your map to see the visualization clearly. + +1. **Customize the visualization** + + 1. In the Geomap options, under `Map Layers`, click `+ Add layer` and select `Heatmap`. + You now see the areas where a taxi driver is most likely to pick up a passenger who wants a + longer ride, and make more money. + + ![Real-time analytics geolocation](https://assets.timescale.com/docs/images/use-case-rta-grafana-heatmap.png) + + + + + +[in-console-editors]: /getting-started/:currentVersion:/run-queries-from-console/ +[postgis]: http://postgis.net/ diff --git a/_partials/_where-to-next.md b/_partials/_where-to-next.md index 5d1124a297..62a9c7ff4c 100644 --- a/_partials/_where-to-next.md +++ b/_partials/_where-to-next.md @@ -1,7 +1,11 @@ -Now you have TimescaleDB running, have a look at the: +What next? [Try the main features offered by Timescale][try-timescale-features], see the [use case tutorials][tutorials], +interact with the data in your $SERVICE_LONG using [your favorite programming language][connect-with-code], integrate +your $SERVICE_LONG with a range of [third-party tools][integrations], plain old [Use Timescale][use-timescale], or dive +into [the API][use-the-api]. -* [Tutorials][tutorials]: walk through a variety of business scenarios using example datasets. -* [Use Timescale][tsdb-docs]: browse the features available with TimescaleDB. - -[tsdb-docs]: /use-timescale/:currentVersion:/ [tutorials]: /tutorials/:currentVersion:/ +[connect-with-code]: /quick-start/:currentVersion:/ +[integrations]: /use-timescale/:currentVersion:/integrations/ +[use-the-api]: /api/:currentVersion:/ +[use-timescale]: /use-timescale/:currentVersion:/ +[try-timescale-features]: /getting-started/:currentVersion:/try-key-features-timescale-products/ diff --git a/_procedure-block.md b/_procedure-block.md index 38921bc352..ef01fc2e8d 100644 --- a/_procedure-block.md +++ b/_procedure-block.md @@ -16,13 +16,13 @@ ``` 3. TimescaleDB is a time-series database, built on top of PostgreSQL. More than that, -however, it's a relational database for time-series. Developers who use TimescaleDB -get the benefit of a purpose-built time-series database, plus a classic relational -database (PostgreSQL), all in one, with full SQL support. + however, it's a relational database for time-series. Developers who use TimescaleDB + get the benefit of a purpose-built time-series database, plus a classic relational + database (PostgreSQL), all in one, with full SQL support. ```python def start: print('start') ``` - + \ No newline at end of file diff --git a/_queries/getting-started-crypto-cagg.md b/_queries/getting-started-crypto-cagg.md new file mode 100644 index 0000000000..2834d8ea02 --- /dev/null +++ b/_queries/getting-started-crypto-cagg.md @@ -0,0 +1,19 @@ +SELECT * FROM assets_candlestick_daily +ORDER BY day DESC, symbol +LIMIT 10; + +-- Output + +day | symbol | high | open | close | low +-----------------------+--------+----------+--------+----------+---------- +2025-01-30 00:00:00+00 | ADA/USD | 0.9708 | 0.9396 | 0.9607 | 0.9365 +2025-01-30 00:00:00+00 | ATOM/USD | 6.114 | 5.825 | 6.063 | 5.776 +2025-01-30 00:00:00+00 | AVAX/USD | 34.1 | 32.8 | 33.95 | 32.44 +2025-01-30 00:00:00+00 | BNB/USD | 679.3 | 668.12 | 677.81 | 666.08 +2025-01-30 00:00:00+00 | BTC/USD | 105595.65 | 103735.84 | 105157.21 | 103298.84 +2025-01-30 00:00:00+00 | CRO/USD | 0.13233 | 0.12869 | 0.13138 | 0.12805 +2025-01-30 00:00:00+00 | DAI/USD | 1 | 1 | 0.9999 | 0.99989998 +2025-01-30 00:00:00+00 | DOGE/USD | 0.33359 | 0.32392 | 0.33172 | 0.32231 +2025-01-30 00:00:00+00 | DOT/USD | 6.01 | 5.779 | 6.004 | 5.732 +2025-01-30 00:00:00+00 | ETH/USD | 3228.9 | 3113.36 | 3219.25 | 3092.92 +(10 rows) diff --git a/_queries/getting-started-crypto-srt-orderby.md b/_queries/getting-started-crypto-srt-orderby.md new file mode 100644 index 0000000000..4aaad26f1c --- /dev/null +++ b/_queries/getting-started-crypto-srt-orderby.md @@ -0,0 +1,20 @@ +SELECT * FROM crypto_ticks srt +WHERE symbol='ETH/USD' +ORDER BY time DESC +LIMIT 10; + +-- Output + +time | symbol | price | day_volume +-----------------------+--------+----------+------------ +2025-01-30 12:05:09+00 | ETH/USD | 3219.25 | 39425 +2025-01-30 12:05:00+00 | ETH/USD | 3219.26 | 39425 +2025-01-30 12:04:42+00 | ETH/USD | 3219.26 | 39459 +2025-01-30 12:04:33+00 | ETH/USD | 3219.91 | 39458 +2025-01-30 12:04:15+00 | ETH/USD | 3219.6 | 39458 +2025-01-30 12:04:06+00 | ETH/USD | 3220.68 | 39458 +2025-01-30 12:03:57+00 | ETH/USD | 3220.68 | 39483 +2025-01-30 12:03:48+00 | ETH/USD | 3220.12 | 39483 +2025-01-30 12:03:20+00 | ETH/USD | 3219.79 | 39482 +2025-01-30 12:03:11+00 | ETH/USD | 3220.06 | 39472 +(10 rows) diff --git a/_queries/getting-started-srt-orderby.md b/_queries/getting-started-srt-orderby.md index a0afde5a41..a1831ca158 100644 --- a/_queries/getting-started-srt-orderby.md +++ b/_queries/getting-started-srt-orderby.md @@ -1,20 +1,20 @@ SELECT * FROM stocks_real_time srt -WHERE symbol='TSLA' and day_volume is not null -ORDER BY time DESC, day_volume desc +WHERE symbol='TSLA' +ORDER BY time DESC LIMIT 10; -- Output -time | symbol | price | day_volume +time | symbol | price | day_volume -----------------------+--------+----------+------------ -2023-07-31 16:52:54+00 | TSLA | 267.88 | 55220135 -2023-07-31 16:52:53+00 | TSLA | 267.9141 | 55218795 -2023-07-31 16:52:52+00 | TSLA | 267.8987 | 55218669 -2023-07-31 16:52:51+00 | TSLA | 267.89 | 55218368 -2023-07-31 16:52:49+00 | TSLA | 267.87 | 55217109 -2023-07-31 16:52:49+00 | TSLA | 267.862 | 55216861 -2023-07-31 16:52:48+00 | TSLA | 267.8649 | 55216275 -2023-07-31 16:52:46+00 | TSLA | 267.88 | 55215737 -2023-07-31 16:52:44+00 | TSLA | 267.88 | 55215060 -2023-07-31 16:52:44+00 | TSLA | 267.88 | 55214966 +2025-01-30 00:51:00+00 | TSLA | 405.32 | NULL +2025-01-30 00:41:00+00 | TSLA | 406.05 | NULL +2025-01-30 00:39:00+00 | TSLA | 406.25 | NULL +2025-01-30 00:32:00+00 | TSLA | 406.02 | NULL +2025-01-30 00:32:00+00 | TSLA | 406.10 | NULL +2025-01-30 00:25:00+00 | TSLA | 405.95 | NULL +2025-01-30 00:24:00+00 | TSLA | 406.04 | NULL +2025-01-30 00:24:00+00 | TSLA | 406.04 | NULL +2025-01-30 00:22:00+00 | TSLA | 406.38 | NULL +2025-01-30 00:21:00+00 | TSLA | 405.77 | NULL (10 rows) diff --git a/_troubleshooting/mst/low-disk-memory-cpu.md b/_troubleshooting/mst/low-disk-memory-cpu.md index 39f03b0eb0..d752c9f8ef 100644 --- a/_troubleshooting/mst/low-disk-memory-cpu.md +++ b/_troubleshooting/mst/low-disk-memory-cpu.md @@ -34,11 +34,11 @@ for. In the `Overview` tab, locate the `Service Plan` section, and click `Upgrade` to enable the additional resources. If you run out of resources regularly, you might need to consider using your -resources more efficiently. Consider enabling [compression][howto-compression], +resources more efficiently. Consider enabling [Hypercore][setup-hypercore], using [continuous aggregates][howto-caggs], or [configuring data retention][howto-dataretention] to reduce the amount of resources your database uses. -[howto-compression]: /use-timescale/:currentVersion:/compression +[setup-hypercore]: /use-timescale/:currentVersion:/hypercore/real-time-analytics-in-hypercore/ [howto-caggs]: /use-timescale/:currentVersion:/continuous-aggregates [howto-dataretention]: /use-timescale/:currentVersion:/data-retention diff --git a/about/changelog.md b/about/changelog.md index 06df29e191..7e29f6a69a 100644 --- a/about/changelog.md +++ b/about/changelog.md @@ -8,6 +8,7 @@ keywords: [changelog, upgrades, updates, releases] All the latest features and updates to Timescale products. + ## Agent Mode for PopSQL and more diff --git a/about/contribute-to-docs.md b/about/contribute-to-docs.md deleted file mode 100644 index d8cd1cff08..0000000000 --- a/about/contribute-to-docs.md +++ /dev/null @@ -1,524 +0,0 @@ ---- -title: Contribute to Timescale documentation -excerpt: Contribute to Timescale documentation following our key guidelines -products: [cloud, mst, self_hosted] -keywords: [contribute] -tags: [docs, style guide] ---- - -# Contributing to Timescale documentation - -Timescale documentation is hosted in a [GitHub repository][github-docs] and is -open for contribution from all community members. If you find errors or would -like to add content to our docs, you can create a pull request using GitHub for -review by our documentation team. This document contains everything you need to -know about our writing style and standards, but don't worry too much if you -aren't sure what to write. Our documentation team helps you craft the -perfect words when you have a PR ready. We also have some automation on our -repository to help you. - -If you want to make minor changes to docs, such as fixing a typo, you can make -corrections and submit pull requests on the GitHub website. Go to the file you -want to correct and click the 'pencil' icon to edit. Make the corrections, and -use the options at the bottom of the page to submit a pull request. - -To make larger changes to the documentation, follow the instructions in -our [Contributors' Guide][contributors]. - -For technical details about the repository, including understanding how the -repository is organized, and the various markup and formatting conventions, see -the [README][readme]. - -Before we accept any contributions, Timescale contributors need to sign the -Contributor License Agreement (CLA). By signing a CLA, we can ensure that the -community is free and confident in its ability to use your contributions. You -are prompted to sign the CLA during the pull request process. - -## Resources - -When making style decisions, consult resources in this order: - -1. This guide: always check this guide first, it contains project-specific - guidance, and in some cases differs from the other resources listed here. -1. The [Google Developer Documentation Style Guide][google-style]: for most - general style guidance, we rely on the style defined here. -1. The Chicago Manual of Style: we use this guide for some formatting decisions - that are not covered in other resources -1. Merriam-Webster: Timescale documentation is written in US English, for - spelling and definitions, consult the dictionary. - -## Language - -We use standard US English, with an emphasis on plain (or classical) language, -in simple present tense, using the second person singular ("you"). We prefer the -active voice, but do not be afraid to use the passive voice if it serves a -purpose. Always choose the simplest and clearest language, regardless of whether -it's passive or active voice. - -For example, here are three ways of writing one sentence: - -* Natural English: In order to perform X installation process, please ensure - that all of the following steps are done ... -* Tech writer's English: To perform the X installation process, verify you - have done the subsequent steps ... -* Plain English: To install X, do these steps ... - -Remember that the order of words is important in English. Put the most important -part of a sentence first, this is usually the actor or the action. Use the -second part of the sentence to give it a focus: what else should the reader -notice? - -Readers are often in an agitated state by the time they get to our -documentation. Stressed readers jump around in the text, skip words, steps, or -paragraphs, and can quickly give up if things seem too complex. To mitigate -this, use short sentences, plain language, and a minimum number of eye-catching -details such as admonitions. - -Never assume that because you've explained something earlier in a document, -readers know it later in the document. You can use cross-references to help -guide readers to further information if they need it. - -## Grammar - -Grammar rules are tacit evolving conventions. They have no implicit value by -themselves, they only gain value because everyone is doing it. - -There are no hard and fast rules about dangling participles, split infinitives, -or ending sentences with prepositions. Obeying these rules can often make -language clearer but, in some cases, they make language more complicated. In -that case, feel free to ignore them. - -## Headings - -All headings should be written in sentence case: capitalize only the first word -in the heading, and proper nouns. - -For top-level page headings, and most section headings, use the simplest noun -phrase possible. For example, instead of "Using hypertables", call the page -"Hypertables". - -For level two sections that contain one or more procedures, use a simple verb -phrase. For example, "Install Timescale". For the associated procedures, use a -level three heading, with the gerund form of the same heading. For example, -"Installing Timescale". - -## Lists - -You can use bulleted or numbered lists to present information. If you are unsure -whether to use bullets or numbers, apply the test, "is the order important?" If -the order is important, use numbers. If the list items can be read in any order, -use bullets. - -Always start each list item with a capital letter. If the list items are short, -incomplete sentences, do not end each item with punctuation. However, if any of -the list items are a longer phrase with more than one full sentence, finish all -list items with a period. - - - -```txt -1. Use numbers -1. If the order of the steps -1. Is important - -* Otherwise -* Use -* Bullets -``` - - - -## Processes and procedures - -We use processes and procedures to provide readers with numbered steps to -achieve a specific goal. - -Processes contain procedures. If the task you are describing is very lengthy, or -has a series of distinct components, break it up into a series of procedures -within a process. - -Procedures contain these elements: - - - -1. Level two section title in simple verb form -1. Short introduction -1. Open `` tag -1. Level three procedure title in gerund verb form -1. Numbered steps -1. Screenshot -1. Close `` tag - - - -For example: - -```txt -## Install Timescale - -This section describes how to install Timescale on premise. - - - -### Installing Timescale - -1. Start each step with a verb, or a location. -1. For example, "Run the `example` command", or "At the command prompt, open the - `example` file." - - - - - -``` - -In general, do not use result statements. However, if you feel it is absolutely -necessary, include it immediately before the closing procedure tag, and do not -put it as a step. - -## Code blocks - -Offset code blocks from the surrounding text by using three backticks and -declaring the language in use in the block. Usually, this is either `sql`, for -queries, or `bash`, for commands given at the command prompt. For a full list of -available languages see [the prismjs site][prismjs]. - -Make sure that the code block is indented at the same level as the surrounding text. - -When writing commands in a code block, do not use `#`, `$`, or any other prompt. -Additionally, for Linux commands, do not use `sudo`, instead, use the stem -sentence to indicate that the command must be run as the root user. - -For example, a bash command: - -```txt - - At the command prompt, as root, use this command: - - ```bash - apt install postgresql - ``` - -``` - -For example, an SQL query: - -```txt - - At the `psql` prompt, use this query: - - ```sql - SELECT * FROM table LIMIT 5; - ``` - -``` - -## Verbs - -* `Click` a button in a graphical user interface using a mouse. Do not `Click on`. -* `Press` a key or key combination on a keyboard. -* `Type` words or numbers using a keyboard. -* `Check` or `uncheck` a checkbox. -* `Select` or `deselect` an item in a menu. -* `Navigate` to a page or location in a graphical user interface. - -## Word usage - -### A - -*Above* -: Avoid all directional words. You cannot guarantee that things will stay in the -same position, or be in the position you expect on an individual reader's -device. - -*Adverbs* -: Do not use. - -❌ Simply install Timescale. - -👍 Install Timescale. - -*AWS* -: Amazon Web Services. No need to expand this acronym. - -*And/Or* -: Do not use. You can usually pick one. If you're not sure, pick "and." - -❌ I like apples and/or oranges. - -👍 I like apples and oranges. - -*Appears* -: Do not use. - -### B - -*Bare metal* -: Noun. - -❌ "Perform a bare metal installation." - -👍 "Install Timescale on bare metal." - -*Bare-metal* -: Adjective. - -❌ "Install Timescale on bare-metal." - -👍 "Perform a bare-metal installation." - -*Backend* -: One word. - -*Below* -: Avoid all directional words. You cannot guarantee that things will stay in the -same position, or be in the position you expect on an individual reader's -device. - -### C - -*Contractions* -: Absolutely fine to use, but try not to overdo it. - -*Cybersecurity* -: One word. - -### D - -*Dataset* -: One word. - -❌ "Ingest the example data set." - -👍 "Ingest the example dataset." - -### E - -### F - -*Fail over* -: Use as a verb - -*Failover* -: Use as a noun or adjective. - -*File system* -: Two words. - -*Frontend* -: One word. - -### G - -*GCP* -: Google Cloud Platform. No need to expand this acronym. - -### H - -*Hypershift* -: One word, always capitalized. Check the spelling of this word -carefully before publishing. - -### I - -### J - -### K - -### L - -*Latin abbreviations* -: Do not use. - -❌ eg - -👍 For example - -❌ I.E: - -👍 That is, - -*Log in* -: Verb. - -❌ "Enter your log in information." - -👍 "Log in to Timescale." - -*Login* -: Adjective or noun. - -❌ "Login to Timescale." - -👍 "Enter your login information." - -### M - -*Managed Service for TimescaleDB* -: The name of the product. Always capitalize the initial letters. Do not -abbreviate. - -❌ "I am using Managed Service for Timescale to manage my time-series data." - -❌ "I am using Managed Service to manage my time-series data." - -👍 "I am using Managed Service for TimescaleDB to manage my time-series -data." - -*Master/Slave* -: Do not use. Use "Primary" and "Secondary" instead. - -*MST* -: Do not use. Use "Managed Service for TimescaleDB" instead. - -### N - -*Next* -: Avoid all directional words. You cannot guarantee that things will stay in the -same position, or be in the position you expect on an individual reader's -device. - -### O - -*Once* -: Do not use. Use "when" instead. - -❌ "Once you have finished the installation, you can..." - -👍 "When you have finished the installation, you can." - -### P - -*Postgres* -: Do not use. Use "PostgreSQL" instead. - -❌ "Run the Postgres query" - -👍 "Run the PostgreSQL query." - -*PostgreSQL* -: Always refer to PostgreSQL by its full name and with correct capitalization, -as shown. - -*Previous* -: Avoid all directional words. You cannot guarantee that things will stay in the -same position, or be in the position you expect on an individual reader's -device. - -### Q - -### R - -### S - -*Simply* -: Don't use. - -❌ Simply install Timescale. - -👍 Install Timescale. - -*Superuser* -: One word. - -### T - -*Thus* -: Do not use. - -*Timescale* -: The name of the company, and the name of the product in a general -sense. Do not use camel case. - -❌ TimeScale - -👍 Timescale is hosting a virtual event. - -👍 I have installed Timescale to manage my time-series data. - -*Timescale* -: The name of the product. Always capitalize the initial letters. - -❌ "I am using timescale to manage my time-series data." - -❌ "I am using Timescale Cloud to manage my time-series data." - -👍 "I am using Timescale to manage my time-series data." - -*TimescaleDB* -: The name of the PostgreSQL extension. Always capitalize the initial letter and -the DB at the end if using in prose. If referring directly to the name of the -extension, use all lowercase. - -❌ "I installed timescaleDB to manage my time-series data." - -👍 "I installed the `timescaledb` extension to manage my time-series data." - -👍 "I installed TimescaleDB to manage my time-series data." - -*tobs* -: The observability suite. Do not use capitalization, even when it begins -a sentence. If possible, rewrite the sentence to avoid this. - -❌ "Tobs can be used to install." - -👍 "Install using tobs" - -*TSDB* -: Do not use. Use "Timescale database" instead. - -### U - -*Update* -: An update is a small or minor improvement, often delivered in a patch. Updates -are done frequently, and require little or no downtime. - -👍 Install the security update to patch this version. - -*Upgrade* -: An upgrade is a large or major improvement, and usually requires a new -version. Upgrades are done less frequently, and could require planning, -prepatory backups, and planned downtime. - -👍 Upgrade from TimescaleDB 1 to TimescaleDB 2. - -👍 Upgrade from TimescaleDB 2.3 to TimescaleDB 2.4. - -*Utilize* -: Do not use. Use "use" instead. - -### V - -*Vanilla PostgreSQL* -: Do not use. If you want to differentiate between regular PostgreSQL tables, -and tables that contain time-series, use "standard PostgreSQL". - -❌ "You can also create services that run vanilla PostgreSQL." - -👍 "You can also create services that run standard PostgreSQL." - -*Via* -: Do not use. There is usually a more accurate English word, like -"through," "with," or "using." - -### W - -*Will* -: Do not use. It usually indicates that you are writing in future tense. -Always write in simple present tense. - -❌ After installation, you will see a message. - -👍 After installation, you see a message. - -### X - -### Y - -### Z - -[contributors]: https://github.com/timescale/docs/blob/latest/CONTRIBUTING.md -[github-docs]: https://github.com/timescale/docs -[google-style]: https://developers.google.com/style -[readme]: https://github.com/timescale/docs#readme -[prismjs]: https://prismjs.com/#supported-languages diff --git a/about/contribute-to-timescale.md b/about/contribute-to-timescale.md index c53468a1df..9be102edd9 100644 --- a/about/contribute-to-timescale.md +++ b/about/contribute-to-timescale.md @@ -6,57 +6,35 @@ keywords: [contribute] tags: [github] --- -# Contributing to Timescale +# Contribute to $COMPANY -There are multiple ways to help make TimescaleDB better. All of the documentation -and source for the PostgreSQL extension are available to use and review on -GitHub. +$TIMESCALE_DB, pgai, pgvectorscale, TimescaleDB Toolkit, and the $COMPANY documentation are all open source. They are available in GitHub for you use, review, and update. This page shows you where you can add to $COMPANY products. -## Contributing to Timescale documentation +## Contribute to the code for $COMPANY products -Timescale documentation is hosted in a [GitHub repository][github-docs] -and is open for contribution from all community members. If you -find errors or would like to add content to the docs, this tutorial -walks you through the process. +$COMPANY appreciates any help the community can provide to make its products better! You can: -### Making minor changes +* Open an issue with a bug report, build issue, feature request or suggestion. +* Fork a corresponding repository and submit a pull request. -If you want to make only minor changes to docs, you can make corrections -and submit pull requests on the GitHub website. Go to the file you want to -correct and click the 'pencil' icon to edit. Once done, GitHub gives you -an option to submit a pull request at the bottom of the page. +Head over to the $COMPANY source repositories to learn, review, and help improve our products! -### Making larger contributions to docs +* [TimescaleDB][timescaledb]: a PostgreSQL extension for high-performance real-time analytics on time-series and event data. +* [pgai][pgai]: a suite of tools to develop RAG, semantic search, and other AI applications more easily with PostgreSQL. +* [pgvectorscale][pgvectorscale]: a complement to pgvector for higher performance embedding search and cost-efficient storage for AI applications. +* [TimescaleDB Toolkit][toolkit]: all things analytics when using $TIMESCALE_DB, with a particular focus on developer ergonomics and performance. -In order to modify documentation, you should have a working knowledge -of [git][install-git] and [Markdown][markdown-tutorial]. You -also need to create a GitHub account. +## Contribute to $COMPANY documentation -Be sure to read the [Timescale docs contribution styleguide][timescale-docs-style]. -You'll see information about how we refer to aspects of Timescale, -how we format our docs, and special Markdown tags available to -you as you author your contribution. +$COMPANY documentation is hosted in the [docs GitHub repository][github-docs] +and open for contribution from all community members. -Before we accept any contributions, Timescale contributors need to -sign the Contributor License Agreement (CLA). By signing a CLA, we -can ensure that the community is free and confident in its -ability to use your contributions. You are prompted to sign the -CLA during the pull request process. - -## Contributing to TimescaleDB code - -Timescale appreciates any help the community can provide to make TimescaleDB better! - -There are multiple ways you can help: - -* Open an issue with a bug report, build issue, feature request, suggestion, etc. -* Fork this repository and submit a pull request - -[Head over to our GitHub repository][github-timescaledb] for TimescaleDB to learn -more about how you can help and to review our coding style guide! +See the [contribution guide][contribution-guide] for details. +[contribution-guide]: https://github.com/timescale/docs/blob/latest/CONTRIBUTING.md [github-docs]: https://github.com/timescale/docs -[github-timescaledb]: https://github.com/timescale/timescaledb/blob/master/CONTRIBUTING.md -[install-git]: https://git-scm.com/book/en/v2/Getting-Started-Installing-Git -[markdown-tutorial]: https://www.markdownguide.org/basic-syntax/ -[timescale-docs-style]: https://github.com/timescale/docs/blob/master/README.md +[timescaledb]: https://github.com/timescale/timescaledb/blob/main/CONTRIBUTING.md +[pgai]: https://github.com/timescale/pgai/blob/main/CONTRIBUTING.md +[pgvectorscale]: https://github.com/timescale/pgvectorscale/blob/main/CONTRIBUTING.md +[toolkit]: https://github.com/timescale/timescaledb-toolkit + diff --git a/about/page-index/page-index.js b/about/page-index/page-index.js index 56d53008db..ea733dbc35 100644 --- a/about/page-index/page-index.js +++ b/about/page-index/page-index.js @@ -31,12 +31,7 @@ module.exports = [ title: "Contribute to Timescale", href: "contribute-to-timescale", excerpt: "Contribute to the Timescale codebase", - }, - { - title: "Contribute to Timescale documentation", - href: "contribute-to-docs", - excerpt: "Contribute to Timescale documentation", - }, + } ], }, ]; diff --git a/about/pricing-and-account-management.md b/about/pricing-and-account-management.md index b7cc42a55c..ce45df56c5 100644 --- a/about/pricing-and-account-management.md +++ b/about/pricing-and-account-management.md @@ -243,7 +243,6 @@ When you get $CLOUD_LONG at AWS Marketplace, the following pricing options are a - **Annual commit**: your consumption is calculated at the end of the month ensuring predictable pricing and seamless billing through your AWS account. We confirm the contract terms with you before finalizing the commitment. [cloud-login]: https://console.cloud.timescale.com/ -[compression]: /use-timescale/:currentVersion:/compression/ [data-tiering]: /use-timescale/:currentVersion:/data-tiering/ [cloud-billing]: https://console.cloud.timescale.com/dashboard/billing/details [commercial-sla]: https://www.timescale.com/legal/timescale-cloud-terms-of-service @@ -252,4 +251,4 @@ When you get $CLOUD_LONG at AWS Marketplace, the following pricing options are a [production-support]: https://www.timescale.com/support [get-in-touch]: https://www.timescale.com/contact [hipaa-compliance]: https://www.hhs.gov/hipaa/for-professionals/index.html -[aws-pricing]: #aws-pricing \ No newline at end of file +[aws-pricing]: /about/:currentVersion:/pricing-and-account-management/#aws-marketplace-pricing diff --git a/about/release-notes.md b/about/release-notes.md index 13425d94fc..43bab8884e 100644 --- a/about/release-notes.md +++ b/about/release-notes.md @@ -16,7 +16,7 @@ notes about our downloadable products, see: * [pgspot](https://github.com/timescale/pgspot/releases) - spot vulnerabilities in PostgreSQL extension scripts. * [live-migration](https://hub.docker.com/r/timescale/live-migration/tags) - a Docker image to migrate data to a Timescale Cloud service. - +This documentation is based on TimescaleDB v2.18.0 and compatible products. diff --git a/about/timescaledb-editions.md b/about/timescaledb-editions.md index 70f9a98c5f..727eeaddd5 100644 --- a/about/timescaledb-editions.md +++ b/about/timescaledb-editions.md @@ -6,6 +6,9 @@ keywords: [Apache, community, license] tags: [learn, contribute] --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; +import Since2180 from "versionContent/_partials/_since_2_18_0.mdx"; + # TimescaleDB Apache 2 and TimescaleDB Community Edition There are two versions of TimescaleDB available: @@ -162,40 +165,55 @@ You can access a hosted version of TimescaleDB Community Edition through - Compression + Hypercore - ALTER TABLE (Compression) + ALTER MATERIALIZED VIEW (Hypercore) ❌ ✅ - add_compression_policy + ALTER TABLE (Hypercore) ❌ ✅ - remove_compression_policy + add_columnstore_policy ❌ ✅ - compress_chunk + remove_columnstore_policy ❌ ✅ - decompress_chunk + convert_to_columnstore ❌ ✅ - hypertable_compression_stats + convert_to_rowstore ❌ ✅ - chunk_compression_stats + hypertable_columnstore_settings + ❌ + ✅ + + + hypertable_columnstore_stats + ❌ + ✅ + + + chunk_columnstore_settings + ❌ + ✅ + + + chunk_columnstore_stats ❌ ✅ @@ -452,6 +470,44 @@ You can access a hosted version of TimescaleDB Community Edition through ✅ ✅ + + Compression replaced by Hypercore + + + ALTER TABLE (Compression) + ❌ + ✅ + + + add_compression_policy + ❌ + ✅ + + + remove_compression_policy + ❌ + ✅ + + + compress_chunk + ❌ + ✅ + + + decompress_chunk + ❌ + ✅ + + + hypertable_compression_stats + ❌ + ✅ + + + chunk_compression_stats + ❌ + ✅ + diff --git a/api/_hyperfunctions/candlestick_agg/examples.md b/api/_hyperfunctions/candlestick_agg/examples.md index 137562588b..bc579d0b13 100644 --- a/api/_hyperfunctions/candlestick_agg/examples.md +++ b/api/_hyperfunctions/candlestick_agg/examples.md @@ -17,7 +17,7 @@ SELECT low(candlestick_agg("time", price, volume)), close(candlestick_agg("time", price, volume)), volume(candlestick_agg("time", price, volume)) -FROM stocks_real_time +FROM crypto_ticks WHERE "time" > now() - '1 day'::interval GROUP BY ts, symbol ; @@ -28,7 +28,7 @@ WITH cs AS ( SELECT time_bucket('1 hour'::interval, "time") AS hourly_bucket, symbol, candlestick_agg("time", price, volume) AS candlestick - FROM stocks_real_time + FROM crypto_ticks WHERE "time" > now() - '1 day'::interval GROUP BY hourly_bucket, symbol ) @@ -53,7 +53,7 @@ WITH (timescaledb.continuous) AS SELECT time_bucket('1 minute'::interval, "time") AS ts, symbol, candlestick_agg("time", price, volume) AS candlestick -FROM stocks_real_time +FROM crypto_ticks GROUP BY ts, symbol ; ``` diff --git a/api/_hyperfunctions/count_min_sketch/approx_count.md b/api/_hyperfunctions/count_min_sketch/approx_count.md index a3c6955a47..5c2a8244da 100644 --- a/api/_hyperfunctions/count_min_sketch/approx_count.md +++ b/api/_hyperfunctions/count_min_sketch/approx_count.md @@ -41,7 +41,7 @@ api_details: code: | WITH t AS ( SELECT toolkit_experimental.count_min_sketch(symbol, 0.01, 0.01) AS symbol_sketch - FROM stocks_real_time + FROM crypto_ticks ) SELECT toolkit_experimental.approx_count('AAPL', symbol_sketch) FROM t; diff --git a/api/add_continuous_aggregate_policy.md b/api/add_continuous_aggregate_policy.md index 0dabce5df1..4bdc766bba 100644 --- a/api/add_continuous_aggregate_policy.md +++ b/api/add_continuous_aggregate_policy.md @@ -40,7 +40,7 @@ While setting `end_offset` to `NULL` is possible, it is not recommended. By default, querying a continuous aggregate returns data between `end_offset` and the current time. There is no need to set `end_offset` to `NULL`. To learn more about how continuous aggregates use real-time aggregation, see the -[real-time aggregation section](/use-timescale/latest/continuous-aggregates/real-time-aggregates/). +[real-time aggregation section](/use-timescale/:currentVersion:/continuous-aggregates/real-time-aggregates/). ### Optional arguments diff --git a/api/add_policies.md b/api/add_policies.md index 3f13169887..1870c07d58 100644 --- a/api/add_policies.md +++ b/api/add_policies.md @@ -25,7 +25,8 @@ timescaledb_experimental.add_policies( refresh_start_offset "any" = NULL, refresh_end_offset "any" = NULL, compress_after "any" = NULL, - drop_after "any" = NULL + drop_after "any" = NULL, + hypercore_use_access_method BOOL = NULL) ) RETURNS BOOL ``` @@ -52,6 +53,7 @@ If you would like to set this add your policies manually (see [`add_continuous_a |`refresh_end_offset`|`INTERVAL` or `INTEGER`|The end of the continuous aggregate refresh window, expressed as an offset from the policy run time. Must be greater than `refresh_start_offset`.| |`compress_after`|`INTERVAL` or `INTEGER`|Continuous aggregate chunks are compressed if they exclusively contain data older than this interval.| |`drop_after`|`INTERVAL` or `INTEGER`|Continuous aggregate chunks are dropped if they exclusively contain data older than this interval.| +| `hypercore_use_access_method` | BOOLEAN | `NULL` | Set to `true` to use hypercore table access metod. If set to `NULL` it will use the value from `timescaledb.default_hypercore_use_access_method`. | For arguments that could be either an `INTERVAL` or an `INTEGER`, use an `INTERVAL` if your time bucket is based on timestamps. Use an `INTEGER` if your @@ -59,7 +61,7 @@ time bucket is based on integers. ## Returns -Returns true if successful. +Returns `true` if successful. ## Sample usage diff --git a/api/add_reorder_policy.md b/api/add_reorder_policy.md index 30d8fce14c..ef93b276b3 100644 --- a/api/add_reorder_policy.md +++ b/api/add_reorder_policy.md @@ -18,12 +18,14 @@ still being inserted into. For more information about reordering chunks, see the [reorder_chunk][reorder_chunk] section. + When a chunk has been reordered by the background worker it is not reordered again. If you insert significant amounts of data in to older chunks that have already been reordered, you might need to manually re-run the -[reorder_chunk](/api/latest/hypertable/reorder_chunk) +[reorder_chunk][reorder_chunk] function on older chunks. Alternatively, you can drop and re-create the policy, which can work better if you have changed a lot of older chunks. + ## Required arguments diff --git a/api/alter_table_compression.md b/api/alter_table_compression.md deleted file mode 100644 index c2990aa5ac..0000000000 --- a/api/alter_table_compression.md +++ /dev/null @@ -1,77 +0,0 @@ ---- -api_name: ALTER TABLE (Compression) -excerpt: Change compression settings on a compressed hypertable -topics: [compression] -keywords: [compression] -tags: [settings, hypertables, alter, change] -api: - license: community - type: command ---- - -# ALTER TABLE (Compression) - -'ALTER TABLE' statement is used to turn on compression and set compression -options. - -By itself, this `ALTER` statement alone does not compress a hypertable. To do so, either create a -compression policy using the [add_compression_policy][add_compression_policy] function or manually -compress a specific hypertable chunk using the [compress_chunk][compress_chunk] function. - -The syntax is: - -``` sql -ALTER TABLE SET (timescaledb.compress, - timescaledb.compress_orderby = ' [ASC | DESC] [ NULLS { FIRST | LAST } ] [, ...]', - timescaledb.compress_segmentby = ' [, ...]', - timescaledb.compress_chunk_time_interval='interval' -); -``` - -## Required arguments - -|Name|Type|Description| -|-|-|-| -|`timescaledb.compress`|BOOLEAN|Enable or disable compression| - -## Optional arguments - -|Name|Type|Description| -|-|-|-| -|`timescaledb.compress_orderby`|TEXT|Order used by compression, specified in the same way as the ORDER BY clause in a SELECT query. The default is the descending order of the hypertable's time column.| -|`timescaledb.compress_segmentby`|TEXT|Column list on which to key the compressed segments. An identifier representing the source of the data such as `device_id` or `tags_id` is usually a good candidate. The default is no `segment by` columns.| -|`timescaledb.compress_chunk_time_interval`|TEXT|EXPERIMENTAL: Set compressed chunk time interval used to roll chunks into. This parameter compresses every chunk, and then irreversibly merges it into a previous adjacent chunk if possible, to reduce the total number of chunks in the hypertable. Note that chunks will not be split up during decompression. It should be set to a multiple of the current chunk interval. This option can be changed independently of other compression settings and does not require the `timescaledb.compress` argument.| - -## Parameters - -|Name|Type|Description| -|-|-|-| -|`table_name`|TEXT|Hypertable that supports compression| -|`column_name`|TEXT|Column used to order by or segment by| -|`interval`|TEXT|Time interval used to roll compressed chunks into| - -## Sample usage - -Configure a hypertable that ingests device data to use compression. Here, if the hypertable -is often queried about a specific device or set of devices, the compression should be -segmented using the `device_id` for greater performance. - -```sql -ALTER TABLE metrics SET (timescaledb.compress, timescaledb.compress_orderby = 'time DESC', timescaledb.compress_segmentby = 'device_id'); -``` - -You can also specify compressed chunk interval without changing other -compression settings: - -```sql -ALTER TABLE metrics SET (timescaledb.compress_chunk_time_interval = '24 hours'); -``` - -To disable the previously set option, set the interval to 0: - -```sql -ALTER TABLE metrics SET (timescaledb.compress_chunk_time_interval = '0'); -``` - -[add_compression_policy]: /api/:currentVersion:/compression/add_compression_policy/ -[compress_chunk]: /api/:currentVersion:/compression/compress_chunk/ diff --git a/api/add_compression_policy.md b/api/compression/add_compression_policy.md similarity index 91% rename from api/add_compression_policy.md rename to api/compression/add_compression_policy.md index 74210e4142..7a154443c3 100644 --- a/api/add_compression_policy.md +++ b/api/compression/add_compression_policy.md @@ -9,8 +9,12 @@ api: type: function --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # add_compression_policy() + Replaced by add_columnstore_policy(). + Allows you to set a policy by which the system compresses a chunk automatically in the background after it reaches a given age. @@ -50,6 +54,9 @@ on the type of the time column of the hypertable or continuous aggregate: |`initial_start`|TIMESTAMPTZ|Time the policy is first run. Defaults to NULL. If omitted, then the schedule interval is the interval from the finish time of the last execution to the next start. If provided, it serves as the origin with respect to which the next_start is calculated | |`timezone`|TEXT|A valid time zone. If `initial_start` is also specified, subsequent executions of the compression policy are aligned on its initial start. However, daylight savings time (DST) changes may shift this alignment. Set to a valid time zone if this is an issue you want to mitigate. If omitted, UTC bucketing is performed. Defaults to `NULL`.| |`if_not_exists`|BOOLEAN|Setting to `true` causes the command to fail with a warning instead of an error if a compression policy already exists on the hypertable. Defaults to false.| +| `hypercore_use_access_method` | BOOLEAN | `NULL` | Set to `true` to use hypercore table access metod. If set to `NULL` it will use the value from `timescaledb.default_hypercore_use_access_method`. | + + @@ -89,3 +96,4 @@ SELECT add_compression_policy('cpu_weekly', INTERVAL '8 weeks'); [compression_continuous-aggregate]: /api/:currentVersion:/continuous-aggregates/alter_materialized_view/ [set_integer_now_func]: /api/:currentVersion:/hypertable/set_integer_now_func [informational-views]: /api/:currentVersion:/informational-views/jobs/ + diff --git a/api/compression/alter_table_compression.md b/api/compression/alter_table_compression.md new file mode 100644 index 0000000000..ae133e2cb6 --- /dev/null +++ b/api/compression/alter_table_compression.md @@ -0,0 +1,83 @@ +--- +api_name: ALTER TABLE (Compression) +excerpt: Change compression settings on a compressed hypertable +topics: [compression] +keywords: [compression] +tags: [settings, hypertables, alter, change] +api: + license: community + type: command +--- + +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + +# ALTER TABLE (Compression) + + Replaced by ALTER TABLE (Hypercore). + +'ALTER TABLE' statement is used to turn on compression and set compression +options. + +By itself, this `ALTER` statement alone does not compress a hypertable. To do so, either create a +compression policy using the [add_compression_policy][add_compression_policy] function or manually +compress a specific hypertable chunk using the [compress_chunk][compress_chunk] function. + +The syntax is: + +``` sql +ALTER TABLE SET (timescaledb.compress, + timescaledb.compress_orderby = ' [ASC | DESC] [ NULLS { FIRST | LAST } ] [, ...]', + timescaledb.compress_segmentby = ' [, ...]', + timescaledb.compress_chunk_time_interval='interval' +); +``` + +## Required arguments + +|Name|Type|Description| +|-|-|-| +|`timescaledb.compress`|BOOLEAN|Enable or disable compression| + +## Optional arguments + +|Name|Type| Description | +|-|-|--| +|`timescaledb.compress_orderby`|TEXT| Order used by compression, specified in the same way as the ORDER BY clause in a SELECT query. The default is the descending order of the hypertable's time column. | +|`timescaledb.compress_segmentby`|TEXT| Column list on which to key the compressed segments. An identifier representing the source of the data such as `device_id` or `tags_id` is usually a good candidate. The default is no `segment by` columns. | +|`timescaledb.enable_segmentwise_recompression`|TEXT| Set to `OFF` to disable segmentwise recompression on compressed chunks. This can be beneficial for some user workloads where segmentwise recompression is slow, and full recompression is more performant. The default is `ON`. | +|`timescaledb.compress_chunk_time_interval`|TEXT| EXPERIMENTAL: Set compressed chunk time interval used to roll chunks into. This parameter compresses every chunk, and then irreversibly merges it into a previous adjacent chunk if possible, to reduce the total number of chunks in the hypertable. Note that chunks will not be split up during decompression. It should be set to a multiple of the current chunk interval. This option can be changed independently of other compression settings and does not require the `timescaledb.compress` argument. | + + +## Parameters + +|Name|Type|Description| +|-|-|-| +|`table_name`|TEXT|Hypertable that supports compression| +|`column_name`|TEXT|Column used to order by or segment by| +|`interval`|TEXT|Time interval used to roll compressed chunks into| + +## Sample usage + +Configure a hypertable that ingests device data to use compression. Here, if the hypertable +is often queried about a specific device or set of devices, the compression should be +segmented using the `device_id` for greater performance. + +```sql +ALTER TABLE metrics SET (timescaledb.compress, timescaledb.compress_orderby = 'time DESC', timescaledb.compress_segmentby = 'device_id'); +``` + +You can also specify compressed chunk interval without changing other +compression settings: + +```sql +ALTER TABLE metrics SET (timescaledb.compress_chunk_time_interval = '24 hours'); +``` + +To disable the previously set option, set the interval to 0: + +```sql +ALTER TABLE metrics SET (timescaledb.compress_chunk_time_interval = '0'); +``` + +[add_compression_policy]: /api/:currentVersion:/compression/add_compression_policy/ +[compress_chunk]: /api/:currentVersion:/compression/compress_chunk/ diff --git a/api/chunk_compression_stats.md b/api/compression/chunk_compression_stats.md similarity index 94% rename from api/chunk_compression_stats.md rename to api/compression/chunk_compression_stats.md index 5f7b995937..6b7014bf3b 100644 --- a/api/chunk_compression_stats.md +++ b/api/compression/chunk_compression_stats.md @@ -9,8 +9,12 @@ api: type: function --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # chunk_compression_stats() Community + Replaced by chunk_columnstore_stats(). + Get chunk-specific statistics related to hypertable compression. All sizes are in bytes. diff --git a/api/compress_chunk.md b/api/compression/compress_chunk.md similarity index 79% rename from api/compress_chunk.md rename to api/compression/compress_chunk.md index 75d5e65c89..b6cbb2efee 100644 --- a/api/compress_chunk.md +++ b/api/compression/compress_chunk.md @@ -9,8 +9,12 @@ api: type: function --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # compress_chunk() Community + Replaced by convert_to_columnstore(). + The `compress_chunk` function is used to compress (or recompress, if necessary) a specific chunk. This is most often used instead of the [`add_compression_policy`][add_compression_policy] function, when a user @@ -38,6 +42,7 @@ You can get a list of chunks belonging to a hypertable using the |Name|Type|Description| |---|---|---| | `if_not_compressed` | BOOLEAN | Disabling this will make the function error out on chunks that are already compressed. Defaults to true.| +| `hypercore_use_access_method` | BOOLEAN | `NULL` |✖| Set to `true` to use hypercore table access metod. If set to `NULL` it will use the value from `timescaledb.default_hypercore_use_access_method`. | ### Returns diff --git a/api/decompress_chunk.md b/api/compression/decompress_chunk.md similarity index 86% rename from api/decompress_chunk.md rename to api/compression/decompress_chunk.md index ef5c833fff..f775c71a92 100644 --- a/api/decompress_chunk.md +++ b/api/compression/decompress_chunk.md @@ -8,18 +8,24 @@ api: type: function --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # decompress_chunk() Community + Replaced by convert_to_rowstore(). + If you need to modify or add a lot of data to a chunk that has already been compressed, you should decompress the chunk first. This is especially useful for backfilling old data. + Before decompressing chunks, stop any compression policy on the hypertable you are decompressing. You can use `SELECT alter_job(JOB_ID, scheduled => false);` to prevent scheduled execution. When you finish backfilling or updating data, turn the policy back on. The database automatically recompresses your chunks in the next scheduled job. + ### Required arguments diff --git a/api/hypertable_compression_stats.md b/api/compression/hypertable_compression_stats.md similarity index 92% rename from api/hypertable_compression_stats.md rename to api/compression/hypertable_compression_stats.md index af91649f01..39f141ff76 100644 --- a/api/hypertable_compression_stats.md +++ b/api/compression/hypertable_compression_stats.md @@ -9,8 +9,12 @@ api: type: function --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # hypertable_compression_stats() Community + Replaced by hypertable_columnstore_stats(). + Get statistics related to hypertable compression. All sizes are in bytes. For more information about using hypertables, including chunk size partitioning, @@ -42,7 +46,9 @@ For more information about compression, see the |`node_name`|TEXT|nodes on which the hypertable is located, applicable only to distributed hypertables| + Returns show `NULL` if the data is currently uncompressed. + ### Sample usage diff --git a/api/compression.md b/api/compression/index.md similarity index 88% rename from api/compression.md rename to api/compression/index.md index 430bc86699..3df9a04737 100644 --- a/api/compression.md +++ b/api/compression/index.md @@ -5,7 +5,13 @@ keywords: [compression] tags: [hypertables] --- -# Compression Community +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + +# Compression (Old API, replaced by Hypercore) Community + + Replaced by Hypercore. + +Compression functionality is included in Hypercore. Before you set up compression, you need to [configure the hypertable for compression][configure-compression] and then @@ -59,3 +65,4 @@ You can also use advanced insert statements like `ON CONFLICT` and `RETURNING`. [compress_chunk]: /api/:currentVersion:/compression/compress_chunk/ [configure-compression]: /api/:currentVersion:/compression/alter_table_compression/ [skipscan]: /use-timescale/:currentVersion:/query-data/skipscan/ +[hypercore]: /api/:currentVersion:/hypercore/ diff --git a/api/recompress_chunk.md b/api/compression/recompress_chunk.md similarity index 92% rename from api/recompress_chunk.md rename to api/compression/recompress_chunk.md index 85d6274881..9287833eb4 100644 --- a/api/recompress_chunk.md +++ b/api/compression/recompress_chunk.md @@ -9,8 +9,12 @@ api: type: function --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # recompress_chunk() + Replaced by convert_to_columnstore(). + Recompresses a compressed chunk that had more data inserted after compression. ```sql diff --git a/api/remove_compression_policy.md b/api/compression/remove_compression_policy.md similarity index 84% rename from api/remove_compression_policy.md rename to api/compression/remove_compression_policy.md index 489d5ec052..3b9c72372c 100644 --- a/api/remove_compression_policy.md +++ b/api/compression/remove_compression_policy.md @@ -9,8 +9,13 @@ api: type: function --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # remove_compression_policy() + + Replaced by remove_columnstore_policy(). + If you need to remove the compression policy. To restart policy-based compression you need to add the policy again. To view the policies that already exist, see [informational views][informational-views]. diff --git a/api/distributed-hypertables.md b/api/distributed-hypertables.md index a4e08ee207..4f3e7837a0 100644 --- a/api/distributed-hypertables.md +++ b/api/distributed-hypertables.md @@ -1,5 +1,5 @@ --- -title: Distributed hypertables +title: Distributed hypertables ( Sunsetted v2.14.x ) excerpt: Sunsetted v2.14.x. Timescale Cloud API reference for dealing with distributed hypertables keywords: [distributed hypertables] --- @@ -8,7 +8,7 @@ import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecati -# Distributed Hypertables Community +# Distributed hypertables ( Sunsetted v2.14.x) Community Distributed hypertables are an extension of regular hypertables, available when using a [multi-node installation][getting-started-multi-node] of TimescaleDB. diff --git a/api/enable_chunk_skipping.md b/api/enable_chunk_skipping.md index 078117e5c8..34f91f261a 100644 --- a/api/enable_chunk_skipping.md +++ b/api/enable_chunk_skipping.md @@ -1,6 +1,6 @@ --- api_name: enable_chunk_skipping() -excerpt: Enable range tracking for columns of chunks from a hypertable +excerpt: Enable range tracking for columns of chunks from a hypertable topics: [hypertables] keywords: [hypertables, chunks, range-tracking, skipping] tags: [columns, ranges, min-max, chunks] @@ -18,7 +18,7 @@ partitioning column. In other words, enable tracking on secondary columns which referenced in the `WHERE` clauses in your queries. TimescaleDB supports min/max range tracking for the `smallint`, `int`, -`bigint`, `serial`, `bigserial`, `date`, `timestamp`, and `timestamptz` data types. The +`bigint`, `serial`, `bigserial`, `date`, `timestamp`, and `timestamptz` data types. The min/max ranges are calculated when a chunk belonging to this hypertable is compressed using the [compress_chunk][compress_chunk] function. The range is stored in start (inclusive) and end (exclusive) form in the @@ -39,7 +39,7 @@ from the `chunk_column_stats` catalog table since now it's available for DML and min/max range values can change on any further data manipulation in the chunk. By default, this feature is disabled. To enable chunk skipping, set `timescaledb.enable_chunk_skipping = on` in -`postgresql.conf`. When you upgrade from a database instance that uses compression but does not support chunk +`postgresql.conf`. When you upgrade from a database instance that uses compression but does not support chunk skipping, you need to recompress the previously compressed chunks for chunk skipping to work. ## Samples diff --git a/api/hypercore/add_columnstore_policy.md b/api/hypercore/add_columnstore_policy.md new file mode 100644 index 0000000000..d5c0eefdfd --- /dev/null +++ b/api/hypercore/add_columnstore_policy.md @@ -0,0 +1,122 @@ +--- +api_name: add_columnstore_policy() +excerpt: Set a policy to automatically move chunks in a hypertable to the columnstore when they reach a given age. +topics: [hypercore, columnstore, jobs] +keywords: [columnstore, hypercore, policies] +tags: [scheduled jobs, background jobs, automation framework] +products: [cloud, self_hosted] +api: + license: community + type: procedure +--- + +import Since2180 from "versionContent/_partials/_since_2_18_0.mdx"; + +# add_columnstore_policy() + +Create a [job][job] that automatically moves chunks in a hypertable to the columnstore after a +specific time interval. + +You enable the columnstore a hypertable or continuous aggregate before you create a columnstore policy. +You do this by calling `ALTER TABLE` for hypertables and `ALTER MATERIALIZED VIEW` for continuous aggregates. + +To view the policies that you set or the policies that already exist, +see [informational views][informational-views], to remove a policy, see [remove_columnstore_policy][remove_columnstore_policy]. + + + +## Samples + +To create a columnstore job: + + + +1. **Enable columnstore** + + * [Use `ALTER TABLE` for a hypertable][compression_alter-table] + ```sql + ALTER TABLE crypto_ticks SET ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'symbol'); + ``` + * [Use ALTER MATERIALIZED VIEW for a continuous aggregate][compression_continuous-aggregate] + ```sql + ALTER MATERIALIZED VIEW assets_candlestick_daily set ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'symbol' ); + ``` + +1. **Add a policy to move chunks to the columnstore at a specific time interval** + + For example: + + * 60 days after the data was added to the table: + ``` sql + CALL add_columnstore_policy('crypto_ticks', after => INTERVAL '60d'); + ``` + * 3 months prior to the moment you run the query: + + ``` sql + CALL add_columnstore_policy('crypto_ticks', created_before => INTERVAL '3 months'); + ``` + * With an integer-based time column: + + ``` sql + CALL add_columnstore_policy('table_with_bigint_time', BIGINT '600000'); + ``` + * Older than eight weeks: + + ``` sql + CALL add_columnstore_policy('cpu_weekly', INTERVAL '8 weeks'); + ``` + + * Older than eight weeks and using the Hypercore table access method: + + ``` sql + CALL add_columnstore_policy( + 'cpu_weekly', + INTERVAL '8 weeks', + hypercore_use_access_method => true); + ``` + +1. **View the policies that you set or the policies that already exist** + + ``` sql + SELECT * FROM timescaledb_information.jobs + WHERE proc_name='policy_compression'; + ``` + See [timescaledb_information.jobs][informational-views]. + + + +## Arguments + +Calls to `add_columnstore_policy` require either `after` or `created_before`, but cannot have both. + + + + +| Name | Type | Default | Required | Description | +|--|--|--|--|--| +| `hypertable` |REGCLASS| - | ✔ | Name of the hypertable or continuous aggregate to run this [job][job] on.| +| `after` |INTERVAL or INTEGER|- | ✖ | Add chunks containing data older than `now - {after}::interval` to the columnstore.
Use an object type that matchs the time column type in `hypertable`:
  • TIMESTAMP, TIMESTAMPTZ, or DATE: use an INTERVAL type.
  • Integer-based timestamps : set an integer type using the [integer_now_func][set_integer_now_func].
`after` is mutually exclusive with `created_before`. | +| `created_before` |INTERVAL| NULL | ✖ | Add chunks with a creation time of `now() - created_before` to the columnstore.
`created_before` is
  • Not supported for continuous aggregates.
  • Mutually exclusive with `after`.
| +| `schedule_interval` |INTERVAL| 12 hours when [chunk_time_interval][chunk_time_interval] >= `1 day` for `hypertable`. Otherwise `chunk_time_interval` / `2`. | ✖ | Set the interval between the finish time of the last execution of this policy and the next start.| +| `initial_start` |TIMESTAMPTZ| The interval from the finish time of the last execution to the [next_start][next-start].| ✖| Set the time this job is first run. This is also the time that `next_start` is calculated from.| +| `timezone` |TEXT| UTC. However, daylight savings time(DST) changes may shift this alignment. | ✖ | Set to a valid time zone to mitigate DST shifting. If `initial_start` is set, subsequent executions of this policy are aligned on `initial_start`.| +| `if_not_exists` |BOOLEAN| `false` | ✖ | Set to `true` so this job fails with a warning rather than an error if a columnstore policy already exists on `hypertable` | +| `hypercore_use_access_method` | BOOLEAN | `NULL` | ✖ | Set to `true` to use hypercore table access metod. If set to `NULL` it will use the value from `timescaledb.default_hypercore_use_access_method`. | + + + + + + +[compression_alter-table]: /api/:currentVersion:/hypercore/alter_table/ +[compression_continuous-aggregate]: /api/:currentVersion:/hypercore/alter_materialized_view/ +[set_integer_now_func]: /api/:currentVersion:/hypertable/set_integer_now_func +[informational-views]: /api/:currentVersion:/informational-views/jobs/ +[chunk_time_interval]: /api/:currentVersion:/hypertable/set_chunk_time_interval/ +[next-start]: /api/:currentVersion:/informational-views/jobs/#arguments +[job]: /api/:currentVersion:/actions/add_job/ +[remove_columnstore_policy]: /api/:currentVersion:/hypercore/remove_columnstore_policy/ diff --git a/api/hypercore/alter_materialized_view.md b/api/hypercore/alter_materialized_view.md new file mode 100644 index 0000000000..f4dc10cfe5 --- /dev/null +++ b/api/hypercore/alter_materialized_view.md @@ -0,0 +1,104 @@ +--- +api_name: ALTER MATERIALIZED VIEW (Hypercore) +excerpt: Change an existing continuous aggregate +topics: [hypercore, continuous aggregates, columnstore,] +keywords: [hypercore, continuous aggregates, columnstore, ] +tags: [materialized views, hypertables, alter, change] +api: + license: community + type: command +--- + +import Since2180 from "versionContent/_partials/_since_2_18_0.mdx"; +import EarlyAccess from "versionContent/_partials/_early_access.mdx"; + +# ALTER MATERIALIZED VIEW (Hypercore) Community + +`ALTER MATERIALIZED VIEW` statement can be used to modify some of the `WITH` +clause [options][create_materialized_view] for the continuous aggregate view. +`ALTER MATERIALIZED VIEW` statement also supports the following +[PostgreSQL clauses][postgres-alterview] on the +continuous aggregate view: + +* `RENAME TO` clause to rename the continuous aggregate view +* `RENAME [COLUMN]` clause to rename the continuous aggregate column +* `SET SCHEMA` clause to set the new schema for the continuous aggregate view +* `SET TABLESPACE` clause to move the materialization of the continuous + aggregate view to the new tablespace +* `OWNER TO` clause to set new owner for the continuous aggregate view + + + +## Samples + +- Disable real-time aggregates for a continuous aggregate: + + ```sql + ALTER MATERIALIZED VIEW contagg_view SET (timescaledb.materialized_only); + ``` + +- Enable hypercore for a continuous aggregate: + + ```sql + ALTER MATERIALIZED VIEW assets_candlestick_daily set ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'symbol' ); + ``` + +- Rename a column for a continuous aggregate: + + ```sql + ALTER MATERIALIZED VIEW contagg_view RENAME COLUMN old_name TO new_name; + ``` + +- **Enable indexing on data in the columnstore** + + ```sql + ALTER MATERIALIZED VIEW assets_candlestick_daily + set access method hypercore, + set (timescaledb.enable_columnstore = true, timescaledb.segmentby = 'symbol' ); + ``` + +- **Enable indexing on a chunk you are adding to the columnstore** + + ```sql + ALTER MATERIALIZED VIEW _timescaledb_internal._hyper_1_21_chunk + set access method hypercore; + ``` + +The only options that currently can be modified with `ALTER +MATERIALIZED VIEW` are `materialized_only` and `compress`. The other options +`continuous` and `create_group_indexes` can only be set when creating +the continuous aggregate. + + +## Arguments + +The syntax is: + + +``` sql +ALTER MATERIALIZED VIEW SET (timescaledb.enable_columnstore, + timescaledb.materialized_only = 'true' | 'false', + timescaledb.orderby = ' [ASC | DESC] [ NULLS { FIRST | LAST } ] [, ...]', + timescaledb.segmentby = ' [, ...]', + timescaledb.compress_chunk_time_interval='interval', + SET ACCESS METHOD { new_access_method | DEFAULT } +); +``` + +| Name | Type | Default | Required | Description | +|------------------------------------------------|--|------------------------------------------------------|--|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `view_name` |TEXT| - | ✖ | The materialized view to enable columstore for. | +| `timescaledb.enable_columnstore` |BOOLEAN| `true` | ✖ | Enable columnstore. | +|`timescaledb.materialized_only`|BOOLEAN| `true` | ✖ | Enable and disable real time aggregation | +| `timescaledb.orderby` |TEXT| Descending order on the time column in `table_name`. | ✖| The order in which items are used in the columnstore. Specified in the same way as an `ORDER BY` clause in a `SELECT` query. | +| `timescaledb.segmentby` |TEXT| No segementation by column. | ✖| Set the list of columns used to segment data in the columnstore for `table`. An identifier representing the source of the data such as `device_id` or `tags_id` is usually a good candidate. | +| `column_name` |TEXT| - | ✖ | The name of the column to `orderby` or `segmentby`. | +| `timescaledb.compress_chunk_time_interval` |TEXT| - | ✖ | EXPERIMENTAL: reduce the total number of chunks in the columnstore for `table`. If you set `compress_chunk_time_interval`, chunks added to the columnstore are merged with the previous adjacent chunk within `chunk_time_interval` whenever possible. These chunks are irreversibly merged. If you call [convert_to_rowstore][convert_to_rowstore], merged chunks are not split up. You can call `compress_chunk_time_interval` independently of other compression settings; `timescaledb.enable_columnstore` is not required. | +|`SET ACCESS METHOD`|TEXT| DEFAULT ([heap][default_table_access_method])| ✖| To enable indexing on the columstore, set to `hypercore` after you [create a continuous aggregate][create-cagg]. | + +[create_materialized_view]: /api/:currentVersion:/continuous-aggregates/create_materialized_view/#parameters +[postgres-alterview]: https://www.postgresql.org/docs/current/sql-alterview.html +[create-cagg]: /use-timescale/:currentVersion:/continuous-aggregates/create-a-continuous-aggregate/ +[default_table_access_method]: https://www.postgresql.org/docs/17/runtime-config-client.html#GUC-DEFAULT-TABLE-ACCESS-METHOD diff --git a/api/hypercore/alter_table.md b/api/hypercore/alter_table.md new file mode 100644 index 0000000000..b7a181c1cd --- /dev/null +++ b/api/hypercore/alter_table.md @@ -0,0 +1,106 @@ +--- +api_name: ALTER TABLE (Hypercore) +excerpt: Enable the columnstore for a hypertable. +topics: [hypercore, columnstore] +keywords: [columnstore, hypercore] +tags: [settings, hypertables, alter, change] +api: + license: community + type: command +products: [cloud, self_hosted] +--- + +import Since2180 from "versionContent/_partials/_since_2_18_0.mdx"; +import EarlyAccess from "versionContent/_partials/_early_access.mdx"; + +# ALTER TABLE (Hypercore) + +Enable the columnstore for a hypertable. + +After you have enabled the columnstore, either: +- [add_columnstore_policy][add_columnstore_policy]: create a [job][job] that automatically moves chunks in a hypertable to the columnstore at a + specific time interval. +- [convert_to_columnstore][convert_to_columnstore]: manually add a specific chunk in a hypertable to the columnstore. + + + +## Samples + +To enable the columnstore: + +- **Configure a hypertable that ingests device data to use the columnstore**: + + In this example, the `metrics` hypertable is often queried about a specific device or set of devices. + Segment the hypertable by `device_id` to improve query performance. + + ```sql + ALTER TABLE metrics SET( + timescaledb.enable_columnstore, + timescaledb.orderby = 'time DESC', + timescaledb.segmentby = 'device_id'); + ``` + +- **Specify the chunk interval without changing other columnstore settings**: + + - Set the time interval when chunks are added to the columnstore: + + ```sql + ALTER TABLE metrics SET (timescaledb.compress_chunk_time_interval = '24 hours'); + ``` + + - To disable the option you set previously, set the interval to 0: + + ```sql + ALTER TABLE metrics SET (timescaledb.compress_chunk_time_interval = '0'); + ``` + +- **Enable secondary indexing on all data you add to the columnstore** + + ```sql + alter table metrics + set access method hypercore, + set (timescaledb.compress_orderby = 'created_at', + timescaledb.compress_segmentby = 'location_id'); + ``` + +- **Enable secondary indexing on a chunk you are adding to the columnstore** + + ```sql + alter table _timescaledb_internal._hyper_1_21_chunk + set access method hypercore; + ``` + + +## Arguments + +The syntax is: + +``` sql +ALTER TABLE SET (timescaledb.enable_columnstore, + timescaledb.orderby = ' [ASC | DESC] [ NULLS { FIRST | LAST } ] [, ...]', + timescaledb.segmentby = ' [, ...]', + timescaledb.compress_chunk_time_interval='interval', + timescaledb.enable_segmentwise_recompression = 'ON' | 'OFF', + SET ACCESS METHOD { new_access_method | DEFAULT } +); +``` + +| Name | Type | Default | Required | Description | +|--|--|------------------------------------------------------|--|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +|`table_name`|TEXT| - | ✖ | The hypertable to enable columstore for. | +|`timescaledb.enable_columnstore`|BOOLEAN| `true` | ✖ | Enable columnstore. | +|`timescaledb.orderby`|TEXT| Descending order on the time column in `table_name`. | ✖| The order in which items are used in the columnstore. Specified in the same way as an `ORDER BY` clause in a `SELECT` query. | +|`timescaledb.segmentby`|TEXT| No segementation by column. | ✖| Set the list of columns used to segment data in the columnstore for `table`. An identifier representing the source of the data such as `device_id` or `tags_id` is usually a good candidate. | +|`column_name`|TEXT| - | ✖ | The name of the column to `orderby` or `segmentby`. | +|`timescaledb.compress_chunk_time_interval`|TEXT| - | ✖ | EXPERIMENTAL: reduce the total number of chunks in the columnstore for `table`. If you set `compress_chunk_time_interval`, chunks added to the columnstore are merged with the previous adjacent chunk within `chunk_time_interval` whenever possible. These chunks are irreversibly merged. If you call [convert_to_rowstore][convert_to_rowstore], merged chunks are not split up. You can call `compress_chunk_time_interval` independently of other compression settings; `timescaledb.enable_columnstore` is not required. | +|`interval`|TEXT| - | ✖ | Set to a multiple of the [chunk_time_interval][chunk_time_interval] for `table`. | +|`timescaledb.enable_segmentwise_recompression`|TEXT| ON | ✖| Set to `OFF` to disable segmentwise recompression on chunks in the columnstore. This can be beneficial for some user workloads where segmentwise recompression is slow, and full recompression is more performant. | +|`SET ACCESS METHOD`|TEXT| DEFAULT ([heap][default_table_access_method])| ✖| To enable indexing on the columstore, set to `hypercore` after you [create a hypertable][create-hypertable]. | + +[chunk_time_interval]: /api/:currentVersion:/hypertable/set_chunk_time_interval/ +[add_columnstore_policy]: /api/:currentVersion:/hypercore/add_columnstore_policy/ +[convert_to_columnstore]: /api/:currentVersion:/hypercore/convert_to_columnstore/ +[convert_to_rowstore]: /api/:currentVersion:/hypercore/convert_to_rowstore/ +[job]: /api/:currentVersion:/actions/add_job/ +[default_table_access_method]: https://www.postgresql.org/docs/17/runtime-config-client.html#GUC-DEFAULT-TABLE-ACCESS-METHOD +[create-hypertable]: /api/:currentVersion:/hypertable/create_hypertable diff --git a/api/hypercore/chunk_columnstore_settings.md b/api/hypercore/chunk_columnstore_settings.md new file mode 100644 index 0000000000..7a2c97d8ea --- /dev/null +++ b/api/hypercore/chunk_columnstore_settings.md @@ -0,0 +1,57 @@ +--- +api_name: timescaledb_information.chunk_columnstore_settings +excerpt: Get information about settings on each chunk in the columnstore +topics: [hypercore, information, columnstore, chunk] +keywords: [columnstore, hypercore, chunk, information] +tags: [chunk, columnstore settings] +api: + license: community + type: view +--- +import Since2180 from "versionContent/_partials/_since_2_18_0.mdx"; + +# timescaledb_information.chunk_columnstore_settings + +Retrieve the compression settings for each chunk in the columnstore. + + + +## Samples + +To retrieve information about settings: + +- **Show settings for all chunks in the columnstore**: + + ```sql + SELECT * FROM timescaledb_information.chunk_columnstore_settings + ``` + Returns: + ```sql + hypertable | chunk | segmentby | orderby + ------------+-------+-----------+--------- + measurements | _timescaledb_internal._hyper_1_1_chunk| | "time" DESC + ``` + +* **Find all chunk columnstore settings for a specific hypertable**: + + ```sql + SELECT * + FROM timescaledb_information.chunk_columnstore_settings + WHERE hypertable::TEXT LIKE 'metrics'; + ``` + Returns: + ```sql + hypertable | chunk | segmentby | orderby + ------------+-------+-----------+--------- + metrics | _timescaledb_internal._hyper_2_3_chunk | metric_id | "time" + ``` + +## Returns + +| Name | Type | Default | Required | Description | +|--|--|--|--|--| +|`hypertable`|`REGCLASS`|-|✖| The name of a hypertable in the columnstore | +|`chunk`|`REGCLASS`|-|✖| The name of a chunk in `hypertable` | +|`segmentby`|`TEXT`|-|✖| A list of columns used to segment `hypertable` | +|`orderby`|`TEXT`|-|✖| A list of columns used to order data in `hypertable`. Along with ordering and NULL ordering information. IAIN, I don't understand the second sentence. | + diff --git a/api/hypercore/chunk_columnstore_stats.md b/api/hypercore/chunk_columnstore_stats.md new file mode 100644 index 0000000000..c903a95760 --- /dev/null +++ b/api/hypercore/chunk_columnstore_stats.md @@ -0,0 +1,109 @@ +--- +api_name: chunk_columnstore_stats() +excerpt: Get statistics about chunks in the columnstore +topics: [hypercore, columnstore] +keywords: [columnstore, hypercore, statistics, chunks, information] +tags: [disk space, schemas, size] +api: + license: community + type: procedure +--- +import Since2180 from "versionContent/_partials/_since_2_18_0.mdx"; + +# chunk_columnstore_stats() Community + +Retrieve statistics about the chunks in the columnstore + +`chunk_columnstore_stats` returns the size of chunks in the columnstore, these values are computed when you call either: +- [add_columnstore_policy][add_columnstore_policy]: create a [job][job] that automatically moves chunks in a hypertable to the columnstore at a + specific time interval. +- [convert_to_columnstore][convert_to_columnstore]: manually add a specific chunk in a hypertable to the columnstore. + + +Inserting into a chunk in the columnstore does not change the chunk size. For more information about how to compute +chunk sizes, see [chunks_detailed_size][chunks_detailed_size]. + + + +## Samples + +To retrieve statistics about chunks: + +- **Show the status of the first two chunks in the `conditions` hypertable**: + ```sql + SELECT * FROM chunk_columnstore_stats('conditions') + ORDER BY chunk_name LIMIT 2; + ``` + Returns: + ```sql + -[ RECORD 1 ]------------------+---------------------- + chunk_schema | _timescaledb_internal + chunk_name | _hyper_1_1_chunk + compression_status | Uncompressed + before_compression_table_bytes | + before_compression_index_bytes | + before_compression_toast_bytes | + before_compression_total_bytes | + after_compression_table_bytes | + after_compression_index_bytes | + after_compression_toast_bytes | + after_compression_total_bytes | + node_name | + -[ RECORD 2 ]------------------+---------------------- + chunk_schema | _timescaledb_internal + chunk_name | _hyper_1_2_chunk + compression_status | Compressed + before_compression_table_bytes | 8192 + before_compression_index_bytes | 32768 + before_compression_toast_bytes | 0 + before_compression_total_bytes | 40960 + after_compression_table_bytes | 8192 + after_compression_index_bytes | 32768 + after_compression_toast_bytes | 8192 + after_compression_total_bytes | 49152 + node_name | + ``` + +- **Use `pg_size_pretty` to return a more human friendly format**: + + ```sql + SELECT pg_size_pretty(after_compression_total_bytes) AS total + FROM chunk_columnstore_stats('conditions') + WHERE compression_status = 'Compressed'; + ``` + Returns: + ```sql + -[ RECORD 1 ]--+------ + total | 48 kB + ``` + + +## Arguments + +| Name | Type | Default | Required | Description | +|--|--|--|--|--| +|`hypertable`|`REGCLASS`|-|✖| The name of a hypertable | + + +## Returns + +|Column|Type| Description | +|-|-|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +|`chunk_schema`|TEXT| Schema name of the chunk. | +|`chunk_name`|TEXT| Name of the chunk. | +|`compression_status`|TEXT| Current compression status of the chunk. | +|`before_compression_table_bytes`|BIGINT| Size of the heap before compression. Returns `NULL` if `compression_status` == `Uncompressed`. | +|`before_compression_index_bytes`|BIGINT| Size of all the indexes before compression. Returns `NULL` if `compression_status` == `Uncompressed`. | +|`before_compression_toast_bytes`|BIGINT| Size the TOAST table before compression. Returns `NULL` if `compression_status` == `Uncompressed`. | +|`before_compression_total_bytes`|BIGINT| Size of the entire chunk table (`before_compression_table_bytes` + `before_compression_index_bytes` + `before_compression_toast_bytes`) before compression. Returns `NULL` if `compression_status` == `Uncompressed`.| +|`after_compression_table_bytes`|BIGINT| Size of the heap after compression. Returns `NULL` if `compression_status` == `Uncompressed`. | +|`after_compression_index_bytes`|BIGINT| Size of all the indexes after compression. Returns `NULL` if `compression_status` == `Uncompressed`. | +|`after_compression_toast_bytes`|BIGINT| Size the TOAST table after compression. Returns `NULL` if `compression_status` == `Uncompressed`. | +|`after_compression_total_bytes`|BIGINT| Size of the entire chunk table (`after_compression_table_bytes` + `after_compression_index_bytes `+ `after_compression_toast_bytes`) after compression. Returns `NULL` if `compression_status` == `Uncompressed`. | +|`node_name`|TEXT| **DEPRECATED**: nodes the chunk is located on, applicable only to distributed hypertables. | + + +[add_columnstore_policy]: /api/:currentVersion:/hypercore/add_columnstore_policy/ +[convert_to_columnstore]: /api/:currentVersion:/hypercore/convert_to_columnstore/ +[job]: /api/:currentVersion:/actions/add_job/ +[chunks_detailed_size]: /api/:currentVersion:/hypertable/chunks_detailed_size/ diff --git a/api/hypercore/convert_to_columnstore.md b/api/hypercore/convert_to_columnstore.md new file mode 100644 index 0000000000..a0daa57e74 --- /dev/null +++ b/api/hypercore/convert_to_columnstore.md @@ -0,0 +1,54 @@ +--- +api_name: convert_to_columnstore() +excerpt: Manually add a chunk to thee columnstore +topics: [hypercore, columnstore, rowstore] +keywords: [columnstore, rowstore, hypercore] +tags: [chunks, hypercore] +api: + license: community + type: procedure +--- + +import Since2180 from "versionContent/_partials/_since_2_18_0.mdx"; + +# convert_to_columnstore() Community + +Manually convert a specific chunk in the hypertable rowstore to the columnstore. + +Although `convert_to_columnstore` gives you more more fine grained control, best practice is to use +[`add_columnstore_policy`][add_columnstore_policy]. You can also add chunks to the columnstore at a specific time +[running the job associated with your columnstore policy][run-job] manually. + +To move a chunk from the columnstore back to the rowstore, use [`convert_to_rowstore`][convert_to_rowstore]. + + + +## Samples + +To convert a single chunk to columnstore: + +``` sql +CALL convert_to_columnstore('_timescaledb_internal._hyper_1_2_chunk'); +``` + +## Arguments + +| Name | Type | Default | Required | Description | +|----------------------|--|---------|--|----------------------------------------------------------------------------------------------------------------| +| `chunk` | REGCLASS | - |✔| Name of the chunk to add to the columnstore. | +| `if_not_columnstore` | BOOLEAN | `true` |✖| Set to `false` so this job fails with an error rather than a warning if `chunk` is already in the columnstore. | +| `recompress` | BOOLEAN | `false` |✖| Set to `true` to add a chunk that had more data inserted after being added to the columnstore. | +| `hypercore_use_access_method` | BOOLEAN | `NULL` |✖| Set to `true` to use hypercore table access metod. If set to `NULL` it will use the value from `timescaledb.default_hypercore_use_access_method`. | + +## Returns + +Calls to `convert_to_columnstore` return: + +| Column | Type | Description | +|-------------------|--------------------|----------------------------------------------------------------------------------------------------| +| `chunk name` or `table` | REGCLASS or String | The name of the chunk added to the columnstore, or a table-like result set with zero or more rows. | + + +[add_columnstore_policy]: /api/:currentVersion:/hypercore/add_columnstore_policy/ +[run-job]: /api/:currentVersion:/actions/run_job/ +[convert_to_rowstore]: /api/:currentVersion:/hypercore/convert_to_rowstore/ diff --git a/api/hypercore/convert_to_rowstore.md b/api/hypercore/convert_to_rowstore.md new file mode 100644 index 0000000000..810f11ea9d --- /dev/null +++ b/api/hypercore/convert_to_rowstore.md @@ -0,0 +1,46 @@ +--- +api_name: convert_to_rowstore() +excerpt: Move a chunk from the columnstore to the rowstore +topics: [hypercore, columnstore] +keywords: [columnstore, hypercore, rowstore, chunks, backfilling] +api: + license: community + type: procedure +--- + +import Since2180 from "versionContent/_partials/_since_2_18_0.mdx"; +import HypercoreManualWorkflow from "versionContent/_partials/_hypercore_manual_workflow.mdx"; + +# convert_to_rowstore() Community + +Manually convert a specific chunk in the hypertable columnstore to the rowstore. + +If you need to modify or add a lot of data to a chunk in the columnstore, best practice is to stop +any [jobs][job] moving chunks to the columnstore, convert the chunk back to the rowstore, then modify the +data. After the update, [convert the chunk to the columnstore][convert_to_columnstore] and restart the jobs. +This workflow is especially useful if you need to backfill old data. + + + +## Samples + +To modify or add a lot of data to a chunk: + + + + + + + +## Arguments + +| Name | Type | Default | Required | Description| +|--|----------|---------|----------|-| +|`chunk`| REGCLASS | - | ✖ | Name of the chunk to be moved to the rowstore. | +|`if_compressed`| BOOLEAN | `true` | ✔ | Set to `false` so this job fails with an error rather than an warning if `chunk` is not in the columnstore | + +[job]: /api/:currentVersion:/actions/ +[alter_job]: /api/:currentVersion:/actions/alter_job/ +[convert_to_columnstore]: /api/:currentVersion:/hypercore/convert_to_columnstore/ +[informational-views]: /api/:currentVersion:/informational-views/jobs/ +[insert]: /use-timescale/:currentVersion:/write-data/insert/ diff --git a/api/hypercore/hypertable_columnstore_settings.md b/api/hypercore/hypertable_columnstore_settings.md new file mode 100644 index 0000000000..0798b73c17 --- /dev/null +++ b/api/hypercore/hypertable_columnstore_settings.md @@ -0,0 +1,62 @@ +--- +api_name: timescaledb_information.hypertable_columnstore_settings +excerpt: Get information about columnstore settings for all hypertables +topics: [hypercore, information, columnstore, hypertable] +keywords: [columnstore, hypercore, hypertable, information] +tags: [hypertable columnstore, columnstore settings] +api: + license: community + type: view +--- +import Since2180 from "versionContent/_partials/_since_2_18_0.mdx"; + +# timescaledb_information.hypertable_columnstore_settings + +Retrieve information about the settings for all hypertables in the columnstore. + + + +## Samples + +To retrieve information about settings: + +- **Show columnstore settings for all hypertables**: + + ```sql + SELECT * FROM timescaledb_information.hypertable_columnstore_settings; + ``` + Returns: + ```sql + hypertable | measurements + segmentby | + orderby | "time" DESC + compress_interval_length | + ``` + +- **Retrieve columnstore settings for a specific hypertable**: + + ```sql + SELECT * FROM timescaledb_information.hypertable_columnstore_settings WHERE hypertable::TEXT LIKE 'metrics'; + ``` + Returns: + ```sql + hypertable | metrics + segmentby | metric_id + orderby | "time" + compress_interval_length | + ``` + +## Returns + +|Name|Type| Description | +|-|-|---------------------------------------------------------------------------------------------------------------------| +|`hypertable`|`REGCLASS`| A hypertable which has the [columnstore enabled][compression_alter-table]. | +|`segmentby`|`TEXT`| The list of columns used to segment data | +|`orderby`|`TEXT`| List of columns used to order the data, along with ordering and NULL ordering information | +|`compress_interval_length`|`TEXT`| Interval used for [rolling up chunks during compression][rollup-compression] IAIN, update when main doc is written. | + + + +[rollup-compression]: /use-timescale/:currentVersion:/compression/manual-compression/#roll-up-uncompressed-chunks-when-compressing +[compression_alter-table]: /api/:currentVersion:/hypercore/alter_table/ + diff --git a/api/hypercore/hypertable_columnstore_stats.md b/api/hypercore/hypertable_columnstore_stats.md new file mode 100644 index 0000000000..a6558e3a59 --- /dev/null +++ b/api/hypercore/hypertable_columnstore_stats.md @@ -0,0 +1,82 @@ +--- +api_name: hypertable_columnstore_stats() +excerpt: Get columnstore statistics related to the columnstore +topics: [hypercore, columnstore] +keywords: [hypercore, columnstore, hypertables, information] +tags: [statistics, size] +api: + license: community + type: procedure +--- +import Since2180 from "versionContent/_partials/_since_2_18_0.mdx"; + +# hypertable_columnstore_stats() Community + +Retrieve compression statistics for the columnstore. + +For more information about using hypertables, including chunk size partitioning, +see [hypertables][hypertable-docs]. + + + +## Samples + +To retrieve compression statistics: + +- **Show the compression status of the `conditions` hypertable**: + + ```sql + SELECT * FROM hypertable_columnstore_stats('conditions'); + ``` + Returns: + ```sql + -[ RECORD 1 ]------------------+------ + total_chunks | 4 + number_compressed_chunks | 1 + before_compression_table_bytes | 8192 + before_compression_index_bytes | 32768 + before_compression_toast_bytes | 0 + before_compression_total_bytes | 40960 + after_compression_table_bytes | 8192 + after_compression_index_bytes | 32768 + after_compression_toast_bytes | 8192 + after_compression_total_bytes | 49152 + node_name | + ``` + +- **Use `pg_size_pretty` get the output in a more human friendly format**: + + ```sql + SELECT pg_size_pretty(after_compression_total_bytes) as total + FROM hypertable_columnstore_stats('conditions'); + ``` + Returns: + ```sql + -[ RECORD 1 ]--+------ + total | 48 kB + ``` + +## Arguments + +|Name|Type|Description| +|-|-|-| +|`hypertable`|REGCLASS|Hypertable to show statistics for| + +## Returns + +|Column|Type|Description| +|-|-|-| +|`total_chunks`|BIGINT|The number of chunks used by the hypertable. Returns `NULL` if `compression_status` == `Uncompressed`. | +|`number_compressed_chunks`|INTEGER|The number of chunks used by the hypertable that are currently compressed. Returns `NULL` if `compression_status` == `Uncompressed`. | +|`before_compression_table_bytes`|BIGINT|Size of the heap before compression. Returns `NULL` if `compression_status` == `Uncompressed`. | +|`before_compression_index_bytes`|BIGINT|Size of all the indexes before compression. Returns `NULL` if `compression_status` == `Uncompressed`. | +|`before_compression_toast_bytes`|BIGINT|Size the TOAST table before compression. Returns `NULL` if `compression_status` == `Uncompressed`. | +|`before_compression_total_bytes`|BIGINT|Size of the entire table (`before_compression_table_bytes` + `before_compression_index_bytes` + `before_compression_toast_bytes`) before compression. Returns `NULL` if `compression_status` == `Uncompressed`.| +|`after_compression_table_bytes`|BIGINT|Size of the heap after compression. Returns `NULL` if `compression_status` == `Uncompressed`. | +|`after_compression_index_bytes`|BIGINT|Size of all the indexes after compression. Returns `NULL` if `compression_status` == `Uncompressed`. | +|`after_compression_toast_bytes`|BIGINT|Size the TOAST table after compression. Returns `NULL` if `compression_status` == `Uncompressed`. | +|`after_compression_total_bytes`|BIGINT|Size of the entire table (`after_compression_table_bytes` + `after_compression_index_bytes `+ `after_compression_toast_bytes`) after compression. Returns `NULL` if `compression_status` == `Uncompressed`. | +|`node_name`|TEXT|nodes on which the hypertable is located, applicable only to distributed hypertables. Returns `NULL` if `compression_status` == `Uncompressed`. | + +[hypertable-docs]: /use-timescale/:currentVersion:/hypertables/ + diff --git a/api/hypercore/index.md b/api/hypercore/index.md new file mode 100644 index 0000000000..d9e21aa783 --- /dev/null +++ b/api/hypercore/index.md @@ -0,0 +1,99 @@ +--- +title: Hypercore +excerpt: Reference information about the TimescaleDB hybrid row-columnar storage engine +keywords: [hypercore] +tags: [hypercore] +products: [cloud, self_hosted] +api: + license: community +--- + +import Since2180 from "versionContent/_partials/_since_2_18_0.mdx"; + +# Hypercore + +Hypercore is the $TIMESCALE_DB hybrid row-columnar storage engine, designed specifically for +real-time analytics and powered by time-series data. The advantage of hypercore is its ability +to seamlessly switch between row-oriented and column-oriented storage. This flexibility enables +$CLOUD_LONG to deliver the best of both worlds, solving the key challenges in real-time analytics. + +Hypercore’s hybrid approach combines the benefits of row-oriented and column-oriented formats +in each $CLOUD_LONG service: + +- **Fast ingest with rowstore**: new data is initially written to the rowstore, which is optimized for + high-speed inserts and updates. + +- **Efficient analytics with columnstore**: you create [columnstore_policies][hypercore_workflow] + that automatically move your data to the columnstore as it _cools_. + +- **Faster queries on compressed data in columnstore**: in columnstore conversion, hypertable + chunks are compressed by more than 90%, and organized for efficient, large-scale queries more suitable + for analytics. This saves on storage costs, and keeps your queries operating at lightning speed. + +- **Full mutability with transactional semantics**: regardless of where data is stored, + hypercore provides full ACID support. + + + +## Hypercore workflow + +Best practice for using Hypercore is to: + + + +1. **Enable columnstore** + + * [Use `ALTER TABLE` for a hypertable][alter_table_hypercore] + ```sql + ALTER TABLE crypto_ticks SET ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'symbol'); + ``` + * [Use ALTER MATERIALIZED VIEW for a continuous aggregate][compression_continuous-aggregate] + ```sql + ALTER MATERIALIZED VIEW assets_candlestick_daily set ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'symbol' ); + ``` + +1. **Add a policy to move chunks to the columnstore at a specific time interval** + + For example, 7 days after the data was added to the table: + ``` sql + CALL add_columnstore_policy('crypto_ticks', after => INTERVAL '7d'); + ``` + See [add_columnstore_policy][add_columnstore_policy]. + +1. **View the policies that you set or the policies that already exist** + + ``` sql + SELECT * FROM timescaledb_information.jobs + WHERE proc_name='policy_compression'; + ``` + See [timescaledb_information.jobs][informational-views]. + + + +You can also [convert_to_columnstore][convert_to_columnstore] and [convert_to_rowstore][convert_to_rowstore] manually +for more fine-grained control over your data. + +## Limitations + +Chunks in the columnstore have the following limitations: + +* `ROW LEVEL SECURITY` is not supported on chunks in the columnstore. +* To add unique constraints on chunks in the columnstore [convert_the chunk to rowstore][convert_to_rowstore], + add the constraints to your data, then [convert the chunk back to the rowstore][convert_to_columnstore]. +* [SkipScan][skipscan] does not currently work on chunks in the columnstore. + + +[alter_table_hypercore]: /api/:currentVersion:/hypercore/alter_table/ +[compression_continuous-aggregate]: /api/:currentVersion:/hypercore/alter_materialized_view/ +[convert_to_rowstore]: /api/:currentVersion:/hypercore/convert_to_rowstore/ +[convert_to_columnstore]: /api/:currentVersion:/hypercore/convert_to_columnstore/ +[informational-views]: /api/:currentVersion:/informational-views/jobs/ +[skipscan]: /use-timescale/:currentVersion:/query-data/skipscan/ +[add_columnstore_policy]: /api/:currentVersion:/hypercore/add_columnstore_policy/ +[hypercore_workflow]: /api/:currentVersion:/hypercore/#hypercore-workflow +[alter_job]: /api/:currentVersion:/actions/alter_job/ +[remove_columnstore_policy]: /api/:currentVersion:/hypercore/remove_columnstore_policy/ diff --git a/api/hypercore/page-index/page-index.js b/api/hypercore/page-index/page-index.js new file mode 100644 index 0000000000..4e2fed1557 --- /dev/null +++ b/api/hypercore/page-index/page-index.js @@ -0,0 +1,55 @@ +module.exports = [ + { + title: "Hypercore", + href: "hypercore", + excerpt: + "Seamlessly switch between row-oriented and column-oriented storage", + children: [ + { + title: "add_columnstore_policy", + href: "add_columnstore_policy", + excerpt: "Convert a chunk to columnstore automatically in the background after it reaches a given age", + }, + { + title: "chunk_columnstore_settings", + href: "chunk_columnstore_settings", + excerpt: "Show the columnstore settings for each chunk that is a columnstore", + }, + { + title: "chunk_columnstore_stats", + href: "chunk_columnstore_stats", + excerpt: "Get statistics for columnstore chunks", + }, + { + title: "columnstore_settings", + href: "columnstore_settings", + excerpt: "Get information about columnstore-related settings for hypertables", + }, + { + title: "convert_to_columnstore", + href: "convert_to_columnstore", + excerpt: "Convert a specific chunk from rowstore to columnstore", + }, + { + title: "convert_to_rowstore", + href: "convert_to_rowstore", + excerpt: "Convert a specific chunk from columnstore to rowstore", + }, + { + title: "hypertable_columnstore_settings", + href: "hypertable_columnstore_settings", + excerpt: "Returns information about the columnstore settings for each hypertable", + }, + { + title: "hypertable_columnstore_stats", + href: "hypertable_columnstore_stats", + excerpt: "Get columnstore statistics for hypertables", + }, + { + title: "remove_columnstore_policy", + href: "remove_columnstore_policy", + excerpt: "Remove the columnstore policy", + }, + ], + }, +]; diff --git a/api/hypercore/remove_columnstore_policy.md b/api/hypercore/remove_columnstore_policy.md new file mode 100644 index 0000000000..08bb519745 --- /dev/null +++ b/api/hypercore/remove_columnstore_policy.md @@ -0,0 +1,47 @@ +--- +api_name: remove_columnstore_policy() +excerpt: Remove a columnstore policy from a hypertable +topics: [hypercore, columnstore, jobs] +keywords: [hypercore, columnstore, policies, remove] +tags: [delete, drop] +api: + license: community + type: procedure +--- +import Since2180 from "versionContent/_partials/_since_2_18_0.mdx"; + +# remove_columnstore_policy() + +Remove a columnstore policy from a hypertable or continuous aggregate. + +To restart automatic chunk migration to the columnstore, you need to call +[add_columnstore_policy][add_columnstore_policy] again. + + + +## Samples + +You see the columnstore policies in the [informational views][informational-views]. + +- **Remove the columnstore policy from the `cpu` table**: + + ``` sql + CALL remove_columnstore_policy('cpu'); + ``` + +- **Remove the columnstore policy from the `cpu_weekly` continuous aggregate**: + + ``` sql + CALL remove_columnstore_policy('cpu_weekly'); + ``` + +## Arguments + +| Name | Type | Default | Required | Description | +|--|--|--|--|-| +|`hypertable`|REGCLASS|-|✔| Name of the hypertable or continuous aggregate to remove the policy from| +| `if_exists` | BOOLEAN | `false` |✖| Set to `true` so this job fails with a warning rather than an error if a columnstore policy does not exist on `hypertable` | + + +[informational-views]: /api/:currentVersion:/informational-views/jobs/ +[add_columnstore_policy]: /api/:currentVersion:/hypercore/add_columnstore_policy/ diff --git a/api/merge_chunks.md b/api/merge_chunks.md new file mode 100644 index 0000000000..73924abae3 --- /dev/null +++ b/api/merge_chunks.md @@ -0,0 +1,56 @@ +--- +api_name: merge_chunks() +excerpt: Merge two or more chunks into one chunk +topics: [hypertables] +keywords: [hypertables, chunk, merge] +api: + license: community + type: procedure +--- + +# merge_chunks() + +Merge two or more chunks into one. + +The partition boundaries for the new chunk is the union of all partitions of the merged chunks. +The new chunk retains the name, constraints, and triggers of the _first_ chunk in the partition order. + +You can only merge chunks that have directly adjacent partitions. It is not possible to merge +chunks that have another chunk, or an empty range between them in any of the partitioning +dimensions. + +In this first release, chunk merging has the following limitations. You cannot: + +* Merge compressed chunks +* Merge chunks using table access methods other than heap +* Merge chunks with tiered data +* Read or write from the chunks while they are being merged + + + +## Samples + +- Merge two chunks: + + ```sql + CALL merge_chunks('_timescaledb_internal._hyper_1_1_chunk', '_timescaledb_internal._hyper_1_2_chunk'); + ``` + +- Merge more than two chunks: + + ```sql + CALL merge_chunks('{_timescaledb_internal._hyper_1_1_chunk, _timescaledb_internal._hyper_1_2_chunk, _timescaledb_internal._hyper_1_3_chunk}'); + ``` + + +## Arguments + +You can merge either two chunks, or an arbitrary number of chunks specified as an array of chunk identifiers. +When you call `merge_chunks`, you must specify either `chunk1` and `chunk2`, or `chunks`. You cannot use both +arguments. + + +| Name | Type | Default | Required | Description | +|--------------------|-------------|--|--|------------------------------------------------| +| `chunk1`, `chunk2` | REGCLASS | - | ✖ | The two chunk to merge in partition order | +| `chunks` | REGCLASS[] |- | ✖ | The array of chunks to merge in partition order | diff --git a/api/page-index/page-index.js b/api/page-index/page-index.js index 408913ed06..dd4b6e046c 100644 --- a/api/page-index/page-index.js +++ b/api/page-index/page-index.js @@ -34,6 +34,10 @@ module.exports = [ title: "reorder_chunk", href: "reorder_chunk", }, + { + title: "merge_chunks", + href: "merge_chunks", + }, { title: "move_chunk", href: "move_chunk", @@ -117,102 +121,59 @@ module.exports = [ ], }, { - title: "Distributed hypertables", - type: "directory", - href: "distributed-hypertables", + title: "Hypercore", + excerpt: "Seamlessly switch between fast row-oriented storage and efficient column-oriented storage", + href: "hypercore", children: [ { - title: "create_distributed_hypertable", - href: "create_distributed_hypertable", - }, - { - title: "add_data_node", - href: "add_data_node", - }, - { - title: "attach_data_node", - href: "attach_data_node", - }, - { - title: "alter_data_node", - href: "alter_data_node", - }, - { - title: "detach_data_node", - href: "detach_data_node", - }, - { - title: "delete_data_node", - href: "delete_data_node", - }, - { - title: "distributed_exec", - href: "distributed_exec", - }, - { - title: "set_number_partitions", - href: "set_number_partitions", - }, - { - title: "set_replication_factor", - href: "set_replication_factor", - }, - { - title: "copy_chunk", - href: "copy_chunk_experimental", - }, - { - title: "move_chunk", - href: "move_chunk_experimental", + title: "ALTER MATERIALIZED VIEW", + href: "alter_materialized_view", + excerpt: "Enable the columnstore for a materialized view.", }, { - title: "cleanup_copy_chunk_operation", - href: "cleanup_copy_chunk_operation_experimental", + title: "ALTER TABLE", + href: "alter_table", + excerpt: "Enable the columnstore for a hypertable.", }, { - title: "create_distributed_restore_point", - href: "create_distributed_restore_point", + title: "add_columnstore_policy", + href: "add_columnstore_policy", + excerpt: "Automatically convert chunks in the hypertable rowstore to the columnstore after a specific time interval", }, - ], - }, - { - title: "Compression", - type: "directory", - href: "compression", - description: - "We highly recommend reading the blog post and tutorial about compression before trying to set it up for the first time.", - children: [ { - title: "ALTER TABLE (Compression)", - href: "alter_table_compression", + title: "remove_columnstore_policy", + href: "remove_columnstore_policy", + excerpt: "Remove a columnstore policy from a hypertable or continuous aggregate", }, { - title: "add_compression_policy", - href: "add_compression_policy", + title: "convert_to_columnstore", + href: "convert_to_columnstore", + excerpt: "Manually convert a specific chunk in the hypertable rowstore to the columnstore", }, { - title: "remove_compression_policy", - href: "remove_compression_policy", + title: "convert_to_rowstore", + href: "convert_to_rowstore", + excerpt: "Manually convert a specific chunk in the hypertable columnstore to the rowstore", }, { - title: "compress_chunk", - href: "compress_chunk", + title: "hypertable_columnstore_settings", + href: "hypertable_columnstore_settings", + excerpt: "Retrieve information about the settings for all hypertables in the columnstore", }, { - title: "decompress_chunk", - href: "decompress_chunk", + title: "hypertable_columnstore_stats", + href: "hypertable_columnstore_stats", + excerpt: "Retrieve compression statistics for the columnstore", }, { - title: "recompress_chunk", - href: "recompress_chunk", + title: "chunk_columnstore_settings", + href: "chunk_columnstore_settings", + excerpt: "Retrieve the compression settings for each chunk in the columnstore", }, { - title: "hypertable_compression_stats", - href: "hypertable_compression_stats", - }, - { - title: "chunk_compression_stats", - href: "chunk_compression_stats", + title: "chunk_columnstore_stats", + href: "chunk_columnstore_stats", + excerpt: "Retrieve statistics about the chunks in the columnstore", }, ], }, @@ -607,6 +568,105 @@ module.exports = [ description: "An overview of what different tags represent in the API section of Timescale Documentation.", }, + { + title: "Compression (Old API, replaced by Hypercore)", + href: "compression", + description: + "We highly recommend reading the blog post and tutorial about compression before trying to set it up for the first time.", + children: [ + { + title: "ALTER TABLE (Compression)", + href: "alter_table_compression", + }, + { + title: "add_compression_policy", + href: "add_compression_policy", + }, + { + title: "remove_compression_policy", + href: "remove_compression_policy", + }, + { + title: "compress_chunk", + href: "compress_chunk", + }, + { + title: "decompress_chunk", + href: "decompress_chunk", + }, + { + title: "recompress_chunk", + href: "recompress_chunk", + }, + { + title: "hypertable_compression_stats", + href: "hypertable_compression_stats", + }, + { + title: "chunk_compression_stats", + href: "chunk_compression_stats", + }, + ], + }, + { + title: "Distributed hypertables (Sunsetted v2.14.x)", + type: "directory", + href: "distributed-hypertables", + children: [ + { + title: "create_distributed_hypertable", + href: "create_distributed_hypertable", + }, + { + title: "add_data_node", + href: "add_data_node", + }, + { + title: "attach_data_node", + href: "attach_data_node", + }, + { + title: "alter_data_node", + href: "alter_data_node", + }, + { + title: "detach_data_node", + href: "detach_data_node", + }, + { + title: "delete_data_node", + href: "delete_data_node", + }, + { + title: "distributed_exec", + href: "distributed_exec", + }, + { + title: "set_number_partitions", + href: "set_number_partitions", + }, + { + title: "set_replication_factor", + href: "set_replication_factor", + }, + { + title: "copy_chunk", + href: "copy_chunk_experimental", + }, + { + title: "move_chunk", + href: "move_chunk_experimental", + }, + { + title: "cleanup_copy_chunk_operation", + href: "cleanup_copy_chunk_operation_experimental", + }, + { + title: "create_distributed_restore_point", + href: "create_distributed_restore_point", + }, + ], + }, ], }, ]; diff --git a/api/remove_continuous_aggregate_policy.md b/api/remove_continuous_aggregate_policy.md index dfc052bf35..46591f2357 100644 --- a/api/remove_continuous_aggregate_policy.md +++ b/api/remove_continuous_aggregate_policy.md @@ -21,8 +21,9 @@ remove_continuous_aggregate_policy( ``` -To view the existing continuous aggregate policies, see the [policies -informational view](/api/latest/informational-views/policies/). + +To view the existing continuous aggregate policies, see the [policies informational view](/api/:currentVersion:/informational-views/policies/). + ## Required arguments diff --git a/api/rollup.md b/api/rollup.md index 2c9b49bd2e..67b49d3883 100644 --- a/api/rollup.md +++ b/api/rollup.md @@ -67,7 +67,7 @@ WITH ohlc AS ( SELECT time_bucket('1 minute'::interval, ts) AS minute_bucket, symbol, toolkit_experimental.ohlc(ts, price) - FROM stocks_real_time + FROM crypto_ticks GROUP BY minute_bucket, symbol ) SELECT time_bucket('1 day'::interval , bucket) AS daily_bucket diff --git a/getting-started/aggregation.md b/getting-started/aggregation.md deleted file mode 100644 index d9850f0f52..0000000000 --- a/getting-started/aggregation.md +++ /dev/null @@ -1,214 +0,0 @@ ---- -title: Continuous aggregation -excerpt: A must have for real-time analytics, continuous aggregates summarize your data and stay up-to-date as new data pours in. Create and query a continuous aggregate in Timescale Cloud -products: [cloud] -keywords: [continuous aggregates, create] -layout_components: [next_prev_large] -content_group: Getting started ---- - -import CaggsIntro from "versionContent/_partials/_caggs-intro.mdx"; -import CaggsTypes from "versionContent/_partials/_caggs-types.mdx"; -import CandlestickIntro from "versionContent/_partials/_candlestick_intro.mdx"; - -# Continuous aggregation - -Aggregation is a way of combing data to get insights from it. At its simplest, -aggregation is something like looking for an average. For example, if you have -data showing temperature changes over time, you can calculate an average of -those temperatures, or a count of how many readings have been taken. Average, -sum, and count are all example of simple aggregates. - -However, aggregation calculations can get big and slow, quickly. If you want to -find the average open and closing values of a range of stocks for each day, then -you need something a little more sophisticated. That's where Timescale -continuous aggregates come in. Continuous aggregates can minimize the number of -records that you need to look up to perform your query. - -## Continuous aggregates - - - - - -In this section, you create a continuous aggregate, and query it for more -information about the trading data. - - - -## Create an aggregate query - - - -In this section, you use a `SELECT` statement to find the high and low values -with `min` and `max` functions, and the open and close values with `first` and -`last` functions. You then aggregate the data into 1 day buckets, like this: - - - -Then, you organize the results by day and symbol: - - - - - -### Creating an aggregate query - -1. At the command prompt, use the `psql` connection string from the cheat sheet - you downloaded to connect to your database. -1. At the `psql` prompt, type this query: - - - -1. Type `q` to return to the `psql` prompt. - - - -## Create a continuous aggregate - -Now that you have an aggregation query, you can use it to create a continuous -aggregate. - -In this section, your query starts by creating a materialized view called -`stock_candlestick_daily`, then converting it into a Timescale continuous -aggregate: - - - -Then, you give the aggregate query you created earlier as the contents for the -continuous aggregate: - - - -When you run this query, you create the view, and populate the view with the -aggregated calculation. This can take a few minutes to run, because it needs to -perform these calculations across all of your stock trade data the first time. - -When you continuous aggregate has been created and the data aggregated for the -first time, you can query your continuous aggregate. For example, you can look -at all the aggregated data, like this: - - - -Or you can look at a single stock, like this: - - - - - -### Creating a continuous aggregate - -1. At the command prompt, use the `psql` connection string from the cheat sheet - you downloaded to connect to your database. -1. At the `psql` prompt, type this query: - - ```sql - CREATE MATERIALIZED VIEW stock_candlestick_daily - WITH (timescaledb.continuous) AS - SELECT - time_bucket('1 day', "time") AS day, - symbol, - max(price) AS high, - first(price, time) AS open, - last(price, time) AS close, - min(price) AS low - FROM stocks_real_time srt - GROUP BY day, symbol; - ``` - -1. Query your continuous aggregate for all stocks: - - - -1. Query your continuous aggregate for Tesla stock: - - - - - -For more information about how continuous aggregates work, see the -[continuous aggregates section][continuous-aggregates]. - -[continuous-aggregates]: /use-timescale/:currentVersion:/continuous-aggregates -[time-buckets]: /use-timescale/:currentVersion:/time-buckets/ diff --git a/getting-started/index.md b/getting-started/index.md index 3e356471c4..5785eec2a1 100644 --- a/getting-started/index.md +++ b/getting-started/index.md @@ -16,21 +16,24 @@ import WhereNext from "versionContent/_partials/_where-to-next.mdx"; This section shows you how to: -1. [Create and connect to a Timescale service][services-create] -1. [Run queries from Timescale Console][run-queries-from-console] -1. [Ingest some real financial data into your database][ingest-data] -1. [Construct some interesting queries][queries] Try out some live queries -1. [Create and query a continuous aggregates][caggs] +1. [Create and connect to a $SERVICE_LONG][services-create]: choose the capabilities that match your business and + engineering needs on $COMPANY's cloud-based PostgreSQL platform. +1. [Run queries from $CONSOLE][run-queries-from-console]: securely interact with your data in the $CONSOLE UI. +1. [Try the main features in Timescale products][test-drive]: rapidly implement the features in $CLOUD_LONG that + enable you to ingest and query data faster while keeping prices low. -Already know the basics? See the -[more advanced tutorials][tutorials], or see how to -[Use Timescale][use-timescale]. +What next? See the [use case tutorials][tutorials], interact with the data in your $SERVICE_LONG using +[your favorite programming language][connect-with-code], integrate your $SERVICE_LONG with a range of +[third-party tools][integrations], plain old [Use Timescale][use-timescale], or dive into [the API][use-the-api]. [tutorials]: /tutorials/:currentVersion:/ +[connect-with-code]: /quick-start/:currentVersion:/ +[integrations]: /use-timescale/:currentVersion:/integrations/ +[use-the-api]: /api/:currentVersion:/ [use-timescale]: /use-timescale/:currentVersion:/ [services-create]: /getting-started/:currentVersion:/services#create-your-timescale-account [services-connect]: /getting-started/:currentVersion:/services/#connect-to-your-service +[test-drive]: /getting-started/:currentVersion:/try-key-features-timescale-products/ [run-queries-from-console]: /getting-started/:currentVersion:/run-queries-from-console/ -[ingest-data]: /getting-started/:currentVersion:/time-series-data/ -[queries]: /getting-started/:currentVersion:/queries/ -[caggs]: /getting-started/:currentVersion:/aggregation/ +[ingest-data]: /getting-started/:currentVersion:/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables + diff --git a/getting-started/next-steps.md b/getting-started/next-steps.md deleted file mode 100644 index 7b0b33bbcc..0000000000 --- a/getting-started/next-steps.md +++ /dev/null @@ -1,25 +0,0 @@ ---- -title: Next steps -excerpt: Explore the features that are available for your real-time analytics workloads after you have created a Timescale Cloud account and launched your first service -products: [cloud] -keywords: [data migration, ingest, visualize, connect] -layout_components: [next_prev_large] -content_group: Getting started ---- - -# Where to next? - -To continue exploring Timescale, here are some things you might like to try: - -* Try some [tutorials][tutorials]. -* Write a program using a Timescale database backend in - [your favorite programming language][connect-with-code]. -* Get your Timescale database integrated with a range of - [third-party tools][integrations] - for tasks like data ingestion, visualization, and logging. -* Extend your Timescale database with [PostgreSQL extensions][extensions]. - -[connect-with-code]: /quick-start/:currentVersion:/ -[integrations]: /use-timescale/:currentVersion:/integrations/ -[extensions]: /use-timescale/:currentVersion:/extensions/ -[tutorials]: /tutorials/:currentVersion:/ diff --git a/getting-started/page-index/page-index.js b/getting-started/page-index/page-index.js index ef51f973f8..01cc595287 100644 --- a/getting-started/page-index/page-index.js +++ b/getting-started/page-index/page-index.js @@ -9,36 +9,16 @@ module.exports = [ href: "services", excerpt: "Create a Timescale service and connect to it", }, + { + title: "Try the key Timescale features", + href: "try-key-features-timescale-products", + excerpt: "Improve database performance with Hypertables, time bucketing, continuous aggregates, compression, data tiering, and high availability", + }, { title: "Run your queries from Timescale Console", href: "run-queries-from-console", excerpt: "Run your queries securely from inside Timescale Console", }, - { - title: "Tables and hypertables", - href: "tables-hypertables", - excerpt: "Create tables and hypertables for your data", - }, - { - title: "Time-series data", - href: "time-series-data", - excerpt: "Add time-series data to your database", - }, - { - title: "Queries", - href: "queries", - excerpt: "Query your data using full SQL", - }, - { - title: "Aggregation", - href: "aggregation", - excerpt: "Query aggregated data, and create a continuous aggregate", - }, - { - title: "Next steps", - href: "next-steps", - excerpt: "Get even more from your Timescale database", - }, ], }, ]; diff --git a/getting-started/queries.md b/getting-started/queries.md deleted file mode 100644 index aa0d812dff..0000000000 --- a/getting-started/queries.md +++ /dev/null @@ -1,217 +0,0 @@ ---- -title: Queries -excerpt: Timescale Cloud supports full SQL, so you don't need to learn a custom query language. Construct and run simple SQL queries on your Timescale Cloud service -products: [cloud] -keywords: [queries] -layout_components: [next_prev_large] -content_group: Getting started ---- - -# Queries - -Timescale supports full SQL, so you don't need to learn a custom query language. -This section contains some simple queries that you can run directly on this -page. When you have constructed the perfect query, use the copy button to use it -on your own database. - -Most of the queries in this section look for the last four days of data. This is -to account for the fact there are no stock trades over the weekends, and to make -sure that you always get some data in your results. - -The main building block of all SQL queries is the `SELECT` statement. It is an -instruction to select data from a database. Doing a quick `SELECT` query is -often the first thing you do with a new database, just to make sure that your -data is stored in your database in the way you expect it to be. - -## Use SELECT to return data - -This first section uses a `SELECT` statement to ask your database to return -every column, represented by the asterisk, from the `stocks_real_time srt` -table, like this: - - - -If your table is very big, you might not want to return every row. You can -limit the number of rows that get returned with a `LIMIT` clause: - - - - - -### Using SELECT to return data - -1. At the command prompt, use the `psql` connection string from the cheat sheet - you downloaded to connect to your database. -1. At the `psql` prompt, type this query. - - - Get a sneak peek at the results by clicking "Run query" below. This runs the - SQL query against a live instance curated by Timescale. - - - - -1. Type `q` to return to the `psql` prompt. - - - -## Use ORDER BY to organize results - -In the previous section, you saw a selection of rows from the table. Usually, -you want to order the rows so that you see the most recent trades. You can -change how your results are displayed using an `ORDER BY` statement. - -In this section, you query Tesla's stock with a `SELECT` query like this, -which asks for all of the trades from the `stocks_real_time srt` table, with the -`TSLA` symbol, and which has day volume data: - - - -Then, you add an `ORDER BY` statement to order the results by time in descending -order, and also by day volume in descending order. The day volume shows the -total number of trades for this stock for the day. Every time another trade -occurs, the day volume figure increases by 1. Here is the `ORDER BY` statement: - - - -Finally, to limit the number of results, you can use a `LIMIT` clause again: - - - - - -### Using ORDER BY to organize results - -1. At the command prompt, use the `psql` connection string from the cheat sheet - you downloaded to connect to your database. -1. At the `psql` prompt, type this query: - - - - There are multiple trades every second, but you know that the order is - correct, because the `day_volume` column is ordered correctly. - - - -## Get the first and last value - -Timescale has custom SQL functions that can help make time-series analysis -easier and faster. In this section, you learn about two common Timescale -functions: `first` to find the earliest value within a group, and `last` to find -the most recent value within a group. - -The `first()` and `last()` functions retrieve the first and last value of one -column when ordered by another. For example, the stock data has a timestamp -column called `time`, and a numeric column called `price`. You can use -`first(price, time)` to get the first value in the `price` column when ordered -with an increasing `time` column. - -In this query, you start by selecting the `first()` and `last()` trading price -for every stock in the `stocks_real_time srt` table for the last four days: - - now() - INTERVAL '4 days' -`} /> - -Then, you organize the results so that you can see the first and last value for -each stock together with a `GROUP BY` statement, and in alphabetical order with -an `ORDER BY` statement, like this: - - - -For more information about these functions, see the API documentation for -[first()][first], and [last()][last]. - - - -### Getting the first and last value - -1. At the command prompt, use the `psql` connection string from the cheat sheet - you downloaded to connect to your database. -1. At the `psql` prompt, type this query: - - - -1. Type `q` to return to the `psql` prompt. - - - -## Use time buckets to get values - -To make it easier to look at numbers over different time ranges, you can use the -Timescale `time_bucket` function. Time buckets are used to group data, so that -you can perform calculations over different time periods. Time buckets represent -a specific point in time, so all the timestamps for data in a single time bucket -use the bucket timestamp. - -In this section, you use the same query as the previous section to find the -`first` and `last` values, but start by organizing the data into 1-hour time -buckets. In the last section, you retrieves the first and last value of a -column, this time, you retrieve the first and last value for a 1-hour time bucket. - -Start by declaring the time bucket interval to use, and give your time bucket a -name: - - - -Then, you can add the query in the same way as you used before: - - now() - INTERVAL '4 days' -`} /> - -Finally, organize the results by time bucket, using the `GROUP BY` statement, -like this: - - - -For more information about time bucketing, see the [time bucket section][time-buckets]. - - - -### Using time buckets to get values - -1. At the command prompt, use the `psql` connection string from the cheat sheet - you downloaded to connect to your database. -1. At the `psql` prompt, type this query: - - - -1. Type `q` to return to the `psql` prompt. - - - When you create a hypertable, Timescale automatically creates an index on - the time column. However, you often need to filter your time-series data on - other columns as well. Using indexes appropriately helps your queries - perform better. For more information about indexing, see the - [about indexing section](https://docs.timescale.com/use-timescale/latest/schema-management/about-indexing/) - - - - -[first]: /api/:currentVersion:/hyperfunctions/first/ -[last]: /api/:currentVersion:/hyperfunctions/last/ -[time-buckets]: /use-timescale/:currentVersion:/time-buckets/ diff --git a/getting-started/run-queries-from-console.md b/getting-started/run-queries-from-console.md index 51ea1b6b80..13c53c8753 100644 --- a/getting-started/run-queries-from-console.md +++ b/getting-started/run-queries-from-console.md @@ -6,6 +6,8 @@ layout_components: [next_prev_large] content_group: Getting started --- +import WhereNext from "versionContent/_partials/_where-to-next.mdx"; + # Run your queries from Timescale Console As Timescale Cloud is based on PostgreSQL, you can use lots of [different tools][integrations] to @@ -16,9 +18,9 @@ In Timescale Console you can use the following ways to run SQL queries against y - [Data mode][run-popsql]: a rich experience powered by PopSQL. You can write queries with autocomplete, save them in folders, share them, create charts/dashboards, and much more. -- [SQL Assistant in Data mode][sql-assistant]: write, fix, and organize SQL faster and more accurately. +- [SQL Assistant in data mode][sql-assistant]: write, fix, and organize SQL faster and more accurately. -- [SQL editor in Ops mode][run-sqleditor]: a simple SQL editor in Ops mode that lets you run ad-hoc ephemeral +- [SQL editor in ops mode][run-sqleditor]: a simple SQL editor in the ops mode that lets you run ad-hoc ephemeral queries. This is useful for quick one-off tasks like creating an index on a small table or inspecting `pg_stat_statements`. @@ -226,6 +228,7 @@ To use SQL editor with Timescale: details. + [readreplica]: /use-timescale/:currentVersion:/ha-replicas/read-scaling/ [run-popsql]: /getting-started/:currentVersion:/run-queries-from-console/#data-mode diff --git a/getting-started/services.md b/getting-started/services.md index 714df524b9..e602e202b5 100644 --- a/getting-started/services.md +++ b/getting-started/services.md @@ -11,6 +11,7 @@ import Connect from "versionContent/_partials/_cloud-connect.mdx"; import CreateAHypertable from "versionContent/_partials/_create-hypertable.mdx"; import ServiceOverview from "versionContent/_partials/_service-overview.mdx"; import CloudIntro from "versionContent/_partials/_cloud-intro.mdx"; +import WhereNext from "versionContent/_partials/_where-to-next.mdx"; # Create your first $SERVICE_LONG @@ -23,47 +24,43 @@ To start using $CLOUD_LONG for your data: 1. [Create a $COMPANY account][create-an-account]: register in $CONSOLE to get a centralized point to administer and interact with your data. 1. [Create a $SERVICE_LONG][create-a-service]: that is, a PostgreSQL database instance, powered by [$TIMESCALE_DB][timescaledb], built for production, and extended with cloud features like transparent data tiering to object storage. 1. [Connect to your $SERVICE_LONG][connect-to-your-service]: to run queries, add and migrate your data from other sources. -1. [Create a hypertable][create-a-hypertable]: create a standard PostgreSQL table in your service, then convert it into a [hypertable][hypertables]. - - Anything you can do with regular PostgreSQL tables, you can do with hypertables, just with better performance and improved user experience for time-series data. ## Create a $SERVICE_LONG -Now that you have an active $COMPANY account, you create and manage your $SERVICE_SHORTs in $CONSOLE. When you create a service, you give a structure for your future data, which you then add manually or migrate from other services. All relevant $CLOUD_LONG features under your pricing plan are automatically available when you create a service. +Now that you have an active $CLOUD_LONG account, you create and manage your $SERVICE_SHORTs in $CONSOLE. When you create a $SERVICE_SHORT, you give a structure for your future data, which you then add manually or migrate from other services. All relevant $CLOUD_LONG features under your pricing plan are automatically available when you create a $SERVICE_SHORT. -1. In the [service creation page][create-service], choose the PostgreSQL service or add `Time-series and analytics` and `AI and Vector` capabilities. Click `Save and continue`. +1. In the [$SERVICE_SHORT creation page][create-service], choose the PostgreSQL capability or add `Time-series and analytics` and `AI and Vector` capabilities. Click `Save and continue`. ![Create Timescale Cloud service](https://assets.timescale.com/docs/images/create-timescale-service.png) -1. Follow the next steps in `Create a service` to configure the compute size, environment, availability, region, and service name. Then click `Create service`. +1. Follow the next steps in `Create a service` to configure the compute size, environment, availability, region, and $SERVICE_SHORT name. Then click `Create service`. - Your service is constructed immediately and is ready to use. + Your $SERVICE_SHORT is constructed immediately and is ready to use. -1. Click `Download the config` and store the configuration information you need to connect to this service in a secure location. +1. Click `Download the config` and store the configuration information you need to connect to this $SERVICE_SHORT in a secure location. - This file contains the passwords and configuration information you need to connect to your service using the + This file contains the passwords and configuration information you need to connect to your $SERVICE_SHORT using the $CONSOLE data mode, from the command line, or using third-party database administration tools. -If you choose to go directly to the service overview, [Check your service and connect to it][connect-to-your-service] +If you choose to go directly to the service overview, [Connect to your $SERVICE_SHORT][connect-to-your-service] shows you how to connect. -## Connect to your service +## Connect to your $SERVICE_SHORT -A Timescale service comes with access control to its data. To be able to run queries and perform other operations, connect to the service with credentials generated during its creation. +A $SERVICE_LONG comes with access control to its data. To be able to run queries and perform other operations, connect to the $SERVICE_SHORT with credentials generated during its creation. -## Create a hypertable +And that is it, you are up and running. Enjoy developing with $COMPANY. - + -And that is it, you are up and running. Enjoy developing with $COMPANY. [tsc-portal]: https://console.cloud.timescale.com/ [services-how-to]: /use-timescale/:currentVersion:/services/ diff --git a/getting-started/tables-hypertables.md b/getting-started/tables-hypertables.md deleted file mode 100644 index 43589aaf13..0000000000 --- a/getting-started/tables-hypertables.md +++ /dev/null @@ -1,119 +0,0 @@ ---- -title: Tables and hypertables -excerpt: Hypertables are PostgreSQL tables designed to boost the performance of your real-time analytical queries. Create a table and then convert it into a hypertable in Timescale Console -products: [cloud] -keywords: [hypertables, create] -layout_components: [next_prev_large] -content_group: Getting started ---- - -import HypertableIntro from "versionContent/_partials/_hypertables-intro.mdx"; - -# Tables and hypertables - - - -Databases are made up of tables that contain your data. In PostgreSQL, these -tables are relational, so the data in one table relates to the data in another -table. In Timescale, you use regular PostgreSQL relational tables, in addition -to special time-series hypertables. - -Hypertables are designed specifically for time-series data, so they have a few -special qualities that makes them different to a regular PostgreSQL table. A -hypertable is always partitioned on time, but can also be partitioned on -additional columns as well. The other special thing about hypertables is that -they are broken down into smaller tables called chunks. - -In this section, you create a hypertable for time-series data, and regular -PostgreSQL tables for relational data. You also create an index on your -hypertable, which isn't required, but can help your queries run more efficiently. -One of the other special qualities of hypertables is that you can also create -indexes later on, if you need to. - -For more information, see -[the hypertables section][hypertable-how-to]. - -## Create your first hypertable - -For the financial dataset used in this guide, create a hypertable named -`stocks_real_time` that contains second-by-second stock-trade data for the top -100 most-traded symbols. - - - -### Creating your first hypertable - -1. At the command prompt, use the `psql` connection string from the cheat sheet - you downloaded to connect to your database. -1. Create a regular PostgreSQL table to store the real-time stock trade data - using `CREATE TABLE`: - - ```sql - CREATE TABLE stocks_real_time ( - time TIMESTAMPTZ NOT NULL, - symbol TEXT NOT NULL, - price DOUBLE PRECISION NULL, - day_volume INT NULL - ); - ``` - -1. Convert the regular table into a hypertable partitioned on the `time` column - using the `create_hypertable()` function provided by Timescale. You must - provide the name of the table (`stocks_real_time`) and the column in that - table that holds the timestamp data to use for partitioning (`time`): - - - -1. Create an index to support efficient queries on the `symbol` and `time` - columns: - - - - - -## Create regular PostgreSQL tables for relational data - -Timescale isn't just for hypertables. When you have other relational data that -enhances your time-series data, you can create regular PostgreSQL tables just as -you would normally. For this dataset, there is one other table of data called -`company`. - - - -### Creating regular PostgreSQL tables - -1. Add a table to store the company name and symbol for the stock trade data: - - ```sql - CREATE TABLE company ( - symbol TEXT NOT NULL, - name TEXT NOT NULL - ); - ``` - -1. You now have two tables within your Timescale database. One hypertable named - `stocks_real_time`, and one normal PostgreSQL table named `company`. You can - check this by running this command at the `psql` prompt: - - - - This command returns information about your tables, like this: - - - - - -[hypertable-how-to]: /use-timescale/:currentVersion:/hypertables/ diff --git a/getting-started/time-series-data.md b/getting-started/time-series-data.md deleted file mode 100644 index b4d7aff625..0000000000 --- a/getting-started/time-series-data.md +++ /dev/null @@ -1,62 +0,0 @@ ---- -title: Time-series data -excerpt: Find out what time-series data is, how it is generated, and how you can run real-time analytics on it with Timescale Cloud -products: [cloud] -keywords: [ingest] -tags: [add, data, time-series] -layout_components: [next_prev_large] -content_group: Getting started ---- - -import TimeseriesIntro from "versionContent/_partials/_timeseries-intro.mdx"; - -# Time-series data - - - -To explore Timescale's features, you need some sample data. This guide -uses real-time stock trade data, also known as tick data, from -[Twelve Data][twelve-data]. - -## About the dataset - -The dataset contains second-by-second stock-trade data for the top 100 -most-traded symbols, in a hypertable named `stocks_real_time`. It also includes -a separate table of company symbols and company names, in a regular PostgreSQL -table named `company`. - -The dataset is updated on a nightly basis and contains data from the last four -weeks, typically ~8 million rows of data. Stock trades are recorded in real-time -Monday through Friday, during normal trading hours of the New York Stock -Exchange (9:30 AM - 4:00 PM EST). - -## Ingest the dataset - -To ingest data into the tables that you created, you need to download the -dataset and copy the data to your database. - - - -1. Unzip [real_time_stock_data.zip](https://assets.timescale.com/docs/downloads/get-started/real_time_stock_data.zip) to your local device. - This archive one `.csv` file with company information, and one with real-time stock trades for - the past month. - -1. At the `psql` prompt, use the `COPY` command to transfer data into your - Timescale instance. If the `.csv` files aren't in your current directory, - specify the file paths in the following commands: - - - - - - Because there are millions of rows of data, the `COPY` process may take a few - minutes depending on your internet connection and local client resources. - - - -[twelve-data]: https://twelvedata.com/ -[console-services]: https://console.cloud.timescale.com/dashboard/services diff --git a/getting-started/try-key-features-timescale-products.md b/getting-started/try-key-features-timescale-products.md new file mode 100644 index 0000000000..0ed161cc2e --- /dev/null +++ b/getting-started/try-key-features-timescale-products.md @@ -0,0 +1,437 @@ +--- +title: Try the key Timescale features +excerpt: Improve database performance with hypertables, time bucketing, compression and continuous aggregates. +products: [cloud] +content_group: Getting started +--- + +import HASetup from 'versionContent/_partials/_high-availability-setup.mdx'; +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; + +# Try the key Timescale features + +$CLOUD_LONG scales PostgreSQL to ingest and query vast amounts of live data. $CLOUD_LONG +provides a range of features and optimizations that supercharge your queries while keeping the +costs down. For example: +* The Hypercore row-columnar engine makes queries up to 350x faster, ingests 44% faster, and reduces storage by 90%. +* Tiered storage seamlessly moves your data from high performance storage for frequently accessed data to low cost bottomless storage for rarely accessed data. + +The following figure shows how $CLOUD_LONG optimizes your data for superfast real-time analytics and reduced +costs: + +![Main features and tiered data](https://assets.timescale.com/docs/images/timescalecloud-service-architecture.png ) + +This page shows you how to rapidly implement the features in $CLOUD_LONG that enable you to +ingest and query data faster while keeping the costs low. + +## Prerequisites + + + +## Optimize time-series data in hypertables + +Time-series data represents how a system, process, or behavior changes over time. Hypertables are PostgreSQL tables +that help you improve insert and query performance by automatically partitioning your data by time. Each hypertable +is made up of child tables called chunks. Each chunk is assigned a range of time, and only +contains data from that range. You can also tune hypertables to increase performance +even more. + +![Main features and tiered data](https://assets.timescale.com/docs/images/hypertable-structure.png ) + +Hypertables exist alongside regular PostgreSQL tables. +You use regular PostgreSQL tables for relational data, and interact with hypertables +and regular PostgreSQL tables in the same way. + +This section shows you how to create regular tables and hypertables, and import +relational and time-series data from external files. + + + +1. **Import some time-series data into your hypertable** + + 1. Unzip [crypto_sample.zip](https://assets.timescale.com/docs/downloads/candlestick/crypto_sample.zip) to a ``. + + This test dataset contains second-by-second trade data for the most-traded crypto-assets + and a regular table of asset symbols and company names. + + To import up to 100GB of data directly from your current PostgreSQL-based database, + [migrate with downtime][migrate-with-downtime] using native PostgreSQL tooling. To seamlessly import 100GB-10TB+ + of data, use the [live migration][migrate-live] tooling supplied by $COMPANY. To add data from non-PostgreSQL + data sources, see [Import and ingest data][data-ingest]. + + 1. Upload data from the CSVs to your $SERVICE_SHORT: + + + + + + The $CONSOLE data upload creates the tables for you from the data you are uploading: + 1. In [$CONSOLE][portal-ops-mode], select the service to add data to, then click `Actions` > `Upload CSV`. + 1. Drag `/tutorial_sample_tick.csv` to `Upload .CSV` and change `New table name` to `crypto_ticks`. + 1. Enable `hypertable partition` for the `time` column and click `Upload CSV`. + + The upload wizard creates a hypertable containing the data from the CSV file. + 1. When the data is uploaded, close `Upload .CSV`. + + If you want to have a quick look at your data, press `Run` . + 1. Repeat the process with `/tutorial_sample_assets.csv` and rename to `crypto_assets`. + + There is no time-series data in this table, so you don't see the `hypertable partition` option. + + + + + + 1. In Terminal, navigate to `` and connect to your $SERVICE_SHORT. + ```bash + psql -d "postgres://:@:/" + ``` + You use your [connection details][connection-info] to fill in this PostgreSQL connection string. + + 2. Create tables for the data to import: + + - For the time-series data: + 1. In your sql client, create a normal PostgreSQL table: + + ```sql + CREATE TABLE crypto_ticks ( + "time" TIMESTAMPTZ, + symbol TEXT, + price DOUBLE PRECISION, + day_volume NUMERIC + ); + ``` + 1. Convert `crypto_ticks` to a hypertable: + ```sql + SELECT create_hypertable('crypto_ticks', by_range('time')); + ``` + To more fully understand how hypertables work, and how to optimize them for performance by + tuning chunk intervals and enabling chunk skipping, see [the hypertables documentation][hypertables-section]. + + - For the relational data: + + In your sql client, create a normal PostgreSQL table: + ```sql + CREATE TABLE crypto_assets ( + symbol TEXT NOT NULL, + name TEXT NOT NULL + ); + ``` + + 3. Upload the dataset to your $SERVICE_SHORT: + + ```sql + \COPY crypto_ticks from './tutorial_sample_tick.csv' DELIMITER ',' CSV HEADER; + ``` + + ```sql + \COPY crypto_assets from './tutorial_sample_assets.csv' DELIMITER ',' CSV HEADER; + ``` + + + + + + To more fully understand how hypertables work, and how to optimize them for performance by + tuning chunk intervals and enabling chunk skipping, see [the hypertables documentation][hypertables-section]. + +1. **Have a quick look at your data** + + You query hypertables in exactly the same way as you would a relational PostgreSQL table. + Use one of the following SQL editors to run a query and see the data you uploaded: + - **Data mode**: write queries, visualize data, and share your results in [$CONSOLE][portal-data-mode] for all your $SERVICE_LONGs. + - **SQL editor**: write, fix, and organize SQL faster and more accurately in [$CONSOLE][portal-ops-mode] for a $SERVICE_LONG. + - **psql**: easily run queries on your $SERVICE_LONGs or self-hosted TimescaleDB deployment from Terminal. + + + + + +## Write fast and efficient analytical queries + +Aggregation is a way of combing data to get insights from it. Average, sum, and count are all +examples of simple aggregates. However, with large amounts of data, aggregation slows things down, quickly. +Continuous aggregates are a kind of hypertable that is refreshed automatically in +the background as new data is added, or old data is modified. Changes to your dataset are tracked, +and the hypertable behind the continuous aggregate is automatically updated in the background. + +![Reduced data calls with Continuous Aggregates](https://assets.timescale.com/docs/images/continuous-aggregate.png ) + +You create continuous aggregates on uncompressed data in high-performance storage. They continue to work +on [data in the columnstore][test-drive-enable-compression] +and [rarely accessed data in tiered storage][test-drive-tiered-storage]. You can even +create [continuous aggregates on top of your continuous aggregates][hierarchical-caggs]. + +You use time buckets to create a continuous aggregate. Time buckets aggregate data in hypertables by time +interval. For example, a 5-minute, 1-hour, or 3-day bucket. The data grouped in a time bucket uses a single +timestamp. Continuous aggregates minimize the number of records that you need to look up to perform your +query. + +This section shows you how to run fast analytical queries using time buckets and continuous aggregates in +$CONSOLE. You can also do this using psql. + + + + + + + +1. **In [$CONSOLE][portal-ops-mode], select the service you uploaded data to, then click `SQL Editor`** + +1. **Create a continuous aggregate** + + For a continuous aggregate, data grouped using a time bucket is stored in a + PostgreSQL `MATERIALIZED VIEW` in a hypertable. `timescaledb.continuous` ensures that this data + is always up to date. + In your SQL editor, use the following code to create a continuous aggregate on the real-time data in + the `crypto_ticks` table: + + ```sql + CREATE MATERIALIZED VIEW assets_candlestick_daily + WITH (timescaledb.continuous) AS + SELECT + time_bucket('1 day', "time") AS day, + symbol, + max(price) AS high, + first(price, time) AS open, + last(price, time) AS close, + min(price) AS low + FROM crypto_ticks srt + GROUP BY day, symbol; + ``` + + This continuous aggregate creates the [candlestick chart][charts] data you use to visualize + the price change of an asset. + +1. **Create a policy to refresh the view every hour** + + ```sql + SELECT add_continuous_aggregate_policy('assets_candlestick_daily', + start_offset => INTERVAL '3 weeks', + end_offset => INTERVAL '24 hours', + schedule_interval => INTERVAL '3 hours'); + ``` + +1. **Have a quick look at your data** + + You query continuous aggregates exactly the same way as your other tables. To query the `assets_candlestick_daily` + continuous aggregate for all assets: + + + + + + + + + + + + +1. **In [$CONSOLE][portal-ops-mode], select the service you uploaded data to**. +1. **Click `Operations` > `Continuous aggregates`, select `crypto_ticks`, then click `Create a Continuous Aggregate`**. + ![Continuous aggregate wizard](https://assets.timescale.com/docs/images/continuous-aggregate-wizard.png ) +1. **Create a view called `assets_candlestick_daily` on the `time` column with an interval of `1 day`, then click `Next step`**. +1. **Update the view SQL with the following functions, then click `Run`** + ```sql + CREATE MATERIALIZED VIEW assets_candlestick_daily + WITH (timescaledb.continuous) AS + SELECT + time_bucket('1 day', "time") AS bucket, + symbol, + max(price) AS high, + first(price, time) AS open, + last(price, time) AS close, + min(price) AS low + FROM "public"."crypto_ticks" srt + GROUP BY bucket, symbol; + ``` +1. **When the view is created, click `Next step`** +1. **Define a refresh policy with the following values:** + - `How far back do you want to materialize?`: `3 weeks` + - `What recent data to exclude?`: `24 hours` + - `How often do you want the job to run?`: `3 hours` +1. **Click `Next step`, then click `Run`** + +$CLOUD_LONG creates the continuous aggregate and displays the aggregate ID in $CONSOLE. Click `DONE` to close the wizard. + + + + + + + +To see the change in terms of query time and data returned between a regular query and +a continuous aggregate, run the query part of the continuous aggregate +( `SELECT ...GROUP BY day, symbol;` ) and compare the results. + + +## Enhance query performance for analytics + +Hypercore is the $TIMESCALE_DB hybrid row-columnar storage engine, designed specifically for real-time analytics and +powered by time-series data. The advantage of Hypercore is its ability to seamlessly switch between row-oriented and +column-oriented storage. This flexibility enables Timescale Cloud to deliver the best of both worlds, solving the key +challenges in real-time analytics. + +![Move from rowstore and columstore in hypercore](https://assets.timescale.com/docs/images/hypercore.png ) + +When you convert chunks from the rowstore to the columnstore, multiple records are grouped into a single row. +The columns of this row hold an array-like structure that stores all the data. Because a single row takes up less disk +space, you can reduce your chunk size by more than 90%, and can also speed up your queries. This saves on storage costs, +and keeps your queries operating at lightning speed. + +Best practice is to compress data that is no longer needed for highest performance queries, but is still accessed +regularly. For example, last week's market data. + + + +1. **Enable Hypercore on a hypertable** + + Create a [job][job] that automatically moves chunks in a hypertable to the columnstore at a specific time interval. + + ```sql + ALTER TABLE crypto_ticks SET ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'symbol'); + ``` + You [segmentby][alter-table-arguments] to speed up queries. + + +1. **Add a policy to convert chunks to the columnstore at a specific time interval** + + For example, yesterday's data: + ``` sql + CALL add_columnstore_policy('crypto_ticks', after => INTERVAL '1d'); + ``` + See [add_columnstore_policy][add_columnstore_policy]. + +1. **View your data space saving** + + When you convert data to the columnstore, as well as being optimized for analytics, it is compressed by more than + 90%. This saves on storage costs and keeps your queries operating at lightning speed. To see the amount of space + saved: + ``` sql + SELECT + pg_size_pretty(before_compression_total_bytes) as before, + pg_size_pretty(after_compression_total_bytes) as after + FROM hypertable_compression_stats('crypto_ticks'); + ``` + You see something like: + + | Before | After | + |--------|---------| + | 32 MB | 3808 KB | + + + + + +## Slash storage charges + +In the previous sections, you used continuous aggregates to make fast analytical queries, and +compression to reduce storage costs on frequently accessed data. To reduce storage costs even more, +you create tiering policies to move rarely accessed data to the object store. The object store is +low-cost bottomless data storage built on Amazon S3. However, no matter the tier, you can +[query your data when you need][querying-tiered-data]. $CLOUD_LONG seamlessly accesses the correct storage +tier and generates the response. + +![Tiered storage](https://assets.timescale.com/docs/images/tiered-storage.png ) + +Data tiering is available in the [scale and enterprise][pricing-plans] pricing plans for $CLOUD_LONG. + +To set up data tiering: + + + +1. **Enable data tiering** + + 1. In [$CONSOLE][portal-ops-mode], select the service to modify. + + You see the `Overview` section. + + 1. Scroll down, then click `Enable tiered storage`. + + ![Enable tiered storage](https://assets.timescale.com/docs/images/console-enable-tiered-storage.png) + + When tiered storage is enabled, you see the amount of data in the tiered object storage. + +1. **Set the time interval when data is tiered** + + In $CONSOLE, click `SQL Editor`, then enable data tiering on a hypertable with the following query: + ```sql + SELECT add_tiering_policy('assets_candlestick_daily', INTERVAL '3 weeks'); + ``` + +1. **Query tiered data** + + You enable reads from tiered data for each query, for a session or for all future + sessions. To run a single query on tiered data: + + 1. Enable reads on tiered data: + ```sql + set timescaledb.enable_tiered_reads = true + ``` + 1. Query the data: + ```sql + SELECT * FROM crypto_ticks srt LIMIT 10 + ``` + 1. Disable reads on tiered data: + ```sql + set timescaledb.enable_tiered_reads = false; + ``` + For more information, see [Querying tiered data][querying-tiered-data]. + + + +## Reduce the risk of downtime and data loss + +By default, all $SERVICE_LONGs have rapid recovery enabled. However, if your app has very low tolerance +for downtime, $CLOUD_LONG offers High Availability (HA) replicas. HA replicas are exact, up-to-date copies +of your database hosted in multiple AWS availability zones (AZ) within the same region as your primary node. +HA replicas automatically take over operations if the original primary data node becomes unavailable. +The primary node streams its write-ahead log (WAL) to the replicas to minimize the chances of +data loss during failover. + +![Move from rowstore and columstore in hypercore](https://assets.timescale.com/docs/images/ha-read-replica.png ) + +High availability is available in the [scale and enterprise][pricing-plans] pricing plans for $CLOUD_LONG. + + + +For more information, see [High availability][high-availability]. + +What next? See the [use case tutorials][tutorials], interact with the data in your $SERVICE_LONG using +[your favorite programming language][connect-with-code], integrate your $SERVICE_LONG with a range of +[third-party tools][integrations], plain old [Use Timescale][use-timescale], or dive into [the API][use-the-api]. + +[tutorials]: /tutorials/:currentVersion:/ +[connect-with-code]: /quick-start/:currentVersion:/ +[integrations]: /use-timescale/:currentVersion:/integrations/ +[use-the-api]: /api/:currentVersion:/ +[use-timescale]: /use-timescale/:currentVersion:/ +[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[create-a-service]: /getting-started/:currentVersion:/services/ +[deploy-self-hosted]: /self-hosted/:currentVersion:/install/ +[connect-to-your-service]: /getting-started/:currentVersion:/run-queries-from-console/ +[portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services +[portal-data-mode]: https://console.cloud.timescale.com/dashboard/services?popsql +[migrate-with-downtime]: /migrate/:currentVersion:/pg-dump-and-restore/ +[migrate-live]: /migrate/:currentVersion:/live-migration/ +[data-ingest]: /use-timescale/:currentVersion:/ingest-data/ +[hypertables-section]: /use-timescale/:currentVersion:/hypertables/ +[test-drive-enable-compression]: /getting-started/:currentVersion:/try-key-features-timescale-products/#enhance-query-performance-for-analytics +[test-drive-tiered-storage]: /getting-started/:currentVersion:/try-key-features-timescale-products/#slash-storage-charges +[data-tiering]: /use-timescale/:currentVersion:/data-tiering/ +[compression]: /use-timescale/:currentVersion:/compression/ +[hierarchical-caggs]: /use-timescale/:currentVersion:/continuous-aggregates/hierarchical-continuous-aggregates/ +[charts]: https://www.investopedia.com/terms/c/candlestick.asp +[hierarchical-storage]: https://en.wikipedia.org/wiki/Hierarchical_storage_management +[querying-tiered-data]: /use-timescale/:currentVersion:/data-tiering/querying-tiered-data/ +[data-tiering]: /use-timescale/:currentVersion:/data-tiering/ +[pricing-plans]: /about/:currentVersion:/pricing-and-account-management +[querying-tiered-data]: /use-timescale/:currentVersion:/data-tiering/querying-tiered-data/ +[high-availability]: /use-timescale/:currentVersion:/ha-replicas/high-availability/ +[sign-up]: https://console.cloud.timescale.com/signup +[job]: /api/:currentVersion:/actions/add_job/ +[alter-table-arguments]: /api/:currentVersion:/hypercore/alter_table/#arguments +[add_columnstore_policy]: /api/:currentVersion:/hypercore/add_columnstore_policy/ diff --git a/lambda/redirects.js b/lambda/redirects.js index 3393be2af9..8703531175 100644 --- a/lambda/redirects.js +++ b/lambda/redirects.js @@ -233,23 +233,23 @@ module.exports = [ }, { from: "/getting-started/exploring-cloud", - to: "https://docs.timescale.com/mst/latest/about-mst/", + to: "https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/", }, { from: "/getting-started/installation/timescale-cloud/installation-timescale-cloud", - to: "https://docs.timescale.com/install/latest/installation-mst/", + to: "https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/", }, { from: "/getting-started/latest/add-data/", - to: "https://docs.timescale.com/getting-started/latest/time-series-data/", + to: "https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables", }, { - from: "/getting-started/latest/compress-data/", - to: "https://docs.timescale.com/use-timescale/latest/compression/", + from: "/getting-started/latest/aggregation/", + to: "https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/#write-fast-analytical-queries-on-frequently-access-data-using-time-buckets-and-continuous-aggregates", }, { from: "/getting-started/latest/create-cagg/", - to: "https://docs.timescale.com/getting-started/latest/aggregation/", + to: "https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/#write-fast-analytical-queries-on-frequently-access-data-using-time-buckets-and-continuous-aggregates", }, { from: "/getting-started/latest/data-retention", @@ -264,8 +264,16 @@ module.exports = [ to: "https://docs.timescale.com/self-hosted/latest/install/installation-macos/", }, { - from: "/getting-started/latest/query-data/", - to: "https://docs.timescale.com/getting-started/latest/queries/", + from: "/getting-started/latest/queries/", + to: "https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables", + }, + { + from: "/getting-started/latest/tables-hypertables/", + to: "https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables", + }, + { + from: "/getting-started/latest/time-series-data/", + to: "https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables", }, { from: "/install/latest", @@ -372,7 +380,7 @@ module.exports = [ }, { from: "/timescaledb/latest/contribute-to-docs", - to: "https://docs.timescale.com/about/latest/contribute-to-docs/", + to: "https://docs.timescale.com/about/latest/contribute-to-timescale/", }, { from: "/timescaledb/latest/contribute-to-timescaledb", @@ -558,6 +566,22 @@ module.exports = [ from: "/tutorials/latest/aws-lambda/", to: "https://docs.timescale.com/tutorials/latest/", }, + { + from: "/tutorials/latest/energy-data/", + to: "https://docs.timescale.com/tutorials/real-time-analytics-energy-consumption/", + }, + { + from: "/tutorials/latest/energy-data/dataset-energy/", + to: "https://docs.timescale.com/tutorials/real-time-analytics-energy-consumption/", + }, + { + from: "/tutorials/latest/energy-data/query-energy/", + to: "https://docs.timescale.com/tutorials/real-time-analytics-energy-consumption/", + }, + { + from: "/tutorials/latest/energy-data/compress-energy/", + to: "https://docs.timescale.com/tutorials/real-time-analytics-energy-consumption/", + }, { from: "/tutorials/latest/financial-candlestick-tick-data/create-candlestick-aggregates/", to: "https://docs.timescale.com/tutorials/latest/financial-tick-data/financial-tick-query/", @@ -578,6 +602,30 @@ module.exports = [ from: "/tutorials/latest/nfl-analytics/", to: "https://docs.timescale.com/tutorials/latest/", }, + { + from: "/tutorials/latest/nyc-taxi-cab/", + to: "https://docs.timescale.com/tutorials/real-time-analytics-transport/", + }, + { + from: "/tutorials/latest/nyc-taxi-cab/dataset-nyc/", + to: "https://docs.timescale.com/tutorials/real-time-analytics-transport/", + }, + { + from: "/tutorials/latest/nyc-taxi-cab/query-nyc/", + to: "https://docs.timescale.com/tutorials/real-time-analytics-transport/", + }, + { + from: "/tutorials/latest/nyc-taxi-cab/compress-nyc/", + to: "https://docs.timescale.com/tutorials/real-time-analytics-transport/", + }, + { + from: "/tutorials/latest/nyc-taxi-geospatial/dataset-nyc/", + to: "https://docs.timescale.com/tutorials/real-time-analytics-transport/", + }, + { + from: "/tutorials/latest/nyc-taxi-geospatial/plot-nyc", + to: "https://docs.timescale.com/tutorials/real-time-analytics-transport/", + }, { from: "/tutorials/latest/prometheus-adapter/", to: "https://docs.timescale.com/tutorials/latest/", diff --git a/migrate/dual-write-and-backfill/dual-write-from-other.md b/migrate/dual-write-and-backfill/dual-write-from-other.md index c32b8e4210..be376ca5a6 100644 --- a/migrate/dual-write-and-backfill/dual-write-from-other.md +++ b/migrate/dual-write-and-backfill/dual-write-from-other.md @@ -79,7 +79,7 @@ continuous aggregate over the whole time range to ensure that there are no holes in the aggregated data. -[tables and hypertables]: /getting-started/:currentVersion:/tables-hypertables/ +[tables and hypertables]: /getting-started/:currentVersion:/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables diff --git a/migrate/dual-write-and-backfill/dual-write-from-postgres.md b/migrate/dual-write-and-backfill/dual-write-from-postgres.md index 4299536479..5442383b1c 100644 --- a/migrate/dual-write-and-backfill/dual-write-from-postgres.md +++ b/migrate/dual-write-and-backfill/dual-write-from-postgres.md @@ -130,7 +130,7 @@ features, such as: - [compression] to reduce the size of your hypertables - [continuous aggregates] to write blisteringly fast aggregate queries on your data -[time-series data]: /getting-started/:currentVersion:/time-series-data/ +[time-series data]: /getting-started/:currentVersion:/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables [create_table API reference]: /api/:currentVersion:/hypertable/create_hypertable/ [hypertable documentation]: /use-timescale/:currentVersion:/hypertables/ [retention policies]: /use-timescale/:currentVersion:/data-retention/ diff --git a/quick-start/golang.md b/quick-start/golang.md index e1c6eac4cd..dcc08e5e92 100644 --- a/quick-start/golang.md +++ b/quick-start/golang.md @@ -927,13 +927,13 @@ your Go application, be sure to check out these advanced TimescaleDB tutorials: [parallel-copy-tool]: https://github.com/timescale/timescaledb-parallel-copy [pgx-docs]: https://pkg.go.dev/github.com/jackc/pgx [pgx-driver-github]: https://github.com/jackc/pgx -[install]: /getting-started/latest/ -[connect]: #connect-to-timescaledb -[create-table]: #create-a-relational-table -[create-a-hypertable]: #create-a-hypertable -[insert]: #insert-rows-of-data -[query]: #execute-a-query +[install]: /getting-started/:currentVersion:/ +[connect]: /quick-start/:currentVersion:/golang/#connect-to-timescaledb +[create-table]: /quick-start/:currentVersion:/golang/#create-a-relational-table +[create-a-hypertable]: /quick-start/:currentVersion:/golang/#generate-a-hypertable +[insert]: /quick-start/:currentVersion:/golang/#insert-rows-of-data +[query]: /quick-start/:currentVersion:/golang/#execute-a-query [create-hypertable-docs]: /use-timescale/:currentVersion:/hypertables/create/ [insert]: /quick-start/:currentVersion:/golang/#insert-a-row-into-your-timescale-database [query]: /quick-start/:currentVersion:/golang/#execute-a-query-on-your-timescale-database -[create-hypertable]: /quick-start/:currentVersion:/python/#create-hypertable +[create-hypertable]: /quick-start/:currentVersion:/python/#generate-a-hypertable diff --git a/self-hosted/configuration/docker-config.md b/self-hosted/configuration/docker-config.md index 71b65e71cc..8e7092f0c1 100644 --- a/self-hosted/configuration/docker-config.md +++ b/self-hosted/configuration/docker-config.md @@ -59,4 +59,4 @@ using the `-c` option. For example: docker run -i -t timescale/timescaledb:latest-pg10 postgres -c max_wal_size=2GB ``` -[docker]: /self-hosted/latest/install/installation-docker/ +[docker]: /self-hosted/:currentVersion:/install/installation-docker/ diff --git a/self-hosted/configuration/timescaledb-config.md b/self-hosted/configuration/timescaledb-config.md index 54c89e1da6..c357981adf 100644 --- a/self-hosted/configuration/timescaledb-config.md +++ b/self-hosted/configuration/timescaledb-config.md @@ -7,6 +7,7 @@ tags: [tune] --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; +import ConfigCloudSelf from "versionContent/_partials/_cloud_self_configuration.mdx"; # TimescaleDB configuration and tuning @@ -15,14 +16,6 @@ settings that may be useful to your specific installation and performance needs. also be set within the `postgresql.conf` file or as command-line parameters when starting PostgreSQL. -## Policies - -### `timescaledb.max_background_workers (int)` - -Max background worker processes allocated to TimescaleDB. Set to at least 1 + -the number of databases loaded with a TimescaleDB extension in a PostgreSQL -instance. Default value is 16. - ## Query Planning and Execution ### `timescaledb.enable_chunkwise_aggregation (bool)` @@ -31,7 +24,6 @@ planning. The first part of the aggregation is executed on a per-chunk basis. Then, these partial results are combined and finalized. Splitting aggregations decreases the size of the created hash tables and increases data locality, which speeds up queries. - ### `timescaledb.vectorized_aggregation (bool)` Enables or disables the vectorized optimizations in the query executor. For @@ -44,6 +36,22 @@ Set to `ON` to dramatically decrease the amount of data written on a continuous in the presence of a small number of changes, reduce the i/o cost of refreshing a [continuous aggregate][continuous-aggregates], and generate fewer Write-Ahead Logs (WAL). Only works for continuous aggregates that don't have compression enabled. +## Policies + +### `timescaledb.max_background_workers (int)` + +Max background worker processes allocated to TimescaleDB. Set to at least 1 + +the number of databases loaded with a TimescaleDB extension in a PostgreSQL +instance. Default value is 16. + + + +## Hypercore features + +### `timescaledb.default_hypercore_use_access_method (bool)` + +The default value for `hypercore_use_access_method` for functions that have this parameter. This function is in `user` context, meaning that any user can set it for the session. The default value is `false`. + ## Distributed hypertables diff --git a/self-hosted/migration/migrate-influxdb.md b/self-hosted/migration/migrate-influxdb.md index 0c123b65f0..5edd7b32fd 100644 --- a/self-hosted/migration/migrate-influxdb.md +++ b/self-hosted/migration/migrate-influxdb.md @@ -43,8 +43,6 @@ and MacOS. -### Installing Outflux - 1. Go to the [releases section][outflux-releases] of the Outflux repository. 1. Download the latest compressed tarball for your platform. 1. Extract it to a preferred location. @@ -167,4 +165,4 @@ outflux migrate --help [discover-validate-and-transfer-schema]: /self-hosted/:currentVersion:/migration/migrate-influxdb/#discover-validate-and-transfer-schema [migrate-data-to-timescale]: /self-hosted/:currentVersion:/migration/migrate-influxdb/#migrate-data-to-timescale [outflux-gitbuh]: https://github.com/timescale/outflux#connection -[outflux-readme]: https://github.com/timescale/outflux/blob/master/README.md \ No newline at end of file +[outflux-readme]: https://github.com/timescale/outflux/blob/master/README.md diff --git a/self-hosted/migration/same-db.md b/self-hosted/migration/same-db.md index 563aaa281a..2070007e3c 100644 --- a/self-hosted/migration/same-db.md +++ b/self-hosted/migration/same-db.md @@ -81,4 +81,4 @@ Migrate your data into Timescale from within the same database. [create_hypertable]: /api/:currentVersion:/hypertable/create_hypertable/ -[install]: /getting-started/latest/ +[install]: /self-hosted/:currentVersion:/install/ diff --git a/self-hosted/multinode-timescaledb/index.md b/self-hosted/multinode-timescaledb/index.md index 600e990e7e..6b18cdd3ed 100644 --- a/self-hosted/multinode-timescaledb/index.md +++ b/self-hosted/multinode-timescaledb/index.md @@ -32,4 +32,4 @@ giving you faster data ingest, and more responsive and efficient queries. [multi-node-grow-shrink]: /self-hosted/:currentVersion:/multinode-timescaledb/multinode-grow-shrink/ [multi-node-ha]: /self-hosted/:currentVersion:/multinode-timescaledb/multinode-ha/ [multi-node-maintenance]: /self-hosted/:currentVersion:/multinode-timescaledb/multinode-maintenance/ -[setup-selfhosted]: /self-hosted/:currentVersion:/multinode-timescaledb/multinode-setup/ \ No newline at end of file +[setup-selfhosted]: /self-hosted/:currentVersion:/multinode-timescaledb/multinode-setup/ diff --git a/self-hosted/page-index/page-index.js b/self-hosted/page-index/page-index.js index f1947f3816..27faa96424 100644 --- a/self-hosted/page-index/page-index.js +++ b/self-hosted/page-index/page-index.js @@ -150,7 +150,89 @@ module.exports = [ excerpt: "Manage storage by moving data between tablespaces", }, { - title: "Multi-node", + title: "Replication and High Availability", + href: "replication-and-ha", + children: [ + { + title: "About high availability", + href: "about-ha", + excerpt: "High availability in self-hosted TimescaleDB", + }, + { + title: "Configure replication", + href: "configure-replication", + excerpt: "Configure replication", + }, + ], + }, + { + title: "Additional tooling", + href: "tooling", + children: [ + { + title: "TimescaleDB Tune", + href: "about-timescaledb-tune", + }, + { + title: "Install and update Timescale Toolkit", + href: "install-toolkit", + excerpt: "Install and update the Timescale Toolkit", + }, + ], + }, + { + title: "Upgrade self-hosted TimescaleDB", + href: "upgrades", + children: [ + { + title: "Upgrade TimescaleDB to a minor version", + href: "minor-upgrade", + excerpt: + "Upgrade self-hosted TimescaleDB to a new minor version", + }, + { + title: "Upgrade TimescaleDB to a major version", + href: "major-upgrade", + excerpt: + "Upgrade self-hosted TimescaleDB to a new major version", + }, + { + title: "Upgrade TimescaleDB running in Docker", + href: "upgrade-docker", + excerpt: + "Upgrade self-hosted TimescaleDB running in a Docker container to a new minor version", + }, + { + title: "Upgrade PostgreSQL", + href: "upgrade-pg", + excerpt: "Upgrade PostgreSQL to a new version", + }, + { + title: "Downgrade TimescaleDB to a minor version", + href: "downgrade", + excerpt: "Downgrade self-hosted TimescaleDB to the previous minor version", + }, + ], + }, + { + title: "Uninstall self-hosted TimescaleDB", + href: "uninstall", + excerpt: "Uninstalling self-hosted TimescaleDB", + children: [ + { + title: "Uninstall self-hosted TimescaleDB on macOS", + href: "uninstall-timescaledb", + excerpt: "Uninstall self-hosted TimescaleDB on macOS", + }, + ], + }, + { + title: "Troubleshooting self-hosted TimescaleDB", + href: "troubleshooting", + type: "placeholder", + }, + { + title: "Multi-node (Sunsetted v2.14.x)", href: "multinode-timescaledb", children: [ { @@ -196,7 +278,7 @@ module.exports = [ ], }, { - title: "Distributed hypertables", + title: "Distributed hypertables (Sunsetted v2.14.x)", href: "distributed-hypertables", excerpt: "Distributed hypertables for multi-node Timescale", children: [ @@ -239,88 +321,6 @@ module.exports = [ }, ], }, - { - title: "Replication and High Availability", - href: "replication-and-ha", - children: [ - { - title: "About high availability", - href: "about-ha", - excerpt: "High availability in self-hosted TimescaleDB", - }, - { - title: "Configure replication", - href: "configure-replication", - excerpt: "Configure replication", - }, - ], - }, - { - title: "Additional tooling", - href: "tooling", - children: [ - { - title: "TimescaleDB Tune", - href: "about-timescaledb-tune", - }, - { - title: "Install and update Timescale Toolkit", - href: "install-toolkit", - excerpt: "Install and update the Timescale Toolkit", - }, - ], - }, - { - title: "Upgrade self-hosted TimescaleDB", - href: "upgrades", - children: [ - { - title: "Upgrade TimescaleDB to a minor version", - href: "minor-upgrade", - excerpt: - "Upgrade self-hosted TimescaleDB to a new minor version", - }, - { - title: "Upgrade TimescaleDB to a major version", - href: "major-upgrade", - excerpt: - "Upgrade self-hosted TimescaleDB to a new major version", - }, - { - title: "Upgrade TimescaleDB running in Docker", - href: "upgrade-docker", - excerpt: - "Upgrade self-hosted TimescaleDB running in a Docker container to a new minor version", - }, - { - title: "Upgrade PostgreSQL", - href: "upgrade-pg", - excerpt: "Upgrade PostgreSQL to a new version", - }, - { - title: "Downgrade TimescaleDB to a minor version", - href: "downgrade", - excerpt: "Downgrade self-hosted TimescaleDB to the previous minor version", - }, - ], - }, - { - title: "Uninstall self-hosted TimescaleDB", - href: "uninstall", - excerpt: "Uninstalling self-hosted TimescaleDB", - children: [ - { - title: "Uninstall self-hosted TimescaleDB on macOS", - href: "uninstall-timescaledb", - excerpt: "Uninstall self-hosted TimescaleDB on macOS", - }, - ], - }, - { - title: "Troubleshooting self-hosted TimescaleDB", - href: "troubleshooting", - type: "placeholder", - }, ], }, ]; diff --git a/self-hosted/tooling/install-toolkit.md b/self-hosted/tooling/install-toolkit.md index be1c9d0239..65b9d7525b 100644 --- a/self-hosted/tooling/install-toolkit.md +++ b/self-hosted/tooling/install-toolkit.md @@ -248,8 +248,8 @@ developer documentation][toolkit-gh-docs]. [brew-install]: https://brew.sh [cloud]: /use-timescale/:currentVersion:/services/ -[debian-install]: /self-hosted/latest/install/installation-linux/ -[docker-install]: /self-hosted/latest/install/installation-docker/ +[debian-install]: /self-hosted/:currentVersion:/install/installation-linux/ +[docker-install]: /self-hosted/:currentVersion:/install/installation-docker/ [mst]: /mst/:currentVersion:/ -[red-hat-install]: /self-hosted/latest/install/installation-linux/ +[red-hat-install]: /self-hosted/:currentVersion:/install/installation-linux/ [toolkit-gh-docs]: https://github.com/timescale/timescaledb-toolkit#-installing-from-source diff --git a/tutorials/OLD_grafana/visualizations/pie-chart.md b/tutorials/OLD_grafana/visualizations/pie-chart.md index 41268df2c4..d7c8ef0656 100644 --- a/tutorials/OLD_grafana/visualizations/pie-chart.md +++ b/tutorials/OLD_grafana/visualizations/pie-chart.md @@ -166,4 +166,4 @@ Pie charts are a great tool for comparing categorized data. They're especially g for visualizing percentages. But they don't work as well if you have too many categories with similar percentages or large amount of data. -[gsg-data]: https://docs.timescale.com/getting-started/latest/time-series-data/ +[gsg-data]: https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables diff --git a/tutorials/blockchain-analyze/blockchain-dataset.md b/tutorials/blockchain-analyze/blockchain-dataset.md index 0c1189424f..d57ceeb7f1 100644 --- a/tutorials/blockchain-analyze/blockchain-dataset.md +++ b/tutorials/blockchain-analyze/blockchain-dataset.md @@ -7,43 +7,14 @@ layout_components: [next_prev_large] content_group: Analyze the Bitcoin blockchain --- -import CreateAndConnect from "versionContent/_partials/_cloud-create-connect-tutorials.mdx"; -import CreateHypertableBlockchain from "versionContent/_partials/_create-hypertable-blockchain.mdx"; -import AddDataBlockchain from "versionContent/_partials/_add-data-blockchain.mdx"; +import IngestData from "versionContent/_partials/_use-case-setup-blockchain-dataset.mdx"; import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; -# Set up the database - -This tutorial uses a dataset that contains Bitcoin blockchain data for -the past five days, in a hypertable named `transactions`. - - - - - - - - - -The dataset contains around 1.5 million Bitcoin transactions, the trades for five days. It includes -information about each transaction, along with the value in [satoshi][satoshi-def]. It also states if a -trade is a [coinbase][coinbase-def] transaction, and the reward a coin miner receives for mining the coin. - - - - - - - - - -The queries in this tutorial are suitable for graphing in Grafana. If you want -to visualize the results of your queries, connect your Grafana account to the -Bitcoin blockchain dataset. + - + [satoshi-def]: https://www.pcmag.com/encyclopedia/term/satoshi [coinbase-def]: https://www.pcmag.com/encyclopedia/term/coinbase-transaction diff --git a/tutorials/blockchain-analyze/index.md b/tutorials/blockchain-analyze/index.md index 7aee648ba8..35b6f0550a 100644 --- a/tutorials/blockchain-analyze/index.md +++ b/tutorials/blockchain-analyze/index.md @@ -7,16 +7,19 @@ layout_components: [next_prev_large] content_group: Analyze the Bitcoin blockchain --- +import FinancialIndustry from "versionContent/_partials/_financial-industry-data-analysis.mdx"; + # Analyze the Bitcoin blockchain -[Blockchains][blockchain-def] are, at their essence, a distributed database. The -[transactions][transactions-def] in a blockchain are an example of time-series -data. You can use Timescale to query transactions on a blockchain, in exactly the -same way as you might query time-series transactions in any other database. + -In this tutorial, you use Timescale hyperfunctions to analyze transactions -on the Bitcoin blockchain. You can use these instructions to query any type of data on a -blockchain, including other cryptocurrencies, smart contracts, or health data. +In this tutorial, you use Timescale to ingest, store, and analyze transactions +on the Bitcoin blockchain. + +[Blockchains][blockchain-def] are, at their essence, a distributed database. The +[transactions][transactions-def] in a blockchain are an example of time-series data. You can use +Timescale to query transactions on a blockchain, in exactly the same way as you +might query time-series transactions in any other database. ## Prerequisites diff --git a/tutorials/blockchain-query/blockchain-dataset.md b/tutorials/blockchain-query/blockchain-dataset.md index d19b09a721..76033d0afd 100644 --- a/tutorials/blockchain-query/blockchain-dataset.md +++ b/tutorials/blockchain-query/blockchain-dataset.md @@ -7,32 +7,7 @@ layout_components: [next_prev_large] content_group: Query the Bitcoin blockchain --- -import CreateAndConnect from "versionContent/_partials/_cloud-create-connect-tutorials.mdx"; -import CreateHypertableBlockchain from "versionContent/_partials/_create-hypertable-blockchain.mdx"; -import AddDataBlockchain from "versionContent/_partials/_add-data-blockchain.mdx"; +import IngestData from "versionContent/_partials/_use-case-setup-blockchain-dataset.mdx"; -# Set up the database -This tutorial uses a dataset that contains Bitcoin blockchain data for -the past five days, in a hypertable named `transactions`. - - - - - - - - - -The dataset contains around 1.5 million Bitcoin transactions, the trades for five days. It includes -information about each transaction, along with the value in [satoshi][satoshi-def]. It also states if a -trade is a [coinbase][coinbase-def] transaction, and the reward a coin miner receives for mining the coin. - - - - - - - -[satoshi-def]: https://www.pcmag.com/encyclopedia/term/satoshi -[coinbase-def]: https://www.pcmag.com/encyclopedia/term/coinbase-transaction + diff --git a/tutorials/blockchain-query/index.md b/tutorials/blockchain-query/index.md index 133f10cb5d..159f3da678 100644 --- a/tutorials/blockchain-query/index.md +++ b/tutorials/blockchain-query/index.md @@ -7,17 +7,20 @@ layout_components: [next_prev_large] content_group: Query the Bitcoin blockchain --- +import FinancialIndustry from "versionContent/_partials/_financial-industry-data-analysis.mdx"; + # Query the Bitcoin blockchain + + +In this tutorial, you use Timescale to ingest, store, and analyze transactions +on the Bitcoin blockchain. + [Blockchains][blockchain-def] are, at their essence, a distributed database. The [transactions][transactions-def] in a blockchain are an example of time-series data. You can use Timescale to query transactions on a blockchain, in exactly the same way as you might query time-series transactions in any other database. -In this tutorial, you use Timescale to ingest, store, and analyze transactions -on the Bitcoin blockchain. You can use these skills to query any data on a -blockchain, including other cryptocurrencies, smart contracts, or health data. - ## Prerequisites Before you begin, make sure you have: diff --git a/tutorials/energy-data/dataset-energy.md b/tutorials/energy-data/dataset-energy.md index 812b5fa05b..88f1b3e095 100644 --- a/tutorials/energy-data/dataset-energy.md +++ b/tutorials/energy-data/dataset-energy.md @@ -8,115 +8,35 @@ layout_components: [next_prev_large] content_group: Analyze energy consumption data --- -import CreateAndConnect from "versionContent/_partials/_cloud-create-connect-tutorials.mdx"; +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; import CreateHypertableEnergy from "versionContent/_partials/_create-hypertable-energy.mdx"; import AddDataEnergy from "versionContent/_partials/_add-data-energy.mdx"; import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; import CreateCaggs from "versionContent/_partials/_caggs-intro.mdx"; +import CreateCaggsOnIOTData from "versionContent/_partials/_use-case-iot-create-cagg.mdx"; -# Set up the database +# Ingest data into a $SERVICE_LONG This tutorial uses the energy consumption data for over a year in a hypertable named `metrics`. - +## Prerequisites - - - - - - -This tutorial uses the energy consumption data for over a year in a typical -household. You can use this data to analyze the energy consumption pattern. + - - - +## Create continuous aggregates -## Create continuous aggregates - -### Creating continuous aggregates for energy consumption by day and hour - -1. Create a continuous aggregate `kwh_day_by_day` for energy consumption on a - day to day basis: - - ```sql - CREATE MATERIALIZED VIEW kwh_day_by_day(time, value) - with (timescaledb.continuous) as - SELECT time_bucket('1 day', created, 'Europe/Berlin') AS "time", - round((last(value, created) - first(value, created)) * 100.) / 100. AS value - FROM metrics - WHERE type_id = 5 - GROUP BY 1; - ``` - -1. Add a refresh policy to keep the continuous aggregate up-to-date: - - ```sql - SELECT add_continuous_aggregate_policy('kwh_day_by_day', - start_offset => NULL, - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); - ``` - -1. Create a continuous aggregate `kwh_hour_by_hour` for energy consumption on - an hourly basis: - - ```sql - CREATE MATERIALIZED VIEW kwh_hour_by_hour(time, value) - with (timescaledb.continuous) as - SELECT time_bucket('01:00:00', metrics.created, 'Europe/Berlin') AS "time", - round((last(value, created) - first(value, created)) * 100.) / 100. AS value - FROM metrics - WHERE type_id = 5 - GROUP BY 1; - ``` - -1. Add a refresh policy to keep the continuous aggregate up-to-date: - - ```sql - SELECT add_continuous_aggregate_policy('kwh_hour_by_hour', - start_offset => NULL, - end_offset => INTERVAL '1 hour', - schedule_interval => INTERVAL '1 hour'); - ``` - -1. You can confirm that the continuous aggregates were created: - - ```sql - SELECT view_name, format('%I.%I', materialization_hypertable_schema,materialization_hypertable_name) AS materialization_hypertable - FROM timescaledb_information.continuous_aggregates; - ``` - - You should see this: - - ```sql - view_name | materialization_hypertable - ------------------+-------------------------------------------------- - kwh_day_by_day | _timescaledb_internal._materialized_hypertable_2 - kwh_hour_by_hour | _timescaledb_internal._materialized_hypertable_3 - - ``` + - - - - -The queries in this tutorial are suitable for visualizing in Grafana. If you -want to visualize the results of your queries, connect your Grafana account to -the energy consumption dataset. - - diff --git a/tutorials/energy-data/index.md b/tutorials/energy-data/index.md index f951d215c7..52eb4d4a46 100644 --- a/tutorials/energy-data/index.md +++ b/tutorials/energy-data/index.md @@ -10,15 +10,13 @@ content_group: Analyze energy consumption data # Analyze energy consumption data -When you are planning to switch to a rooftop solar system it isn't easy, even +When you are planning to switch to a rooftop solar system, it isn't easy, even with a specialist at hand. You need details of your power consumption, typical -usage hours, or distribution over a year. Collecting consumption data at the -granularity of a few seconds is key to finding all the answers for more -precision. This tutorial uses energy consumption data from a typical household -for over a year. Because nearly all of this data is time-series data, proper -analysis requires a purpose-built time-series database, like Timescale. +usage hours, distribution over a year, and other information. Collecting consumption data at the +granularity of a few seconds and then getting insights on it is key - and this is what Timescale is best at. -In this tutorial you can construct queries that look at how many watts were +This tutorial uses energy consumption data from a typical household +for over a year. You construct queries that look at how many watts were consumed, and when. Additionally, you can visualize the energy consumption data in Grafana. diff --git a/tutorials/financial-ingest-real-time/financial-ingest-dataset.md b/tutorials/financial-ingest-real-time/financial-ingest-dataset.md index d75ba5fe11..0997c010b8 100644 --- a/tutorials/financial-ingest-real-time/financial-ingest-dataset.md +++ b/tutorials/financial-ingest-real-time/financial-ingest-dataset.md @@ -8,40 +8,36 @@ layout_components: [next_prev_large] content_group: Ingest real-time financial websocket data --- -import CreateAndConnect from "versionContent/_partials/_cloud-create-connect-tutorials.mdx"; +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; import CreateHypertable from "versionContent/_partials/_create-hypertable-twelvedata-stocks.mdx"; -import CreateHypertableStocks from "versionContent/_partials/_create-hypertable-twelvedata-stocks.mdx"; +import CreateHypertableCrypo from "versionContent/_partials/_create-hypertable-twelvedata-crypto.mdx"; import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; -# Set up the database +# Ingest data into a $SERVICE_LONG This tutorial uses a dataset that contains second-by-second stock-trade data for the top 100 most-traded symbols, in a hypertable named `stocks_real_time`. It also includes a separate table of company symbols and company names, in a regular PostgreSQL table named `company`. - +## Prerequisites - + - - - +## Connect to the websocket server When you connect to the Twelve Data API through a websocket, you create a persistent connection between your computer and the websocket server. You set up a Python environment, and pass two arguments to create a websocket object and establish the connection. -## Set up a new Python environment +### Set up a new Python environment Create a new Python virtual environment for this project and activate it. All the packages you need to complete for this tutorial are installed in this environment. -### Setting up a new Python environment - 1. Create and activate a Python virtual environment: ```bash @@ -67,13 +63,13 @@ the packages you need to complete for this tutorial are installed in this enviro -## Create the websocket connection +### Create the websocket connection A persistent connection between your computer and the websocket server is used to receive data for as long as the connection is maintained. You need to pass two arguments to create a websocket object and establish connection. -### Websocket arguments +#### Websocket arguments * `on_event` @@ -99,7 +95,7 @@ two arguments to create a websocket object and establish connection. -### Connecting to the websocket server +### Connect to the websocket server 1. Create a new Python file called `websocket_test.py` and connect to the Twelve Data servers using the ``: @@ -163,22 +159,8 @@ two arguments to create a websocket object and establish connection. - - - - -To ingest the data into your Timescale service, you need to implement the -`on_event` function. - -After the websocket connection is set up, you can use the `on_event` function -to ingest data into the database. This is a data pipeline that ingests real-time -financial data into your Timescale service. - -Stock trades are ingested in real-time Monday through Friday, typically during -normal trading hours of the New York Stock Exchange (9:30 AM to -4:00 PM EST). - + When you ingest data into a transactional database like Timescale, it is more efficient to insert data in batches rather than inserting data row-by-row. Using @@ -195,6 +177,13 @@ universal, but you can experiment with different batch sizes Using batching is a fairly common pattern when ingesting data into TimescaleDB from Kafka, Kinesis, or websocket connections. +To ingest the data into your Timescale service, you need to implement the +`on_event` function. + +After the websocket connection is set up, you can use the `on_event` function +to ingest data into the database. This is a data pipeline that ingests real-time +financial data into your Timescale service. + You can implement a batching solution in Python with Psycopg2. You can implement the ingestion logic within the `on_event` function that you can then pass over to the websocket object. @@ -207,7 +196,7 @@ This function needs to: 1. Add it to the in-memory batch, which is a list in Python. 1. If the batch reaches a certain size, insert the data, and reset or empty the list. -## Ingesting data in real-time +## Ingest data in real-time @@ -323,17 +312,8 @@ If you see an error message similar to this: Then check that you use a proper API key received from Twelve Data. - - - - -The queries in this tutorial are suitable for visualizing in Grafana. If you -want to visualize the results of your queries, connect your Grafana account to -the energy consumption dataset. - - [twelve-wrapper]: https://github.com/twelvedata/twelvedata-python [psycopg2]: https://www.psycopg.org/docs/ diff --git a/tutorials/financial-ingest-real-time/financial-ingest-query.md b/tutorials/financial-ingest-real-time/financial-ingest-query.md index 116668b4ea..c24909c34c 100644 --- a/tutorials/financial-ingest-real-time/financial-ingest-query.md +++ b/tutorials/financial-ingest-real-time/financial-ingest-query.md @@ -38,7 +38,7 @@ the last two hours' worth of data. MIN(price) AS low, LAST(price, time) AS "close", LAST(day_volume, time) AS day_volume - FROM stocks_real_time + FROM crypto_ticks GROUP BY bucket, symbol; ``` diff --git a/tutorials/financial-ingest-real-time/index.md b/tutorials/financial-ingest-real-time/index.md index cff6156daf..8b7231c798 100644 --- a/tutorials/financial-ingest-real-time/index.md +++ b/tutorials/financial-ingest-real-time/index.md @@ -1,5 +1,5 @@ --- -title: Ingest real-time financial websocket data +title: Ingest real-time financial data using WebSocket excerpt: Ingest time-series data into Timescale Cloud using a websocket connection products: [cloud] keywords: [finance, analytics, websockets, data pipeline] @@ -9,8 +9,11 @@ content_group: Ingest real-time financial websocket data --- import CandlestickIntro from "versionContent/_partials/_candlestick_intro.mdx"; +import FinancialIndustry from "versionContent/_partials/_financial-industry-data-analysis.mdx"; -# Ingest real-time financial websocket data +# Ingest real-time financial data using WebSocket + + This tutorial shows you how to ingest real-time time-series data into TimescaleDB using a websocket connection. The tutorial sets up a data pipeline diff --git a/tutorials/financial-tick-data/financial-tick-compress.md b/tutorials/financial-tick-data/financial-tick-compress.md index e8493cd746..d4f29c8388 100644 --- a/tutorials/financial-tick-data/financial-tick-compress.md +++ b/tutorials/financial-tick-data/financial-tick-compress.md @@ -49,7 +49,7 @@ memory. order-by column using the `ALTER TABLE` command: ```sql - ALTER TABLE stocks_real_time + ALTER TABLE crypto_ticks SET ( timescaledb.compress, timescaledb.compress_segmentby='symbol', @@ -64,7 +64,7 @@ memory. `compress_chunk` in this manner: ```sql - SELECT compress_chunk(c) from show_chunks('stocks_real_time') c; + SELECT compress_chunk(c) from show_chunks('crypto_ticks') c; ``` You can also [automate compression][automatic-compression] by adding a [compression policy][add_compression_policy] which will @@ -76,7 +76,7 @@ memory. SELECT pg_size_pretty(before_compression_total_bytes) as before, pg_size_pretty(after_compression_total_bytes) as after - FROM hypertable_compression_stats('stocks_real_time'); + FROM hypertable_compression_stats('crypto_ticks'); ``` This shows a significant improvement in data usage: @@ -97,7 +97,7 @@ allows you to compress data that is older than a particular age, for example, to compress all chunks that are older than 8 days: ```sql -SELECT add_compression_policy('stocks_real_time', INTERVAL '8 days'); +SELECT add_compression_policy('crypto_ticks', INTERVAL '8 days'); ``` Compression policies run on a regular schedule, by default once every @@ -141,7 +141,7 @@ timing query times in psql by running: To decompress the whole dataset, run: ```sql - SELECT decompress_chunk(c) from show_chunks('stocks_real_time') c; + SELECT decompress_chunk(c) from show_chunks('crypto_ticks') c; ``` On an example setup, speedup performance observed was significant, diff --git a/tutorials/financial-tick-data/financial-tick-dataset.md b/tutorials/financial-tick-data/financial-tick-dataset.md index d654ae50bc..d6dab2ce25 100644 --- a/tutorials/financial-tick-data/financial-tick-dataset.md +++ b/tutorials/financial-tick-data/financial-tick-dataset.md @@ -8,42 +8,28 @@ layout_components: [next_prev_large] content_group: Analyze financial tick data --- -import CreateAndConnect from "versionContent/_partials/_cloud-create-connect-tutorials.mdx"; -import CreateHypertable from "versionContent/_partials/_create-hypertable-twelvedata-stocks.mdx"; -import AddData from "versionContent/_partials/_add-data-twelvedata-stocks.mdx"; +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; +import CreateHypertableCrypto from "versionContent/_partials/_create-hypertable-twelvedata-crypto.mdx"; +import AddDataCrypto from "versionContent/_partials/_add-data-twelvedata-crypto.mdx"; +import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; -# Set up the database +# Ingest data into a $SERVICE_LONG -This tutorial uses a dataset that contains second-by-second stock-trade data for -the top 100 most-traded symbols, in a hypertable named `stocks_real_time`. It -also includes a separate table of company symbols and company names, in a -regular PostgreSQL table named `company`. - - - - - - - - +This tutorial uses a dataset that contains second-by-second trade data for +the most-traded crypto-assets. You optimize this time-series data in a a hypertable called `assets_real_time`. +You also create a separate table of asset symbols in a regular PostgreSQL table named `assets`. The dataset is updated on a nightly basis and contains data from the last four -weeks, typically around 8 million rows of data. Stock trades are recorded in -real-time Monday through Friday, typically during normal trading hours of the -New York Stock Exchange (9:30 AM - 4:00 PM EST). - - +weeks, typically around 8 million rows of data. Trades are recorded in +real-time from 180+ cryptocurrency exchanges. - +## Prerequisites - + - + -The queries in this tutorial are suitable for visualizing in Grafana. If you -want to visualize the results of your queries, connect your Grafana account to -the energy consumption dataset. + - diff --git a/tutorials/financial-tick-data/financial-tick-query.md b/tutorials/financial-tick-data/financial-tick-query.md index 1fa9f7b78d..8a71337c50 100644 --- a/tutorials/financial-tick-data/financial-tick-query.md +++ b/tutorials/financial-tick-data/financial-tick-query.md @@ -61,7 +61,7 @@ the last two days' worth of data. MIN(price) AS low, LAST(price, time) AS "close", LAST(day_volume, time) AS day_volume - FROM stocks_real_time + FROM crypto_ticks GROUP BY bucket, symbol; ``` diff --git a/tutorials/financial-tick-data/index.md b/tutorials/financial-tick-data/index.md index 8fcc4fceb0..e2733e90a3 100644 --- a/tutorials/financial-tick-data/index.md +++ b/tutorials/financial-tick-data/index.md @@ -8,9 +8,13 @@ layout_components: [next_prev_large] content_group: Analyze financial tick data --- +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; import CandlestickIntro from "versionContent/_partials/_candlestick_intro.mdx"; +import FinancialIndustry from "versionContent/_partials/_financial-industry-data-analysis.mdx"; -# Analyze financial tick data with TimescaleDB +# Analyze financial tick data + + To analyze financial data, you can chart the open, high, low, close, and volume (OHLCV) information for a financial asset. Using this data, you can create @@ -24,25 +28,7 @@ aggregated data, and visualize the data in Grafana. ## Prerequisites -Before you begin, make sure you have: - -* Signed up for a [free Timescale account][cloud-install]. - -## Steps in this tutorial - -This tutorial covers: - -1. [Setting up your dataset][financial-tick-dataset]: Load data from - [Twelve Data][twelve-data] into your TimescaleDB database. -1. [Querying your dataset][financial-tick-query]: Create candlestick views, query - the aggregated data, and visualize the data in Grafana. -1. [Bonus: Store data efficiently][financial-tick-compress]: Learn how to store and query -your financial tick data more efficiently using compression feature of Timescale. - - This tutorial shows you how to ingest real-time time-series data into a Timescale - database. To create candlestick views, query the - aggregated data, and visualize the data in Grafana, see the - [ingest real-time websocket data section][advanced-websocket]. + ## About OHLCV data and candlestick charts @@ -58,6 +44,23 @@ these stories from some Timescale community members: * [How Messari uses data to open the cryptoeconomy to everyone][messari] * [How I power a (successful) crypto trading bot with TimescaleDB][bot] +## Steps in this tutorial + +This tutorial shows you how to ingest real-time time-series data into a Timescale +database: + +1. [Setting up your dataset][financial-tick-dataset]: Load data from + [Twelve Data][twelve-data] into your TimescaleDB database. +1. [Querying your dataset][financial-tick-query]: Create candlestick views, query + the aggregated data, and visualize the data in Grafana. +1. [Bonus: Store data efficiently][financial-tick-compress]: Learn how to store and query +your financial tick data more efficiently using compression feature of Timescale. + + +To create candlestick views, query the aggregated data, and visualize the data in Grafana, see the +[ingest real-time websocket data section][advanced-websocket]. + + [advanced-websocket]: /tutorials/:currentVersion:/financial-ingest-real-time/ [cloud-install]: /getting-started/:currentVersion:/#create-your-timescale-account [financial-tick-dataset]: /tutorials/:currentVersion:/financial-tick-data/financial-tick-dataset/ diff --git a/tutorials/index.md b/tutorials/index.md index 53adf0ac44..69e22acf85 100644 --- a/tutorials/index.md +++ b/tutorials/index.md @@ -9,14 +9,26 @@ products: [cloud, mst, self_hosted] Timescale tutorials are designed to help you get up and running with Timescale fast. They walk you through a variety of scenarios using example datasets, to teach you how to construct interesting queries, find out what information your -database has hidden in it, and even gives you options for visualizing and +database has hidden in it, and even give you options for visualizing and graphing your results. -|🔐 Cryptocurrency|🔋 Energy|💰 Finance|🚘 Transport| 💡 IoT | -|-|-|-|-|------------------------------------------------------------| -|🟢 [Part 1][beginner-crypto]
Do your own research on the Bitcoin blockchain|🟢 [Part 1][beginner-energy]
Optimize your energy consumption for a rooftop solar PV system|🟢 [Part 1][beginner-finance]
Chart the trading highs and lows for your favorite stock|🟢 [Part 1][beginner-fleet]
Find out about taxi rides taken in and around NYC| 🟢 [Part 1][iot]
Simulate an IoT sensor dataset | -|⭐ [Part 2][intermediate-crypto]
Discover the relationship between transactions, blocks, fees, and miner revenue|⭐ *Coming Soon!*|⭐ [Part 2][advanced-finance]
Use a websocket connection to visualize the trading highs and lows for your favorite stock|⭐ [Part 2][intermediate-fleet]
Map the longest taxi rides in NYC| | +- **Real-time analytics** + - [Analytics on energy consumption][rta-energy]: make data-driven decisions using energy consumption data. + - [Analytics on transport and geospatial data][rta-transport]: optimize profits using geospatial transport data. +- **Cryptocurrency** + - [Query the Bitcoin blockchain][beginner-crypto]: do your own research on the Bitcoin blockchain. + - [Analyze the Bitcoin blockchain][intermediate-crypto]: discover the relationship between transactions, blocks, fees, and miner revenue. +- **Finance** + - [Analyze financial tick data][beginner-finance]: chart the trading highs and lows for your favorite stock. + - [Ingest real-time financial data using WebSocket][advanced-finance]: use a websocket connection to visualize the trading highs and lows for your favorite stock. +- **IoT** + - [Simulate an IoT sensor dataset][iot]: simulate an IoT sensor dataset and run simple queries on it. +- **Cookbooks** + - [Timescale community cookbook][cookbooks]: get suggestions from the TimescaleDB Community about how to resolve common issues. + +[rta-energy]: /tutorials/:currentVersion:/real-time-analytics-energy-consumption +[rta-transport]: /tutorials/:currentVersion:/real-time-analytics-transport [beginner-fleet]: /tutorials/:currentVersion:/nyc-taxi-cab/ [beginner-finance]: /tutorials/:currentVersion:/financial-tick-data/ [beginner-crypto]: /tutorials/:currentVersion:/blockchain-query/ @@ -25,3 +37,4 @@ graphing your results. [intermediate-crypto]: /tutorials/:currentVersion:/blockchain-analyze/ [advanced-finance]: /tutorials/:currentVersion:/financial-ingest-real-time/ [iot]: /tutorials/:currentVersion:/simulate-iot-sensor-data/ +[cookbooks]: /tutorials/:currentVersion:/cookbook/ diff --git a/tutorials/ingest-real-time-websocket-data.md b/tutorials/ingest-real-time-websocket-data.md index ce4f604d95..c06d4450df 100644 --- a/tutorials/ingest-real-time-websocket-data.md +++ b/tutorials/ingest-real-time-websocket-data.md @@ -1,5 +1,5 @@ --- -title: Ingest real-time financial websocket data +title: Ingest real-time financial data using WebSocket excerpt: Set up a data pipeline to get data from different financial APIs products: [cloud, mst, self_hosted] keywords: [finance, analytics, websockets, data pipeline] @@ -8,7 +8,7 @@ keywords: [finance, analytics, websockets, data pipeline] import CreateHypertableStocks from "versionContent/_partials/_create-hypertable-twelvedata-stocks.mdx"; import GraphOhlcv from "versionContent/_partials/_graphing-ohlcv-data.mdx"; -# Ingest real-time financial websocket data +# Ingest real-time financial data using WebSocket This tutorial shows you how to ingest real-time time-series data into TimescaleDB using a websocket connection. The tutorial sets up a data pipeline diff --git a/tutorials/nyc-taxi-cab/dataset-nyc.md b/tutorials/nyc-taxi-cab/dataset-nyc.md index fceb651151..06219aaa41 100644 --- a/tutorials/nyc-taxi-cab/dataset-nyc.md +++ b/tutorials/nyc-taxi-cab/dataset-nyc.md @@ -11,31 +11,22 @@ import CreateAndConnect from "versionContent/_partials/_cloud-create-connect-tut import CreateHypertableNyc from "versionContent/_partials/_create-hypertable-nyctaxis.mdx"; import AddDataNyc from "versionContent/_partials/_add-data-nyctaxis.mdx"; import PreloadedData from "versionContent/_partials/_preloaded-data.mdx"; +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; -# Set up the database -This tutorial uses a dataset that contains historical data from New York's -yellow taxi network, in a hypertable named `rides`. It also includes a separate +# Ingest data into a $SERVICE_LONG + +This tutorial uses a dataset that contains historical data from the New York City Taxi and Limousine +Commission [NYC TLC][nyc-tlc], in a hypertable named `rides`. It also includes a separate tables of payment types and rates, in a regular PostgreSQL table named `payment_types`, and `rates`. - - - - - +## Prerequisites - - - - -This tutorial uses historical data from New York's yellow taxi network, provided -by the New York City Taxi and Limousine Commission [NYC TLC][nyc-tlc]. + - - [nyc-tlc]: https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page diff --git a/tutorials/nyc-taxi-geospatial/dataset-nyc.md b/tutorials/nyc-taxi-geospatial/dataset-nyc.md index eb7e335897..8a6d82e93a 100644 --- a/tutorials/nyc-taxi-geospatial/dataset-nyc.md +++ b/tutorials/nyc-taxi-geospatial/dataset-nyc.md @@ -12,39 +12,25 @@ import CreateAndConnect from "versionContent/_partials/_cloud-create-connect-tut import CreateHypertableNyc from "versionContent/_partials/_create-hypertable-nyctaxis.mdx"; import AddDataNyc from "versionContent/_partials/_add-data-nyctaxis.mdx"; import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; -# Set up the database +# Ingest data into a $SERVICE_LONG -This tutorial uses a dataset that contains historical data from New York's -yellow taxi network, in a hypertable named `rides`. It also includes a separate +This tutorial uses a dataset that contains historical data from the New York City Taxi and Limousine +Commission [NYC TLC][nyc-tlc], in a hypertable named `rides`. It also includes a separate tables of payment types and rates, in a regular PostgreSQL table named `payment_types`, and `rates`. - +## Prerequisites - - - - - - -This tutorial uses historical data from New York's yellow taxi network, provided -by the New York City Taxi and Limousine Commission [NYC TLC][nyc-tlc]. + - - - - -The queries in this tutorial are suitable for visualizing in Grafana. If you -want to visualize the results of your queries, connect your Grafana account to -the NYC taxi cab dataset. - - + [nyc-tlc]: https://www1.nyc.gov/site/tlc/about/tlc-trip-record-data.page diff --git a/tutorials/page-index/page-index.js b/tutorials/page-index/page-index.js index 8b2281dca2..a8a77f25df 100644 --- a/tutorials/page-index/page-index.js +++ b/tutorials/page-index/page-index.js @@ -5,18 +5,38 @@ module.exports = [ excerpt: "Learn about common scenarios and use cases for Timescale", children: [ { - title: "Cryptocurrency - part 1", + title: "Analytics on transport and geospatial data", + href: "real-time-analytics-transport", + excerpt: "Analyse your data in real-time", + }, + { + title: "Analytics on energy consumption", + href: "real-time-analytics-energy-consumption", + excerpt: "Analyse your data in real-time", + }, + { + title: "Simulate an IoT sensor dataset", + href: "simulate-iot-sensor-data", + excerpt: "Simulate and query an IoT sensor dataset", + }, + { + title: "Timescale community cookbook", + href: "cookbook", + excerpt: "Code examples from the community that help you with loads of common conundrums.", + }, + { + title: "Query the Bitcoin blockchain", href: "blockchain-query", excerpt: "Query the Bitcoin blockchain", children: [ { - title: "Set up", + title: "Ingest data into a service", href: "blockchain-dataset", excerpt: "Set up a dataset so you can query the Bitcoin blockchain", }, { - title: "Query data", + title: "Query the data", href: "beginner-blockchain-query", excerpt: "Query the Bitcoin blockchain dataset", }, @@ -29,58 +49,35 @@ module.exports = [ ], }, { - title: "Cryptocurrency - part 2", + title: "Analyze the Bitcoin blockchain", href: "blockchain-analyze", excerpt: "Analyze the Bitcoin blockchain with Timescale hyperfunctions", children: [ { - title: "Set up", + title: "Ingest data into a service", href: "blockchain-dataset", excerpt: "Set up a dataset so you can analyze the Bitcoin blockchain", }, { - title: "Query data", + title: "Analyse the data", href: "analyze-blockchain-query", excerpt: "Analyze the Bitcoin blockchain dataset with Timescale hyperfunctions", }, ], }, { - title: "Energy - part 1", - href: "energy-data", - excerpt: "Learn how to analyze energy consumption data", - children: [ - { - title: "Set up", - href: "dataset-energy", - excerpt: "Set up a dataset so you can analyze energy consumption data", - }, - { - title: "Query data", - href: "query-energy", - excerpt: "Queries energy consumption data", - }, - { - title: "Bonus: set up compression", - href: "compress-energy", - excerpt: - "Compress the dataset so you can store the data more efficiently", - }, - ] - }, - { - title: "Finance - part 1", + title: "Analyze financial tick data ", href: "financial-tick-data", excerpt: "Use Timescale to store financial tick data", children: [ { - title: "Set up", + title: "Ingest data into a service", href: "financial-tick-dataset", excerpt: "Set up a dataset so you can query financial tick data", }, { - title: "Query data", + title: "Query the data", href: "financial-tick-query", excerpt: "Query and visualize financial tick data", }, @@ -93,72 +90,22 @@ module.exports = [ ], }, { - title: "Finance - part 2", + title: "Ingest real-time financial data", href: "financial-ingest-real-time", excerpt: "Ingest real-time financial data with websocket", children: [ { - title: "Set up", + title: "Ingest data into a service", href: "financial-ingest-dataset", excerpt: "Set up a dataset so you can query the real-time data", }, { - title: "Query data", + title: "Query the data", href: "financial-ingest-query", excerpt: "Query and visualize real-time data", }, ], }, - { - title: "Transport - part 1", - href: "nyc-taxi-cab", - excerpt: "An introduction to time-series using NYC taxi data", - children: [ - { - title: "Set up", - href: "dataset-nyc", - excerpt: "Set up a dataset so you can query NYC data", - }, - { - title: "Query data", - href: "query-nyc", - excerpt: "Query NYC data", - }, - { - title: "Bonus: set up compression", - href: "compress-nyc", - excerpt: - "Compress the dataset so you can store the data more efficiently", - }, - ], - }, - { - title: "Transport - part 2", - href: "nyc-taxi-geospatial", - excerpt: "Learn how to plot geospatial time-series data with NYC taxi cabs", - children: [ - { - title: "Set up", - href: "dataset-nyc", - excerpt: "Set up a dataset so you can plot geospatial NYC taxi data", - }, - { - title: "Query data", - href: "plot-nyc", - excerpt: "Plot geospatial NYC taxi data", - }, - ], - }, - { - title: "Internet of things", - href: "simulate-iot-sensor-data", - excerpt: "Simulate and query an IoT sensor dataset", - }, - { - title: "Timescale community cookbook", - href: "cookbook", - excerpt: "Code examples from the community that help you with loads of common conundrums.", - }, ], }, ]; diff --git a/tutorials/real-time-analytics-energy-consumption.md b/tutorials/real-time-analytics-energy-consumption.md new file mode 100644 index 0000000000..746048b634 --- /dev/null +++ b/tutorials/real-time-analytics-energy-consumption.md @@ -0,0 +1,226 @@ +--- +title: Real-time analytics with Timescale Cloud and Grafana +excerpt: Simulate an IOT dataset in your Timescale Cloud service +products: [cloud, mst, self_hosted] +keywords: [IoT, simulate] +--- + + +import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; +import ImportDataEnergy from "versionContent/_partials/_import-data-iot.mdx"; +import CreateCaggsOnIOTData from "versionContent/_partials/_use-case-iot-create-cagg.mdx"; +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; + +# Analytics on energy consumption + +Energy providers understand that customers tend to lose patience when there is not enough power for them +to complete day-to-day activities. Task one is keeping the lights on. If you are transitioning to renewable energy, +it helps to know when you need to produce energy so you can choose a suitable energy source. + +Real-time analytics refers to the process of collecting, analyzing, and interpreting data instantly as it is generated. +This approach enables you to track and monitor activity, make the decisions based on real-time insights on data stored in +a $SERVICE_LONG and keep those lights on. + + +[Grafana][grafana-docs] is a popular data visualization tool that enables you to create customizable dashboards +and effectively monitor your systems and applications. + +![Grafana real-time analytics](https://assets.timescale.com/docs/images/use-case-rta-grafana-timescale-energy-cagg.png) + +This page shows you how to integrate Grafana with a $SERVICE_LONG and make insights based on visualization of +data optimized for size and speed in the columnstore. + +## Prerequisites + + + +* Install and run [self-managed Grafana][grafana-self-managed], or sign up for [Grafana Cloud][grafana-cloud]. + +## Optimize time-series data in hypertables + + + +## Write fast analytical queries + +Aggregation is a way of combining data to get insights from it. Average, sum, and count are all examples of simple +aggregates. However, with large amounts of data aggregation slows things down, quickly. Continuous aggregates +are a kind of hypertable that is refreshed automatically in the background as new data is added, or old data is +modified. Changes to your dataset are tracked, and the hypertable behind the continuous aggregate is automatically +updated in the background. + +By default, querying continuous aggregates provides you with real-time data. Pre-aggregated data from the materialized +view is combined with recent data that hasn't been aggregated yet. This gives you up-to-date results on every query. + +You create continuous aggregates on uncompressed data in high-performance storage. They continue to work +on [data in the columnstore][test-drive-enable-compression] +and [rarely accessed data in tiered storage][test-drive-tiered-storage]. You can even +create [continuous aggregates on top of your continuous aggregates][hierarchical-caggs]. + + + + + + + +## Optimize your data for real-time analytics + +[Hypercore][hypercore] is the $COMPANY hybrid row-columnar storage engine used by hypertables. Hypertables partition your data in +chunks. Chunks stored in the rowstore use a row-oriented data format optimized for high-speed inserts and updates. +Chunks stored in the columnstore use a columnar data format optimized for analytics. You ingest `hot` data into the +rowstore. As data cools and becomes more suited for analytics, $CLOUD_LONG automatically converts these chunks of data +to the columnstore. You define the moment when data is converted using a columnstore policy. + +When $CLOUD_LONG converts a chunk to the columnstore, TimescaleDB automatically creates a different schema for your +data. $TIMESCALE_DB creates and uses custom indexes to incorporate the `segmentby` and `orderby` parameters when +you write to and read from the columstore. + +To increase the speed of your analytical queries by a factor of 10 and reduce storage costs by up to 90%, convert data +to the columnstore: + + + +1. **Connect to your $SERVICE_LONG** + + In [$CONSOLE][services-portal] open an [SQL editor][in-console-editors]. The in-Console editors display the query speed. + You can also connect to your service using [psql][connect-using-psql]. + +1. **Enable columnstore on a hypertable** + + Create a [job][job] that automatically moves chunks in a hypertable to the columnstore at a specific time interval. + By default, your table is `orderedby` the time column. For efficient queries on columnstore data, remember to + `segmentby` the column you will use most often to filter your data: + + ```sql + ALTER TABLE metrics SET ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'type_id', + timescaledb.orderby = 'created DESC' + ); + ``` + +1. **Add a policy to convert chunks to the columnstore at a specific time interval** + + For example, 60 days after the data was added to the table: + ``` sql + CALL add_columnstore_policy('metrics', INTERVAL '8 days'); + ``` + See [add_columnstore_policy][add_columnstore_policy]. + +1. **View your data space saving** + + When you convert data to the columnstore, as well as being optimized for analytics, it is compressed by more than 90%. + This saves on storage costs and keeps your queries operating at lightning speed. To see the amount of space saved: + ```sql + SELECT + pg_size_pretty(before_compression_total_bytes) as before, + pg_size_pretty(after_compression_total_bytes) as after + FROM hypertable_compression_stats('metrics'); + ``` + You see something like: + + | before | after | + |--------|-------| + | 181 MB | 16 MB | + +1. **Faster analytical queries on data in the columnstore** + + Now run the analytical query again: + ```sql + SELECT time_bucket('1 day', created, 'Europe/Berlin') AS "time", + round((last(value, created) - first(value, created)) * 100.) / 100. AS value + FROM metrics + WHERE type_id = 5 + GROUP BY 1; + ``` + On this amount of data, this analytical query on data in the columnstore takes about 0.8 seconds. + + + +Just to hit this one home, by converting cooling data to the columnstore, you have increased the speed of your analytical +queries by a factor of 10, and reduced storage by up to 90%. + + + +## Visualize energy consumption + +A Grafana dashboard represents a view into the performance of a system, and each dashboard consists of one or +more panels, which represent information about a specific metric related to that system. + +To visually monitor the volume of energy consumption over time: + + + +1. **Create the dashboard** + + 1. On the `Dashboards` page, click `New` and select `New dashboard`. + + 1. Click `Add visualization`, then select the data source that connects to your $SERVICE_LONG and the `Bar chart` + visualization. + + ![Grafana create dashboard](https://assets.timescale.com/docs/images/use-case-rta-grafana-timescale-configure-dashboard.png) + 1. In the `Queries` section, select `Code`, then run the following query based on your continuous aggregate: + + ```sql + WITH per_hour AS ( + SELECT + time, + value + FROM kwh_hour_by_hour + WHERE "time" at time zone 'Europe/Berlin' > date_trunc('month', time) - interval '1 year' + ORDER BY 1 + ), hourly AS ( + SELECT + extract(HOUR FROM time) * interval '1 hour' as hour, + value + FROM per_hour + ) + SELECT + hour, + approx_percentile(0.50, percentile_agg(value)) as median, + max(value) as maximum + FROM hourly + GROUP BY 1 + ORDER BY 1; + ``` + + This query averages the results for households in a specific time zone by hour and orders them by time. + Because you use a continuous aggregate, this data is always correct in real time. + + ![Grafana real-time analytics](https://assets.timescale.com/docs/images/use-case-rta-grafana-timescale-energy-cagg.png) + + You see that energy consumption is highest in the evening and at breakfast time. You also know that the wind + drops off in the evening. This data proves that you need to supply a supplementary power source for peak times, + or plan to store energy during the day for peak times. + +1. **Click `Save dashboard`** + + + + +You have integrated Grafana with a Timescale Cloud service and made insights based on visualization of your data. + +[grafana-docs]: https://grafana.com/docs/ +[grafana-self-managed]: https://grafana.com/get/?tab=self-managed +[grafana-cloud]: https://grafana.com/get/ +[use-time-buckets]: /use-timescale/:currentVersion:/time-buckets/use-time-buckets/ + +[test-drive-enable-compression]: /getting-started/:currentVersion:/try-key-features-timescale-products/#enhance-query-performance-for-analytics +[test-drive-tiered-storage]: /getting-started/:currentVersion:/try-key-features-timescale-products/#slash-storage-charges +[data-tiering]: /use-timescale/:currentVersion:/data-tiering/ +[compression]: /use-timescale/:currentVersion:/compression/ +[hierarchical-caggs]: /use-timescale/:currentVersion:/continuous-aggregates/hierarchical-continuous-aggregates/ +[job]: /api/:currentVersion:/actions/add_job/ +[alter_table_hypercore]: /api/:currentVersion:/hypercore/alter_table/ +[compression_continuous-aggregate]: /api/:currentVersion:/hypercore/alter_materialized_view/ +[convert_to_rowstore]: /api/:currentVersion:/hypercore/convert_to_rowstore/ +[convert_to_columnstore]: /api/:currentVersion:/hypercore/convert_to_columnstore/ +[informational-views]: /api/:currentVersion:/informational-views/jobs/ +[add_columnstore_policy]: /api/:currentVersion:/hypercore/add_columnstore_policy/ +[hypercore_workflow]: /api/:currentVersion:/hypercore/#hypercore-workflow +[alter_job]: /api/:currentVersion:/actions/alter_job/ +[remove_columnstore_policy]: /api/:currentVersion:/hypercore/remove_columnstore_policy/ +[in-console-editors]: /getting-started/:currentVersion:/run-queries-from-console/ +[services-portal]: https://console.cloud.timescale.com/dashboard/services +[connect-using-psql]: /use-timescale/:currentVersion:/integrations/query-admin/psql#connect-to-your-service +[insert]: /use-timescale/:currentVersion:/write-data/insert/ +[hypercore]: /use-timescale/:currentVersion:/hypercore/ diff --git a/tutorials/real-time-analytics-transport.md b/tutorials/real-time-analytics-transport.md new file mode 100644 index 0000000000..94c5c34418 --- /dev/null +++ b/tutorials/real-time-analytics-transport.md @@ -0,0 +1,189 @@ +--- +title: Analytics on transport and geospatial data +excerpt: Simulate and analyze a transport dataset in your Timescale Cloud service +products: [cloud, mst, self_hosted] +keywords: [IoT, simulate] +--- + +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; +import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; +import ImportData from "versionContent/_partials/_import-data-nyc-taxis.mdx"; +import GeolocationAnalytics from "versionContent/_partials/_use-case-transport-geolocation.mdx"; + +# Analytics on transport and geospatial data + +Real-time analytics refers to the process of collecting, analyzing, and interpreting data instantly as it +is generated. This approach enables you track and monitor activity, and make decisions based on real-time +insights on data stored in a $SERVICE_LONG. + +![Real-time analytics geolocation](https://assets.timescale.com/docs/images/use-case-rta-grafana-heatmap.png) + +This page shows you how to integrate [Grafana][grafana-docs] with a $SERVICE_LONG and make insights based on visualization +of data optimized for size and speed in the columnstore. + +## Prerequisites + + + +* Install and run [self-managed Grafana][grafana-self-managed], or sign up for [Grafana Cloud][grafana-cloud]. + +## Optimize time-series data in hypertables + + + +## Optimize your data for real-time analytics + +[Hypercore][hypercore] is the $COMPANY hybrid row-columnar storage engine used by hypertables. Hypertables partition your data in +chunks. Chunks stored in the rowstore use a row-oriented data format optimized for high-speed inserts and updates. +Chunks stored in the columnstore use a columnar data format optimized for analytics. You ingest `hot` data into the +rowstore. As data cools and becomes more suited for analytics, $CLOUD_LONG automatically converts these chunks of data +to the columnstore. You define the moment when data is converted using a columnstore policy. + +When $CLOUD_LONG converts a chunk to the columnstore, TimescaleDB automatically creates a different schema for your +data. $TIMESCALE_DB creates and uses custom indexes to incorporate the `segmentby` and `orderby` parameters when +you write to and read from the columstore. + +To increase the speed of your analytical queries by a factor of 10 and reduce storage costs by up to 90%, convert data +to the columnstore: + + + +1. **Connect to your $SERVICE_LONG** + + In [$CONSOLE][services-portal] open an [SQL editor][in-console-editors]. The in-Console editors display the query speed. + You can also connect to your service using [psql][connect-using-psql]. + +1. **Enable columnstore on a hypertable** + + Create a [job][job] that automatically moves chunks in a hypertable to the columnstore at a specific time interval. + By default, your table is `orderedby` the time column. For efficient queries on columnstore data, remember to + `segmentby` the column you will use most often to filter your data: + + ```sql + ALTER TABLE rides SET ( + timescaledb.enable_columnstore = true, + timescaledb.segmentby = 'vendor_id', + timescaledb.orderby = 'pickup_datetime DESC' + ); + ``` + +1. **Add a policy to convert chunks to the columnstore at a specific time interval** + + For example, convert data older than 8 days old to the columstore: + ``` sql + CALL add_columnstore_policy('rides', INTERVAL '8 days'); + ``` + See [add_columnstore_policy][add_columnstore_policy]. + +1. **View your data space saving** + + When you convert data to the columnstore, as well as being optimized for analytics, it is compressed by more than 90%. + This saves on storage costs and keeps your queries operating at lightning speed. To see the amount of space saved: + ```sql + SELECT + pg_size_pretty(before_compression_total_bytes) as before, + pg_size_pretty(after_compression_total_bytes) as after + FROM hypertable_compression_stats('rides'); + ``` + You see something like: + + | before | after | + |--|--| + |2818 MB | 673 MB | + +1. **Faster analytical queries on data in the columnstore** + + Now run the analytical query again: + ```sql + SELECT rates.description, COUNT(vendor_id) AS num_trips + FROM rides + JOIN rates ON rides.rate_code = rates.rate_code + WHERE pickup_datetime < '2016-01-08' + GROUP BY rates.description + ORDER BY LOWER(rates.description); + ``` + On this large amount of data, this analytical query on data in the columnstore takes about 6 seconds. + + + +Just to hit this one home, by converting cooling data to the columnstore, you have increased the speed of your analytical +queries by a factor of 10, and reduced storage by up to 90%. + + + + +## Monitor performance over time + +A Grafana dashboard represents a view into the performance of a system, and each dashboard consists of one or +more panels, which represent information about a specific metric related to that system. + +To visually monitor the volume of taxi rides over time: + + + +1. **Create the dashboard** + + 1. On the `Dashboards` page, click `New` and select `New dashboard`. + + 1. Click `Add visualization`. + 1. Select the data source that connects to your $SERVICE_LONG. + The `Time series` visualization is chosen by default. + ![Grafana create dashboard](https://assets.timescale.com/docs/images/use-case-rta-grafana-timescale-configure-dashboard.png) + 1. In the `Queries` section, select `Code`, then select `Time series` in `Format`. + 1. Select the data range for your visualization: + the data set is from 2016. Click the date range above the panel and set: + - From: ```2016-01-01 01:00:00``` + - To: ```2016-01-30 01:00:00``` + +1. **Combine $TIMESCALE_DB and Grafana functionality to analyze your data** + + Combine a $TIMESCALE_DB [time_bucket][use-time-buckets], with the Grafana `$__timefilter()` function to set the + `pickup_datetime` column as the filtering range for your visualizations. + ```sql + SELECT + time_bucket('1 day', pickup_datetime) AS "time", + COUNT(*) + FROM rides + WHERE $__timeFilter(pickup_datetime) + GROUP BY time + ORDER BY time; + ``` + This query groups the results by day and orders them by time. + + ![Grafana real-time analytics](https://assets.timescale.com/docs/images/use-case-rta-grafana-timescale-final-dashboard.png) + +1. **Click `Save dashboard`** + + + + +## Optimize revenue potential + +Having all this data is great but how do you use it? Monitoring data is useful to check what +has happened, but how can you analyse this information to your advantage? This section explains +how to create a visualization that shows how you can maximize potential revenue. + + + +You have integrated Grafana with a $SERVICE_LONG and made insights based on visualization of +your data. + +[grafana-docs]: https://grafana.com/docs/ +[grafana-self-managed]: https://grafana.com/get/?tab=self-managed +[grafana-cloud]: https://grafana.com/get/ +[use-time-buckets]: /use-timescale/:currentVersion:/time-buckets/use-time-buckets/ +[job]: /api/:currentVersion:/actions/add_job/ +[alter_table_hypercore]: /api/:currentVersion:/hypercore/alter_table/ +[compression_continuous-aggregate]: /api/:currentVersion:/hypercore/alter_materialized_view/ +[convert_to_rowstore]: /api/:currentVersion:/hypercore/convert_to_rowstore/ +[convert_to_columnstore]: /api/:currentVersion:/hypercore/convert_to_columnstore/ +[informational-views]: /api/:currentVersion:/informational-views/jobs/ +[add_columnstore_policy]: /api/:currentVersion:/hypercore/add_columnstore_policy/ +[hypercore_workflow]: /api/:currentVersion:/hypercore/#hypercore-workflow +[alter_job]: /api/:currentVersion:/actions/alter_job/ +[remove_columnstore_policy]: /api/:currentVersion:/hypercore/remove_columnstore_policy/ +[in-console-editors]: /getting-started/:currentVersion:/run-queries-from-console/ +[services-portal]: https://console.cloud.timescale.com/dashboard/services +[connect-using-psql]: /use-timescale/:currentVersion:/integrations/query-admin/psql#connect-to-your-service +[insert]: /use-timescale/:currentVersion:/write-data/insert/ +[hypercore]: /use-timescale/:currentVersion:/hypercore/ diff --git a/use-timescale/alerting.md b/use-timescale/alerting.md index 1f711d6061..dafb4a11c2 100644 --- a/use-timescale/alerting.md +++ b/use-timescale/alerting.md @@ -7,6 +7,8 @@ keywords: [alert, integration, Grafana, Datadog, Nagios, Zabbix] # Alerting +Early issue detecting and prevention, ensuring high availability, and performance optimization are only a few of the reasons why alerting plays a major role for modern applications, databases, and services. + There are a variety of different alerting solutions you can use in conjunction with Timescale that are part of the PostgreSQL ecosystem. Regardless of whether you are creating custom alerts embedded in your applications, or using @@ -15,7 +17,7 @@ are a wide selection of tools available. ## Grafana -Grafana is a great way to visualize and explore time-series data and has a +Grafana is a great way to visualize your analytical queries, and it has a first-class integration with Timescale. Beyond data visualization, Grafana also provides alerting functionality to keep you notified of anomalies. @@ -33,7 +35,9 @@ it is designed to take advantage of the database's time-series capabilities. From there, proceed to your dashboard and set up alert rules as described above. + Alerting is only available in Grafana v4.0 and later. + ## Other alerting tools diff --git a/use-timescale/compression/about-compression.md b/use-timescale/compression/about-compression.md index 3a5acd4e71..3889de65c7 100644 --- a/use-timescale/compression/about-compression.md +++ b/use-timescale/compression/about-compression.md @@ -4,11 +4,13 @@ excerpt: When you compress data in a hypertable, multiple records are grouped in products: [self_hosted] keywords: [compression, hypertables] --- - +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; import CompressionIntro from 'versionContent/_partials/_compression-intro.mdx'; # About compression + Replaced by Hypercore. + This section explains how to enable native compression, and then goes into diff --git a/use-timescale/compression/compression-design.md b/use-timescale/compression/compression-design.md index 54100d2d3d..68fffa6163 100644 --- a/use-timescale/compression/compression-design.md +++ b/use-timescale/compression/compression-design.md @@ -5,8 +5,13 @@ products: [cloud, mst, self_hosted] keywords: [compression, schema, tables] --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # Designing for compression + Replaced by Hypercore. + + Time-series data can be unique, in that it needs to handle both shallow and wide queries, such as "What's happened across the deployment in the last 10 minutes," and deep and narrow, such as "What is the average CPU usage for this server diff --git a/use-timescale/compression/compression-methods.md b/use-timescale/compression/compression-methods.md index 5fe3449fcc..a15abfe536 100644 --- a/use-timescale/compression/compression-methods.md +++ b/use-timescale/compression/compression-methods.md @@ -5,8 +5,13 @@ products: [cloud, mst, self_hosted] keywords: [compression] --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # About compression methods + Replaced by Hypercore. + + TimescaleDB uses different compression algorithms, depending on the data type that is being compressed. diff --git a/use-timescale/compression/compression-policy.md b/use-timescale/compression/compression-policy.md index 578a8fdc7d..6fdc5c9526 100644 --- a/use-timescale/compression/compression-policy.md +++ b/use-timescale/compression/compression-policy.md @@ -4,11 +4,14 @@ excerpt: Timescale Cloud can compress your data automatically, based on the cond products: [cloud, mst, self_hosted] keywords: [compression, hypertables, policy] --- - +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; import CompressionIntro from 'versionContent/_partials/_compression-intro.mdx'; # Compression policy + Replaced by Optimize your data for real-time analytics. + + You can enable compression on individual hypertables, by declaring which column you want to segment by. diff --git a/use-timescale/compression/decompress-chunks.md b/use-timescale/compression/decompress-chunks.md index 6a5dff47a1..9f6412a183 100644 --- a/use-timescale/compression/decompress-chunks.md +++ b/use-timescale/compression/decompress-chunks.md @@ -6,8 +6,12 @@ keywords: [compression, hypertables, backfilling] tags: [decompression] --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # Decompression + Replaced by Modify your data in Hypercore. + Timescale automatically supports `INSERT`s into compressed chunks. But if you need to insert a lot of data, for example as part of a bulk backfilling operation, you should first decompress the chunk. Inserting data into a @@ -15,10 +19,12 @@ compressed chunk is more computationally expensive than inserting data into an uncompressed chunk. This adds up over a lot of rows. + When compressing your data, you can reduce the amount of storage space for your Timescale instance. But you should always leave some additional storage capacity. This gives you the flexibility to decompress chunks when necessary, for actions such as bulk inserts. + This section describes commands to use for decompressing chunks. You can filter diff --git a/use-timescale/compression/index.md b/use-timescale/compression/index.md index 7532f19960..c932d0bf28 100644 --- a/use-timescale/compression/index.md +++ b/use-timescale/compression/index.md @@ -4,10 +4,12 @@ excerpt: With data compression, you can achieve a significant improvement in the products: [cloud, mst, self_hosted] keywords: [compression, hypertables] --- - +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; import UsageBasedStorage from "versionContent/_partials/_usage-based-storage-intro.mdx"; -# Compression +# Compression (Replaced by [Hypercore][hypercore]) + + Replaced by Hypercore. Time-series data can be compressed to reduce the amount of storage required, and increase the speed of some queries. This is a cornerstone feature of @@ -17,3 +19,6 @@ data to the form of compressed columns. This occurs across chunks of Timescale hypertables. + + +[hypercore]: /use-timescale/:currentVersion:/hypercore/ diff --git a/use-timescale/compression/manual-compression.md b/use-timescale/compression/manual-compression.md index e8b45c75a9..a967e1b970 100644 --- a/use-timescale/compression/manual-compression.md +++ b/use-timescale/compression/manual-compression.md @@ -4,9 +4,12 @@ excerpt: Timescale Cloud provides automated and manual data compression. Learn t products: [self_hosted] keywords: [compression, hypertables] --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; # Manually compress chunks + Replaced by Modify your data in Hypercore. + In most cases, an [automated compression policy][add_compression_policy] is sufficient to automatically compress your chunks. However, if you want more control over compression, you can also manually compress specific chunks. diff --git a/use-timescale/compression/modify-a-schema.md b/use-timescale/compression/modify-a-schema.md index 35a482bb2e..5111132d0e 100644 --- a/use-timescale/compression/modify-a-schema.md +++ b/use-timescale/compression/modify-a-schema.md @@ -5,8 +5,12 @@ products: [cloud, mst, self_hosted] keywords: [compression, schemas, hypertables] --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; + # Schema modifications + Replaced by Modify your data in Hypercore. + You can modify the schema of compressed hypertables in recent versions of Timescale. diff --git a/use-timescale/compression/modify-compressed-data.md b/use-timescale/compression/modify-compressed-data.md index f6f25983a9..8e5cad4808 100644 --- a/use-timescale/compression/modify-compressed-data.md +++ b/use-timescale/compression/modify-compressed-data.md @@ -4,9 +4,12 @@ excerpt: In Timescale Cloud, compressed data can still be modified. Learn to ins products: [cloud, mst, self_hosted] keywords: [compression, backfilling, hypertables] --- +import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; # Insert and modify compressed data + Replaced by Modify your data in Hypercore. + In TimescaleDB 2.11 and later, you can insert data into compressed chunks, and modify data in compressed rows. diff --git a/use-timescale/configuration/advanced-parameters.md b/use-timescale/configuration/advanced-parameters.md index a9b4e687e5..17f6d2812c 100644 --- a/use-timescale/configuration/advanced-parameters.md +++ b/use-timescale/configuration/advanced-parameters.md @@ -9,6 +9,8 @@ cloud_ui: - [services, :serviceId, operations, database_parameters] --- +import ConfigCloudSelf from "versionContent/_partials/_cloud_self_configuration.mdx"; + # Advanced parameters It is possible to configure a wide variety of service database parameters by @@ -23,9 +25,9 @@ prompting you to confirm changes before the service is modified. ## Multiple databases -To create more than one Timescale database, you need to create a new -service for each database. Timescale does not support multiple -databases within the same service. Having a separate service for each database +To create more than one database, you need to create a new +$SERVICE_SHORT for each database. $CLOUD_LONG does not support multiple +databases within the same service. Having a separate $SERVICE_SHORT for each database affords each database its own isolated resources. You can also use [schemas][schemas] to organize tables into logical groups. A @@ -33,6 +35,9 @@ single database can contain multiple schemas, which in turn contain tables. The main difference between isolating with databases versus schemas is that a user can access objects in any of the schemas in the database they are connected to, so long as they have the corresponding privileges. Schemas can help isolate -smaller use cases that do not warrant their own service. +smaller use cases that do not warrant their own $SERVICE_SHORT. + + + [schemas]: /use-timescale/:currentVersion:/schema-management/ diff --git a/use-timescale/continuous-aggregates/hierarchical-continuous-aggregates.md b/use-timescale/continuous-aggregates/hierarchical-continuous-aggregates.md index 985b2ce6cf..957edea425 100644 --- a/use-timescale/continuous-aggregates/hierarchical-continuous-aggregates.md +++ b/use-timescale/continuous-aggregates/hierarchical-continuous-aggregates.md @@ -6,9 +6,9 @@ keywords: [continuous aggregates, hierarchical, create] # Hierarchical continuous aggregates -You can create continuous aggregates on top of other continuous aggregates. This -allows you to summarize data at different levels of granularity. For example, -you might have an hourly continuous aggregate that summarizes minute-by-minute +The more data you have, the more likely you are to run a more sophisticated analysis on it. When a simple one-level aggregation is not enough, $CLOUD_LONG lets you create continuous aggregates on top of other continuous aggregates. This way, you summarize data at different levels of granularity, while still saving resources with precomputing. + +For example, you might have an hourly continuous aggregate that summarizes minute-by-minute data. To get a daily summary, you can create a new continuous aggregate on top of your hourly aggregate. This is more efficient than creating the daily aggregate on top of the original hypertable, because you can reuse the diff --git a/use-timescale/continuous-aggregates/index.md b/use-timescale/continuous-aggregates/index.md index 84f701c6dd..902cf0187e 100644 --- a/use-timescale/continuous-aggregates/index.md +++ b/use-timescale/continuous-aggregates/index.md @@ -7,22 +7,20 @@ keywords: [continuous aggregates] # Continuous aggregates -Continuous aggregates are designed to make queries on very large datasets run -faster. Timescale continuous aggregates use -PostgreSQL [materialized views][postgres-materialized-views] to continuously and -incrementally refresh a query in the background, so that when you run the query, -only the data that has changed needs to be computed, not the entire dataset. +From real-time dashboards to performance monitoring and historical trend analysis, data aggregation is a must-have for any sort of analytical application. To address this need, $CLOUD_LONG continuous aggregates precompute and store aggregate data for you. Using PostgreSQL [materialized views][postgres-materialized-views], continuous aggregates incrementally refresh the aggregation query in the background, so that when you do run it, only the data that has changed needs to be computed, not the entire dataset. This means you always have the latest aggregate data at your fingertips - and spend as little resources on it, as possible. + +In this section you: * [Learn about continuous aggregates][about-caggs] to understand how it works before you begin using it. * [Create a continuous aggregate][cagg-create] and query it. -* [Create a continuous aggregate on top of another continuous aggregate][cagg-on-cagg] +* [Create a continuous aggregate on top of another continuous aggregate][cagg-on-cagg]. * [Add refresh policies][cagg-autorefresh] to an existing continuous aggregate. * [Manage time][cagg-time] in your continuous aggregates. * [Drop data][cagg-drop] from your continuous aggregates. * [Manage materialized hypertables][cagg-mat-hypertables]. * [Use real-time aggregates][cagg-realtime]. -* [Compression with continuous aggregates][cagg-compression]. +* [Use compression with continuous aggregates][cagg-compression]. * [Migrate your continuous aggregates][cagg-migrate] from old to new format. Continuous aggregates created in Timescale 2.7 and later are in the new format, unless explicitly created in the old format. diff --git a/use-timescale/continuous-aggregates/real-time-aggregates.md b/use-timescale/continuous-aggregates/real-time-aggregates.md index cd861e20f5..4a7863b80b 100644 --- a/use-timescale/continuous-aggregates/real-time-aggregates.md +++ b/use-timescale/continuous-aggregates/real-time-aggregates.md @@ -7,35 +7,36 @@ keywords: [continuous aggregates, real-time aggregates] import CaggsRealTimeHistoricalDataRefreshes from 'versionContent/_partials/_caggs-real-time-historical-data-refreshes.mdx'; -# Real time aggregates +# Real-time aggregates -Continuous aggregates do not include the most recent data chunk from the -underlying hypertable. Real time aggregates use the aggregated data and add the -most recent raw data to it to provide accurate and up to date results, without -needing to aggregate data as it is being written. In Timescale versions 1.7 to 2.12, -real time aggregates are enabled by default; when you create a continuous +Rapidly growing data means you need more control over what to aggregate and how to aggregate it. With this in mind, $CLOUD_LONG equips you with tools for more fine-tuned data analysis. + +By default, continuous aggregates do not include the most recent data chunk from the +underlying hypertable. Real-time aggregates, however, use the aggregated data **and** add the +most recent raw data to it. This provides accurate and up-to-date results, without +needing to aggregate data as it is being written. + +In Timescale 2.13 and later real-time aggregates are *DISABLED* by default. In Timescale versions 1.7 to 2.12, real-time aggregates are enabled by default; when you create a continuous aggregate view, queries to that view include the most recent data, even if -it has not yet been aggregated. In Timescale 2.13 and later real time aggregates are *DISABLED* by default. +it has not yet been aggregated. -For more detail on the comparison between continuous and real time aggregates, +For more detail on the comparison between continuous and real-time aggregates, see our [real time aggregate blog post][blog-rtaggs]. -## Use real time aggregates +## Use real-time aggregates -You can enable and disable real time aggregation by setting the +You can enable and disable real-time aggregation by setting the `materialized_only` parameter when you create or alter the view. -### Using real time aggregation - -1. For an existing table, at the `psql` prompt, disable real time aggregation: +1. For an existing table, at the `psql` prompt, disable real-time aggregation: ```sql ALTER MATERIALIZED VIEW table_name set (timescaledb.materialized_only = true); ``` -1. Re-enable real time aggregation: +1. Re-enable real-time aggregation: ```sql ALTER MATERIALIZED VIEW table_name set (timescaledb.materialized_only = false); diff --git a/use-timescale/data-retention/about-data-retention.md b/use-timescale/data-retention/about-data-retention.md index 331e4ccaeb..ee879d975b 100644 --- a/use-timescale/data-retention/about-data-retention.md +++ b/use-timescale/data-retention/about-data-retention.md @@ -9,15 +9,12 @@ import UsageBasedStorage from "versionContent/_partials/_usage-based-storage-int # About data retention -In time-series applications, data often becomes less useful as it gets older. If -you don't need your historical data, you can delete it once it reaches a certain -age. Timescale lets you set up -[automatic data retention policies][retention-policy] to discard old data. You -can also fine-tune data retention by [manually dropping chunks][manual-drop]. - -Often, you want to keep summaries of your historical data, but you don't need -the raw data. You can downsample your older data by -[combining data retention with continuous aggregates][retention-with-caggs]. +In modern applications, data grows exponentially. As data gets older, it often becomes less useful in day-to-day operations. +However, you still need it for analysis. Timescale elegantly solves this problem with +[automated data retention policies][retention-policy]. + +Data retention policies delete raw old data for you on a schedule that you define. +By [combining retention policies with continuous aggregates][retention-with-caggs], you can downsample your data and keep useful summaries of it instead. This lets you analyze historical data - while also saving on storage. diff --git a/use-timescale/data-tiering/enabling-data-tiering.md b/use-timescale/data-tiering/enabling-data-tiering.md index 4ae589a219..fb7f56ce5a 100644 --- a/use-timescale/data-tiering/enabling-data-tiering.md +++ b/use-timescale/data-tiering/enabling-data-tiering.md @@ -34,14 +34,14 @@ You enable tiered storage from the `Overview` tab in Console. When tiered storage is enabled, you see the amount of data in the tiered object storage. - - - Data tiering is available in [Scale and Enterprise][pricing-plans] pricing plans only. - - - + + +Data tiering is available in [Scale and Enterprise][pricing-plans] pricing plans only. + + + ## Automate tiering with policies A tiering policy automatically moves any chunks that only contain data diff --git a/use-timescale/ha-replicas/high-availability.md b/use-timescale/ha-replicas/high-availability.md index bfb3c56766..3834fe5749 100644 --- a/use-timescale/ha-replicas/high-availability.md +++ b/use-timescale/ha-replicas/high-availability.md @@ -9,6 +9,8 @@ cloud_ui: - [services, :serviceId, operations, replication] --- +import HASetup from 'versionContent/_partials/_high-availability-setup.mdx'; + # Manage high availability @@ -71,26 +73,12 @@ The `High` and `Highest` HA strategies are available with the [Scale and the Ent To enable HA for a Timescale Cloud Service: - - -1. In [Timescale Console][cloud-login], select the service to enable replication for. -1. Click `Operations`, then select `High availability`. -1. Choose your replication strategy, then click `Change configuration`. - Creating a database replica in Timescale - -1. In `Change high availability configuration`, click `Change config`. + To change your HA replica strategy, click `Change configuration`, choose a strategy and click `Change configuration`. To download the connection information for the HA replica, either click the link next to the replica `Active configuration`, or find the information in the `Overview` tab for this service. - - - ## Test failover for your HA replicas To test the failover mechanism, you can trigger a switchover. A switchover is a diff --git a/use-timescale/hypercore/best-practice-for-large-amounts-of-data.md b/use-timescale/hypercore/best-practice-for-large-amounts-of-data.md new file mode 100644 index 0000000000..e01b8e802f --- /dev/null +++ b/use-timescale/hypercore/best-practice-for-large-amounts-of-data.md @@ -0,0 +1,188 @@ +--- +title: Indexing data in Hypercore +excerpt: Use indexes on hybrid row-columnar data +products: [cloud,] +keywords: [hypertable, compression, row-columnar storage, hypercore] +--- + +import Prereq from "versionContent/_partials/_prereqs-cloud-and-self.mdx"; +import HypercoreManualWorkflow from "versionContent/_partials/_hypercore_manual_workflow.mdx"; + +# SUGGESTION: Best practice for large amounts of data + +IAIN: Hi Mats, I left this in here so you do no lose your work. + +Indexes are a central component to designing efficient and performant +databases: they allow unstructured data to be efficiently searched to +find the data of interests and in that way allow efficient execution +of queries. + +Indexes are similar to a alpabetical book index: you can quickly find +the right name in the index at the end of the book because it is +alphabetical and it will give you the page and maybe the paragraph +where the name is. + +Indexes uses a *key*—for example the pair first-last name "John, +Doe"—to search through the index and find the *page*. + +## Indexes and Table Access Methods in PostgreSQL + +The keys of an index can be either be a single column, or consist of +multiple colums. For example, using the example hypertable from the +previous chapter: + +```sql +create table metrics( + metric_id serial, + created_at timestamptz not null unique, + location_id smallint, + device_id bigint, + temp float8, + humidity float4 +); +``` + +You can then define an index with the name `metrics_device_id_idx` for +the devices, which will allow you to efficiently find the device +information in the `metrics` table. + +```sql +create index metrics_device_id_idx on metrics (device_id); +``` + +In the same way as indexes for books, indexes in PostgreSQL are data +structures that translate keys to a *disk page* and an *offset*, which +are in PostgreSQL called a *tuple identifier* (or just *TID*). The TID +corresponds to the page for the data in the book analogy. You can +imagine that the index is a sorted list of rows like this, with the +keys each in one column and the TID in a separate column with the page +and the offset: + +| `device_id` | TID | +|-------------|------| +| . | . | +| . | . | +| . | . | +| 2 | 5:32 | +| 3 | 6:19 | +| 3 | 7:23 | +| 3 | 4:99 | +| 4 | 1:79 | +| 4 | 4:51 | +| 5 | 2:45 | +| 6 | 1:23 | +| . | . | +| . | . | +| . | . | + +When PostgreSQL then want to use the index, it can locate the TIDs for +a particular key. For example, if you execute the query: + +```sql +select * from metrics where device_id = 3; +``` + +PostgreSQL will first use the index to locate all the TIDs for the key +`3` (that is, `6:19`, `7:23`, and `4:99`), but then it need to find +the tuples (the entire row with data) for those TIDs, so how is that +done? The answer is that it is the responsibility of the *table access +method* for the table to figure that out. + +Each table in PostgreSQL has an associated table access method. The +default one is called the "heap". The TID is passed to the table +access method in question, which then returns the entire tuple. + +You can see the table access method for a table in this manner: + +```sql +example=# \d+ metrics + Table "public.metrics" + Column | Type | Collation | Nullable | Default | Storage | Compression | Stats target | Description +-------------+--------------------------+-----------+----------+--------------------------------------------+---------+-------------+--------------+------------- + metric_id | integer | | not null | nextval('metrics_metric_id_seq'::regclass) | plain | | | + created_at | timestamp with time zone | | not null | | plain | | | + location_id | smallint | | | | plain | | | + device_id | bigint | | | | plain | | | + temp | double precision | | | | plain | | | + humidity | real | | | | plain | | | +Indexes: + "metrics_created_at_key" UNIQUE CONSTRAINT, btree (created_at) +Access method: heap +``` + +Since data for the Hypercore columnstore is stored in a combined row +and columnar format (that is, some data is in row format and some are +in columnar format), Timescale has a dedicated table access method +called `hypercore` that allow chunks of a hypertable to maintain +indexes over the combined row-columnar format that Hypercore uses. + +## Creating a new hypertable using Hypercore TAM + +This case is normally used if you want to create a new hypertable +where all chunks uses the `hypercore` table access method. + +You can create a new hypertable using the Hypercore TAM using the +following steps: + +1. Create the table using `CREATE TABLE`, for example, the table + above. +2. Transform the table into a hypertable using `create_hypertable` and + use the `created_at` column for partitioning the hypertable. + + ```sql + select create_hypertable('readings', by_range('created_at')); + ``` + +3. Configure compression using `alter table` as in the previous + chapter, but set the table access method to `hypercore`. + + ```sql + alter table metrics + set access method hypercore, + set (timescaledb.compress_orderby = 'created_at', + timescaledb.compress_segmentby = 'location_id'); + ``` + + Setting the table access method in this manner for a hypertable + means that any new chunks created when inserting data will use the + `hypercore` table access method. + +4. Set up a policy as in the previous chapter, but tell the policy to + use the (table) access method to compress the chunks using the + `hypercore_use_access_method` parameter. + + ```sql + CALL add_columnstore_policy('metrics', interval '1 day', + hypercore_use_access_method => true); + ``` + +## Changing a single chunk to replaced by Hypercore TAM + +If you have an existing hypertable, and want to just change a single +chunk to use the `hypercore` table access method, you can change the +table access method for that chunk only either using `alter table` or +the procedure `convert_to_columnstore`. + +This can be useful if you want to experiment with indexing on +compressed data, or if you just want to index some chunks of the +hypertable. + +As an example, if you have a chunk +`_timescaledb_internal._hyper_1_21_chunk` (in this case from the +`metrics` table above), you can convert it to a columnstore chunk +using the table access method either by setting the access method: + +```sql +alter table _timescaledb_internal._hyper_1_21_chunk + set access method hypercore; +``` + +Or using the `convert_to_columnstore` procedure with +`hypercore_use_access_method`: + +```sql +call convert_to_columnstore('_timescaledb_internal._hyper_1_21_chunk', + hypercore_use_access_method => true); +``` + +This will also rebuild indexes for the chunk if it was compressed, s diff --git a/use-timescale/hypercore/efficient-data-access-in-hypercore.md b/use-timescale/hypercore/efficient-data-access-in-hypercore.md new file mode 100644 index 0000000000..308742d27a --- /dev/null +++ b/use-timescale/hypercore/efficient-data-access-in-hypercore.md @@ -0,0 +1,75 @@ +--- +title: Efficient data access in Hypercore +excerpt: Read and write data efficiently in Hypercore. +products: [cloud,self_hosted] +keywords: [hypertable, compression, row-columnar storage, hypercore] +--- +import Prereq from "versionContent/_partials/_prereqs-cloud-and-self.mdx"; + +# Access your data efficiently in Hypercore + + +Intro + + + +This page shows you how to ... . + +## Prerequisites + + + +- [Prepare your data for real-time analytics in Hypercore][setup-hypercore] + + +## Read data efficiently + +Explain why and how to read data efficiently. + + + +1. **Do this** + + Why and how to do this. + + ``` sql + Code example + ``` + +1. **Do that** + + Why and how to do that. + + ``` sql + Code example + ``` + + + +## Write data efficiently + + +Explain why and how to write data efficiently. + + + +1. **Do this** + + Why and how to do this. + + ``` sql + Code example + ``` + +1. **Do that** + + Why and how to do that. + + ``` sql + Code example + ``` + + + + +[setup-hypercore]: /use-timescale/:currentVersion:/hypercore/real-time-analytics-in-hypercore/ diff --git a/use-timescale/hypercore/index.md b/use-timescale/hypercore/index.md new file mode 100644 index 0000000000..834276cc04 --- /dev/null +++ b/use-timescale/hypercore/index.md @@ -0,0 +1,60 @@ +--- +title: Hypercore +excerpt: The Timescale hybrid row-columnar storage engine for real-time analytics, powered by time-series data +products: [cloud,] +keywords: [hypercore, hypertable, compression, row-columnar storage] +--- + +import UsageBasedStorage from "versionContent/_partials/_usage-based-storage-intro.mdx"; + +# Hypercore + +Hypercore is the $TIMESCALE_DB hybrid row-columnar storage engine, designed specifically for +real-time analytics and powered by time-series data. The advantage of Hypercore is its ability +to seamlessly switch between row-oriented and column-oriented storage. This flexibility enables +$CLOUD_LONG to deliver the best of both worlds, solving the key challenges in real-time analytics: + +- High ingest throughput +- Low-latency ingestion +- Fast query performance +- Efficient handling of data updates and late-arriving data +- Streamlined data management + +Hypercore’s hybrid approach combines the benefits of row-oriented and column-oriented formats +in each $SERVICE_LONG: + +- **Fast ingest with rowstore**: new data is initially written to the rowstore, which is optimized for + high-speed inserts and updates. This process ensures that real-time applications easily handle + rapid streams of incoming data. Mutability—upserts, updates, and deletes happen seamlessly. + +- **Efficient analytics with columnstore**: as the data _cools_ and becomes more suited for + analytics, it is automatically migrated to the columnstore. This columnar format enables + fast scanning and aggregation, optimizing performance for analytical workloads while also + saving significant storage space. + +- **Faster queries on compressed data in columnstore**: in columnstore conversion, hypertable + chunks are compressed by more than 90%, and organized for efficient, large-scale queries. This + saves on storage costs, and keeps your queries operating at lightning speed. + +- **Full mutability with transactional semantics**: regardless of where data is stored, + Hypercore provides full ACID support. Like in a vanilla Postgres database, inserts and updates + to the rowstore and columnstore are always consistent, and available to queries as soon as they are + completed. + + +![Hypercore workflow](https://assets.timescale.com/docs/images/hypercore-overview.png) + +In Timescale Cloud you only pay for what you use. Data moved to the columnstore is compressed, which +immediately translates into cost savings. + + +This section shows you how to: + +* [Optimize your data for real-time analytics][setup-hypercore] +* [Modify data in the columnstore][modify-data-in-hypercore] + +[setup-hypercore]: /use-timescale/:currentVersion:/hypercore/real-time-analytics-in-hypercore/ +[modify-data-in-hypercore]: /use-timescale/:currentVersion:/hypercore/modify-data-in-hypercore/ +[indexing-data-in-hypercore]: /use-timescale/:currentVersion:/hypercore/best-practice-for-large-amounts-of-data/ +[compression]: /use-timescale/:currentVersion:/compression/ + diff --git a/use-timescale/hypercore/modify-data-in-hypercore.md b/use-timescale/hypercore/modify-data-in-hypercore.md new file mode 100644 index 0000000000..b0162628ec --- /dev/null +++ b/use-timescale/hypercore/modify-data-in-hypercore.md @@ -0,0 +1,110 @@ +--- +title: Modify data in Hypercore +excerpt: Update data and the table schema in the columnstore +products: [cloud,] +keywords: [hyperscore, hypertable, compression, row-columnar storage, hypercore] +--- + +import PrereqCloud from "versionContent/_partials/_prereqs-cloud-and-self.mdx"; +import HypercoreManualWorkflow from "versionContent/_partials/_hypercore_manual_workflow.mdx"; + +# Modify your data in Hypercore + +You [set up Hypercore][setup-hypercore] to automatically convert data between the rowstore and columnstore +when it reaches a certain age. After you have optimized data in the columnstore, you may need to modify it. +For example, to make small changes, or backfill large amounts of data. You may even have to update the schema to +accommodate these changes to the data. + +This page shows you how to update small and large amounts of new data, and update the schema in columnstore. + +## Prerequisites + + + +- [Optimize your data for real-time analytics][setup-hypercore] + +## Modify small amounts of data + +You can [`INSERT` `UPDATE` and `DELETE`][write] data in the columnstore, even if the data you are +inserting has unique constraints. When you insert data into a chunk in the columnstore, a small amount +of data is decompressed to allow a speculative insertion, and block any inserts which could violate +constraints. + +When you `DELETE` whole segments of data, filter your deletes using the column you `segment_by` +instead of separate deletes. This considerably increases performance. + +## Modify large amounts of data + +If you need to modify or add a lot of data to a chunk in the columnstore, best practice is to stop +any [jobs][job] moving chunks to the columnstore, convert the chunk back to the rowstore, then modify the +data. After the update, [convert the chunk to the columnstore][convert_to_columnstore] and restart the jobs. +This workflow is especially useful if you need to backfill old data. + + + + + + + +## Modify a table schema for data in the columnstore + +You can modify the schema of a table in the columnstore. To do this, you need to: + + + +1. **Stop the jobs that are automatically adding chunks to the columnstore** + + Retrieve the list of jobs from the [timescaledb_information.jobs][informational-views] view + to find the job you need to [alter_job][alter_job]. + + ``` sql + SELECT alter_job(JOB_ID, scheduled => false); + ``` + +1. **Convert a chunk to update back to the rowstore** + + ``` sql + CALL convert_to_rowstore('_timescaledb_internal._hyper_2_2_chunk'); + ``` + +2. **Modify the schema**: + + Possible modifications are: + + - Add a nullable column: + + `ALTER TABLE ADD COLUMN ;` + - Add a column with a default value and a `NOT NULL` constraint: + + `ALTER TABLE ADD COLUMN NOT NULL DEFAULT ;` + - Rename a column: + + `ALTER TABLE RENAME TO ;` + - Drop a column: + + `ALTER TABLE DROP COLUMN ;` + + You cannot change the data type of an existing column. + +1. **Convert the updated chunks back to the columnstore** + + ``` sql + CALL convert_to_columnstore('_timescaledb_internal._hyper_1_2_chunk'); + ``` + +1. **Restart the jobs that are automatically converting chunks to the columnstore** + + ``` sql + SELECT alter_job(JOB_ID, scheduled => true); + ``` + + + +[write]: /use-timescale/:currentVersion:/write-data/ +[setup-hypercore]: /use-timescale/:currentVersion:/hypercore/real-time-analytics-in-hypercore/ +[job]: /api/:currentVersion:/actions/ +[alter_job]: /api/:currentVersion:/actions/alter_job/ +[convert_to_columnstore]: /api/:currentVersion:/hypercore/convert_to_columnstore/ +[informational-views]: /api/:currentVersion:/informational-views/jobs/ +[insert]: /use-timescale/:currentVersion:/write-data/insert/ +[setup-hypercore]: /use-timescale/:currentVersion:/hypercore/real-time-analytics-in-hypercore/ diff --git a/use-timescale/hypercore/real-time-analytics-in-hypercore.md b/use-timescale/hypercore/real-time-analytics-in-hypercore.md new file mode 100644 index 0000000000..70119a97e3 --- /dev/null +++ b/use-timescale/hypercore/real-time-analytics-in-hypercore.md @@ -0,0 +1,83 @@ +--- +title: Optimize your data for real-time analytics +excerpt: Reduce your chunk size by more than 90% and speed up your queries by automatically converting data between the rowstore and columnstore. +products: [cloud,] +keywords: [hyperscore, hypertable, compression, row-columnar storage, hypercore] +--- + +import PrereqCloud from "versionContent/_partials/_prereqs-cloud-only.mdx"; +import HCConversionOverview from "versionContent/_partials/_hypercore-conversion-overview.mdx"; +import HCPolicyWorkflow from "versionContent/_partials/_hypercore_policy_workflow.mdx"; +import UsageBasedStorage from "versionContent/_partials/_usage-based-storage-intro.mdx"; +import EarlyAccess from "versionContent/_partials/_early_access.mdx"; + +# Optimize your data for real-time analytics + +Hypercore is the TimescaleDB hybrid row-columnar storage engine used by hypertables. +Hypertables partition your data in chunks. Chunks stored in the rowstore use a row-oriented data format optimized for high-speed inserts and updates. Chunks stored in the columnstore use a columnar data format optimized for analytics. +You ingest _hot_ data into the rowstore. As data _cools_ and becomes more suited for analytics, +$CLOUD_LONG automatically converts these chunks of data to the columnstore. You define the moment when data is converted +using a columnstore policy. + + + +This page shows you how to get the best results when you set a policy to automatically convert chunks in a hypertable +from the rowstore to the columnstore. + +## Prerequisites + + + +The code samples in this page use the [crypto_sample.zip](https://assets.timescale.com/docs/downloads/candlestick/crypto_sample.zip) data from [Try the key Timescale features][ingest-data]. + +## Optimize your data with columnstore policies + +The compression ratio and query performance of data in the columnstore is dependent on the order and structure of your +data. Rows that change over a dimension should be close to each other. With time-series data, you `orderby` the time +dimension. For example, `Timestamp`: + +| Timestamp | Device ID | Device Type | CPU |Disk IO| +|---|---|---|---|---| +|12:00:01|A|SSD|70.11|13.4| + +This ensures that records are compressed and accessed in the same order. However, you would always have to +access the data using the time dimension, then filter all the rows using other criteria. To make your queries more +efficient, you segment your data based on the way you want to access it. For example, to rapidly access data about a +single device, you `segmentby` the `Device ID` column. This enables you to run much faster analytical queries on +data in the columnstore. + +When $CLOUD_LONG converts a chunk to the columnstore, TimescaleDB automatically creates a different schema for your +data. $TIMESCALE_DB creates and uses custom indexes to incorporate the `segmentby` and `orderby` parameters when +you write to and read from the columstore. + +To set up your Hypercore automation: + + + + + +## Reference + +For integers, timestamps, and other integer-like types, data is compressed using [delta encoding][delta], +[delta-of-delta][delta-delta], [simple-8b][simple-8b], and [run-length encoding][run-length]. For columns with few +repeated values,[XOR-based][xor] and [dictionary compression][dictionary] is used. For all other types, +[dictionary compression][dictionary] is used. + + + +[create-hypertable]: /use-timescale/:currentVersion:/compression/ +[add_columnstore_policy]: /api/:currentVersion:/hypercore/add_columnstore_policy/ +[delta]: /use-timescale/:currentVersion:/compression/compression-methods/#delta-encoding +[delta-delta]: /use-timescale/:currentVersion:/compression/compression-methods/#delta-of-delta-encoding +[simple-8b]: /use-timescale/:currentVersion:/compression/compression-methods/#simple-8b +[run-length]: /use-timescale/:currentVersion:/compression/compression-methods/#run-length-encoding +[xor]: /use-timescale/:currentVersion:/compression/compression-methods/#xor-based-encoding +[dictionary]: /use-timescale/:currentVersion:/compression/compression-methods/#dictionary-compression +[ingest-data]: /getting-started/:currentVersion:/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables +[add_columnstore_policy]: /api/:currentVersion:/hypercore/add_columnstore_policy/ +[run-job]: /api/:currentVersion:/actions/run_job/ +[convert_to_rowstore]: /api/:currentVersion:/hypercore/convert_to_rowstore/ +[alter_job]: /api/:currentVersion:/actions/alter_job/ +[informational-views]: /api/:currentVersion:/informational-views/jobs/ +[insert]: /use-timescale/:currentVersion:/write-data/insert/ +[modify-data-in-hypercore]: /use-timescale/:currentVersion:/hypercore/modify-data-in-hypercore/ diff --git a/use-timescale/hyperfunctions/about-hyperfunctions.md b/use-timescale/hyperfunctions/about-hyperfunctions.md index 4e27f5d3e3..248e2a7dc2 100644 --- a/use-timescale/hyperfunctions/about-hyperfunctions.md +++ b/use-timescale/hyperfunctions/about-hyperfunctions.md @@ -11,25 +11,18 @@ import ExperimentalUpgrade from "versionContent/_partials/_experimental-schema-u # About Timescale hyperfunctions -Timescale hyperfunctions are a specialized set of functions that allow you to -analyze time-series data. You can use hyperfunctions to analyze anything you -have stored as time-series data, including IoT devices, IT systems, marketing -analytics, user behavior, financial metrics, and cryptocurrency. +Timescale hyperfunctions are a specialized set of functions that power real-time analytics on time series and events. +IoT devices, IT systems, marketing analytics, user behavior, financial metrics, cryptocurrency - these are only a few examples of domains where +hyperfunctions can make a huge difference. Hyperfunctions provide you with meaningful, actionable insights in real time. -Hyperfunctions allow you to perform critical time-series queries quickly, -analyze time-series data, and extract meaningful information. They aim to -identify, build, and combine all of the functionality SQL needs to perform -time-series analysis into a single extension. +$CLOUD_LONG includes all hyperfunctions by default, while self-hosted $TIMESCALE_DB includes a subset of them. For +additional hyperfunctions, install the [Timescale Toolkit][install-toolkit] PostgreSQL extension. -Some hyperfunctions are included in the default TimescaleDB product. For -additional hyperfunctions, you need to install the -[Timescale Toolkit][install-toolkit] PostgreSQL extension. - -## Hyperfunctions available with TimescaleDB and Timescale Toolkit +## Available hyperfunctions Here is a list of all the hyperfunctions provided by Timescale. Hyperfunctions -marked 'Toolkit' require an installation of Timescale Toolkit. Hyperfunctions -marked 'experimental' are still under development. +with a tick in the `Toolkit` column require an installation of Timescale Toolkit for self-hosted deployments. Hyperfunctions +with a tick in the `Experimental` column are still under development. diff --git a/use-timescale/hyperfunctions/heartbeat-agg.md b/use-timescale/hyperfunctions/heartbeat-agg.md index 6a51f8263a..f6279bf089 100644 --- a/use-timescale/hyperfunctions/heartbeat-agg.md +++ b/use-timescale/hyperfunctions/heartbeat-agg.md @@ -7,11 +7,12 @@ keywords: [hyperfunctions, Toolkit, heartbeat, liveness] # Heartbeat aggregation Given a series of timestamped health checks, it can be tricky to determine the -overall health of a system over a given interval. PostgresQL provides window -functions which can be used to get a sense of where unhealthy gaps are, but can -be somewhat awkward to use efficiently. The heartbeat aggregate is part of the -Timescale Toolkit, and can be used to solve this problem in a simpler, more -accessible manner. +overall health of a system over a given interval. PostgreSQL provides window +functions that you use to get a sense of where unhealthy gaps are, but they can +be somewhat awkward to use efficiently. + +This is one of the many cases where hyperfunctions provide an efficient, simple solution for +a frequently occurring problem. Heartbeat aggregation helps analyze event-based time-series data with intermittent or irregular signals. This example uses the [SustData public dataset][sustdata]. This dataset tracks the power usage of a small number of apartments and houses over four different @@ -19,7 +20,7 @@ deployment intervals. The data is collected in one minute samples from each unit. When you have loaded the data into hypertables, you can create a materialized -view containing weekly heartbeat aggregates for each of the units. +view containing weekly heartbeat aggregates for each of the units: ```sql CREATE MATERIALIZED VIEW weekly_heartbeat AS @@ -69,7 +70,7 @@ SELECT live_ranges(rollup(heartbeat_agg)) FROM weekly_heartbeat WHERE unit = 17; ("2012-03-25 03:00:51+00","2012-04-11 00:01:00+00") ``` -You can do also do more elaborate queries, like looking for the 5 units with the +You can construct more elaborate queries. For example, to return the 5 units with the lowest uptime during the third deployment: ```sql @@ -90,7 +91,7 @@ ORDER BY uptime LIMIT 5; 30 | 222 days 22:05:00 ``` -You can also combine aggregates from different units to get the combined +Combine aggregates from different units to get the combined coverage. This example queries the interval where any part of a deployment was active: @@ -115,11 +116,12 @@ FROM weekly_heartbeat group by deploy order by deploy; 4 | ("2014-03-30 03:00:01+00","2014-04-25 00:01:00+00") ``` -You can use this data to make some observations. First, it looks like the second -deployment had a lot more problems than the other ones. Second, it looks like -there were some readings from February 2013 that were incorrectly categorized as -a second deployment. And finally, it looks like the timestamps are given in a -local time without time zone, resulting in some missing hours around springtime +Then use this data to make observations and draw conclusions: + +- The second deployment had a lot more problems than the other ones. +- There were some readings from February 2013 that were incorrectly categorized as +a second deployment. +- The timestamps are given in a local time without time zone, resulting in some missing hours around springtime daylight savings time changes. For more information about heartbeat aggregation API calls, see the diff --git a/use-timescale/hyperfunctions/index.md b/use-timescale/hyperfunctions/index.md index 41168901f3..1887625014 100644 --- a/use-timescale/hyperfunctions/index.md +++ b/use-timescale/hyperfunctions/index.md @@ -7,12 +7,10 @@ keywords: [hyperfunctions, Toolkit, analytics] # Hyperfunctions -Hyperfunctions allow you to perform critical time-series queries quickly, -analyze time-series data, and extract meaningful information. +Timescale hyperfunctions are designed to efficiently process, aggregate, and analyze significant volumes of data while maintaining high performance. +They let you run sophisticated analytical queries and get meaningful data in real time. -Some hyperfunctions are included by default in Timescale. For -additional hyperfunctions, you need to install the -[Timescale Toolkit][install-toolkit] PostgreSQL extension. +$CLOUD_LONG includes all hyperfunctions by default, while self-hosted $TIMESCALE_DB includes a subset of them. To include all hyperfunctions with $TIMESCALE_DB, install the [Timescale Toolkit][install-toolkit] PostgreSQL extension on your self-hosted PostgreSQL deployment. For more information, read the [hyperfunctions blog post][hyperfunctions-blog]. @@ -21,7 +19,7 @@ For more information, read the [hyperfunctions blog post][hyperfunctions-blog]. * [Learn about hyperfunctions][about-hyperfunctions] to understand how they work before using them. * Install the [Toolkit extension][install-toolkit] to access more - hyperfunctions. + hyperfunctions on self-hosted $TIMESCALE_DB. ## Browse hyperfunctions and Toolkit features by category diff --git a/use-timescale/hypertables/about-hypertables.md b/use-timescale/hypertables/about-hypertables.md index 1c77c0bac3..5ca8132a3c 100644 --- a/use-timescale/hypertables/about-hypertables.md +++ b/use-timescale/hypertables/about-hypertables.md @@ -17,10 +17,7 @@ Inheritance is not supported for hypertables and may lead to unexpected behavior ## Hypertable partitioning -When you create and use a hypertable, it automatically partitions data by time, -and optionally by space. - -Each hypertable is made up of child tables called chunks. Each chunk is assigned +Each hypertable is partitioned into child tables called chunks. Each chunk is assigned a range of time, and only contains data from that range. If the hypertable is also partitioned by space, each chunk is also assigned a subset of the space values. @@ -36,7 +33,7 @@ it makes sense. ### Time partitioning Each chunk of a hypertable only holds data from a specific time range. When you -insert data from a time range that doesn't yet have a chunk, Timescale +insert data from a time range that doesn't yet have a chunk, $CLOUD_LONG automatically creates a chunk to store it. By default, each chunk covers 7 days. You can change this to better suit your @@ -50,7 +47,7 @@ alt="A normal table compared to a hypertable. The normal table holds data for 3 /> -Timescale divides time into potential chunk ranges, based on the +$CLOUD_LONG divides time into potential chunk ranges, based on the `chunk_time_interval`. If data exists for a potential chunk range, that chunk is created. @@ -70,7 +67,7 @@ affect query planning time and compression. Best practice is to set `chunk_time_interval` so that prior to processing, one chunk of data takes up 25% of main memory, including the indexes from each active hypertable. -For example, if your write approximately 2 GB of data per day to a database with 64 GB of +For example, if you write approximately 2 GB of data per day to a database with 64 GB of memory, set `chunk_time_interval` to 1 week. If you write approximately 10 GB of data per day on the same machine, set the time interval to 1 day. diff --git a/use-timescale/hypertables/improve-query-performance.md b/use-timescale/hypertables/improve-query-performance.md index e6d1d8219b..be08f1c0ca 100644 --- a/use-timescale/hypertables/improve-query-performance.md +++ b/use-timescale/hypertables/improve-query-performance.md @@ -7,6 +7,7 @@ keywords: [hypertables, indexes, chunks] # Improve query performance +One of the key purposes of hypertables is to make your analytical queries run with the lowest latency possible - and they have been designed accordingly. When you execute a query on a hypertable, you do not parse the whole table; you only access the chunks necessary to satisfy the query. This works well when the `WHERE` clause of a query uses the column by which a hypertable is partitioned. For example, in a hypertable where every day of the year is a separate chunk, a query for September 1 diff --git a/use-timescale/hypertables/index.md b/use-timescale/hypertables/index.md index d179cb74df..101225c528 100644 --- a/use-timescale/hypertables/index.md +++ b/use-timescale/hypertables/index.md @@ -7,10 +7,8 @@ keywords: [hypertables] # Hypertables -Hypertables are PostgreSQL tables with special features that make it easy to -handle time-series data. Anything you can do with regular PostgreSQL tables, you -can do with hypertables. In addition, you get the benefits of improved -performance and user experience for time-series data. +Hypertables are PostgreSQL tables designed to simplify and accelerate data analysis. Anything you can do with regular PostgreSQL tables, you +can do with hypertables - but much faster and more conveniently. In this section, you: * [Learn about hypertables][about-hypertables] * [Create a hypertable][create-hypertables] diff --git a/use-timescale/integrations/apache-airflow.md b/use-timescale/integrations/apache-airflow.md index e675e70af9..15642bc195 100644 --- a/use-timescale/integrations/apache-airflow.md +++ b/use-timescale/integrations/apache-airflow.md @@ -26,7 +26,7 @@ This page shows you how to use a Python connector in a DAG to integrate Apache A Ensure that your Airflow instance has network access to $CLOUD_LONG. -This example DAG uses the `company` table you create in [Create regular PostgreSQL tables for relational data][create-a-table-in-timescale] +This example DAG uses the `company` table you create in [Optimize time-series data in hypertables][create-a-table-in-timescale] ## Install python connectivity libraries @@ -98,10 +98,10 @@ To exchange data between Airflow and your $SERVICE_LONG: This could be any query. This example inserts data into the table you create in: - https://docs.timescale.com/getting-started/latest/tables-hypertables/#create-regular-postgresql-tables-for-relational-data + https://docs.timescale.com/getting-started/latest/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables """ - cursor.execute("INSERT INTO company (symbol, name) VALUES (%s, %s)", - ('new_company_symbol', 'New Company Name')) + cursor.execute("INSERT INTO crypto_assets (symbol, name) VALUES (%s, %s)", + ('NEW/Asset','New Asset Name')) conn.commit() cursor.close() conn.close() @@ -138,7 +138,7 @@ To exchange data between Airflow and your $SERVICE_LONG: You have successfully integrated Apache Airflow with $CLOUD_LONG and created a data pipeline. -[create-a-table-in-timescale]: /getting-started/:currentVersion:/tables-hypertables/#create-regular-postgresql-tables-for-relational-data +[create-a-table-in-timescale]: /getting-started/:currentVersion:/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables [install-apache-airflow]: https://airflow.apache.org/docs/apache-airflow/stable/start.html [install-python-pip]: https://docs.python.org/3/using/index.html [console]: https://console.cloud.timescale.com/ diff --git a/use-timescale/integrations/datadog.md b/use-timescale/integrations/datadog.md index df76c78268..3dc934a47c 100644 --- a/use-timescale/integrations/datadog.md +++ b/use-timescale/integrations/datadog.md @@ -39,4 +39,4 @@ tool. You create an exporter on the [project level][projects], in the same AWS r [datadog-signup]: https://www.datadoghq.com/ [projects]: /use-timescale/:currentVersion:/members/ [datadog-api-key]: https://docs.datadoghq.com/account_management/api-app-keys/#add-an-api-key-or-client-token -[pricing-plan-features]: /about/:currentVersion:/pricing-and-account-management/#features-included-in-each-plan \ No newline at end of file +[pricing-plan-features]: /about/:currentVersion:/pricing-and-account-management/#features-included-in-each-plan diff --git a/use-timescale/integrations/grafana.md b/use-timescale/integrations/grafana.md index a42b3556a4..102f6fbe70 100644 --- a/use-timescale/integrations/grafana.md +++ b/use-timescale/integrations/grafana.md @@ -4,7 +4,7 @@ excerpt: Grafana enables you to query, visualize, alert on, and explore your met products: [cloud] keywords: [Grafana, visualizations, analytics, monitoring] --- - +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; # Integrate Grafana and Timescale Cloud @@ -13,6 +13,12 @@ import GrafanaConnect from "versionContent/_partials/_grafana-connect.mdx"; This page shows you how to integrate Grafana with a $SERVICE_LONG, create a dashboard and panel, then visualize geospatial data. +## Prerequisites + + + +* Install [self-managed Grafana][grafana-self-managed] or sign up for [Grafana Cloud][grafana-cloud]. + ## Create a Grafana dashboard and panel @@ -177,4 +183,5 @@ tutorial as a starting point. [nyc-taxi]: /tutorials/:currentVersion:/nyc-taxi-cab [grafana-website]: https://www.grafana.com [time-buckets]: /use-timescale/:currentVersion:/time-buckets/ - +[grafana-self-managed]: https://grafana.com/get/?tab=self-managed +[grafana-cloud]: https://grafana.com/get/ diff --git a/use-timescale/integrations/index.md b/use-timescale/integrations/index.md index c73fa372e3..454a086e1f 100644 --- a/use-timescale/integrations/index.md +++ b/use-timescale/integrations/index.md @@ -46,6 +46,7 @@ Some of the most in-demand integrations for $CLOUD_LONG are listed below, with l + ## Data engineering and extract, transform, load | Name | Description | diff --git a/use-timescale/integrations/pgadmin.md b/use-timescale/integrations/pgadmin.md index b2cba48513..008374d037 100644 --- a/use-timescale/integrations/pgadmin.md +++ b/use-timescale/integrations/pgadmin.md @@ -43,4 +43,4 @@ You have successfully integrated pgAdmin with $CLOUD_LONG. [pgadmin]: https://www.pgadmin.org/ [download-pgadmin]: https://www.pgadmin.org/download/ [connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ -[ssl-mode]: /use-timescale/:currentVersion:/security/strict-ssl/ \ No newline at end of file +[ssl-mode]: /use-timescale/:currentVersion:/security/strict-ssl/ diff --git a/use-timescale/integrations/supabase.md b/use-timescale/integrations/supabase.md index 55b0ae8b25..f6fb4e86e5 100644 --- a/use-timescale/integrations/supabase.md +++ b/use-timescale/integrations/supabase.md @@ -48,7 +48,7 @@ To setup a $SERVICE_LONG optimized for analytics to receive data from Supabase: ``` 1. **Optimize cooling data for analytics** - Hypercore is the Timescale hybrid row-columnar storage engine, designed specifically for real-time analytics + Hypercore is the $TIMESCALE_DB hybrid row-columnar storage engine, designed specifically for real-time analytics and powered by time-series data. The advantage of Hypercore is its ability to seamlessly switch between row-oriented and column-oriented storage. This flexibility enables $CLOUD_LONG to deliver the best of both worlds, solving the key challenges in real-time analytics. diff --git a/use-timescale/page-index/page-index.js b/use-timescale/page-index/page-index.js index c0ba3a0ba3..8762b38d8e 100644 --- a/use-timescale/page-index/page-index.js +++ b/use-timescale/page-index/page-index.js @@ -192,6 +192,24 @@ module.exports = [ }, ], }, + { + title: "Hypercore", + href: "hypercore", + excerpt: + "Seamlessly switch between row-oriented and column-oriented storage", + children: [ + { + title: "Optimize your data for real-time analytics", + href: "real-time-analytics-in-hypercore", + excerpt: "Automate", + }, + { + title: "Modify data in Hypercore", + href: "modify-data-in-hypercore", + excerpt: "Update data stored in the columnstore", + }, + ], + }, { title: "Schema management", href: "schema-management", @@ -247,57 +265,6 @@ module.exports = [ }, ], }, - { - title: "Compression", - href: "compression", - children: [ - { - title: "About compression", - href: "about-compression", - excerpt: "Learn about how compression works", - }, - { - title: "Compression design", - href: "compression-design", - excerpt: "The design of TimescaleDB compression", - }, - { - title: "About compression methods", - href: "compression-methods", - excerpt: "Learn about the different compression methods", - }, - { - title: "Enable a compression policy", - href: "compression-policy", - excerpt: "Create a compression policy on a hypertable", - }, - { - title: "Manual compression", - href: "manual-compression", - excerpt: "Compress data chunks", - }, - { - title: "Modify compressed data", - href: "modify-compressed-data", - excerpt: "Insert and modify data in compressed chunks", - }, - { - title: "Decompress chunks", - href: "decompress-chunks", - excerpt: "Decompress data chunks", - }, - { - title: "Modify a schema", - href: "modify-a-schema", - excerpt: "Change the data schema in compressed chunks", - }, - { - title: "Troubleshooting", - href: "troubleshooting", - type: "placeholder", - }, - ], - }, { title: "Configuration", href: "configuration", @@ -975,6 +942,57 @@ module.exports = [ href: "troubleshoot-timescaledb", excerpt: "Troubleshooting Timescale", }, + { + title: "Compression (Old API, replaced by Hypercore)", + href: "compression", + children: [ + { + title: "About compression", + href: "about-compression", + excerpt: "Learn about how compression works", + }, + { + title: "Compression design", + href: "compression-design", + excerpt: "The design of TimescaleDB compression", + }, + { + title: "About compression methods", + href: "compression-methods", + excerpt: "Learn about the different compression methods", + }, + { + title: "Enable a compression policy", + href: "compression-policy", + excerpt: "Create a compression policy on a hypertable", + }, + { + title: "Manual compression", + href: "manual-compression", + excerpt: "Compress data chunks", + }, + { + title: "Modify compressed data", + href: "modify-compressed-data", + excerpt: "Insert and modify data in compressed chunks", + }, + { + title: "Decompress chunks", + href: "decompress-chunks", + excerpt: "Decompress data chunks", + }, + { + title: "Modify a schema", + href: "modify-a-schema", + excerpt: "Change the data schema in compressed chunks", + }, + { + title: "Troubleshooting", + href: "troubleshooting", + type: "placeholder", + }, + ], + }, ], }, ]; diff --git a/use-timescale/time-buckets/about-time-buckets.md b/use-timescale/time-buckets/about-time-buckets.md index c2ff30d373..0dab19f29e 100644 --- a/use-timescale/time-buckets/about-time-buckets.md +++ b/use-timescale/time-buckets/about-time-buckets.md @@ -7,19 +7,17 @@ keywords: [time buckets] # About time buckets -The [`time_bucket`][time_bucket] function allows you to aggregate data in a -[hypertable][create-hypertable] into buckets of time. For example: 5 minutes, 1 hour, or 3 days. +Time bucketing is essential for real-time analytics. The [`time_bucket`][time_bucket] function enables you to aggregate data in a [hypertable][create-hypertable] into buckets of time. For example, 5 minutes, 1 hour, or 3 days. It's similar to PostgreSQL's [`date_bin`][date_bin] function, but it gives you more -flexibility in bucket size and start time. +flexibility in the bucket size and start time. -Time bucketing is essential to working with time-series data. You can use it to -roll up data for analysis or downsampling. For example, you can calculate +You can use it to roll up data for analysis or downsampling. For example, you can calculate 5-minute averages for a sensor reading over the last day. You can perform these rollups as needed, or pre-calculate them in [continuous aggregates][caggs]. This section explains how time bucketing works. For examples of the `time_bucket` function, see the section on -[using time buckets][use-time-buckets]. +[Aggregate time-series data with `time_bucket`][use-time-buckets]. ## How time bucketing works diff --git a/use-timescale/time-buckets/index.md b/use-timescale/time-buckets/index.md index d766900317..d728b6d522 100644 --- a/use-timescale/time-buckets/index.md +++ b/use-timescale/time-buckets/index.md @@ -10,7 +10,7 @@ keywords: [time buckets] Time buckets enable you to aggregate data in [hypertables][create-hypertable] by time interval. For example, you can group data into 5-minute, 1-hour, and 3-day buckets to calculate summary values. -* [Learn how time buckets work][about-time-buckets] in Timescale +* [Learn how time buckets work][about-time-buckets] in $CLOUD_LONG * [Use time buckets][use-time-buckets] to aggregate data [about-time-buckets]: /use-timescale/:currentVersion:/time-buckets/about-time-buckets/ diff --git a/use-timescale/troubleshoot-timescaledb.md b/use-timescale/troubleshoot-timescaledb.md index 13f3ff7e3e..395dd666bd 100644 --- a/use-timescale/troubleshoot-timescaledb.md +++ b/use-timescale/troubleshoot-timescaledb.md @@ -132,7 +132,7 @@ tsdb=> \dp _timescaledb_internal._compressed_hypertable_2 This means that the `relacl` column of `pg_class` needs to be updated and the offending user removed, but it is not possible to drop a user by numerical value. Instead, you can use the internal function -`repair_relation_acls` in `_timescaledb_function` schema: +`repair_relation_acls` in the `_timescaledb_function` schema: ```sql tsdb=> CALL _timescaledb_functions.repair_relation_acls(); diff --git a/use-timescale/write-data/about-writing-data.md b/use-timescale/write-data/about-writing-data.md index 1e40441c0b..1d28585a5d 100644 --- a/use-timescale/write-data/about-writing-data.md +++ b/use-timescale/write-data/about-writing-data.md @@ -12,7 +12,7 @@ Timescale supports writing data in the same way as PostgreSQL, using `INSERT`, `UPDATE`, `INSERT ... ON CONFLICT`, and `DELETE`. -Because Timescale is a time-series database, hypertables are optimized for +$CLOUD_LONG is optimized for running real-time analytics workloads on time-series data. For this reason, hypertables are optimized for inserts to the most recent time intervals. Inserting data with recent time values gives [excellent performance](https://www.timescale.com/blog/timescaledb-vs-6a696248104e/). From 69b87f49f88b2ecd82e8bbc984aa7209e1cec6f5 Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Thu, 13 Mar 2025 11:57:21 +0200 Subject: [PATCH 19/90] link fix (#3921) --- api/compression/decompress_chunk.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/compression/decompress_chunk.md b/api/compression/decompress_chunk.md index f775c71a92..3ec7ac6fdf 100644 --- a/api/compression/decompress_chunk.md +++ b/api/compression/decompress_chunk.md @@ -12,7 +12,7 @@ import Deprecated2180 from "versionContent/_partials/_deprecated_2_18_0.mdx"; # decompress_chunk() Community - Replaced by convert_to_rowstore(). + Replaced by convert_to_rowstore(). If you need to modify or add a lot of data to a chunk that has already been compressed, you should decompress the chunk first. This is especially From c9a23297f3103b42f9158821a8d49874ddc2a928 Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Thu, 13 Mar 2025 12:28:47 +0200 Subject: [PATCH 20/90] Redirect fixes (#3922) * link fix * redirect fix --- lambda/redirects.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/lambda/redirects.js b/lambda/redirects.js index 8703531175..9cf4b25eb9 100644 --- a/lambda/redirects.js +++ b/lambda/redirects.js @@ -568,19 +568,19 @@ module.exports = [ }, { from: "/tutorials/latest/energy-data/", - to: "https://docs.timescale.com/tutorials/real-time-analytics-energy-consumption/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-energy-consumption/", }, { from: "/tutorials/latest/energy-data/dataset-energy/", - to: "https://docs.timescale.com/tutorials/real-time-analytics-energy-consumption/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-energy-consumption/", }, { from: "/tutorials/latest/energy-data/query-energy/", - to: "https://docs.timescale.com/tutorials/real-time-analytics-energy-consumption/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-energy-consumption/", }, { from: "/tutorials/latest/energy-data/compress-energy/", - to: "https://docs.timescale.com/tutorials/real-time-analytics-energy-consumption/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-energy-consumption/", }, { from: "/tutorials/latest/financial-candlestick-tick-data/create-candlestick-aggregates/", @@ -604,27 +604,27 @@ module.exports = [ }, { from: "/tutorials/latest/nyc-taxi-cab/", - to: "https://docs.timescale.com/tutorials/real-time-analytics-transport/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-transport/", }, { from: "/tutorials/latest/nyc-taxi-cab/dataset-nyc/", - to: "https://docs.timescale.com/tutorials/real-time-analytics-transport/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-transport/", }, { from: "/tutorials/latest/nyc-taxi-cab/query-nyc/", - to: "https://docs.timescale.com/tutorials/real-time-analytics-transport/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-transport/", }, { from: "/tutorials/latest/nyc-taxi-cab/compress-nyc/", - to: "https://docs.timescale.com/tutorials/real-time-analytics-transport/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-transport/", }, { from: "/tutorials/latest/nyc-taxi-geospatial/dataset-nyc/", - to: "https://docs.timescale.com/tutorials/real-time-analytics-transport/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-transport/", }, { from: "/tutorials/latest/nyc-taxi-geospatial/plot-nyc", - to: "https://docs.timescale.com/tutorials/real-time-analytics-transport/", + to: "https://docs.timescale.com/tutorials/latest/real-time-analytics-transport/", }, { from: "/tutorials/latest/prometheus-adapter/", From 658b7a5cb6af8a3a7f7234fe82cbcebd87a2ff87 Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Thu, 13 Mar 2025 14:00:55 +0200 Subject: [PATCH 21/90] Tiered data billing explained (#3910) --- _partials/_tiered-storage-billing.md | 1 + about/pricing-and-account-management.md | 4 +++- use-timescale/data-tiering/about-data-tiering.md | 9 +++++++-- use-timescale/data-tiering/enabling-data-tiering.md | 4 ++++ 4 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 _partials/_tiered-storage-billing.md diff --git a/_partials/_tiered-storage-billing.md b/_partials/_tiered-storage-billing.md new file mode 100644 index 0000000000..81674e4786 --- /dev/null +++ b/_partials/_tiered-storage-billing.md @@ -0,0 +1 @@ +$COMPANY charges only for the storage that your data occupies in S3, regardless of whether it was compressed in $CLOUD_LONG before tiering. There are no additional expenses, such as data transfer or compute. \ No newline at end of file diff --git a/about/pricing-and-account-management.md b/about/pricing-and-account-management.md index ce45df56c5..78bd4c427f 100644 --- a/about/pricing-and-account-management.md +++ b/about/pricing-and-account-management.md @@ -9,7 +9,7 @@ cloud_ui: - [billing] --- -import UsageBasedStorage from "versionContent/_partials/_usage-based-storage-intro.mdx"; +import TieredStorageBilling from "versionContent/_partials/_tiered-storage-billing.mdx"; # Pricing plans and account management @@ -111,6 +111,8 @@ storage tier and a low-cost bottomless storage tier. You can keep up to 16 TB co (typically 80-100 TBs uncompressed) in the high-performance storage tier, and configure less-frequently accessed data to be moved to our low-cost storage tier built on S3. + + Timescale Cloud is very efficient and generally needs less compute than other databases to deliver the same performance. The best way to size your needs is to signup for a free trial and to test with a realistic workload. diff --git a/use-timescale/data-tiering/about-data-tiering.md b/use-timescale/data-tiering/about-data-tiering.md index ec6aa3e656..5a6921eb8e 100644 --- a/use-timescale/data-tiering/about-data-tiering.md +++ b/use-timescale/data-tiering/about-data-tiering.md @@ -11,12 +11,16 @@ cloud_ui: # About the object storage tier -Timescale's tiered storage architecture includes a standard high-performance storage tier and a low-cost object storage tier built on Amazon S3. You can use the standard tier for data that requires quick access, and the object tier for rarely used historical data. Chunks from a single hypertable, including compressed chunks, can stretch across these two storage tiers. A compressed chunk uses a different storage representation after tiering. +$COMPANY's tiered storage architecture includes a standard high-performance storage tier and a low-cost object storage tier built on Amazon S3. You can use the standard tier for data that requires quick access, and the object tier for rarely used historical data. Chunks from a single hypertable, including compressed chunks, can stretch across these two storage tiers. -In the high-performance storage, chunks are stored in the block format. In the object storage, they are stored in a compressed, columnar format. For better interoperability across various platforms, this format is different from that of the internals of the database. It allows for more efficient columnar scans across longer time periods, and Timescale Cloud uses other metadata and query optimizations to reduce the amount of data that needs to be fetched from the object storage tier to satisfy a query. +In the high-performance storage, your data is stored in the block format. In the object storage, it is stored in [Apache Parquet][parquet]. The original size of the data in your $SERVICE_SHORT, compressed or uncompressed, does not correspond directly to its size in S3. A compressed hypertable may even take more space in S3 than it does in $CLOUD_LONG. + +Apache Parquet allows for more efficient scans across longer time periods, and $CLOUD_LONG uses other metadata and query optimizations to reduce the amount of data that needs to be fetched from the object storage tier to satisfy a query. Regardless of where your data is stored, you can still query it with standard SQL. A single SQL query transparently pulls data from the appropriate chunks using the chunk exclusion algorithms. You can `JOIN` against tiered data, build views, and even define continuous aggregates on it. In fact, because the implementation of continuous aggregates also uses hypertables, they can be tiered to low-cost storage as well. +$COMPANY charges only for the storage that your data occupies in S3, regardless of whether it was compressed in $CLOUD_LONG before tiering. There are no additional expenses, such as data transfer or compute. + ## Benefits of the object storage tier The object storage tier is more than an archiving solution. It is also: @@ -119,3 +123,4 @@ for `device_uuid`, `sensor_id`, and `observed_at` as the query needs only these [blog-data-tiering]: https://www.timescale.com/blog/expanding-the-boundaries-of-postgresql-announcing-a-bottomless-consumption-based-object-storage-layer-built-on-amazon-s3/ [querying-tiered-data]: /use-timescale/:currentVersion:/data-tiering/querying-tiered-data/ +[parquet]: https://parquet.apache.org/ \ No newline at end of file diff --git a/use-timescale/data-tiering/enabling-data-tiering.md b/use-timescale/data-tiering/enabling-data-tiering.md index fb7f56ce5a..ce4ad620a4 100644 --- a/use-timescale/data-tiering/enabling-data-tiering.md +++ b/use-timescale/data-tiering/enabling-data-tiering.md @@ -9,6 +9,8 @@ cloud_ui: - [services, :serviceId, overview] --- +import TieredStorageBilling from "versionContent/_partials/_tiered-storage-billing.mdx"; + # Manage automated and manual tiering You use tiered storage to save on storage costs. Specifically, you can migrate rarely used data from @@ -18,6 +20,8 @@ or [manually tier and untier data][manual-tier]. You can query the data on the object storage tier, but you cannot modify it. Make sure that you are not tiering data that needs to be **actively modified**. + + ## Enable tiered storage You enable tiered storage from the `Overview` tab in Console. From 349f18d6bb54d80347306a860c0d772dad4e352b Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Thu, 13 Mar 2025 14:13:21 +0200 Subject: [PATCH 22/90] noindex (#3924) Co-authored-by: Iain Cox --- api/add_data_node.md | 2 ++ api/alter_data_node.md | 2 ++ api/attach_data_node.md | 2 ++ api/cleanup_copy_chunk_operation_experimental.md | 2 ++ api/copy_chunk_experimental.md | 2 ++ api/create_distributed_hypertable.md | 2 ++ api/create_distributed_restore_point.md | 2 ++ api/delete_data_node.md | 2 ++ api/detach_data_node.md | 2 ++ api/distributed-hypertables.md | 2 ++ api/distributed_exec.md | 2 ++ api/move_chunk.md | 2 ++ api/set_number_partitions.md | 2 ++ api/set_replication_factor.md | 2 ++ .../distributed-hypertables/about-distributed-hypertables.md | 2 ++ .../alter-drop-distributed-hypertables.md | 2 ++ .../distributed-hypertables/create-distributed-hypertables.md | 2 ++ self-hosted/distributed-hypertables/foreign-keys.md | 2 ++ self-hosted/distributed-hypertables/index.md | 2 ++ self-hosted/distributed-hypertables/insert.md | 2 ++ self-hosted/distributed-hypertables/query.md | 2 ++ self-hosted/distributed-hypertables/triggers.md | 2 ++ self-hosted/multinode-timescaledb/about-multinode.md | 2 ++ self-hosted/multinode-timescaledb/index.md | 2 ++ self-hosted/multinode-timescaledb/multinode-administration.md | 2 ++ self-hosted/multinode-timescaledb/multinode-auth.md | 2 ++ self-hosted/multinode-timescaledb/multinode-config.md | 2 ++ self-hosted/multinode-timescaledb/multinode-grow-shrink.md | 2 ++ self-hosted/multinode-timescaledb/multinode-ha.md | 2 ++ self-hosted/multinode-timescaledb/multinode-maintenance.md | 2 ++ self-hosted/multinode-timescaledb/multinode-setup.md | 2 ++ 31 files changed, 62 insertions(+) diff --git a/api/add_data_node.md b/api/add_data_node.md index d65620e2b3..e787fa8a88 100644 --- a/api/add_data_node.md +++ b/api/add_data_node.md @@ -7,6 +7,8 @@ tags: [data nodes, distributed hypertables, cluster] api: license: community type: function +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/api/alter_data_node.md b/api/alter_data_node.md index ba93a30488..3f5b20fd92 100644 --- a/api/alter_data_node.md +++ b/api/alter_data_node.md @@ -7,6 +7,8 @@ tags: [data nodes, distributed hypertables, cluster] api: license: community type: function +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/api/attach_data_node.md b/api/attach_data_node.md index 408931dfa9..5e4148cac5 100644 --- a/api/attach_data_node.md +++ b/api/attach_data_node.md @@ -7,6 +7,8 @@ tags: [distributed hypertables, data nodes, attach] api: license: community type: function +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/api/cleanup_copy_chunk_operation_experimental.md b/api/cleanup_copy_chunk_operation_experimental.md index d04981d292..d3d88d9cc3 100644 --- a/api/cleanup_copy_chunk_operation_experimental.md +++ b/api/cleanup_copy_chunk_operation_experimental.md @@ -6,6 +6,8 @@ keywords: [chunks, multi-node, distributed hypertables, move, copy] api: license: community type: function +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/api/copy_chunk_experimental.md b/api/copy_chunk_experimental.md index 404f8a07c0..5d6fdfb205 100644 --- a/api/copy_chunk_experimental.md +++ b/api/copy_chunk_experimental.md @@ -8,6 +8,8 @@ api: license: community type: function experimental: true +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/api/create_distributed_hypertable.md b/api/create_distributed_hypertable.md index 20530239c6..d6dfe47ea0 100644 --- a/api/create_distributed_hypertable.md +++ b/api/create_distributed_hypertable.md @@ -7,6 +7,8 @@ tags: [cluster] api: license: community type: function +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/api/create_distributed_restore_point.md b/api/create_distributed_restore_point.md index 63ec9d7a74..ed9d2c0117 100644 --- a/api/create_distributed_restore_point.md +++ b/api/create_distributed_restore_point.md @@ -7,6 +7,8 @@ tags: [clusters, write-ahead logs, recovery] api: license: community type: function +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/api/delete_data_node.md b/api/delete_data_node.md index 231df2d906..c8fbe181ae 100644 --- a/api/delete_data_node.md +++ b/api/delete_data_node.md @@ -7,6 +7,8 @@ tags: [distributed hypertables, data nodes, detach, delete] api: license: community type: function +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/api/detach_data_node.md b/api/detach_data_node.md index b08a2ac83a..64330cfc32 100644 --- a/api/detach_data_node.md +++ b/api/detach_data_node.md @@ -7,6 +7,8 @@ tags: [distributed hypertables, cluster, data nodes] api: license: community type: function +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/api/distributed-hypertables.md b/api/distributed-hypertables.md index 4f3e7837a0..601e6d55c5 100644 --- a/api/distributed-hypertables.md +++ b/api/distributed-hypertables.md @@ -2,6 +2,8 @@ title: Distributed hypertables ( Sunsetted v2.14.x ) excerpt: Sunsetted v2.14.x. Timescale Cloud API reference for dealing with distributed hypertables keywords: [distributed hypertables] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/api/distributed_exec.md b/api/distributed_exec.md index 998ad8f69d..39ecaeb5b7 100644 --- a/api/distributed_exec.md +++ b/api/distributed_exec.md @@ -7,6 +7,8 @@ tags: [data nodes, cluster, procedure, roles, permissions] api: license: community type: procedure +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/api/move_chunk.md b/api/move_chunk.md index bcdb354ab1..fd089ac950 100644 --- a/api/move_chunk.md +++ b/api/move_chunk.md @@ -6,6 +6,8 @@ keywords: [chunks, hypertables, tablespaces, move] api: license: community type: function +seo: + robots: noindex --- # move_chunk() Community diff --git a/api/set_number_partitions.md b/api/set_number_partitions.md index bc02303d43..ecc6503b92 100644 --- a/api/set_number_partitions.md +++ b/api/set_number_partitions.md @@ -6,6 +6,8 @@ keywords: [hypertables, partitions] api: license: community type: function +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/api/set_replication_factor.md b/api/set_replication_factor.md index 40e62a4907..5ab854bdb1 100644 --- a/api/set_replication_factor.md +++ b/api/set_replication_factor.md @@ -7,6 +7,8 @@ tags: [cluster] api: license: community type: function +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/distributed-hypertables/about-distributed-hypertables.md b/self-hosted/distributed-hypertables/about-distributed-hypertables.md index 0a13985bd1..35c02a1afe 100644 --- a/self-hosted/distributed-hypertables/about-distributed-hypertables.md +++ b/self-hosted/distributed-hypertables/about-distributed-hypertables.md @@ -3,6 +3,8 @@ title: About distributed hypertables excerpt: Sunsetted v2.14.x. Distributed hypertables are hypertables that span multiple nodes products: [self_hosted] keywords: [distributed hypertables, multi-node] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/distributed-hypertables/alter-drop-distributed-hypertables.md b/self-hosted/distributed-hypertables/alter-drop-distributed-hypertables.md index 829ce73156..88596aa977 100644 --- a/self-hosted/distributed-hypertables/alter-drop-distributed-hypertables.md +++ b/self-hosted/distributed-hypertables/alter-drop-distributed-hypertables.md @@ -4,6 +4,8 @@ excerpt: Sunsetted v2.14.x. Alter and drop distributed hypertables in your self- products: [self_hosted] keywords: [distributed hypertables, alter, delete] tags: [change, delete] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/distributed-hypertables/create-distributed-hypertables.md b/self-hosted/distributed-hypertables/create-distributed-hypertables.md index 6cb6f9e827..b094deaea4 100644 --- a/self-hosted/distributed-hypertables/create-distributed-hypertables.md +++ b/self-hosted/distributed-hypertables/create-distributed-hypertables.md @@ -3,6 +3,8 @@ title: Create distributed hypertables excerpt: Sunsetted v2.14.x. Create a distributed hypertable in a self-hosted multi-node TimescaleDB instance products: [self_hosted] keywords: [distributed hypertables, multi-node, create] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/distributed-hypertables/foreign-keys.md b/self-hosted/distributed-hypertables/foreign-keys.md index 5a013fe574..ffe77b00bd 100644 --- a/self-hosted/distributed-hypertables/foreign-keys.md +++ b/self-hosted/distributed-hypertables/foreign-keys.md @@ -4,6 +4,8 @@ excerpt: Sunsetted v2.14.x. Add foreign keys to the nodes of a distributed hyper products: [self_hosted] keywords: [distributed hypertable, foreign keys] tags: [constraints] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/distributed-hypertables/index.md b/self-hosted/distributed-hypertables/index.md index be1af846e9..3554ecb303 100644 --- a/self-hosted/distributed-hypertables/index.md +++ b/self-hosted/distributed-hypertables/index.md @@ -3,6 +3,8 @@ title: Distributed hypertables excerpt: Sunsetted v2.14.x. Distributed hypertables help you horizontally scale your data storage in multi-node clusters products: [self_hosted] keywords: [distributed hypertables, multi-node] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/distributed-hypertables/insert.md b/self-hosted/distributed-hypertables/insert.md index 7b23b4bf6a..418d50234f 100644 --- a/self-hosted/distributed-hypertables/insert.md +++ b/self-hosted/distributed-hypertables/insert.md @@ -4,6 +4,8 @@ excerpt: Sunsetted v2.14.x. Insert data into distributed hypertables in your sel products: [self_hosted] keywords: [write, distributed hypertables] tags: [ingest, insert] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/distributed-hypertables/query.md b/self-hosted/distributed-hypertables/query.md index aea21f251f..08fb1c3d04 100644 --- a/self-hosted/distributed-hypertables/query.md +++ b/self-hosted/distributed-hypertables/query.md @@ -3,6 +3,8 @@ title: Query data in distributed hypertables excerpt: Sunsetted v2.14.x. Query data in distributed hypertables in your self-hosted TimescaleDB installation products: [self_hosted] keywords: [distributed hypertables, multi-node, queries] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/distributed-hypertables/triggers.md b/self-hosted/distributed-hypertables/triggers.md index 8020fcb0a0..1f44c6d55b 100644 --- a/self-hosted/distributed-hypertables/triggers.md +++ b/self-hosted/distributed-hypertables/triggers.md @@ -3,6 +3,8 @@ title: Use triggers on distributed hypertables excerpt: Sunsetted v2.14.x. Set up triggers on a distributed hypertable in your self-hosted TimescaleDB installation products: [self_hosted] keywords: [distributed hypertables, triggers, multi-node] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/multinode-timescaledb/about-multinode.md b/self-hosted/multinode-timescaledb/about-multinode.md index 66bcf2240a..53c8f8f9c8 100644 --- a/self-hosted/multinode-timescaledb/about-multinode.md +++ b/self-hosted/multinode-timescaledb/about-multinode.md @@ -3,6 +3,8 @@ title: About multi-node excerpt: Sunsetted in v2.14.x. Multi-node TimescaleDB enables you to run and manage a cluster of databases, which can result in faster data ingest, and more responsive and efficient queries for large workloads products: [self_hosted] keywords: [multi-node] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/multinode-timescaledb/index.md b/self-hosted/multinode-timescaledb/index.md index 6b18cdd3ed..114443cd10 100644 --- a/self-hosted/multinode-timescaledb/index.md +++ b/self-hosted/multinode-timescaledb/index.md @@ -3,6 +3,8 @@ title: Multi-node excerpt: Learn about using multi-node TimescaleDB to scale your database horizontally, including setup, authentication, configuration, and managing your cluster products: [self_hosted] keywords: [multi-node, scaling] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/multinode-timescaledb/multinode-administration.md b/self-hosted/multinode-timescaledb/multinode-administration.md index 1f0161b539..149e02cf7b 100644 --- a/self-hosted/multinode-timescaledb/multinode-administration.md +++ b/self-hosted/multinode-timescaledb/multinode-administration.md @@ -4,6 +4,8 @@ excerpt: Sunsetted v2.14.x. Administer your multi-node TimescaleDB cluster direc products: [self_hosted] keywords: [multi-node, admin] tags: [manage] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/multinode-timescaledb/multinode-auth.md b/self-hosted/multinode-timescaledb/multinode-auth.md index 28bee57df8..47d7e4f38b 100644 --- a/self-hosted/multinode-timescaledb/multinode-auth.md +++ b/self-hosted/multinode-timescaledb/multinode-auth.md @@ -4,6 +4,8 @@ excerpt: Sunsetted v2.14.x. Configure authentication between access nodes and da products: [self_hosted] keywords: [multi-node, authenticate] tags: [admin] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/multinode-timescaledb/multinode-config.md b/self-hosted/multinode-timescaledb/multinode-config.md index 3059c60608..4e8504af9a 100644 --- a/self-hosted/multinode-timescaledb/multinode-config.md +++ b/self-hosted/multinode-timescaledb/multinode-config.md @@ -3,6 +3,8 @@ title: Multi-node configuration excerpt: Sunsetted v2.14.x. Configure the main settings in your multi-node TimescaleDB instance products: [self_hosted] keywords: [configuration, settings, multi-node] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/multinode-timescaledb/multinode-grow-shrink.md b/self-hosted/multinode-timescaledb/multinode-grow-shrink.md index 4e3dbdfda7..0ddf99cd6e 100644 --- a/self-hosted/multinode-timescaledb/multinode-grow-shrink.md +++ b/self-hosted/multinode-timescaledb/multinode-grow-shrink.md @@ -4,6 +4,8 @@ excerpt: Sunsetted v2.14.x. Add and remove data nodes from your self-hosted mult products: [self_hosted] keywords: [multi-node, data nodes] tags: [add, remove] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/multinode-timescaledb/multinode-ha.md b/self-hosted/multinode-timescaledb/multinode-ha.md index 49467a4001..ed04821abb 100644 --- a/self-hosted/multinode-timescaledb/multinode-ha.md +++ b/self-hosted/multinode-timescaledb/multinode-ha.md @@ -3,6 +3,8 @@ title: High availability with multi-node excerpt: Sunsetted v2.14.x. Configure your self-hosted multi-node TimescaleDB for high availability products: [self_hosted] keywords: [multi-node, high availability] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/multinode-timescaledb/multinode-maintenance.md b/self-hosted/multinode-timescaledb/multinode-maintenance.md index cd82026cd3..fb6e946913 100644 --- a/self-hosted/multinode-timescaledb/multinode-maintenance.md +++ b/self-hosted/multinode-timescaledb/multinode-maintenance.md @@ -4,6 +4,8 @@ excerpt: Sunsetted v2.14.x. Run maintenance on your self-hosted multi-node Times products: [self_hosted] keywords: [multi-node, maintenance] tags: [manage] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; diff --git a/self-hosted/multinode-timescaledb/multinode-setup.md b/self-hosted/multinode-timescaledb/multinode-setup.md index 448bea394d..ec0746a6a1 100644 --- a/self-hosted/multinode-timescaledb/multinode-setup.md +++ b/self-hosted/multinode-timescaledb/multinode-setup.md @@ -3,6 +3,8 @@ title: Set up multi-node on self-hosted TimescaleDB excerpt: Sunsetted v2.14.x. Learn how to set up a self-hosted multi-node TimescaleDB instance products: [self_hosted] keywords: [multi-node, self-hosted] +seo: + robots: noindex --- import MultiNodeDeprecation from "versionContent/_partials/_multi-node-deprecation.mdx"; From 4908142c4c88e5ee6d0cdcea7e3fda6109987c34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADzio=20de=20Royes=20Mello?= Date: Fri, 14 Mar 2025 07:17:56 -0300 Subject: [PATCH 23/90] New add_continuous_aggregate_policy API options (#3913) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * New add_continuous_aggregate_policy API options * Addressed @philkra review Signed-off-by: Fabrízio de Royes Mello --- api/add_continuous_aggregate_policy.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/api/add_continuous_aggregate_policy.md b/api/add_continuous_aggregate_policy.md index 4bdc766bba..6a36cffdf5 100644 --- a/api/add_continuous_aggregate_policy.md +++ b/api/add_continuous_aggregate_policy.md @@ -50,7 +50,14 @@ about how continuous aggregates use real-time aggregation, see the |`if_not_exists`|BOOLEAN|Set to `true` to issue a notice instead of an error if the job already exists. Defaults to false.| |`timezone`|TEXT|A valid time zone. If you specify `initial_start`, subsequent executions of the refresh policy are aligned on `initial_start`. However, daylight savings time (DST) changes may shift this alignment. If this is an issue you want to mitigate, set `timezone` to a valid time zone. Default is `NULL`, [UTC bucketing](https://docs.timescale.com/use-timescale/latest/time-buckets/about-time-buckets/) is performed.| | `include_tiered_data` | BOOLEAN | Enable/disable reading tiered data. This setting helps override the current settings for the`timescaledb.enable_tiered_reads` GUC. The default is NULL i.e we use the current setting for `timescaledb.enable_tiered_reads` GUC | | +| `buckets_per_batch` | INTEGER | Number of buckets to be refreshed by a _batch_. This value is multiplied by the CAgg bucket width to determine the size of the batch range. Default value is `0`, single batch execution. Values of less than `0` are not allowed. | | +| `max_batches_per_execution` | INTEGER | Limit the maximum number of batches to run when a policy executes. If some batches remain, they are processed the next time the policy runs. Default value is `10`, each job processes a maximum of 10 batches. Set to `0` for the number of batches to be unlimited. Values of less than `0` are not allowed. | | + + +Setting `buckets_per_batch` greater than zero means that the refresh window is split in batches of `bucket width` * `buckets per batch`. For example, a given Continuous Aggregate with `bucket width` of `1 day` and `buckets_per_batch` of 10 has a batch size of `10 days` to process the refresh. +Because each `batch` is an individual transaction, executing a policy in batches make the data visible for the users before the entire job is executed. Batches are processed from the most recent data to the oldest. + ### Returns From 286535d63d1233aedbfad4505f8d7c8b3b9ad7bd Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Fri, 14 Mar 2025 13:52:02 +0200 Subject: [PATCH 24/90] Link fix --- .github/ISSUE_TEMPLATE/bug_report.md | 5 +---- .github/ISSUE_TEMPLATE/rfc.md | 5 +---- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index b615e478f2..90deb95b3c 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -35,7 +35,4 @@ _Use this template for reporting bugs in the docs._ ## Contributing to documentation -We welcome documentation contributions! - -* For information about how to propose a change, see the [contributing guide](https://github.com/timescale/docs/blob/latest/CONTRIBUTING.md) in our GitHub repository. -* For information on style and word usage, see the [style guide](https://docs.timescale.com/about/latest/contribute-to-docs) +We welcome documentation contributions! For guidelines, see the [contributing guide](https://github.com/timescale/docs/blob/latest/CONTRIBUTING.md). diff --git a/.github/ISSUE_TEMPLATE/rfc.md b/.github/ISSUE_TEMPLATE/rfc.md index bcfc053546..1ca2efd1fb 100644 --- a/.github/ISSUE_TEMPLATE/rfc.md +++ b/.github/ISSUE_TEMPLATE/rfc.md @@ -25,7 +25,4 @@ assignees: '' ## Contributing to documentation -We welcome documentation contributions! - -* For information about how to suggest a change, see the [contributing guide](https://github.com/timescale/docs/blob/latest/CONTRIBUTING.md) in our GitHub repository. -* For information on style and word usage, see the [style guide](https://docs.timescale.com/about/latest/contribute-to-docs) +We welcome documentation contributions! For guidelines, see the [contributing guide](https://github.com/timescale/docs/blob/latest/CONTRIBUTING.md). From eed406278eb0ffb96d793d97483403a6c24e650d Mon Sep 17 00:00:00 2001 From: Eric Clymer Date: Fri, 14 Mar 2025 10:18:18 -0500 Subject: [PATCH 25/90] Update changelog.md (#3932) --- about/changelog.md | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/about/changelog.md b/about/changelog.md index 7e29f6a69a..df1a069533 100644 --- a/about/changelog.md +++ b/about/changelog.md @@ -8,11 +8,21 @@ keywords: [changelog, upgrades, updates, releases] All the latest features and updates to Timescale products. +## 💾 16K dimensions on pgvectorscale plus new pgai Vectorizer support + -## Agent Mode for PopSQL and more +### pgvectorscale 0.6 — store up to 16K dimension embeddings + +pgvectorscale 0.6.0 now supports storing vectors with up to 16,000 dimensions, removing the previous limitation of 2,000 from pgvector. This lets you use larger embedding models like OpenAI's text-embedding-3-large (3072 dim) with PostgreSQL as your vector database. This release also includes key performance and capability enhancements, including NEON support for SIMD distance calculations on aarch64 processors, improved inner product distance metric implementation, and improved index statistics. See the release details [here](https://github.com/timescale/pgvectorscale/releases/tag/0.6.0). + +### pgai Vectorizer supports models from AWS Bedrock, Azure AI, Google Vertex via LiteLLM + +Access embedding models from popular cloud model hubs like AWS Bedrock, Azure AI Foundry, Google Vertex, as well as HuggingFace and Cohere as part of the LiteLLM integration with pgai Vectorizer. To use these models with pgai Vectorizer on Timescale Cloud, select `Other` when adding the API key in the credentials section of Timescale Console. + +## 🤖 Agent Mode for PopSQL and more -### 🤖 Agent Mode for PopSQL +### Agent Mode for PopSQL Introducing Agent Mode, a new feature in $CONSOLE SQL Assistant. SQL Assistant lets you query your database using natural language. However, if you ran into errors, you have to approve the implementation of the Assistant's suggestions. From 1e838c349e1e56f50b09717f76892d62f27e2cb8 Mon Sep 17 00:00:00 2001 From: Danil Zhigalin Date: Fri, 14 Mar 2025 18:34:31 +0100 Subject: [PATCH 26/90] Update how upgrades can be triggered by user (#3933) * Update how upgrades can be triggered by user Cloud docs used to instruct users to click a non-existing `Apply upgrades` button. On cloud other ways exist to apply upgrades than on MST. * Update the latest PG version to 17 --------- Signed-off-by: Danil Zhigalin Co-authored-by: Iain Cox --- use-timescale/upgrades.md | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/use-timescale/upgrades.md b/use-timescale/upgrades.md index 672e70f4ca..f97d95a13a 100644 --- a/use-timescale/upgrades.md +++ b/use-timescale/upgrades.md @@ -71,14 +71,15 @@ For more information about replicas, see the ## Non-critical maintenance updates -Non-critical upgrades are made available before the upgrade is performed -automatically. During this time you can click `Apply upgrades` to start the -upgrade at any time. However, after the time expires, usually around a week, -the upgrade is triggered automatically in the next available maintenance window -for your service. You can configure the maintenance window so that these -upgrades are started only at a particular time, on a set day of the week. If -there are no pending upgrades available during a regular maintenance window, no -changes are performed. +Non-critical upgrades are available before the upgrade is performed +automatically by $CLOUD_LONG. To upgrade $TIMESCALE_DB manually, run `ALTER +EXTENSION timescaledb UPDATE` in your $SERVICE_LONG, or `Pause` and `Resume`. +If no action is taken by the user, the upgrade is triggered in the next +available maintenance window. You can configure the maintenance window so that +these upgrades are started at a particular time, on a set day of the week. + +If there are no pending upgrades available during a regular maintenance window, +no changes are performed. When you are considering your maintenance window schedule, you might prefer to choose a day and time that usually has very low activity, such as during the @@ -132,7 +133,7 @@ overview page. --> You can also manually upgrade to the newest supported PostgreSQL version -(PostgreSQL 16) from the service overview page. +(PostgreSQL 17) from the service overview page. Upgrading to a newer version of PostgreSQL allows you to take advantage of new features, enhancements, and security fixes. It also ensures that you are using a From 71aeb8126beee111277b72fac12c8afeabb19a2a Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Mon, 17 Mar 2025 11:13:33 +0200 Subject: [PATCH 27/90] Partial rework for Transit gateway (#3914) --- _partials/_transit-gateway-others.md | 43 ------------------- ...sit-gateway-aws.md => _transit-gateway.md} | 0 use-timescale/integrations/aws.md | 8 +++- .../integrations/corporate-data-center.md | 8 +++- use-timescale/integrations/google-cloud.md | 8 +++- use-timescale/integrations/microsoft-azure.md | 8 +++- use-timescale/security/transit-gateway.md | 27 +++++++++--- 7 files changed, 45 insertions(+), 57 deletions(-) delete mode 100644 _partials/_transit-gateway-others.md rename _partials/{_transit-gateway-aws.md => _transit-gateway.md} (100%) diff --git a/_partials/_transit-gateway-others.md b/_partials/_transit-gateway-others.md deleted file mode 100644 index b11cea5367..0000000000 --- a/_partials/_transit-gateway-others.md +++ /dev/null @@ -1,43 +0,0 @@ -2. **Create a Peering $VPC in [$CONSOLE][console-login]** - - 1. In `Security` > `VPC`, click `Create a VPC`: - - ![$CLOUD_LONG new $VPC](https://assets.timescale.com/docs/images/add-peering-vpc.png) - - 1. Choose your region and IP range, name your VPC, then click `Create VPC`: - - ![Create a new VPC in $CLOUD_LONG](https://assets.timescale.com/docs/images/configure-peering-vpc.png) - - Your $SERVICE_SHORT and Peering $VPC must be in the same AWS region. The number of Peering $VPCs you can create in your project depends on your [price tier][pricing-plans]. If you need another Peering $VPC, either contact [support@timescale.com](mailto:support@timescale.com) or change your tier in [$CONSOLE][console-login]. - - 1. Add a peering connection: - - 1. In the `VPC Peering` column, click `Add`. - 1. Provide your AWS account ID, Transit Gateway ID, CIDR ranges, and AWS region. - 1. Click `Add connection`. - - ![Add peering](https://assets.timescale.com/docs/images/add-peering.png) - -1. **Accept and configure peering connection in your AWS account** - - Once your peering connection appears as `Processing`, you can accept and configure it in AWS: - - 1. Accept the peering request coming from $CLOUD_LONG. The request can take up to 5 min to arrive. Within 5 more minutes after accepting, the peering should appear as `Connected` in $CONSOLE. - - 1. Configure at least the following in yur AWS account networking: - - - Your subnet route table to route traffic to your Transit Gateway for the Peering VPC CIDRs. - - Your Transit Gateway route table to route traffic to the newly created Transit Gateway peering attachment for the Peering VPC CIDRs. - - Security groups to allow outbound TCP 5432. - -1. **Attach a $CLOUD_LONG service to the Peering VPC In [$CONSOLE][console-services]** - - 1. Select the $SERVICE_SHORT you want to connect to the Peering VPC. - 1. Click `Operations` > `Security` > `VPC`. - 1. Select the VPC, then click `Attach VPC`. - - You cannot attach a $SERVICE_LONG to multiple $CLOUD_LONG $VPCs at the same time. - -[console-login]: https://console.cloud.timescale.com/ -[console-services]: https://console.cloud.timescale.com/dashboard/services -[pricing-plans]: /about/:currentVersion:/pricing-and-account-management/ diff --git a/_partials/_transit-gateway-aws.md b/_partials/_transit-gateway.md similarity index 100% rename from _partials/_transit-gateway-aws.md rename to _partials/_transit-gateway.md diff --git a/use-timescale/integrations/aws.md b/use-timescale/integrations/aws.md index c9bcf52a03..62df7a6ff4 100644 --- a/use-timescale/integrations/aws.md +++ b/use-timescale/integrations/aws.md @@ -6,7 +6,7 @@ keywords: [AWS, integrations] --- import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; -import TransitGatewayAWS from "versionContent/_partials/_transit-gateway-aws.mdx"; +import TransitGateway from "versionContent/_partials/_transit-gateway.mdx"; # Integrate Amazon Web Services with $CLOUD_LONG @@ -24,7 +24,11 @@ This page explains how to integrate your AWS infrastructure with $CLOUD_LONG usi To connect to $CLOUD_LONG: - + + + + + You have successfully integrated your AWS infrastructure with $CLOUD_LONG. diff --git a/use-timescale/integrations/corporate-data-center.md b/use-timescale/integrations/corporate-data-center.md index ff158d9496..a6af17ad5c 100644 --- a/use-timescale/integrations/corporate-data-center.md +++ b/use-timescale/integrations/corporate-data-center.md @@ -6,7 +6,7 @@ keywords: [on-premise, integrations] --- import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; -import TransitGatewayOthers from "versionContent/_partials/_transit-gateway-others.mdx"; +import TransitGateway from "versionContent/_partials/_transit-gateway.mdx"; # Integrate your data center with $CLOUD_LONG @@ -22,11 +22,15 @@ This page explains how to integrate your corporate on-premise infrastructure wit To connect to $CLOUD_LONG: + + 1. **Connect your infrastructure to AWS Transit Gateway** Establish connectivity between your on-premise infrastructure and AWS. See the [Centralize network connectivity using AWS Transit Gateway][aws-onprem]. - + + + You have successfully integrated your Microsoft Azure infrastructure with $CLOUD_LONG. diff --git a/use-timescale/integrations/google-cloud.md b/use-timescale/integrations/google-cloud.md index e4777dcddb..486f5cb878 100644 --- a/use-timescale/integrations/google-cloud.md +++ b/use-timescale/integrations/google-cloud.md @@ -6,7 +6,7 @@ keywords: [Google Cloud, integrations] --- import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; -import TransitGatewayOthers from "versionContent/_partials/_transit-gateway-others.mdx"; +import TransitGateway from "versionContent/_partials/_transit-gateway.mdx"; # Integrate Google Cloud with $CLOUD_LONG @@ -24,11 +24,15 @@ This page explains how to integrate your Google Cloud infrastructure with $CLOUD To connect to $CLOUD_LONG: + + 1. **Connect your infrastructure to AWS Transit Gateway** Establish connectivity between Google Cloud and AWS. See [Connect HA VPN to AWS peer gateways][gcp-aws]. - + + + You have successfully integrated your Google Cloud infrastructure with $CLOUD_LONG. diff --git a/use-timescale/integrations/microsoft-azure.md b/use-timescale/integrations/microsoft-azure.md index 6686a802e6..1582970653 100644 --- a/use-timescale/integrations/microsoft-azure.md +++ b/use-timescale/integrations/microsoft-azure.md @@ -6,7 +6,7 @@ keywords: [Azure, integrations] --- import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; -import TransitGatewayOthers from "versionContent/_partials/_transit-gateway-others.mdx"; +import TransitGateway from "versionContent/_partials/_transit-gateway.mdx"; # Integrate Microsoft Azure with $CLOUD_LONG @@ -24,11 +24,15 @@ This page explains how to integrate your Microsoft Azure infrastructure with $CL To connect to $CLOUD_LONG: + + 1. **Connect your infrastructure to AWS Transit Gateway** Establish connectivity between Azure and AWS. See the [AWS architectural documentation][azure-aws] for details. - + + + You have successfully integrated your Microsoft Azure infrastructure with $CLOUD_LONG. diff --git a/use-timescale/security/transit-gateway.md b/use-timescale/security/transit-gateway.md index 2cd90e7217..2011eaa3b3 100644 --- a/use-timescale/security/transit-gateway.md +++ b/use-timescale/security/transit-gateway.md @@ -9,8 +9,7 @@ cloud_ui: - [services, :serviceId, operations, vpc] --- -import TransitGatewayAWS from "versionContent/_partials/_transit-gateway-aws.mdx"; -import TransitGatewayOthers from "versionContent/_partials/_transit-gateway-others.mdx"; +import TransitGateway from "versionContent/_partials/_transit-gateway.mdx"; # Securely connect to $CLOUD_LONG using AWS Transit Gateway @@ -35,37 +34,53 @@ AWS Transit Gateway enable you to connect from almost any environment, this page - + + + + + + + 1. **Connect your infrastructure to AWS Transit Gateway** Establish connectivity between Azure and AWS. See the [AWS architectural documentation][azure-aws] for details. - + + + + + 1. **Connect your infrastructure to AWS Transit Gateway** Establish connectivity between Google Cloud and AWS. See [Connect HA VPN to AWS peer gateways][gcp-aws]. - + + + + + 1. **Connect your infrastructure to AWS Transit Gateway** Establish connectivity between your on-premise infrastructure and AWS. See the [Centralize network connectivity using AWS Transit Gateway][aws-onprem]. - + + + From 5da6032505e392a35ba506aa7b245c3480c9512d Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Mon, 17 Mar 2025 12:50:52 +0200 Subject: [PATCH 28/90] Chunk skipping clarification (#3929) Co-authored-by: Iain Cox --- api/enable_chunk_skipping.md | 5 +++-- use-timescale/hypercore/index.md | 5 ++--- use-timescale/hypertables/improve-query-performance.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/api/enable_chunk_skipping.md b/api/enable_chunk_skipping.md index 34f91f261a..22976dda04 100644 --- a/api/enable_chunk_skipping.md +++ b/api/enable_chunk_skipping.md @@ -11,7 +11,8 @@ api: # enable_chunk_skipping() -Enable range statistics for a specific column in a **compressed** hypertable. This tracks a range of values for that column per chunk. Used for chunk pruning during query optimization. +Enable range statistics for a specific column in a **compressed** hypertable. This tracks a range of values for that column per chunk. +Used for chunk skipping during query optimization and applies only to the chunks created after chunk skipping is enabled. Best practice is to enable range tracking on columns that are correlated to the partitioning column. In other words, enable tracking on secondary columns which are @@ -27,7 +28,7 @@ The range is stored in start (inclusive) and end (exclusive) form in the This way you store the min/max values for such columns in this catalog table at the per-chunk level. These min/max range values do not participate in partitioning of the data. These ranges are -used for chunk pruning when the `WHERE` clause of an SQL query specifies +used for chunk skipping when the `WHERE` clause of an SQL query specifies ranges on the column. A [DROP COLUMN](https://www.postgresql.org/docs/current/sql-altertable.html#SQL-ALTERTABLE-DESC-DROP-COLUMN) diff --git a/use-timescale/hypercore/index.md b/use-timescale/hypercore/index.md index 834276cc04..2734840dc5 100644 --- a/use-timescale/hypercore/index.md +++ b/use-timescale/hypercore/index.md @@ -33,8 +33,7 @@ in each $SERVICE_LONG: saving significant storage space. - **Faster queries on compressed data in columnstore**: in columnstore conversion, hypertable - chunks are compressed by more than 90%, and organized for efficient, large-scale queries. This - saves on storage costs, and keeps your queries operating at lightning speed. + chunks are compressed by more than 90%, and organized for efficient, large-scale queries. Combined with [chunk skipping][chunk-skipping], this helps you save on storage costs and keeps your queries operating at lightning speed. - **Full mutability with transactional semantics**: regardless of where data is stored, Hypercore provides full ACID support. Like in a vanilla Postgres database, inserts and updates @@ -57,4 +56,4 @@ This section shows you how to: [modify-data-in-hypercore]: /use-timescale/:currentVersion:/hypercore/modify-data-in-hypercore/ [indexing-data-in-hypercore]: /use-timescale/:currentVersion:/hypercore/best-practice-for-large-amounts-of-data/ [compression]: /use-timescale/:currentVersion:/compression/ - +[chunk-skipping]: /use-timescale/:currentVersion:/hypertables/improve-query-performance/ diff --git a/use-timescale/hypertables/improve-query-performance.md b/use-timescale/hypertables/improve-query-performance.md index be08f1c0ca..c266332cf7 100644 --- a/use-timescale/hypertables/improve-query-performance.md +++ b/use-timescale/hypertables/improve-query-performance.md @@ -19,7 +19,7 @@ performance. To improve query performance, TimescaleDB enables you to skip chunks on non-partitioning columns in hypertables. -Chunk skipping can only be enabled for **compressed** hypertables. +Chunk skipping can only be enabled for **compressed** hypertables and applies to the chunks created **after** it as enabled. ## How chunk skipping works From d5cd12216da5cf8492974c02807e4c50a39da485 Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Tue, 18 Mar 2025 12:20:32 +0200 Subject: [PATCH 29/90] Added config parameters to alter_job (#3936) --- api/alter_job.md | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/api/alter_job.md b/api/alter_job.md index 252ea02610..374c26e741 100644 --- a/api/alter_job.md +++ b/api/alter_job.md @@ -9,6 +9,8 @@ api: type: function --- +import EarlyAccess from "versionContent/_partials/_early_access.mdx"; + # alter_job() Community Actions scheduled using the TimescaleDB automation framework run periodically in @@ -28,20 +30,21 @@ other useful statistics for deciding what the new schedule should be. ### Optional arguments -|Name|Type|Description| -|-|-|-| -|`schedule_interval`|`INTERVAL`|The interval at which the job runs. Defaults to 24 hours.| -|`max_runtime`|`INTERVAL`|The maximum amount of time the job is allowed to run by the background worker scheduler before it is stopped.| -|`max_retries`|`INTEGER`|The number of times the job is retried if it fails.| -|`retry_period`|`INTERVAL`|The amount of time the scheduler waits between retries of the job on failure.| -|`scheduled`|`BOOLEAN`|Set to `FALSE` to exclude this job from being run as background job.| -|`config`|`JSONB`|Job-specific configuration, passed to the function when it runs.| -|`next_start`|`TIMESTAMPTZ`|The next time at which to run the job. The job can be paused by setting this value to `infinity`, and restarted with a value of `now()`.| -|`if_exists`|`BOOLEAN`|Set to `true`to issue a notice instead of an error if the job does not exist. Defaults to false.| -|`check_config`|`REGPROC`|A function that takes a single argument, the `JSONB` `config` structure. The function is expected to raise an error if the configuration is not valid, and return nothing otherwise. Can be used to validate the configuration when updating a job. Only functions, not procedures, are allowed as values for `check_config`.| -|`fixed_schedule`|`BOOLEAN`|To enable fixed scheduled job runs, set to `TRUE`.| -|`initial_start`|`TIMESTAMPTZ`|Set the time when the `fixed_schedule` job run starts. For example, `19:10:25-07`.| -|`timezone`|`TEXT`|Address the 1-hour shift in start time when clocks change from [Daylight Saving Time to Standard Time](https://en.wikipedia.org/wiki/Daylight_saving_time). For example, `America/Sao_Paulo`.| +|Name|Type| Description | +|-|-|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +|`schedule_interval`|`INTERVAL`| The interval at which the job runs. Defaults to 24 hours. | +|`max_runtime`|`INTERVAL`| The maximum amount of time the job is allowed to run by the background worker scheduler before it is stopped. | +|`max_retries`|`INTEGER`| The number of times the job is retried if it fails. | +|`retry_period`|`INTERVAL`| The amount of time the scheduler waits between retries of the job on failure. | +|`scheduled`|`BOOLEAN`| Set to `FALSE` to exclude this job from being run as background job. | +|`config`|`JSONB`| Job-specific configuration, passed to the function when it runs. This includes:
  • verbose_log: boolean, defaults to false. Enable verbose logging output when running the compression policy.
  • maxchunks_to_compress: integer, defaults to 0 (no limit). The maximum number of chunks to compress during a policy run.
  • recompress: boolean, defaults to true. Recompress partially compressed chunks.
  • compress_after: see [add_compression_policy][add-policy].
  • compress_created_before: see [add_compression_policy][add-policy].
  • hypercore_use_access_method: boolean, defaults to false. Use hypercore TAM to compress chunks.
    • | +|`next_start`|`TIMESTAMPTZ`| The next time at which to run the job. The job can be paused by setting this value to `infinity`, and restarted with a value of `now()`. | +|`if_exists`|`BOOLEAN`| Set to `true`to issue a notice instead of an error if the job does not exist. Defaults to false. | +|`check_config`|`REGPROC`| A function that takes a single argument, the `JSONB` `config` structure. The function is expected to raise an error if the configuration is not valid, and return nothing otherwise. Can be used to validate the configuration when updating a job. Only functions, not procedures, are allowed as values for `check_config`. | +|`fixed_schedule`|`BOOLEAN`| To enable fixed scheduled job runs, set to `TRUE`. | +|`initial_start`|`TIMESTAMPTZ`| Set the time when the `fixed_schedule` job run starts. For example, `19:10:25-07`. | +|`timezone`|`TEXT`| Address the 1-hour shift in start time when clocks change from [Daylight Saving Time to Standard Time](https://en.wikipedia.org/wiki/Daylight_saving_time). For example, `America/Sao_Paulo`. | + When a job begins, the `next_start` parameter is set to `infinity`. This prevents the job from attempting to be started again while it is running. When @@ -107,3 +110,6 @@ Finally, there is a distinction between runtime failures that do not cause the j In the event of a job crash, the next start calculation still follows the above formula, but it is always at least 5 minutes after the job's last finish, to give an operator enough time to disable it before another crash.     + + +[add-policy]: /api/:currentVersion:/compression/add_compression_policy/#required-arguments \ No newline at end of file From ef8f0c6697dae05e915f85c4d6e222c1c40790f1 Mon Sep 17 00:00:00 2001 From: Iain Cox Date: Tue, 18 Mar 2025 19:03:44 +0100 Subject: [PATCH 30/90] Secondary index docs (#3943) * chore: ready for first review. --- use-timescale/hypercore/secondary-indexes.md | 295 +++++++++++++++++++ use-timescale/page-index/page-index.js | 5 + 2 files changed, 300 insertions(+) create mode 100644 use-timescale/hypercore/secondary-indexes.md diff --git a/use-timescale/hypercore/secondary-indexes.md b/use-timescale/hypercore/secondary-indexes.md new file mode 100644 index 0000000000..b8d3dbc18a --- /dev/null +++ b/use-timescale/hypercore/secondary-indexes.md @@ -0,0 +1,295 @@ +--- +title: Improve query and upsert performance using secondary indexes +excerpt: Using secondary indexes on data in the columnstore make lookup queries 1,185x and upserts 224x faster +products: [cloud, mst, self_hosted] +keywords: [hypertable, compression, row-columnar storage, hypercore] +--- + +import EarlyAccess from "versionContent/_partials/_early_access.mdx"; + +# Improve query and upsert performance using secondary indexes + +Real-time analytics applications require more than fast inserts and analytical queries. They also need high performance +when retrieving individual records, enforcing constraints, or performing upserts, something that OLAP/columnar databases +lack. + +$TIMESCALE_DB supports and accelerates real-time analytics using [Hypercore][hypercore] without missing out on important +PostgreSQL features, including support for standard PostgreSQL indexes. Hypercore is a hybrid storage engine +because it supports deep analytics while staying true to PostgreSQL. Full support for B-tree and hash indexes +on columnstore data enables you to perform point lookups 1,185x faster, enforce unique constraints, and execute +upserts 224x faster—all while maintaining columnstore compression and analytics performance. + + + +## Choose the best indexing method + +[Indexes are a fundamental part of database performance optimization][blog-perf-tuning], they enable queries to +quickly locate and retrieve data without scanning entire tables. B-tree and hash indexes are among PostgreSQL’s +most widely used index types. However, they are designed for different query types: + +- [B-tree indexes][b-tree-overview]: keep data sorted in a hierarchical structure ideal for queries that involve + range (>, <, BETWEEN) and equality (=) lookups. A B-tree index enables you to check unique constraints or for + range-based filtering. You quickly retrieve and filter the relevant rows without scanning all the data. + + When a query searches for a specific value or a range of values in an indexed column, the B-tree structure + enables the database to quickly traverse the tree and find the relevant records in logarithmic time (`O(log n)`), + significantly improving performance compared to a full table scan. + +- [Hash indexes][hash-overview]: designed for exact-match lookups (=) and use a hashing function to map values to + unique disk locations. When searching for a specific ID, a hash index allows direct access to the data with + minimal overhead, and provides the fastest results + + When a query searches for a single value, such as searching for transaction by transaction ID, hash + indexes can be even faster than B-tree indexes as they don’t require tree traversal and have an amortized constant + (`O(1)`) lookup. Hash indexes don’t support range queries, they are ideal specifically for cases with + frequently queried, unique keys. + +The performance advantage from these indexing methods comes from optimized data structures designed for efficient key +searching. This results in fewer disk page reads, which in turn reduces I/O spikes when locating specific data points +or enforcing uniqueness. + +## How B-tree and hash indexes work + +PostgreSQL offers [multiple index types][postgres-index-types], For example, the default B-tree, hash, GIN, and BRIN. +all implemented as Index Access Methods (IAMs). PostgreSQL supplies the [table access method (TAM)][postgres-tam-methods] +interface for table storage. + +![TAM architecture](https://assets.timescale.com/docs/images/tam_architecture.png) + +By default, $TIMESCALE_DB stores data in the rowstore in standard PostgreSQL row-oriented tables, using the default heap +TAM. To make the heap TAM work with the columnstore, $TIMESCALE_DB integrates PostgreSQL [TOAST][storage-toast] to store +columnar data as compressed arrays. However, querying columnized data returns compressed, opaque data. To support +normal queries, $TIMESCALE_DB adds the `DecompressChunk` scan node to the PostgreSQL query plan in order to decompress data +on-the-fly. However, the heap TAM only indexes the compressed values, not the original data. + +Hypercore TAM handles decompression behind the scenes. This enables PostgreSQL to use standard interfaces for +indexing, to collect statistics, enforce constraints and lock tuples by reference. This also allows PostgreSQL’s built-in +scan nodes, such as sequential and index scans, to operate on the columnstore. Custom scan nodes are used for +analytical query performance optimizations, including vectorized filtering and aggregation. + +Hypercore TAM supports B-tree and hash indexes, making point lookups, upserts, and unique constraint +enforcement more efficient on the columnstore. Our benchmarks demonstrate substantial performance improvements: + +* 1,185x faster point lookup queries to retrieve a single record +* 224.3x faster inserts when checking unique constraints +* 2.6x faster upserts +* 4.5x faster range queries + +## Enable secondary indexing + +To speed up your queries using secondary indexes you enable hypercore TAM on your hypertable in the columnstore: + + + +1. **Create a table with the desired columns and constraints** + ```sql + create table readings ( + metric_uuid uuid default gen_random_uuid(), + created_at timestamptz not null, + uploaded_at timestamptz not null, + location_id integer references locations (location_id), + device_id integer references devices (device_id), + temperature float, + humidity float + ); + ``` + +1. **Convert the table to a [hypertable][convert-to-hypertable]** + + ```sql + select create_hypertable ( + 'readings', + by_range('uploaded_at') + ); + ``` + +1. **Enable hypercore TAM for the hypertable** + ```sql + alter table readings + set access method hypercore + set ( + timescaledb.orderby = 'created_at', + timescaledb.segmentby = 'location_id' + ); + ``` + This enables the columnstore on the table. Hypercore TAM is applied to chunks created after you set the access + method. Existing chunks continue to use the default `heap`. + + To return to the `heap` TAM, call `set access method heap`. You can also change the table access method for an + existing chunk with a call like `ALTER TABLE _timescaledb_internal._hyper_1_1_chunk SET ACCESS METHOD hypercore;` + +1. **Move chunks from rowstore to columnstore as they age** + + ```sql + CALL add_columnstore_policy( + readings, + interval '1 day' + ); + ``` + + + +Hypercore TAM is now active on all new chunks created in the hypertable. + +## Create b-tree and hash indexes + +Once you have enabled hypercore TAM in your hypertable, the indexes are rebuilt when the table chunks are converted from +the rowstore to the columnstore. When you query data, these indexes are used by the PostgreSQL query planner over the +rowstore and columnstore. + +You add hash and B-tree indexes to a hypertable the same way as a regular PostgreSQL table: + +- **Hash index** + ```sql + CREATE INDEX readings_metric_uuid_hash_idx ON readings USING hash (metric_uuid); + ``` +- **B-tree index** + ```sql + CREATE UNIQUE INDEX readings_metric_uuid_metric_uuid_uploaded_at_idx + ON readings (metric_uuid, uploaded_at); + ``` + +If you have existing chunks that have not been updated to use the hypercore TAM, to use B-tree and hash indexes, you +change the table access method for an existing chunk with a call like `ALTER TABLE _timescaledb_internal._hyper_1_1_chunk SET ACCESS METHOD hypercore;` + +## Point lookups + +Indexes are particularly useful for highly selective queries, such as retrieving a unique event by its identifier. +For example: + +```sql +SELECT + created_at, + device_id, + temperature +FROM readings +WHERE metric_uuid = 'dd19f5d3-d04b-4afc-a78f-9b231fb29e52'; +``` + +Without an index, a query scans and filters the entire dataset, leading to slow execution and high I/O usage. +Min/max sparse indexes help when with incremental numeric IDs but do not work well for random numeric values or UUIDs. + +Set a hash index on `metric_uuid` to enable a direct lookup. This significantly improves performance by decompressing +only the relevant data segment. + +```sql +CREATE INDEX readings_metric_uuid_hash_idx ON readings USING hash (metric_uuid); +``` + +With a hash index and hypercore TAM enabled, the same SELECT query performs 1,185x faster; hash comes in at 10.9 ms vs. +12,915 ms and B-tree at 12.57. + + +## Backfill and updates to historical data + +A common use case in real-time applications is backfilling or updating old data. For example, a sensor fails +during a batch upload or gets temporarily disconnected, it resends the data later. To avoid possible duplicate +records, you have to check if the data already exists in the database before storing it. + +To prevent duplicate entries, you enforce uniqueness using a primary key. Primary constraints are enforced through +unique indexes, making conflict checks fast. Without an index, verifying uniqueness involves scanning and decompressing +potentially large amounts of data. This significantly slows inserts and consuming excessive [IOPS][iops]. A `UNIQUE` +constraint on a hypertable must also include the hypertable partition key. + +The following `UNIQUE` uses a B-tree index. + +```sql +CREATE UNIQUE INDEX readings_metric_uuid_metric_uuid_created_at_idx + ON readings (metric_uuid, created_at); +``` + +Possible strategies for backfilling historic data include: + +- **Insert data when the row does not exist**: + + An insert statement ensuring no duplicates looks like: + + ```sql + INSERT INTO readings VALUES (...) ON CONFLICT (device_id, created_at) DO NOTHING; + ``` + Our benchmarks showed this makes inserts 224.3x faster, reducing the execution time from 289,139 ms to 1,289 ms. + +- **Insert missing data or update existing data**: + + An upsert is a database operation that inserts a new row if it does not already exist, or updates the existing row + if a conflict occurs. This enables you to re-ingest new versions of rows instead of performing separate update + statements. Without an index, the system needs to scan and decompress data, considerably slowing ingestion speed. + With a primary key index, conflicting rows are directly located within the compressed data segment in + the columnstore. + + The following query attempts to insert a new record. If a record for the same `metric_uuid` and `created_at` values + already exists, it updates `temperature` with the corresponding value from the new record. + + ```sql + INSERT INTO readings VALUES (...) ON CONFLICT DO UPDATE SET temperature = EXCLUDED.temperature; + ``` + + $COMPANY benchmarks showed this makes upserts 2.6x faster, reducing the execution time from 24,805 ms to 9,520 ms. + + +## Fast anomaly detection + +To regularly report the number of times a device has exceeded a critical temperature, you +count the number of times the temperature was exceeded, and group by device ID. + +```sql +SELECT + device_id, + COUNT(temperature) +FROM readings +WHERE temperature > 52.5 +GROUP BY device_id; +``` + +You see something like: + +| device_id | count | +| -- | -- | +68 | 1 | +| 258 | 1 | +| 192 | 1 | +| 276 | 1 | +| 114 | 1 | +| 227 | 1 | +| 153 | 1 | +| 210 | 1 | +| 266 | 1 | +| 165 | 1 | +| 296 | 1 | +| 144 | 1 | +| 93 | 1 | +| 285 | 1 | +| 221 | 1 | +| 167 | 1 | +| 14 | 1 | +| 123 | 1 | +| 152 | 1 | +| 206 | 1 | +| 230 | 1 | +| 136 | 1 | +| 256 | 2 +| 1 | 1 | + +To make this query faster, use a partial B-tree index. That is, a B-tree index that only includes rows satisfying a +specific WHERE condition. This makes a smaller index that is more efficient for queries that match that +condition. For example, to create a partial B-tree index for temperature readings over 52.5: + +```sql +CREATE INDEX ON readings (temperature) where temperature > 52.5; +``` + +Compared with using a sparse min/max index in columnstore, $COMPANY benchmarks show that the B-tree index query is +4.5x faster. + + + +[hypercore]: /use-timescale/:currentVersion:/hypercore/ +[blog-perf-tuning]: https://www.timescale.com/learn/postgresql-performance-tuning-optimizing-database-indexes +[create-hypertable]: /use-timescale/:currentVersion:/compression/ +[b-tree-overview]: https://www.timescale.com/learn/postgresql-performance-tuning-optimizing-database-indexes#:~:text=a%20quick%20summary%3A-,B%2DTree%20indexes%20(default%20index%20type%20in%20PostgreSQL),-CREATE%20INDEX%20index_product_id +[hash-overview]: https://www.timescale.com/learn/postgresql-performance-tuning-optimizing-database-indexes#:~:text=in%20ascending%20order.-,Hash%20indexes,-CREATE%20INDEX%20index_product_id +[storage-toast]: https://www.postgresql.org/docs/current/storage-toast.html +[postgres-index-types]: https://www.timescale.com/learn/database-indexes-in-postgres +[postgres-tam-methods]: https://www.postgresql.org/docs/current/tableam.html +[convert-to-hypertable]: /use-timescale/:currentVersion:/hypertables/create/ +[iops]: https://en.wikipedia.org/wiki/IOPS diff --git a/use-timescale/page-index/page-index.js b/use-timescale/page-index/page-index.js index 8762b38d8e..bf7deac314 100644 --- a/use-timescale/page-index/page-index.js +++ b/use-timescale/page-index/page-index.js @@ -208,6 +208,11 @@ module.exports = [ href: "modify-data-in-hypercore", excerpt: "Update data stored in the columnstore", }, + { + title: "Improve query and upsert performance", + href: "secondary-indexes", + excerpt: "Automate", + }, ], }, { From 5c84b4d18777b365db19406c147e09d7b8c53094 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Erik=20Nordstr=C3=B6m?= <819732+erimatnor@users.noreply.github.com> Date: Wed, 19 Mar 2025 11:53:35 +0100 Subject: [PATCH 31/90] Update merge chunks docs (#3939) In 2.19 it will be possible to merge compressed chunks, so the API docs should be updated to reflect that. Co-authored-by: Iain Cox --- api/merge_chunks.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/api/merge_chunks.md b/api/merge_chunks.md index 73924abae3..0067af55ef 100644 --- a/api/merge_chunks.md +++ b/api/merge_chunks.md @@ -19,10 +19,9 @@ You can only merge chunks that have directly adjacent partitions. It is not poss chunks that have another chunk, or an empty range between them in any of the partitioning dimensions. -In this first release, chunk merging has the following limitations. You cannot: +Chunk merging has the following limitations. You cannot: -* Merge compressed chunks -* Merge chunks using table access methods other than heap +* Merge chunks using an unsupported table access method * Merge chunks with tiered data * Read or write from the chunks while they are being merged From f7e30d88711df4c633ae233fcde2725d3887dcc6 Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Thu, 20 Mar 2025 10:07:52 +0200 Subject: [PATCH 32/90] Implement variables in the get started section (#3931) --- _partials/_cloud-connect.md | 18 +- _partials/_cloud-intro.md | 22 +- _partials/_service-overview.md | 49 +---- getting-started/index.md | 8 +- getting-started/run-queries-from-console.md | 191 +++++++++--------- getting-started/services.md | 17 +- .../try-key-features-timescale-products.md | 141 ++++++------- 7 files changed, 201 insertions(+), 245 deletions(-) diff --git a/_partials/_cloud-connect.md b/_partials/_cloud-connect.md index 97180eecca..66688b1f43 100644 --- a/_partials/_cloud-connect.md +++ b/_partials/_cloud-connect.md @@ -7,23 +7,23 @@ To connect to a $SERVICE_SHORT: In [$CONSOLE][services-portal], check that your $SERVICE_SHORT is marked as `Running`. - ![Check service is running](https://assets.timescale.com/docs/images/console-services-view.png) + ![Check $SERVICE_SHORT is running](https://assets.timescale.com/docs/images/console-services-view.png) 1. **Connect to your $SERVICE_SHORT** Use either: - - [Data mode][popsql] in $CONSOLE: - 1. In the [data mode][portal-data-mode] in $CONSOLE, select a $SERVICE_SHORT and enter your password, then click **Connect**. + - [$DATA_MODE_CAP][popsql] in $CONSOLE: + 1. In the [$DATA_MODE][portal-data-mode] in $CONSOLE, select a $SERVICE_SHORT and enter your password, then click `Connect`. You find your password in the config file you just downloaded. 2. Select a query to edit: ![Select a query to edit](https://assets.timescale.com/docs/images/data-mode-query-window.png) - - [SQL editor][run-sqleditor] in $CONSOLE: + - [$SQL_EDITOR][run-sqleditor] in $CONSOLE: - In the [ops mode][portal-ops-mode] in $CONSOLE, select a $SERVICE_SHORT, then click **SQL editor**. + In the [$OPS_MODE][portal-ops-mode] in $CONSOLE, select a $SERVICE_SHORT, then click `SQL editor`. - ![Check service is running](https://assets.timescale.com/docs/images/ops-view-sql-editor.png) + ![Check $SERVICE_SHORT is running](https://assets.timescale.com/docs/images/ops-view-sql-editor.png) - [psql][install-psql] on the command line: @@ -37,9 +37,9 @@ To connect to a $SERVICE_SHORT: You can now run queries for this $SERVICE_SHORT. Quick recap. You: -- Manage your $SERVICE_SHORTs in the [ops mode][portal-ops-mode] in $CONSOLE: add read replicas and enable - high-availability, configure compression, change parameters, and so on. -- Analyze your data in the [data mode][portal-data-mode] in $CONSOLE: write queries with +- Manage your $SERVICE_SHORTs in the [$OPS_MODE][portal-ops-mode] in $CONSOLE: add $READ_REPLICAs and enable + high availability, configure compression, change parameters, and so on. +- Analyze your data in the [$DATA_MODE][portal-data-mode] in $CONSOLE: write queries with autocomplete, save them in folders, share them, create charts/dashboards, and much more. - Store configuration and security information in your config file. diff --git a/_partials/_cloud-intro.md b/_partials/_cloud-intro.md index 1303373253..3af1b08a71 100644 --- a/_partials/_cloud-intro.md +++ b/_partials/_cloud-intro.md @@ -1,26 +1,26 @@ -$CLOUD_LONG is the modern PostgreSQL data platform for all your applications. It enhances PostgreSQL to handle time series, events, +$CLOUD_LONG is the modern $PG data platform for all your applications. It enhances $PG to handle time series, events, real-time analytics, and vector search—all in a single database alongside transactional workloads. -You get one system that handles live data ingestion, late and out-of-order updates, and low latency queries, with the performance, reliability, and scalability your app needs. Ideal for IoT, crypto, finance, SaaS, and a myriad other domains, $CLOUD_LONG allows you to build data-heavy, mission-critical apps while retaining the familiarity and reliability of PostgreSQL. +You get one system that handles live data ingestion, late and out-of-order updates, and low latency queries, with the performance, reliability, and scalability your app needs. Ideal for IoT, crypto, finance, SaaS, and a myriad other domains, $CLOUD_LONG allows you to build data-heavy, mission-critical apps while retaining the familiarity and reliability of $PG. -A $SERVICE_LONG is a single optimized 100% PostgreSQL database instance that you +A $SERVICE_LONG is a single optimized 100% $PG database instance that you use as is, or extend with the capabilities specific to your business needs: -- **Time-series and analytics**: PostgreSQL with TimescaleDB. The PostgreSQL you know and love, +- **Time-series and analytics**: $PG with $TIMESCALE_DB. The $PG you know and love, supercharged with functionality for storing and querying [time-series data][what-is-time-series] at scale for - real-time analytics and other use cases. Get faster time-based queries with hypertables, continuous aggregates, and columnar storage. Save on storage with native compression, data retention policies, and bottomless data tiering to Amazon S3. -- **AI and vector**: PostgreSQL with vector extensions. Use PostgreSQL as a vector database with + real-time analytics and other use cases. Get faster time-based queries with $HYPERTABLEs, $CAGGs, and columnar storage. Save on storage with native compression, data retention policies, and bottomless data tiering to Amazon S3. +- **AI and vector**: $PG with vector extensions. Use $PG as a vector database with purpose built extensions for building AI applications from start to scale. Get fast and accurate similarity search - with the pgvector and pgvectorscale extensions. Create vector embeddings and perform LLM reasoning on your data with - the pgai extension. + with the pgvector and $PGVECTORSCALE extensions. Create vector embeddings and perform LLM reasoning on your data with + the $PGAI_SHORT extension. - **PostgreSQL**: the trusted industry-standard RDBMS. Ideal for applications requiring strong data consistency, complex relationships, and advanced querying capabilities. Get ACID compliance, extensive SQL support, JSON handling, and extensibility through custom functions, data types, and extensions. $CLOUD_LONG continuously - monitors your $SERVICE_SHORTs and prevents common PostgreSQL out-of-memory crashes. + monitors your $SERVICE_SHORTs and prevents common $PG out-of-memory crashes. All $SERVICE_LONGs include the tooling you expect for production and developer environments: [live migration][live-migration], -[automatic backups and PITR][automatic-backups], [high availability][high-availability], [read replicas][readreplica], [data forking][operations-forking], [connection pooling][connection-pooling], [tiered storage][data-tiering], -[usage-based storage][how-plans-work], secure in-Console [SQL editing][in-console-editors], service [metrics][metrics] +[automatic backups and PITR][automatic-backups], [high availability][high-availability], [$READ_REPLICAs][readreplica], [data forking][operations-forking], [connection pooling][connection-pooling], [tiered storage][data-tiering], +[usage-based storage][how-plans-work], secure in-$CONSOLE [SQL editing][in-console-editors], $SERVICE_SHORT [metrics][metrics] and [insights][insights], [streamlined maintenance][maintain-upgrade], and much more. [what-is-time-series]: https://www.timescale.com/blog/what-is-a-time-series-database/#what-is-a-time-series-database diff --git a/_partials/_service-overview.md b/_partials/_service-overview.md index 9588148b44..25d576dd21 100644 --- a/_partials/_service-overview.md +++ b/_partials/_service-overview.md @@ -1,46 +1,9 @@ You manage your $SERVICE_LONGs and interact with your data in $CONSOLE using the following modes: - - - - - - - - - +| **$OPS_MODE_CAP** | **$DATA_MODE_CAP** | +|-------------------|---------------------| +| ![$CONSOLE $OPS_MODE][ops-mode] | ![$CONSOLE $DATA_MODE][data-mode] | +| **You use the $OPS_MODE to:**
    • Ensure data security with high availability and $READ_REPLICAs
    • Save money using compressed and tiered storage
    • Enable $PG extensions to add extra functionality
    • Increase security using $VPCs
    • Perform day-to-day administration
    | **Powered by $POPSQL, you use the $DATA_MODE to:**
    • Write queries with autocomplete
    • Visualize data with charts and dashboards
    • Schedule queries and dashboards for alerts or recurring reports
    • Share queries and dashboards
    | - - - - - - -
    Ops modeData mode
    - Timescale Console ops mode - - Timescale Console data mode -
    -You use the ops mode to: -
      -
    • Ensure data security with high availability and read replicas
      • -
    • Save money using compressed and tiered storage
      • -
    • Enable PostgreSQL extensions to add extra functionality
      • -
    • Increase security using VPCs
      • -
    • Perform day-to-day administration
      • -
    -    		 -Powered by PopSQL, you use the data mode to: -
      -
    • Write queries with autocomplete
      • -
    • Visualize data with charts and dashboards
      • -
    • Schedule queries and dashboards for alerts or recurring reports
      • -
    • Share queries and dashboards
      • -
    -
    - -[how-plans-work]: /about/:currentVersion:/pricing-and-account-management/#how-plans-work +[ops-mode]: https://assets.timescale.com/docs/images/ops-mode-overview.png +[data-mode]: https://assets.timescale.com/docs/images/data-mode-overview.png \ No newline at end of file diff --git a/getting-started/index.md b/getting-started/index.md index 5785eec2a1..e486f08d7e 100644 --- a/getting-started/index.md +++ b/getting-started/index.md @@ -10,17 +10,15 @@ import CloudIntro from "versionContent/_partials/_cloud-intro.mdx"; import TimescaleIntro from "versionContent/_partials/_timescale-intro.mdx"; import WhereNext from "versionContent/_partials/_where-to-next.mdx"; -# Get started with Timescale +# Get started with $CLOUD_LONG This section shows you how to: -1. [Create and connect to a $SERVICE_LONG][services-create]: choose the capabilities that match your business and - engineering needs on $COMPANY's cloud-based PostgreSQL platform. +1. [Create and connect to a $SERVICE_LONG][services-create]: choose the capabilities that match your business and engineering needs on $COMPANY's cloud-based $PG platform. 1. [Run queries from $CONSOLE][run-queries-from-console]: securely interact with your data in the $CONSOLE UI. -1. [Try the main features in Timescale products][test-drive]: rapidly implement the features in $CLOUD_LONG that - enable you to ingest and query data faster while keeping prices low. +1. [Try the main features in $COMPANY products][test-drive]: rapidly implement the features in $CLOUD_LONG that enable you to ingest and query data faster while keeping the costs low. What next? See the [use case tutorials][tutorials], interact with the data in your $SERVICE_LONG using [your favorite programming language][connect-with-code], integrate your $SERVICE_LONG with a range of diff --git a/getting-started/run-queries-from-console.md b/getting-started/run-queries-from-console.md index 13c53c8753..a516bd2d43 100644 --- a/getting-started/run-queries-from-console.md +++ b/getting-started/run-queries-from-console.md @@ -8,84 +8,81 @@ content_group: Getting started import WhereNext from "versionContent/_partials/_where-to-next.mdx"; -# Run your queries from Timescale Console +# Run your queries from $CONSOLE -As Timescale Cloud is based on PostgreSQL, you can use lots of [different tools][integrations] to -connect to your service and interact with your data. +As $CLOUD_LONG is based on $PG, you can use lots of [different tools][integrations] to +connect to your $SERVICE_SHORT and interact with your data. -In Timescale Console you can use the following ways to run SQL queries against your database: +In $CONSOLE you can use the following ways to run SQL queries against your $SERVICE_SHORT: -- [Data mode][run-popsql]: a rich experience powered by PopSQL. You can write queries with +- [$DATA_MODE_CAP][run-popsql]: a rich experience powered by $POPSQL. You can write queries with autocomplete, save them in folders, share them, create charts/dashboards, and much more. -- [SQL Assistant in data mode][sql-assistant]: write, fix, and organize SQL faster and more accurately. +- [$SQL_ASSISTANT_SHORT in the $DATA_MODE][sql-assistant]: write, fix, and organize SQL faster and more accurately. -- [SQL editor in ops mode][run-sqleditor]: a simple SQL editor in the ops mode that lets you run ad-hoc ephemeral - queries. This is useful for quick one-off tasks like creating an index on a small table or +- [$SQL_EDITOR in the $OPS_MODE][run-sqleditor]: a simple $SQL_EDITOR in the $OPS_MODE that lets you run ad-hoc ephemeral + queries. This is useful for quick one-off tasks like creating an index on a small table or inspecting `pg_stat_statements`. +If you prefer the command line to the $OPS_MODE $SQL_EDITOR in $CONSOLE, use [psql][install-psql]. -If you prefer the command line to the ops mode SQL editor in Timescale Console, use [psql][install-psql]. +## $DATA_MODE_CAP -## Data mode +You use the $DATA_MODE in $CONSOLE to write queries, visualize data, and share your results. -You use the data mode in Timescale Console to write queries, visualize data, and share your results. - -![Screenshot of data mode](https://assets.timescale.com/docs/images/data-mode-schema-chart.png) +![$CONSOLE $DATA_MODE](https://assets.timescale.com/docs/images/data-mode-schema-chart.png) Available features are: -- **Real-time collaboration**: work with your team directly in the data mode query editor with live presence and multiple +- **Real-time collaboration**: work with your team directly in the $DATA_MODE query editor with live presence and multiple cursors. -- **[Schema browser](https://docs.popsql.com/docs/schema)**: understand the structure of your database and see usage data on tables and columns. -- **[SQL Assistant][sql-assistant]**: write, fix, and organize SQL faster and more accurately using AI. +- **[Schema browser](https://docs.popsql.com/docs/schema)**: understand the structure of your $SERVICE_SHORT and see usage data on tables and columns. +- **[$SQL_ASSISTANT_SHORT][sql-assistant]**: write, fix, and organize SQL faster and more accurately using AI. - **Autocomplete**: get suggestions as you type your queries. -- **[Version history](https://docs.popsql.com/docs/version-history)**: access previous versions of a query from the built-in revision history, or connect to a +- **[Version history](https://docs.popsql.com/docs/version-history)**: access previous versions of a query from the built-in revision history, or connect to a git repo. - **[Charts](https://docs.popsql.com/docs/creating-charts)**: visualize data from inside the UI rather than switch to Sheets or Excel. - **[Schedules](https://docs.popsql.com/docs/scheduled-queries)**: automatically refresh queries and dashboards to create push alerts. - **[Query variables](https://docs.popsql.com/docs/query-variables)**: use Liquid to parameterize your queries or use `if` statements. -- **Cross platform**: works from [Timescale Console][portal-data-mode] or download the +- **Cross-platform**: works from [$CONSOLE][portal-data-mode] or download the [desktop](https://popsql.com/download) app for macOS, Windows, and Linux. -- **Easy connection**: to Timescale Cloud, PostgreSQL, Redshift, Snowflake, BigQuery, MySQL, SQL Server, [and more](https://popsql.com/connections). +- **Easy connection**: to $CLOUD_LONG, $PG, Redshift, Snowflake, BigQuery, MySQL, SQL Server, [and more](https://popsql.com/connections). -### Connect to your Timescale Cloud service in the data mode +### Connect to your $SERVICE_LONG in the $DATA_MODE -To connect to a service: +To connect to a $SERVICE_SHORT: -1. **Check your service is running correctly** +1. **Check your $SERVICE_SHORT is running correctly** - In [Timescale Console][services-portal], check that your service is marked as `Running`. + In [$CONSOLE][services-portal], check that your $SERVICE_SHORT is marked as `Running`. - ![Check service is running](https://assets.timescale.com/docs/images/console-services-view.png) + ![Check $SERVICE_LONG is running](https://assets.timescale.com/docs/images/console-services-view.png) -1. **Connect to your service** +1. **Connect to your $SERVICE_SHORT** - 1. In the [data mode][portal-data-mode] in $CONSOLE, select a service in the connection drop-down. - 1. Click the pencil icon next to the service name. + 1. In the [$DATA_MODE][portal-data-mode] in $CONSOLE, select a $SERVICE_SHORT in the connection drop-down. + 1. Click the pencil icon next to the $SERVICE_SHORT name. ![Select a connection](https://assets.timescale.com/docs/images/data-mode-connections.png) - - 1. Click `Edit` next to `Username/Password` and enter [your connection details][connection-info] for this service, then click `Connect`. + + 1. Click `Edit` next to `Username/Password` and enter [your connection details][connection-info] for this $SERVICE_SHORT, then click `Connect`. 1. **Run a test query** - Type `SELECT CURRENT_DATE;` in `Scratchpad` and press `Run`: + Type `SELECT CURRENT_DATE;` in `Scratchpad` and press `Run`: ![Run a simple query](https://assets.timescale.com/docs/images/data-mode-scratchpad.png) - Quick recap. You: -- Manage your services in the [ops mode in Timescale Console][portal-ops-mode] -- Manage your data in the [data mode in Timescale Console][portal-data-mode] +- Manage your $SERVICE_SHORTs in the [$OPS_MODE in $CONSOLE][portal-ops-mode] +- Manage your data in the [$DATA_MODE in $CONSOLE][portal-data-mode] - Store configuration and security information in your config file. - -Now you have used the data mode in Timescale Console, see how to easily do the following: +Now you have used the $DATA_MODE in $CONSOLE, see how to easily do the following: - [Write a query](https://docs.popsql.com/docs/writing-a-query) - [Share a query with your @@ -97,44 +94,41 @@ Now you have used the data mode in Timescale Console, see how to easily do the f - [Create schedules for your queries](https://docs.popsql.com/docs/scheduled-queries) -### Data Mode FAQ +### $DATA_MODE_CAP FAQ -#### What if my service is within a VPC? +#### What if my $SERVICE_SHORT is within a $VPC? -If your Timescale Service runs inside a VPC, do one of the following to enable access for the PopSQL desktop app: +If your $SERVICE_LONG runs inside a $VPC, do one of the following to enable access for the $POPSQL desktop app: -- Use PopSQL's [bridge connector](https://docs.popsql.com/docs/bridge-connector) -- Use an SSH tunnel - - When you configure the connection in PopSQL, under `Advanced Options`, enable `Connect over SSH` -- Add PopSQL's static IPs (`23.20.131.72, 54.211.234.135`) to your allowlist +- Use $POPSQL's [bridge connector](https://docs.popsql.com/docs/bridge-connector). +- Use an SSH tunnel: when you configure the connection in $POPSQL, under `Advanced Options` enable `Connect over SSH`. +- Add $POPSQL's static IPs (`23.20.131.72, 54.211.234.135`) to your allowlist. -#### What happens if another member of my Timescale project uses data mode? +#### What happens if another member of my $COMPANY project uses the $DATA_MODE? -The number of data mode seats you are allocated depends on your -[Pricing Plan][pricing-plan-features]. +The number of $DATA_MODE seats you are allocated depends on your [$PRICING_PLAN][pricing-plan-features]. -#### Will using the data mode affect the performance of my Timescale Cloud service? +#### Will using the $DATA_MODE affect the performance of my $SERVICE_LONG? There are a few factors to consider: -1. What instance size is your database? +1. What instance size is your $SERVICE_SHORT? 1. How many users are running queries? 1. How computationally intensive are the queries? If you have a small number of users running performant SQL queries against a -service with sufficient resources, then there should be no degradation to +$SERVICE_SHORT with sufficient resources, then there should be no degradation to performance. However, if you have a large number of users running queries, or if the queries are computationally expensive, best practice is to create -a [read replica][readreplica] and send analytical queries there. - -If you'd like to prevent write operations such as insert or update. Instead -of using the `tsdbadmin` superuser, create a read-only user for your service and -use that in the data mode. +a [$READ_REPLICA][readreplica] and send analytical queries there. +If you'd like to prevent write operations such as insert or update, instead +of using the `tsdbadmin` user, create a read-only user for your $SERVICE_SHORT and +use that in the $DATA_MODE. -## SQL Assistant +## $SQL_ASSISTANT_SHORT -SQL Assistant in [$CONSOLE][portal-data-mode] helps you write, fix, and organize SQL faster and more accurately. +$SQL_ASSISTANT_SHORT in [$CONSOLE][portal-data-mode] helps you write, fix, and organize SQL faster and more accurately.
    @@ -142,89 +136,89 @@ SQL Assistant in [$CONSOLE][portal-data-mode] helps you write, fix, and organize
    -### Key Capabilities +### Key capabilities -SQL Assistant offers a range of features to improve your SQL workflow, including: +$SQL_ASSISTANT_SHORT offers a range of features to improve your SQL workflow, including: -- **Real-Time Help**: SQL Assistant provides in-context help for writing and understanding SQL. Use it to: +- **Real-time help**: $SQL_ASSISTANT_SHORT provides in-context help for writing and understanding SQL. Use it to: - - **Understand functions**: need to know how functions like `LAG()` or `ROW_NUMBER()` work? SQL Assistant explains it with examples. - - **Interpret complex queries**: SQL Assistant breaks down dense queries, giving you a clear view of each part. + - **Understand functions**: need to know how functions like `LAG()` or `ROW_NUMBER()` work? $SQL_ASSISTANT_SHORT explains it with examples. + - **Interpret complex queries**: $SQL_ASSISTANT_SHORT breaks down dense queries, giving you a clear view of each part. -- **Error resolution**: SQL Assistant diagnoses errors as they happen, you can resolve issues without leaving your editor. Features include: +- **Error resolution**: $SQL_ASSISTANT_SHORT diagnoses errors as they happen, you can resolve issues without leaving your editor. Features include: - - **Error debugging**: if your query fails, SQL Assistant identifies the issue and suggests a fix. - - **Performance tuning**: for slow queries, SQL Assistant provides optimization suggestions to improve performance immediately. + - **Error debugging**: if your query fails, $SQL_ASSISTANT_SHORT identifies the issue and suggests a fix. + - **Performance tuning**: for slow queries, $SQL_ASSISTANT_SHORT provides optimization suggestions to improve performance immediately. - **Query organization**: to keep your query library organized, and help your team understand the - purpose of each query, SQL Assistant automatically adds titles and summaries to your queries. + purpose of each query, $SQL_ASSISTANT_SHORT automatically adds titles and summaries to your queries. ### Limitations to keep in mind -For best results with SQL Assistant: +For best results with $SQL_ASSISTANT_SHORT: -* **Schema Awareness**: SQL Assistant references schema data but may need extra context +* **Schema awareness**: $SQL_ASSISTANT_SHORT references schema data but may need extra context in complex environments. Specify tables, columns, or joins as needed. -* **Business Logic**: SQL Assistant does not inherently know specific business terms - such as _active user_. Define these terms clearly to improve results. +* **Business logic**: $SQL_ASSISTANT_SHORT does not inherently know specific business terms + such as active user. Define these terms clearly to improve results. -### Security, privacy and data usage +### Security, privacy, and data usage -Security and privacy is prioritized in $CONSOLE. In [Data Mode][portal-data-mode], project members -manage SQL Assistant settings under [**User name** > **Settings** > **SQL Assistant**][sql-editor-settings]. +Security and privacy is prioritized in $CONSOLE. In [$DATA_MODE][portal-data-mode], project members +manage $SQL_ASSISTANT_SHORT settings under [`User name` > `Settings` > `SQL Assistant`][sql-editor-settings]. -![SQL Assistant settings](https://assets.timescale.com/docs/images/sql-editor-preferences.png) +![$SQL_ASSISTANT_SHORT settings](https://assets.timescale.com/docs/images/sql-editor-preferences.png) -SQL Assistant settings are: -* **Opt-In Features**: all AI features are off by default. Only [members][project-members] of your $CLOUD_LONG project +$SQL_ASSISTANT_SHORT settings are: +* **Opt-in features**: all AI features are off by default. Only [members][project-members] of your $CLOUD_LONG project can enable them. -* **Data Protection**: your queries and data remain private and are not used for AI training. - SQL Assistant operates with strict security protocols. -* **Data Usage**: to provide AI support, $COMPANY may share the query title, description and body, also the - database connection type such as PostgreSQL, and the schema. +* **Data protection**: your queries and data remain private and are not used for AI training. + $SQL_ASSISTANT_SHORT operates with strict security protocols. +* **Data usage**: to provide AI support, $COMPANY may share the query title, description and body, also the + database connection type such as $PG, and the schema. * **Sample data**: to give the LLM more context so you have better SQL suggestions, enable sample data sharing in the - SQL assistant preferences. -* **Telemetry**: to improve SQL Assistant, $COMPANY collects telemetry and usage data, including prompts, responses, + $SQL_ASSISTANT_SHORT preferences. +* **Telemetry**: to improve $SQL_ASSISTANT_SHORT, $COMPANY collects telemetry and usage data, including prompts, responses, and query metadata. -## Ops mode SQL editor +## $OPS_MODE_CAP $SQL_EDITOR -SQL editor is an integrated secure UI that you use to run queries and see the results -for an Timescale Cloud service. +$SQL_EDITOR is an integrated secure UI that you use to run queries and see the results +for a $SERVICE_LONG. -![Screenshot of SQL editor](https://assets.timescale.com/docs/images/sqleditor.png) +![$CONSOLE $SQL_EDITOR](https://assets.timescale.com/docs/images/sqleditor.png) -To enable or disable SQL editor in your service, click `Operations` > `Service management`, then -update the setting for SQL editor. +To enable or disable $SQL_EDITOR in your $SERVICE_SHORT, click `Operations` > `Service management`, then +update the setting for $SQL_EDITOR. -### Run queries from the ops mode in Timescale Console +### Run queries from the $OPS_MODE in $CONSOLE -To use SQL editor with Timescale: +To use $SQL_EDITOR: -1. **Open SQL editor from Timescale Console** +1. **Open $SQL_EDITOR from $CONSOLE** - In the [ops mode][portal-ops-mode] in Timescale Console, select a service, then click **SQL editor**. + In the [$OPS_MODE][portal-ops-mode] in $CONSOLE, select a $SERVICE_SHORT, then click `SQL editor`. - ![Check service is running](https://assets.timescale.com/docs/images/ops-view-sql-editor.png) + ![Check $SERVICE_SHORT is running](https://assets.timescale.com/docs/images/ops-view-sql-editor.png) -4. **Run a test query** +1. **Run a test query** Type your query in the UI, then click `Run`. The results appear in the lower window. -## Cloud SQL editor licenses +## Cloud $SQL_EDITOR licenses -* **SQL editor in the ops mode**: free for anyone with a [Timescale Cloud account][create-cloud-account]. -* **Data mode**: the number of seats you are allocated depends on your [Pricing Plan][pricing-plan-features]. - [SQL Assistant][sql-assistant] is currently free for all users. In the future, limits or paid options may be +* **$SQL_EDITOR in the $OPS_MODE**: free for anyone with a [$CLOUD_LONG account][create-cloud-account]. +* **Data mode**: the number of seats you are allocated depends on your [$PRICING_PLAN][pricing-plan-features]. + [$SQL_ASSISTANT_SHORT][sql-assistant] is currently free for all users. In the future, limits or paid options may be introduced as we work to build the best experience. -* **PopSQL standalone**: there is a free plan available to everyone, as well as paid plans. See [PopSQL Pricing](https://popsql.com/pricing) for full +* **$POPSQL standalone**: there is a free plan available to everyone, as well as paid plans. See [$POPSQL Pricing][popsql-pricing] for full details. @@ -246,4 +240,5 @@ To use SQL editor with Timescale: [sql-editor-settings]: https://console.cloud.timescale.com/dashboard/settings?popsql=%2Fpreferences%2Fai [portal-data-mode]: https://console.cloud.timescale.com/dashboard/services?popsql [sql-assistant]: /getting-started/:currentVersion:/run-queries-from-console/#sql-assistant -[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ \ No newline at end of file +[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ +[popsql-pricing]: https://popsql.com/pricing diff --git a/getting-started/services.md b/getting-started/services.md index e602e202b5..ccc0a93db4 100644 --- a/getting-started/services.md +++ b/getting-started/services.md @@ -8,7 +8,6 @@ content_group: Getting started import Install from "versionContent/_partials/_cloud-installation.mdx"; import Connect from "versionContent/_partials/_cloud-connect.mdx"; -import CreateAHypertable from "versionContent/_partials/_create-hypertable.mdx"; import ServiceOverview from "versionContent/_partials/_service-overview.mdx"; import CloudIntro from "versionContent/_partials/_cloud-intro.mdx"; import WhereNext from "versionContent/_partials/_where-to-next.mdx"; @@ -21,32 +20,32 @@ import WhereNext from "versionContent/_partials/_where-to-next.mdx"; To start using $CLOUD_LONG for your data: -1. [Create a $COMPANY account][create-an-account]: register in $CONSOLE to get a centralized point to administer and interact with your data. -1. [Create a $SERVICE_LONG][create-a-service]: that is, a PostgreSQL database instance, powered by [$TIMESCALE_DB][timescaledb], built for production, and extended with cloud features like transparent data tiering to object storage. +1. [Create a $COMPANY account][create-an-account]: register to get access to $CONSOLE as a centralized point to administer and interact with your data. +1. [Create a $SERVICE_LONG][create-a-service]: that is, a $PG database instance, powered by [$TIMESCALE_DB][timescaledb], built for production, and extended with cloud features like transparent data tiering to object storage. 1. [Connect to your $SERVICE_LONG][connect-to-your-service]: to run queries, add and migrate your data from other sources. ## Create a $SERVICE_LONG -Now that you have an active $CLOUD_LONG account, you create and manage your $SERVICE_SHORTs in $CONSOLE. When you create a $SERVICE_SHORT, you give a structure for your future data, which you then add manually or migrate from other services. All relevant $CLOUD_LONG features under your pricing plan are automatically available when you create a $SERVICE_SHORT. +Now that you have an active $CLOUD_LONG account, you create and manage your $SERVICE_SHORTs in $CONSOLE. When you create a $SERVICE_SHORT, you give a structure for your future data, which you then add manually or migrate from other services. All relevant $CLOUD_LONG features under your $PRICING_PLAN are automatically available when you create a $SERVICE_SHORT. -1. In the [$SERVICE_SHORT creation page][create-service], choose the PostgreSQL capability or add `Time-series and analytics` and `AI and Vector` capabilities. Click `Save and continue`. +1. In the [$SERVICE_SHORT creation page][create-service], choose the $PG capability or add `Time-series and analytics` and `AI and Vector` capabilities. Click `Save and continue`. - ![Create Timescale Cloud service](https://assets.timescale.com/docs/images/create-timescale-service.png) + ![Create a $SERVICE_LONG](https://assets.timescale.com/docs/images/create-timescale-service.png) 1. Follow the next steps in `Create a service` to configure the compute size, environment, availability, region, and $SERVICE_SHORT name. Then click `Create service`. - Your $SERVICE_SHORT is constructed immediately and is ready to use. + Your $SERVICE_SHORT is constructed and ready to use in a few seconds. 1. Click `Download the config` and store the configuration information you need to connect to this $SERVICE_SHORT in a secure location. This file contains the passwords and configuration information you need to connect to your $SERVICE_SHORT using the - $CONSOLE data mode, from the command line, or using third-party database administration tools. + $CONSOLE $DATA_MODE, from the command line, or using third-party database administration tools. -If you choose to go directly to the service overview, [Connect to your $SERVICE_SHORT][connect-to-your-service] +If you choose to go directly to the $SERVICE_SHORT overview, [Connect to your $SERVICE_SHORT][connect-to-your-service] shows you how to connect. diff --git a/getting-started/try-key-features-timescale-products.md b/getting-started/try-key-features-timescale-products.md index 0ed161cc2e..61e3108683 100644 --- a/getting-started/try-key-features-timescale-products.md +++ b/getting-started/try-key-features-timescale-products.md @@ -8,12 +8,12 @@ content_group: Getting started import HASetup from 'versionContent/_partials/_high-availability-setup.mdx'; import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; -# Try the key Timescale features +# Try the key $COMPANY features -$CLOUD_LONG scales PostgreSQL to ingest and query vast amounts of live data. $CLOUD_LONG +$CLOUD_LONG scales $PG to ingest and query vast amounts of live data. $CLOUD_LONG provides a range of features and optimizations that supercharge your queries while keeping the costs down. For example: -* The Hypercore row-columnar engine makes queries up to 350x faster, ingests 44% faster, and reduces storage by 90%. +* The $HYPERCORE row-columnar engine makes queries up to 350x faster, ingests 44% faster, and reduces storage by 90%. * Tiered storage seamlessly moves your data from high performance storage for frequently accessed data to low cost bottomless storage for rarely accessed data. The following figure shows how $CLOUD_LONG optimizes your data for superfast real-time analytics and reduced @@ -28,35 +28,35 @@ ingest and query data faster while keeping the costs low. -## Optimize time-series data in hypertables +## Optimize time-series data in $HYPERTABLEs -Time-series data represents how a system, process, or behavior changes over time. Hypertables are PostgreSQL tables -that help you improve insert and query performance by automatically partitioning your data by time. Each hypertable -is made up of child tables called chunks. Each chunk is assigned a range of time, and only -contains data from that range. You can also tune hypertables to increase performance +Time-series data represents how a system, process, or behavior changes over time. $HYPERTABLE_CAPs are $PG tables +that help you improve insert and query performance by automatically partitioning your data by time. Each $HYPERTABLE +is made up of child tables called $CHUNKs. Each $CHUNK is assigned a range of time, and only +contains data from that range. When you run a query, $CLOUD_LONG identifies the correct $CHUNK and runs the query on it, instead of going through the entire table. You can also tune $HYPERTABLEs to increase performance even more. ![Main features and tiered data](https://assets.timescale.com/docs/images/hypertable-structure.png ) -Hypertables exist alongside regular PostgreSQL tables. -You use regular PostgreSQL tables for relational data, and interact with hypertables -and regular PostgreSQL tables in the same way. +$HYPERTABLE_CAPs exist alongside regular $PG tables. +You use regular $PG tables for relational data, and interact with $HYPERTABLEs +and regular $PG tables in the same way. -This section shows you how to create regular tables and hypertables, and import +This section shows you how to create regular tables and $HYPERTABLEs, and import relational and time-series data from external files. -1. **Import some time-series data into your hypertable** +1. **Import some time-series data into your $HYPERTABLE** 1. Unzip [crypto_sample.zip](https://assets.timescale.com/docs/downloads/candlestick/crypto_sample.zip) to a ``. This test dataset contains second-by-second trade data for the most-traded crypto-assets and a regular table of asset symbols and company names. - To import up to 100GB of data directly from your current PostgreSQL-based database, - [migrate with downtime][migrate-with-downtime] using native PostgreSQL tooling. To seamlessly import 100GB-10TB+ - of data, use the [live migration][migrate-live] tooling supplied by $COMPANY. To add data from non-PostgreSQL + To import up to 100GB of data directly from your current $PG-based database, + [migrate with downtime][migrate-with-downtime] using native $PG tooling. To seamlessly import 100GB-10TB+ + of data, use the [live migration][migrate-live] tooling supplied by $COMPANY. To add data from non-$PG data sources, see [Import and ingest data][data-ingest]. 1. Upload data from the CSVs to your $SERVICE_SHORT: @@ -66,11 +66,11 @@ relational and time-series data from external files. The $CONSOLE data upload creates the tables for you from the data you are uploading: - 1. In [$CONSOLE][portal-ops-mode], select the service to add data to, then click `Actions` > `Upload CSV`. + 1. In [$CONSOLE][portal-ops-mode], select the $SERVICE_SHORT to add data to, then click `Actions` > `Upload CSV`. 1. Drag `/tutorial_sample_tick.csv` to `Upload .CSV` and change `New table name` to `crypto_ticks`. 1. Enable `hypertable partition` for the `time` column and click `Upload CSV`. - The upload wizard creates a hypertable containing the data from the CSV file. + The upload wizard creates a $HYPERTABLE containing the data from the CSV file. 1. When the data is uploaded, close `Upload .CSV`. If you want to have a quick look at your data, press `Run` . @@ -86,12 +86,13 @@ relational and time-series data from external files. ```bash psql -d "postgres://:@:/" ``` - You use your [connection details][connection-info] to fill in this PostgreSQL connection string. + You use your [connection details][connection-info] to fill in this $PG connection string. 2. Create tables for the data to import: - For the time-series data: - 1. In your sql client, create a normal PostgreSQL table: + + 1. In your sql client, create a normal $PG table: ```sql CREATE TABLE crypto_ticks ( @@ -101,16 +102,16 @@ relational and time-series data from external files. day_volume NUMERIC ); ``` - 1. Convert `crypto_ticks` to a hypertable: + 1. Convert `crypto_ticks` to a $HYPERTABLE: ```sql SELECT create_hypertable('crypto_ticks', by_range('time')); ``` - To more fully understand how hypertables work, and how to optimize them for performance by - tuning chunk intervals and enabling chunk skipping, see [the hypertables documentation][hypertables-section]. + To more fully understand how $HYPERTABLEs work, and how to optimize them for performance by + tuning $CHUNK intervals and enabling $CHUNK_SKIPPING, see [the $HYPERTABLEs documentation][hypertables-section]. - For the relational data: - In your sql client, create a normal PostgreSQL table: + In your sql client, create a normal $PG table: ```sql CREATE TABLE crypto_assets ( symbol TEXT NOT NULL, @@ -132,16 +133,16 @@ relational and time-series data from external files. - To more fully understand how hypertables work, and how to optimize them for performance by - tuning chunk intervals and enabling chunk skipping, see [the hypertables documentation][hypertables-section]. + To more fully understand how $HYPERTABLEs work, and how to optimize them for performance by + tuning $CHUNK intervals and enabling $CHUNK_SKIPPING, see [the $HYPERTABLEs documentation][hypertables-section]. 1. **Have a quick look at your data** - You query hypertables in exactly the same way as you would a relational PostgreSQL table. + You query $HYPERTABLEs in exactly the same way as you would a relational $PG table. Use one of the following SQL editors to run a query and see the data you uploaded: - **Data mode**: write queries, visualize data, and share your results in [$CONSOLE][portal-data-mode] for all your $SERVICE_LONGs. - **SQL editor**: write, fix, and organize SQL faster and more accurately in [$CONSOLE][portal-ops-mode] for a $SERVICE_LONG. - - **psql**: easily run queries on your $SERVICE_LONGs or self-hosted TimescaleDB deployment from Terminal. + - **psql**: easily run queries on your $SERVICE_LONGs or $SELF_LONG_LC deployment from Terminal. @@ -151,23 +152,23 @@ relational and time-series data from external files. Aggregation is a way of combing data to get insights from it. Average, sum, and count are all examples of simple aggregates. However, with large amounts of data, aggregation slows things down, quickly. -Continuous aggregates are a kind of hypertable that is refreshed automatically in +$CAGG_CAPs are a kind of $HYPERTABLE that is refreshed automatically in the background as new data is added, or old data is modified. Changes to your dataset are tracked, -and the hypertable behind the continuous aggregate is automatically updated in the background. +and the $HYPERTABLE behind the $CAGG is automatically updated in the background. -![Reduced data calls with Continuous Aggregates](https://assets.timescale.com/docs/images/continuous-aggregate.png ) +![Reduced data calls with $CAGGs](https://assets.timescale.com/docs/images/continuous-aggregate.png ) -You create continuous aggregates on uncompressed data in high-performance storage. They continue to work -on [data in the columnstore][test-drive-enable-compression] +You create $CAGGs on uncompressed data in high-performance storage. They continue to work +on [data in the $COLUMNSTORE][test-drive-enable-compression] and [rarely accessed data in tiered storage][test-drive-tiered-storage]. You can even -create [continuous aggregates on top of your continuous aggregates][hierarchical-caggs]. +create [$CAGGs on top of your $CAGGs][hierarchical-caggs]. -You use time buckets to create a continuous aggregate. Time buckets aggregate data in hypertables by time -interval. For example, a 5-minute, 1-hour, or 3-day bucket. The data grouped in a time bucket uses a single -timestamp. Continuous aggregates minimize the number of records that you need to look up to perform your +You use $TIME_BUCKETs to create a $CAGG. $TIME_BUCKET_CAPs aggregate data in $HYPERTABLEs by time +interval. For example, a 5-minute, 1-hour, or 3-day bucket. The data grouped in a $TIME_BUCKET uses a single +timestamp. $CAGG_CAPs minimize the number of records that you need to look up to perform your query. -This section shows you how to run fast analytical queries using time buckets and continuous aggregates in +This section shows you how to run fast analytical queries using $TIME_BUCKETs and $CAGG in $CONSOLE. You can also do this using psql. @@ -176,14 +177,14 @@ $CONSOLE. You can also do this using psql. -1. **In [$CONSOLE][portal-ops-mode], select the service you uploaded data to, then click `SQL Editor`** +1. **In [$CONSOLE][portal-ops-mode], select the $SERVICE_SHORT you uploaded data to, then click `SQL Editor`** -1. **Create a continuous aggregate** +1. **Create a $CAGG** - For a continuous aggregate, data grouped using a time bucket is stored in a - PostgreSQL `MATERIALIZED VIEW` in a hypertable. `timescaledb.continuous` ensures that this data + For a $CAGG, data grouped using a $TIME_BUCKET is stored in a + $PG `MATERIALIZED VIEW` in a $HYPERTABLE. `timescaledb.continuous` ensures that this data is always up to date. - In your SQL editor, use the following code to create a continuous aggregate on the real-time data in + In your SQL editor, use the following code to create a $CAGG on the real-time data in the `crypto_ticks` table: ```sql @@ -200,7 +201,7 @@ $CONSOLE. You can also do this using psql. GROUP BY day, symbol; ``` - This continuous aggregate creates the [candlestick chart][charts] data you use to visualize + This $CAGG creates the [candlestick chart][charts] data you use to visualize the price change of an asset. 1. **Create a policy to refresh the view every hour** @@ -214,8 +215,8 @@ $CONSOLE. You can also do this using psql. 1. **Have a quick look at your data** - You query continuous aggregates exactly the same way as your other tables. To query the `assets_candlestick_daily` - continuous aggregate for all assets: + You query $CAGGs exactly the same way as your other tables. To query the `assets_candlestick_daily` + $CAGG for all assets: @@ -228,9 +229,9 @@ $CONSOLE. You can also do this using psql. -1. **In [$CONSOLE][portal-ops-mode], select the service you uploaded data to**. +1. **In [$CONSOLE][portal-ops-mode], select the $SERVICE_SHORT you uploaded data to**. 1. **Click `Operations` > `Continuous aggregates`, select `crypto_ticks`, then click `Create a Continuous Aggregate`**. - ![Continuous aggregate wizard](https://assets.timescale.com/docs/images/continuous-aggregate-wizard.png ) + ![$CAGG wizard](https://assets.timescale.com/docs/images/continuous-aggregate-wizard.png ) 1. **Create a view called `assets_candlestick_daily` on the `time` column with an interval of `1 day`, then click `Next step`**. 1. **Update the view SQL with the following functions, then click `Run`** ```sql @@ -253,7 +254,7 @@ $CONSOLE. You can also do this using psql. - `How often do you want the job to run?`: `3 hours` 1. **Click `Next step`, then click `Run`** -$CLOUD_LONG creates the continuous aggregate and displays the aggregate ID in $CONSOLE. Click `DONE` to close the wizard. +$CLOUD_LONG creates the $CAGG and displays the aggregate ID in $CONSOLE. Click `DONE` to close the wizard. @@ -262,22 +263,22 @@ $CLOUD_LONG creates the continuous aggregate and displays the aggregate ID in $C To see the change in terms of query time and data returned between a regular query and -a continuous aggregate, run the query part of the continuous aggregate +a $CAGG, run the query part of the $CAGG ( `SELECT ...GROUP BY day, symbol;` ) and compare the results. ## Enhance query performance for analytics -Hypercore is the $TIMESCALE_DB hybrid row-columnar storage engine, designed specifically for real-time analytics and -powered by time-series data. The advantage of Hypercore is its ability to seamlessly switch between row-oriented and -column-oriented storage. This flexibility enables Timescale Cloud to deliver the best of both worlds, solving the key +$HYPERCORE_CAP is the $CLOUD_LONG hybrid row-columnar storage engine, designed specifically for real-time analytics and +powered by time-series data. The advantage of $HYPERCORE is its ability to seamlessly switch between row-oriented and +column-oriented storage. This flexibility enables $CLOUD_LONG to deliver the best of both worlds, solving the key challenges in real-time analytics. -![Move from rowstore and columstore in hypercore](https://assets.timescale.com/docs/images/hypercore.png ) +![Move from rowstore to columstore in $HYPERCORE](https://assets.timescale.com/docs/images/hypercore.png ) -When you convert chunks from the rowstore to the columnstore, multiple records are grouped into a single row. +When you convert $CHUNKs from the $ROWSTORE to the $COLUMNSTORE, multiple records are grouped into a single row. The columns of this row hold an array-like structure that stores all the data. Because a single row takes up less disk -space, you can reduce your chunk size by more than 90%, and can also speed up your queries. This saves on storage costs, +space, you can reduce your $CHUNK size by more than 90%, and can also speed up your queries. This helps you save on storage costs, and keeps your queries operating at lightning speed. Best practice is to compress data that is no longer needed for highest performance queries, but is still accessed @@ -285,9 +286,9 @@ regularly. For example, last week's market data. -1. **Enable Hypercore on a hypertable** +1. **Enable $HYPERCORE on a $HYPERTABLE** - Create a [job][job] that automatically moves chunks in a hypertable to the columnstore at a specific time interval. + Create a [job][job] that automatically moves $CHUNKs in a $HYPERTABLE to the $COLUMNSTORE at a specific time interval. ```sql ALTER TABLE crypto_ticks SET ( @@ -297,7 +298,7 @@ regularly. For example, last week's market data. You [segmentby][alter-table-arguments] to speed up queries. -1. **Add a policy to convert chunks to the columnstore at a specific time interval** +1. **Add a policy to convert $CHUNKs to the $COLUMNSTORE at a specific time interval** For example, yesterday's data: ``` sql @@ -307,8 +308,8 @@ regularly. For example, last week's market data. 1. **View your data space saving** - When you convert data to the columnstore, as well as being optimized for analytics, it is compressed by more than - 90%. This saves on storage costs and keeps your queries operating at lightning speed. To see the amount of space + When you convert data to the $COLUMNSTORE, as well as being optimized for analytics, it is compressed by more than + 90%. This helps you save on storage costs and keeps your queries operating at lightning speed. To see the amount of space saved: ``` sql SELECT @@ -328,8 +329,8 @@ regularly. For example, last week's market data. ## Slash storage charges -In the previous sections, you used continuous aggregates to make fast analytical queries, and -compression to reduce storage costs on frequently accessed data. To reduce storage costs even more, +In the previous sections, you used $CAGGs to make fast analytical queries, and +$HYPERCORE to reduce storage costs on frequently accessed data. To reduce storage costs even more, you create tiering policies to move rarely accessed data to the object store. The object store is low-cost bottomless data storage built on Amazon S3. However, no matter the tier, you can [query your data when you need][querying-tiered-data]. $CLOUD_LONG seamlessly accesses the correct storage @@ -337,7 +338,7 @@ tier and generates the response. ![Tiered storage](https://assets.timescale.com/docs/images/tiered-storage.png ) -Data tiering is available in the [scale and enterprise][pricing-plans] pricing plans for $CLOUD_LONG. +Data tiering is available in the [$SCALE and $ENTERPRISE][pricing-plans] $PRICING_PLANs for $CLOUD_LONG. To set up data tiering: @@ -345,7 +346,7 @@ To set up data tiering: 1. **Enable data tiering** - 1. In [$CONSOLE][portal-ops-mode], select the service to modify. + 1. In [$CONSOLE][portal-ops-mode], select the $SERVICE_SHORT to modify. You see the `Overview` section. @@ -357,7 +358,7 @@ To set up data tiering: 1. **Set the time interval when data is tiered** - In $CONSOLE, click `SQL Editor`, then enable data tiering on a hypertable with the following query: + In $CONSOLE, click `SQL Editor`, then enable data tiering on a $HYPERTABLE with the following query: ```sql SELECT add_tiering_policy('assets_candlestick_daily', INTERVAL '3 weeks'); ``` @@ -386,15 +387,15 @@ To set up data tiering: ## Reduce the risk of downtime and data loss By default, all $SERVICE_LONGs have rapid recovery enabled. However, if your app has very low tolerance -for downtime, $CLOUD_LONG offers High Availability (HA) replicas. HA replicas are exact, up-to-date copies +for downtime, $CLOUD_LONG offers $HA_REPLICAs. $HA_REPLICA_SHORTs are exact, up-to-date copies of your database hosted in multiple AWS availability zones (AZ) within the same region as your primary node. -HA replicas automatically take over operations if the original primary data node becomes unavailable. +$HA_REPLICA_SHORTa automatically take over operations if the original primary data node becomes unavailable. The primary node streams its write-ahead log (WAL) to the replicas to minimize the chances of data loss during failover. -![Move from rowstore and columstore in hypercore](https://assets.timescale.com/docs/images/ha-read-replica.png ) +![$SERVICE_LONG replicas](https://assets.timescale.com/docs/images/ha-read-replica.png ) -High availability is available in the [scale and enterprise][pricing-plans] pricing plans for $CLOUD_LONG. +High availability is available in the [$SCALE and $ENTERPRISE][pricing-plans] $PRICING_PLANs for $CLOUD_LONG. From f9bb176d0df3b268b9a5cba52ddacdf3c67c8b0a Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Thu, 20 Mar 2025 11:33:01 +0200 Subject: [PATCH 33/90] Revert variables (#3950) --- _partials/_cloud-intro.md | 14 ++--- _partials/_service-overview.md | 2 +- getting-started/run-queries-from-console.md | 58 +++++++++---------- .../try-key-features-timescale-products.md | 55 +++++++++--------- 4 files changed, 63 insertions(+), 66 deletions(-) diff --git a/_partials/_cloud-intro.md b/_partials/_cloud-intro.md index 3af1b08a71..08de9fc2be 100644 --- a/_partials/_cloud-intro.md +++ b/_partials/_cloud-intro.md @@ -1,22 +1,22 @@ -$CLOUD_LONG is the modern $PG data platform for all your applications. It enhances $PG to handle time series, events, +$CLOUD_LONG is the modern PostgreSQL data platform for all your applications. It enhances PostgreSQL to handle time series, events, real-time analytics, and vector search—all in a single database alongside transactional workloads. -You get one system that handles live data ingestion, late and out-of-order updates, and low latency queries, with the performance, reliability, and scalability your app needs. Ideal for IoT, crypto, finance, SaaS, and a myriad other domains, $CLOUD_LONG allows you to build data-heavy, mission-critical apps while retaining the familiarity and reliability of $PG. +You get one system that handles live data ingestion, late and out-of-order updates, and low latency queries, with the performance, reliability, and scalability your app needs. Ideal for IoT, crypto, finance, SaaS, and a myriad other domains, $CLOUD_LONG allows you to build data-heavy, mission-critical apps while retaining the familiarity and reliability of PostgreSQL. -A $SERVICE_LONG is a single optimized 100% $PG database instance that you +A $SERVICE_LONG is a single optimized 100% PostgreSQL database instance that you use as is, or extend with the capabilities specific to your business needs: -- **Time-series and analytics**: $PG with $TIMESCALE_DB. The $PG you know and love, +- **Time-series and analytics**: PostgreSQL with $TIMESCALE_DB. The PostgreSQL you know and love, supercharged with functionality for storing and querying [time-series data][what-is-time-series] at scale for real-time analytics and other use cases. Get faster time-based queries with $HYPERTABLEs, $CAGGs, and columnar storage. Save on storage with native compression, data retention policies, and bottomless data tiering to Amazon S3. -- **AI and vector**: $PG with vector extensions. Use $PG as a vector database with +- **AI and vector**: PostgreSQL with vector extensions. Use PostgreSQL as a vector database with purpose built extensions for building AI applications from start to scale. Get fast and accurate similarity search - with the pgvector and $PGVECTORSCALE extensions. Create vector embeddings and perform LLM reasoning on your data with + with the pgvector and pgvectorscale extensions. Create vector embeddings and perform LLM reasoning on your data with the $PGAI_SHORT extension. - **PostgreSQL**: the trusted industry-standard RDBMS. Ideal for applications requiring strong data consistency, complex relationships, and advanced querying capabilities. Get ACID compliance, extensive SQL support, JSON handling, and extensibility through custom functions, data types, and extensions. $CLOUD_LONG continuously - monitors your $SERVICE_SHORTs and prevents common $PG out-of-memory crashes. + monitors your $SERVICE_SHORTs and prevents common PostgreSQL out-of-memory crashes. All $SERVICE_LONGs include the tooling you expect for production and developer environments: [live migration][live-migration], [automatic backups and PITR][automatic-backups], [high availability][high-availability], [$READ_REPLICAs][readreplica], [data forking][operations-forking], [connection pooling][connection-pooling], [tiered storage][data-tiering], diff --git a/_partials/_service-overview.md b/_partials/_service-overview.md index 25d576dd21..7457efc781 100644 --- a/_partials/_service-overview.md +++ b/_partials/_service-overview.md @@ -3,7 +3,7 @@ You manage your $SERVICE_LONGs and interact with your data in $CONSOLE using the | **$OPS_MODE_CAP** | **$DATA_MODE_CAP** | |-------------------|---------------------| | ![$CONSOLE $OPS_MODE][ops-mode] | ![$CONSOLE $DATA_MODE][data-mode] | -| **You use the $OPS_MODE to:**
    • Ensure data security with high availability and $READ_REPLICAs
    • Save money using compressed and tiered storage
    • Enable $PG extensions to add extra functionality
    • Increase security using $VPCs
    • Perform day-to-day administration
    | **Powered by $POPSQL, you use the $DATA_MODE to:**
    • Write queries with autocomplete
    • Visualize data with charts and dashboards
    • Schedule queries and dashboards for alerts or recurring reports
    • Share queries and dashboards
    | +| **You use the $OPS_MODE to:**
    • Ensure data security with high availability and $READ_REPLICAs
    • Save money using compressed and tiered storage
    • Enable PostgreSQL extensions to add extra functionality
    • Increase security using $VPCs
    • Perform day-to-day administration
    | **Powered by $POPSQL, you use the $DATA_MODE to:**
    • Write queries with autocomplete
    • Visualize data with charts and dashboards
    • Schedule queries and dashboards for alerts or recurring reports
    • Share queries and dashboards
    | [ops-mode]: https://assets.timescale.com/docs/images/ops-mode-overview.png [data-mode]: https://assets.timescale.com/docs/images/data-mode-overview.png \ No newline at end of file diff --git a/getting-started/run-queries-from-console.md b/getting-started/run-queries-from-console.md index a516bd2d43..2e560d87f3 100644 --- a/getting-started/run-queries-from-console.md +++ b/getting-started/run-queries-from-console.md @@ -10,7 +10,7 @@ import WhereNext from "versionContent/_partials/_where-to-next.mdx"; # Run your queries from $CONSOLE -As $CLOUD_LONG is based on $PG, you can use lots of [different tools][integrations] to +As $CLOUD_LONG is based on PostgreSQL, you can use lots of [different tools][integrations] to connect to your $SERVICE_SHORT and interact with your data. In $CONSOLE you can use the following ways to run SQL queries against your $SERVICE_SHORT: @@ -18,7 +18,7 @@ In $CONSOLE you can use the following ways to run SQL queries against your $SERV - [$DATA_MODE_CAP][run-popsql]: a rich experience powered by $POPSQL. You can write queries with autocomplete, save them in folders, share them, create charts/dashboards, and much more. -- [$SQL_ASSISTANT_SHORT in the $DATA_MODE][sql-assistant]: write, fix, and organize SQL faster and more accurately. +- [SQL assistant in the $DATA_MODE][sql-assistant]: write, fix, and organize SQL faster and more accurately. - [$SQL_EDITOR in the $OPS_MODE][run-sqleditor]: a simple $SQL_EDITOR in the $OPS_MODE that lets you run ad-hoc ephemeral queries. This is useful for quick one-off tasks like creating an index on a small table or @@ -37,7 +37,7 @@ Available features are: - **Real-time collaboration**: work with your team directly in the $DATA_MODE query editor with live presence and multiple cursors. - **[Schema browser](https://docs.popsql.com/docs/schema)**: understand the structure of your $SERVICE_SHORT and see usage data on tables and columns. -- **[$SQL_ASSISTANT_SHORT][sql-assistant]**: write, fix, and organize SQL faster and more accurately using AI. +- **[SQL assistant][sql-assistant]**: write, fix, and organize SQL faster and more accurately using AI. - **Autocomplete**: get suggestions as you type your queries. - **[Version history](https://docs.popsql.com/docs/version-history)**: access previous versions of a query from the built-in revision history, or connect to a git repo. @@ -46,7 +46,7 @@ Available features are: - **[Query variables](https://docs.popsql.com/docs/query-variables)**: use Liquid to parameterize your queries or use `if` statements. - **Cross-platform**: works from [$CONSOLE][portal-data-mode] or download the [desktop](https://popsql.com/download) app for macOS, Windows, and Linux. -- **Easy connection**: to $CLOUD_LONG, $PG, Redshift, Snowflake, BigQuery, MySQL, SQL Server, [and more](https://popsql.com/connections). +- **Easy connection**: to $CLOUD_LONG, PostgreSQL, Redshift, Snowflake, BigQuery, MySQL, SQL Server, [and more](https://popsql.com/connections). ### Connect to your $SERVICE_LONG in the $DATA_MODE @@ -106,7 +106,7 @@ If your $SERVICE_LONG runs inside a $VPC, do one of the following to enable acce #### What happens if another member of my $COMPANY project uses the $DATA_MODE? -The number of $DATA_MODE seats you are allocated depends on your [$PRICING_PLAN][pricing-plan-features]. +The number of $DATA_MODE seats you are allocated depends on your [pricing plan][pricing-plan-features]. #### Will using the $DATA_MODE affect the performance of my $SERVICE_LONG? @@ -126,9 +126,9 @@ If you'd like to prevent write operations such as insert or update, instead of using the `tsdbadmin` user, create a read-only user for your $SERVICE_SHORT and use that in the $DATA_MODE. -## $SQL_ASSISTANT_SHORT +## SQL assistant -$SQL_ASSISTANT_SHORT in [$CONSOLE][portal-data-mode] helps you write, fix, and organize SQL faster and more accurately. +SQL assistant in [$CONSOLE][portal-data-mode] helps you write, fix, and organize SQL faster and more accurately.
    @@ -138,49 +138,47 @@ $SQL_ASSISTANT_SHORT in [$CONSOLE][portal-data-mode] helps you write, fix, and o ### Key capabilities -$SQL_ASSISTANT_SHORT offers a range of features to improve your SQL workflow, including: +SQL assistant offers a range of features to improve your SQL workflow, including: -- **Real-time help**: $SQL_ASSISTANT_SHORT provides in-context help for writing and understanding SQL. Use it to: +- **Real-time help**: SQL assistant provides in-context help for writing and understanding SQL. Use it to: - - **Understand functions**: need to know how functions like `LAG()` or `ROW_NUMBER()` work? $SQL_ASSISTANT_SHORT explains it with examples. - - **Interpret complex queries**: $SQL_ASSISTANT_SHORT breaks down dense queries, giving you a clear view of each part. + - **Understand functions**: need to know how functions like `LAG()` or `ROW_NUMBER()` work? SQL assistant explains it with examples. + - **Interpret complex queries**: SQL assistant breaks down dense queries, giving you a clear view of each part. -- **Error resolution**: $SQL_ASSISTANT_SHORT diagnoses errors as they happen, you can resolve issues without leaving your editor. Features include: +- **Error resolution**: SQL assistant diagnoses errors as they happen, you can resolve issues without leaving your editor. Features include: - - **Error debugging**: if your query fails, $SQL_ASSISTANT_SHORT identifies the issue and suggests a fix. - - **Performance tuning**: for slow queries, $SQL_ASSISTANT_SHORT provides optimization suggestions to improve performance immediately. + - **Error debugging**: if your query fails, SQL assistant identifies the issue and suggests a fix. + - **Performance tuning**: for slow queries, SQL assistant provides optimization suggestions to improve performance immediately. - **Query organization**: to keep your query library organized, and help your team understand the - purpose of each query, $SQL_ASSISTANT_SHORT automatically adds titles and summaries to your queries. + purpose of each query, SQL assistant automatically adds titles and summaries to your queries. ### Limitations to keep in mind -For best results with $SQL_ASSISTANT_SHORT: +For best results with SQL assistant: -* **Schema awareness**: $SQL_ASSISTANT_SHORT references schema data but may need extra context +* **Schema awareness**: SQL assistant references schema data but may need extra context in complex environments. Specify tables, columns, or joins as needed. -* **Business logic**: $SQL_ASSISTANT_SHORT does not inherently know specific business terms +* **Business logic**: SQL assistant does not inherently know specific business terms such as active user. Define these terms clearly to improve results. ### Security, privacy, and data usage Security and privacy is prioritized in $CONSOLE. In [$DATA_MODE][portal-data-mode], project members -manage $SQL_ASSISTANT_SHORT settings under [`User name` > `Settings` > `SQL Assistant`][sql-editor-settings]. +manage SQL assistant settings under [`User name` > `Settings` > `SQL Assistant`][sql-editor-settings]. -![$SQL_ASSISTANT_SHORT settings](https://assets.timescale.com/docs/images/sql-editor-preferences.png) +![SQL assistant settings](https://assets.timescale.com/docs/images/sql-editor-preferences.png) -$SQL_ASSISTANT_SHORT settings are: +SQL assistant settings are: * **Opt-in features**: all AI features are off by default. Only [members][project-members] of your $CLOUD_LONG project can enable them. -* **Data protection**: your queries and data remain private and are not used for AI training. - $SQL_ASSISTANT_SHORT operates with strict security protocols. +* **Data protection**: your queries and data remain private and are not used for AI training. + SQL assistant operates with strict security protocols. * **Data usage**: to provide AI support, $COMPANY may share the query title, description and body, also the - database connection type such as $PG, and the schema. -* **Sample data**: to give the LLM more context so you have better SQL suggestions, enable sample data sharing in the - $SQL_ASSISTANT_SHORT preferences. -* **Telemetry**: to improve $SQL_ASSISTANT_SHORT, $COMPANY collects telemetry and usage data, including prompts, responses, - and query metadata. + database connection type such as PostgreSQL, and the schema. +* **Sample data**: to give the LLM more context so you have better SQL suggestions, enable sample data sharing in the SQL assistant preferences. +* **Telemetry**: to improve SQL assistant, $COMPANY collects telemetry and usage data, including prompts, responses, and query metadata. ## $OPS_MODE_CAP $SQL_EDITOR @@ -215,8 +213,8 @@ To use $SQL_EDITOR: ## Cloud $SQL_EDITOR licenses * **$SQL_EDITOR in the $OPS_MODE**: free for anyone with a [$CLOUD_LONG account][create-cloud-account]. -* **Data mode**: the number of seats you are allocated depends on your [$PRICING_PLAN][pricing-plan-features]. - [$SQL_ASSISTANT_SHORT][sql-assistant] is currently free for all users. In the future, limits or paid options may be +* **Data mode**: the number of seats you are allocated depends on your [pricing plan][pricing-plan-features]. + [SQL assistant][sql-assistant] is currently free for all users. In the future, limits or paid options may be introduced as we work to build the best experience. * **$POPSQL standalone**: there is a free plan available to everyone, as well as paid plans. See [$POPSQL Pricing][popsql-pricing] for full details. diff --git a/getting-started/try-key-features-timescale-products.md b/getting-started/try-key-features-timescale-products.md index 61e3108683..af7f6502c4 100644 --- a/getting-started/try-key-features-timescale-products.md +++ b/getting-started/try-key-features-timescale-products.md @@ -10,10 +10,10 @@ import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.md # Try the key $COMPANY features -$CLOUD_LONG scales $PG to ingest and query vast amounts of live data. $CLOUD_LONG +$CLOUD_LONG scales PostgreSQL to ingest and query vast amounts of live data. $CLOUD_LONG provides a range of features and optimizations that supercharge your queries while keeping the costs down. For example: -* The $HYPERCORE row-columnar engine makes queries up to 350x faster, ingests 44% faster, and reduces storage by 90%. +* The hypercore row-columnar engine makes queries up to 350x faster, ingests 44% faster, and reduces storage by 90%. * Tiered storage seamlessly moves your data from high performance storage for frequently accessed data to low cost bottomless storage for rarely accessed data. The following figure shows how $CLOUD_LONG optimizes your data for superfast real-time analytics and reduced @@ -30,7 +30,7 @@ ingest and query data faster while keeping the costs low. ## Optimize time-series data in $HYPERTABLEs -Time-series data represents how a system, process, or behavior changes over time. $HYPERTABLE_CAPs are $PG tables +Time-series data represents how a system, process, or behavior changes over time. $HYPERTABLE_CAPs are PostgreSQL tables that help you improve insert and query performance by automatically partitioning your data by time. Each $HYPERTABLE is made up of child tables called $CHUNKs. Each $CHUNK is assigned a range of time, and only contains data from that range. When you run a query, $CLOUD_LONG identifies the correct $CHUNK and runs the query on it, instead of going through the entire table. You can also tune $HYPERTABLEs to increase performance @@ -38,9 +38,9 @@ even more. ![Main features and tiered data](https://assets.timescale.com/docs/images/hypertable-structure.png ) -$HYPERTABLE_CAPs exist alongside regular $PG tables. -You use regular $PG tables for relational data, and interact with $HYPERTABLEs -and regular $PG tables in the same way. +$HYPERTABLE_CAPs exist alongside regular PostgreSQL tables. +You use regular PostgreSQL tables for relational data, and interact with $HYPERTABLEs +and regular PostgreSQL tables in the same way. This section shows you how to create regular tables and $HYPERTABLEs, and import relational and time-series data from external files. @@ -54,10 +54,9 @@ relational and time-series data from external files. This test dataset contains second-by-second trade data for the most-traded crypto-assets and a regular table of asset symbols and company names. - To import up to 100GB of data directly from your current $PG-based database, - [migrate with downtime][migrate-with-downtime] using native $PG tooling. To seamlessly import 100GB-10TB+ - of data, use the [live migration][migrate-live] tooling supplied by $COMPANY. To add data from non-$PG - data sources, see [Import and ingest data][data-ingest]. + To import up to 100GB of data directly from your current PostgreSQL-based database, + [migrate with downtime][migrate-with-downtime] using native PostgreSQL tooling. To seamlessly import 100GB-10TB+ + of data, use the [live migration][migrate-live] tooling supplied by $COMPANY. To add data from non-PostgreSQL data sources, see [Import and ingest data][data-ingest]. 1. Upload data from the CSVs to your $SERVICE_SHORT: @@ -86,13 +85,13 @@ relational and time-series data from external files. ```bash psql -d "postgres://:@:/" ``` - You use your [connection details][connection-info] to fill in this $PG connection string. + You use your [connection details][connection-info] to fill in this PostgreSQL connection string. 2. Create tables for the data to import: - For the time-series data: - 1. In your sql client, create a normal $PG table: + 1. In your sql client, create a normal PostgreSQL table: ```sql CREATE TABLE crypto_ticks ( @@ -111,7 +110,7 @@ relational and time-series data from external files. - For the relational data: - In your sql client, create a normal $PG table: + In your sql client, create a normal PostgreSQL table: ```sql CREATE TABLE crypto_assets ( symbol TEXT NOT NULL, @@ -138,11 +137,11 @@ relational and time-series data from external files. 1. **Have a quick look at your data** - You query $HYPERTABLEs in exactly the same way as you would a relational $PG table. + You query $HYPERTABLEs in exactly the same way as you would a relational PostgreSQL table. Use one of the following SQL editors to run a query and see the data you uploaded: - **Data mode**: write queries, visualize data, and share your results in [$CONSOLE][portal-data-mode] for all your $SERVICE_LONGs. - **SQL editor**: write, fix, and organize SQL faster and more accurately in [$CONSOLE][portal-ops-mode] for a $SERVICE_LONG. - - **psql**: easily run queries on your $SERVICE_LONGs or $SELF_LONG_LC deployment from Terminal. + - **psql**: easily run queries on your $SERVICE_LONGs or self-hosted TimescaleDB deployment from Terminal. @@ -159,7 +158,7 @@ and the $HYPERTABLE behind the $CAGG is automatically updated in the background. ![Reduced data calls with $CAGGs](https://assets.timescale.com/docs/images/continuous-aggregate.png ) You create $CAGGs on uncompressed data in high-performance storage. They continue to work -on [data in the $COLUMNSTORE][test-drive-enable-compression] +on [data in the columnstore][test-drive-enable-compression] and [rarely accessed data in tiered storage][test-drive-tiered-storage]. You can even create [$CAGGs on top of your $CAGGs][hierarchical-caggs]. @@ -182,7 +181,7 @@ $CONSOLE. You can also do this using psql. 1. **Create a $CAGG** For a $CAGG, data grouped using a $TIME_BUCKET is stored in a - $PG `MATERIALIZED VIEW` in a $HYPERTABLE. `timescaledb.continuous` ensures that this data + PostgreSQL `MATERIALIZED VIEW` in a $HYPERTABLE. `timescaledb.continuous` ensures that this data is always up to date. In your SQL editor, use the following code to create a $CAGG on the real-time data in the `crypto_ticks` table: @@ -269,14 +268,14 @@ a $CAGG, run the query part of the $CAGG ## Enhance query performance for analytics -$HYPERCORE_CAP is the $CLOUD_LONG hybrid row-columnar storage engine, designed specifically for real-time analytics and -powered by time-series data. The advantage of $HYPERCORE is its ability to seamlessly switch between row-oriented and +Hypercore is the $CLOUD_LONG hybrid row-columnar storage engine, designed specifically for real-time analytics and +powered by time-series data. The advantage of hypercore is its ability to seamlessly switch between row-oriented and column-oriented storage. This flexibility enables $CLOUD_LONG to deliver the best of both worlds, solving the key challenges in real-time analytics. -![Move from rowstore to columstore in $HYPERCORE](https://assets.timescale.com/docs/images/hypercore.png ) +![Move from rowstore to columstore in hypercore](https://assets.timescale.com/docs/images/hypercore.png ) -When you convert $CHUNKs from the $ROWSTORE to the $COLUMNSTORE, multiple records are grouped into a single row. +When you convert $CHUNKs from the rowstore to the columnstore, multiple records are grouped into a single row. The columns of this row hold an array-like structure that stores all the data. Because a single row takes up less disk space, you can reduce your $CHUNK size by more than 90%, and can also speed up your queries. This helps you save on storage costs, and keeps your queries operating at lightning speed. @@ -286,9 +285,9 @@ regularly. For example, last week's market data. -1. **Enable $HYPERCORE on a $HYPERTABLE** +1. **Enable hypercore on a $HYPERTABLE** - Create a [job][job] that automatically moves $CHUNKs in a $HYPERTABLE to the $COLUMNSTORE at a specific time interval. + Create a [job][job] that automatically moves $CHUNKs in a $HYPERTABLE to the columnstore at a specific time interval. ```sql ALTER TABLE crypto_ticks SET ( @@ -298,7 +297,7 @@ regularly. For example, last week's market data. You [segmentby][alter-table-arguments] to speed up queries. -1. **Add a policy to convert $CHUNKs to the $COLUMNSTORE at a specific time interval** +1. **Add a policy to convert $CHUNKs to the columnstore at a specific time interval** For example, yesterday's data: ``` sql @@ -308,7 +307,7 @@ regularly. For example, last week's market data. 1. **View your data space saving** - When you convert data to the $COLUMNSTORE, as well as being optimized for analytics, it is compressed by more than + When you convert data to the columnstore, as well as being optimized for analytics, it is compressed by more than 90%. This helps you save on storage costs and keeps your queries operating at lightning speed. To see the amount of space saved: ``` sql @@ -330,7 +329,7 @@ regularly. For example, last week's market data. ## Slash storage charges In the previous sections, you used $CAGGs to make fast analytical queries, and -$HYPERCORE to reduce storage costs on frequently accessed data. To reduce storage costs even more, +hypercore to reduce storage costs on frequently accessed data. To reduce storage costs even more, you create tiering policies to move rarely accessed data to the object store. The object store is low-cost bottomless data storage built on Amazon S3. However, no matter the tier, you can [query your data when you need][querying-tiered-data]. $CLOUD_LONG seamlessly accesses the correct storage @@ -338,7 +337,7 @@ tier and generates the response. ![Tiered storage](https://assets.timescale.com/docs/images/tiered-storage.png ) -Data tiering is available in the [$SCALE and $ENTERPRISE][pricing-plans] $PRICING_PLANs for $CLOUD_LONG. +Data tiering is available in the [$SCALE and $ENTERPRISE][pricing-plans] pricing plans for $CLOUD_LONG. To set up data tiering: @@ -395,7 +394,7 @@ data loss during failover. ![$SERVICE_LONG replicas](https://assets.timescale.com/docs/images/ha-read-replica.png ) -High availability is available in the [$SCALE and $ENTERPRISE][pricing-plans] $PRICING_PLANs for $CLOUD_LONG. +High availability is available in the [$SCALE and $ENTERPRISE][pricing-plans] pricing plans for $CLOUD_LONG. From 8a33bac394e4071cc5b30589355f1a91c2655dd8 Mon Sep 17 00:00:00 2001 From: Iain Cox Date: Thu, 20 Mar 2025 11:15:27 +0100 Subject: [PATCH 34/90] chore: add when and when not section for b-tree and hash indexes (#3949) * chore: add when and when not section. Co-authored-by: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> --- use-timescale/hypercore/secondary-indexes.md | 26 +++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/use-timescale/hypercore/secondary-indexes.md b/use-timescale/hypercore/secondary-indexes.md index b8d3dbc18a..08f774a545 100644 --- a/use-timescale/hypercore/secondary-indexes.md +++ b/use-timescale/hypercore/secondary-indexes.md @@ -70,10 +70,28 @@ analytical query performance optimizations, including vectorized filtering and a Hypercore TAM supports B-tree and hash indexes, making point lookups, upserts, and unique constraint enforcement more efficient on the columnstore. Our benchmarks demonstrate substantial performance improvements: -* 1,185x faster point lookup queries to retrieve a single record -* 224.3x faster inserts when checking unique constraints -* 2.6x faster upserts -* 4.5x faster range queries +* 1,185x faster point lookup queries to retrieve a single record. +* 224.3x faster inserts when checking unique constraints. +* 2.6x faster upserts. +* 4.5x faster range queries. + +## When to use B-tree and hash indexes + +Adding B-tree and hash indexes to compressed data enables dramatically faster lookups and inserts, but it comes with +a trade-off: increased storage usage due to additional indexing structures. + +B-tree and hash indexes are particularly helpful when: + +- You need fast lookups on non-`SEGMENTBY` keys. For example, querying specific records by UUID. +- Query latency on compressed data is a bottleneck for your application. +- You perform frequent updates to historical data and need efficient uniqueness enforcement. + +However, consider the storage trade-off when: + +- Your queries already benefit from columnstore min/max indexes or `SEGMENTBY` optimizations. +- Your workloads prioritize compression efficiency over lookup speed. +- You primarily run aggregations and range scans, where indexes may not provide meaningful speedups. + ## Enable secondary indexing From a727f193d953ac191786a23632ea316f6dcdd510 Mon Sep 17 00:00:00 2001 From: Iain Cox Date: Fri, 21 Mar 2025 14:50:38 +0100 Subject: [PATCH 35/90] Whitepaper explaining Timescale architecture for real-time analytics (#3956) * Whitepaper commit Authored-by: James Sewell --- _partials/_cloud-installation.md | 2 +- about/index.md | 2 +- about/page-index/page-index.js | 7 +- about/whitepaper.md | 463 +++++++++++++++++++++++++++++++ 4 files changed, 471 insertions(+), 3 deletions(-) create mode 100644 about/whitepaper.md diff --git a/_partials/_cloud-installation.md b/_partials/_cloud-installation.md index 805caff377..a8d8130c40 100644 --- a/_partials/_cloud-installation.md +++ b/_partials/_cloud-installation.md @@ -69,4 +69,4 @@ To set up $CLOUD_LONG via AWS: [aws-marketplace]: https://aws.amazon.com/marketplace [aws-paygo]: https://aws.amazon.com/marketplace/pp/prodview-iestawpo5ihca?applicationId=AWSMPContessa&ref_=beagle&sr=0-1 [aws-annual-commit]: https://aws.amazon.com/marketplace/pp/prodview-ezxwlmjyr6x4u?applicationId=AWSMPContessa&ref_=beagle&sr=0-2 -[timescale-signup]: https://console.cloud.timescale.com/signup \ No newline at end of file +[timescale-signup]: https://console.cloud.timescale.com/signup diff --git a/about/index.md b/about/index.md index c8fddbabbe..01afd5180e 100644 --- a/about/index.md +++ b/about/index.md @@ -1,5 +1,5 @@ --- -title: About Timescale +title: About Timescale products excerpt: Learn about Timescale Cloud and TimescaleDB, including pricing, release notes, feature overview, and contribution guidelines --- diff --git a/about/page-index/page-index.js b/about/page-index/page-index.js index ea733dbc35..40af9ab0ef 100644 --- a/about/page-index/page-index.js +++ b/about/page-index/page-index.js @@ -1,12 +1,17 @@ module.exports = [ { - title: "About Timescale", + title: "About Timescale products", href: "about", filePath: "index.md", pageComponents: ["featured-cards"], excerpt: "Additional information about Timescale, including how to contribute, and release notes", children: [ + { + title: "Timescale architecture for real-time analytics", + href: "whitepaper", + excerpt: "A whitepaper detailing the architectural choices and optimizations for real-time analytics that power Timescale" + }, { title: "Changelog", href: "changelog", diff --git a/about/whitepaper.md b/about/whitepaper.md new file mode 100644 index 0000000000..0854567eea --- /dev/null +++ b/about/whitepaper.md @@ -0,0 +1,463 @@ +--- +title: Timescale architecture for real-time analytics +excerpt: A whitepaper detailing the architectural choices and optimizations for real-time analytics that power Timescale +products: [cloud, mst, self_hosted] +keywords: [real-time analytics, timescale cloud, timescaledb, time-series, whitepaper] +--- +# Timescale architecture for real-time analytics + +Timescale provides a powerful application database for real-time analytics on time-series data. It integrates seamlessly with the PostgreSQL ecosystem and enhances it with automatic time-based partitioning, hybrid row-columnar storage, and vectorized execution—enabling high-ingest performance, sub-second queries, and full SQL support at scale. + +By making use of incrementally updated materialized views and advanced analytical functions, Timescale reduces compute overhead and improves query efficiency. Developers can continue using familiar SQL workflows and tools, while benefiting from a database purpose-built for fast, scalable analytics. + +Timescale combines TimescaleDB, an open-source PostgreSQL extension, and Timescale Cloud, a cloud-native database service. + +This document outlines the architectural choices and optimizations that power Timescale’s performance and scalability while preserving PostgreSQL’s reliability and transactional guarantees. + +Want to read this whitepaper from the comfort of your own computer? Download [Timescale architecture for real-time analytics (PDF)](https://assets.timescale.com/docs/downloads/Timescale_Architecture_for_Real-time_Analytics.pdf). + +## Introduction + +### What is real-time analytics? + +Real-time analytics enables applications to process and query data as it is generated and as it accumulates, delivering immediate and ongoing insights for decision-making. Unlike traditional analytics, which relies on batch processing and delayed reporting, real-time analytics supports *both* instant queries on fresh data and fast exploration of historical trends—powering applications with sub-second query performance across vast, continuously growing datasets. + +Many modern applications depend on real-time analytics to drive critical functionality: + + + +* **IoT monitoring systems** track sensor data over time, identifying long-term performance patterns while still surfacing anomalies as they arise. This allows businesses to optimize maintenance schedules, reduce costs, and improve reliability. +* **Financial and business intelligence platforms** analyze both current and historical data to detect trends, assess risk, and uncover opportunities—from tracking stock performance over a day, week, or year to identifying spending patterns across millions of transactions. +* **Interactive customer dashboards** empower users to explore live and historical data in a seamless experience—whether it's a SaaS product providing real-time analytics on business operations, a media platform analyzing content engagement, or an e-commerce site surfacing personalized recommendations based on recent and past behavior. + +Real-time analytics isn't just about reacting to the latest data, although that is critically important. It's also about delivering fast, interactive, and scalable insights across all your data, enabling better decision-making and richer user experiences. Unlike traditional ad-hoc analytics used by analysts, real-time analytics powers applications—driving dynamic dashboards, automated decisions, and user-facing insights at scale. + +To achieve this, real-time analytics systems must meet several key requirements: + +* **Low-latency queries** ensure sub-second response times even under high load, enabling fast insights for dashboards, monitoring, and alerting. +* **Low-latency ingest** minimizes the lag between when data is created and when it becomes available for analysis, ensuring fresh and accurate insights. +* **Data mutability** allows for efficient updates, corrections, and backfills, ensuring analytics reflect the most accurate state of the data. +* **Concurrency and scalability** enable systems to handle high query volumes and growing workloads without degradation in performance. +* **Seamless access to both recent and historical data** ensures fast queries across time, whether analyzing live, streaming data, or running deep historical queries on days or months of information. +* **Query flexibility** provides full SQL support, allowing for complex queries with joins, filters, aggregations, and analytical functions. + + +### Timescale: real-time analytics from PostgreSQL + +Timescale is a high-performance database that brings real-time analytics to applications. It combines fast queries, high ingest performance, and full SQL support—all while ensuring scalability and reliability. Timescale extends PostgreSQL with the TimescaleDB extension. It enables sub-second queries on vast amounts of incoming data while providing optimizations designed for continuously updating datasets. + +Timescale achieves this through the following optimizations: + + + +* **Efficient data partitioning:** automatically and transparently partitioning data into chunks, ensuring fast queries, minimal indexing overhead, and seamless scalability +* **Row-columnar storage:** providing the flexibility of a row store for transactions and the performance of a column store for analytics +* **Optimized query execution: **using techniques like chunk and batch exclusion, columnar storage, and vectorized execution to minimize latency +* **Continuous aggregates:** precomputing analytical results for fast insights without expensive reprocessing +* **Cloud-native operation: **compute/compute separation, elastic usage-based storage, horizontal scale out, data tiering to object storage +* **Operational simplicity: **offering high availability, connection pooling, and automated backups for reliable and scalable real-time applications + +With Timescale, developers can build low-latency, high-concurrency applications that seamlessly handle streaming data, historical queries, and real-time analytics while leveraging the familiarity and power of PostgreSQL. + + +## Data model + +Today's applications demand a database that can handle real-time analytics and transactional queries without sacrificing speed, flexibility, or SQL compatibility (including joins between tables). Timescale achieves this with **hypertables**, which provide an automatic partitioning engine, and **hypercore**, a hybrid row-columnar storage engine designed to deliver high-performance queries and efficient compression (up to 95 %) within PostgreSQL. + + +### Efficient data partitioning + +Timescale provides hypertables, a table abstraction that automatically partitions data into chunks in real time (using time stamps or incrementing IDs) to ensure fast queries and predictable performance as datasets grow. Unlike traditional relational databases that require manual partitioning, hypertables automate all aspects of partition management, keeping locking minimal even under high ingest load. + +At ingest time, hypertables ensure that PostgreSQL can deal with a constant stream of data without suffering from table bloat and index degradation by automatically partitioning data across time. Because each chunk is ordered by time and has its own indexes and storage, writes are usually isolated to small, recent chunks—keeping index sizes small, improving cache locality, and reducing the overhead of vacuum and background maintenance operations. This localized write pattern minimizes write amplification and ensures consistently high ingest performance, even as total data volume grows. + +At query time, hypertables efficiently exclude irrelevant chunks from the execution plan when the partitioning column is used in a `WHERE` clause. This architecture ensures fast query execution, avoiding the gradual slowdowns that affect non-partitioned tables as they accumulate millions of rows. Chunk-local indexes keep indexing overhead minimal, ensuring index operations scans remain efficient regardless of dataset size. + +
    + +
    + +Hypertables are the foundation for all of TimescaleDB’s real-time analytics capabilities. They enable seamless data ingestion, high-throughput writes, optimized query execution, and chunk-based lifecycle management—including automated data retention (drop a chunk) and data tiering (move a chunk to object storage). + + + +### Row-columnar storage + +Traditional databases force a trade-off between fast inserts (row-based storage) and efficient analytics (columnar storage). Hypercore eliminates this trade-off, allowing real-time analytics without sacrificing transactional capabilities. + +Hypercore dynamically stores data in the most efficient format for its lifecycle: + + + +* **Row-based storage for recent data**: the most recent chunk (and possibly more) is always stored in the rowstore, ensuring fast inserts, updates, and low-latency single record queries. Additionally, row-based storage is used as a writethrough for inserts and updates to columnar storage. +* **Columnar storage for analytical performance**: chunks are automatically compressed into the columnstore, optimizing storage efficiency and accelerating analytical queries. + +Unlike traditional columnar databases, hypercore allows data to be inserted or modified at any stage, making it a flexible solution for both high-ingest transactional workloads and real-time analytics—within a single database. + + +### Columnar storage layout + +TimescaleDB’s columnar storage layout optimizes analytical query performance by structuring data efficiently on disk, reducing scan times, and maximizing compression rates. Unlike traditional row-based storage, where data is stored sequentially by row, columnar storage organizes and compresses data by column, allowing queries to retrieve only the necessary fields in batches rather than scanning entire rows. But unlike many column store implementations, Timescale’s columnstore supports full mutability—inserts, upserts, updates, and deletes, even at the individual record level—with transactional guarantees. Data is also immediately visible to queries as soon as it is written. + + +
    + +
    + + +#### Columnar batches + +TimescaleDB uses columnar collocation and columnar compression within row-based storage to optimize analytical query performance while maintaining full PostgreSQL compatibility. This approach ensures efficient storage, high compression ratios, and rapid query execution. + + +
    + +
    + +A rowstore chunk is converted to a columnstore chunk by successfully grouping together sets of rows (typically up to 1000) into a single batch, then converting the batch into columnar form. + +Each compressed batch does the following: + +* Encapsulates columnar data in compressed arrays of up to 1,000 values per column, stored as a single entry in the underlying compressed table +* Uses a column-major format within the batch, enabling efficient scans by co-locating values of the same column and allowing the selection of individual columns without reading the entire batch +* Applies advanced compression techniques at the column level, including run-length encoding, delta encoding, and Gorilla compression, to significantly reduce storage footprint (by up to 95%) and improve I/O performance. + +While the chunk interval of rowstore and columnstore batches usually remains the same, Timescale can also combine columnstore batches so they use a different chunk interval. + +This architecture provides the benefits of columnar storage—optimized scans, reduced disk I/O, and improved analytical performance—while seamlessly integrating with PostgreSQL’s row-based execution model. + + +#### Segmenting and ordering data + +To optimize query performance, TimescaleDB allows explicit control over how data is physically organized within columnar storage. By structuring data effectively, queries can minimize disk reads and execute more efficiently, using vectorized execution for parallel batch processing where possible. + + +
    + +
    + + + +* **Group related data together to improve scan efficiency**: organizing rows into logical segments ensures that queries filtering by a specific value only scan relevant data sections. For example, in the above, querying for a specific ID is particularly fast. *(Implemented with SEGMENTBY.)* +* **Sort data within segments to accelerate range queries**: defining a consistent order reduces the need for post-query sorting, making time-based queries and range scans more efficient. *(Implemented with ORDERBY.)* +* **Reduce disk reads and maximize vectorized execution**: a well-structured storage layout enables efficient batch processing (Single Instruction, Multiple Data, or SIMD vectorization) and parallel execution, optimizing query performance. + +By combining segmentation and ordering, Timescale ensures that columnar queries are not only fast but also resource-efficient, enabling high-performance real-time analytics. + + +### Data mutability + +Traditional databases force a trade-off between fast updates and efficient analytics. Fully immutable storage is impractical in real-world applications, where data needs to change. Asynchronous mutability—where updates only become visible after batch processing—introduces delays that break real-time workflows. In-place mutability, while theoretically ideal, is prohibitively slow in columnar storage, requiring costly decompression, segmentation, ordering, and recompression cycles. + +Hypercore navigates these trade-offs with a hybrid approach that enables immediate updates without modifying compressed columnstore data in place. By staging changes in an interim rowstore chunk, hypercore allows updates and deletes to happen efficiently while preserving the analytical performance of columnar storage. + + +
    + +
    + + +#### Real-time writes without delays + +All new data which is destined for a columnstore chunk is first written to an interim rowstore chunk, ensuring high-speed ingestion and immediate queryability. Unlike fully columnar systems that require ingestion to go through compression pipelines, hypercore allows fresh data to remain in a fast row-based structure before being later compressed into columnar format in ordered batches as normal. + +Queries transparently access both the rowstore and columnstore chunks, meaning applications always see the latest data instantly, regardless of its storage format. + + +#### Efficient updates and deletes without performance penalties + +When modifying or deleting existing data, hypercore avoids the inefficiencies of both asynchronous updates and in-place modifications. Instead of modifying compressed storage directly, affected batches are decompressed and staged in the interim rowstore chunk, where changes are applied immediately. + +These modified batches remain in row storage until they are recompressed and reintegrated into the columnstore (which happens automatically via a background process). This approach ensures updates are immediately visible, but without the expensive overhead of decompressing and rewriting entire chunks. This approach avoids: + + + +* The rigidity of immutable storage, which requires workarounds like versioning or copy-on-write strategies +* The delays of asynchronous updates, where modified data is only visible after batch processing +* The performance hit of in-place mutability, which makes compressed storage prohibitively slow for frequent updates +* The restrictions some databases have on not altering the segmentation or ordering keys + + +## Query optimizations + +Real-time analytics isn’t just about raw speed—it’s about executing queries efficiently, reducing unnecessary work, and maximizing performance. Timescale optimizes every step of the query lifecycle to ensure that queries scan only what’s necessary, make use of data locality, and execute in parallel for sub-second response times over large datasets. + + +### Skip unnecessary data + +Timescale minimizes the amount of data a query touches, reducing I/O and improving execution speed: + + +#### Primary partition exclusion (row and columnar) + +Queries automatically skip irrelevant partitions (chunks) based on the primary partitioning key (usually a timestamp), ensuring they only scan relevant data. + + +
    + +
    + + +#### Secondary partition exclusion (columnar) + +Min/max metadata allows queries filtering on correlated dimensions (e.g., `order_id` or secondary timestamps) to exclude chunks that don’t contain relevant data. + + +
    + +
    + + +#### PostgreSQL indexes (row and columnar) + +Unlike many databases, Timescale supports standard PostgreSQL indexes on columnstore data (B-tree and hash currently, when using the hypercore table access method), allowing queries to efficiently locate specific values within both row-based and compressed columnar storage. These indexes enable fast lookups, range queries, and filtering operations that further reduce unnecessary data scans. + + +
    + +
    + + +#### Batch-level filtering (columnar) + +Within each chunk, compressed columnar batches are organized using `SEGMENTBY` keys and ordered by `ORDERBY` columns. Indexes and min/max metadata can be used to quickly exclude batches that don’t match the query criteria. + + +
    + +
    + + +### Maximize locality + +Organizing data for efficient access ensures queries are read in the most optimal order, reducing unnecessary random reads and reducing scans of unneeded data. + + +
    + +
    + + + +* **Segmentation**: Columnar batches are grouped using `SEGMENTBY` to keep related data together, improving scan efficiency. +* **Ordering**: Data within each batch is physically sorted using `ORDERBY`, increasing scan efficiency (and reducing I/O operations), enabling efficient range queries, and minimizing post-query sorting. +* **Column selection**: Queries read only the necessary columns, reducing disk I/O, decompression overhead, and memory usage. + + +### Parallelize execution + +Once a query is scanning only the required columnar data in the optimal order, Timescale is able to maximize performance through parallel execution. As well as using multiple workers, Timescale accelerates columnstore query execution by using Single Instruction, Multiple Data (SIMD) vectorization, allowing modern CPUs to process multiple data points in parallel. + + + +
    + +
    + +The Timescale implementation of SIMD vectorization currently allows: + + + +* **Vectorized decompression**, which efficiently restores compressed data into a usable form for analysis. +* **Vectorized filtering**, which rapidly applies filter conditions across data sets. +* **Vectorized aggregation**, which performs aggregate calculations, such as sum or average, across multiple data points concurrently. + + +## Accelerating queries with continuous aggregates + +Aggregating large datasets in real time can be expensive, requiring repeated scans and calculations that strain CPU and I/O. While some databases attempt to brute-force these queries at runtime, compute and I/O are always finite resources—leading to high latency, unpredictable performance, and growing infrastructure costs as data volume increases. + +**Continuous aggregates**, Timescale’s implementation of incrementally updated materialized views, solve this by shifting computation from every query run to a single, asynchronous step after data is ingested. Only the time buckets that receive new or modified data are updated, and queries read precomputed results instead of scanning raw data—dramatically improving performance and efficiency. + + +
    + +
    + +When you know the types of queries you'll need ahead of time, continuous aggregates allow you to pre-aggregate data along meaningful time intervals—such as per-minute, hourly, or daily summaries—delivering instant results without on-the-fly computation. + +Continuous aggregates also avoid the time-consuming and error-prone process of maintaining manual rollups, while continuing to offer data mutability to support efficient updates, corrections, and backfills. Whenever new data is inserted or modified in chunks which have been materialized, Timescale stores invalidation records reflecting that these results are stale and need to be recomputed. Then, an asynchronous process re-computes regions that include invalidated data, and updates the materialized results. Timescale tracks the lineage and dependencies between continuous aggregates and their underlying data, to ensure the continuous aggregates are regularly kept up-to-date. This happens in a resource-efficient manner, and where multiple invalidations can be coalesced into a single refresh (as opposed to refreshing any dependencies at write time, such as via a trigger-based approach). + +Continuous aggregates themselves are stored in hypertables, and they can be converted to columnar storage for compression, and raw data can be dropped, reducing storage footprint and processing cost. Continuous aggregates also support hierarchical rollups (e.g., hourly to daily to monthly) and real-time mode, which merges precomputed results with the latest ingested data to ensure accurate, up-to-date analytics. + +This architecture enables scalable, low-latency analytics while keeping resource usage predictable—ideal for dashboards, monitoring systems, and any workload with known query patterns. + + +### Hyperfunctions for real-time analytics + +Real-time analytics requires more than basic SQL functions—efficient computation is essential as datasets grow in size and complexity. Hyperfunctions, available through the `timescaledb_toolkit` extension, provide high-performance, SQL-native functions tailored for time-series analysis. These include advanced tools for gap-filling, percentile estimation, time-weighted averages, counter correction, and state tracking, among others. + +A key innovation of hyperfunctions is their support for partial aggregation, which allows Timescale to store intermediate computational states rather than just final results. These partials can later be merged to compute rollups efficiently, avoiding expensive reprocessing of raw data and reducing compute overhead. This is especially effective when combined with continuous aggregates. + +Consider a real-world example: monitoring request latencies across thousands of application instances. You might want to compute p95 latency per minute, then roll that up into hourly and daily percentiles for dashboards or alerts. With traditional SQL, calculating percentiles requires a full scan and sort of all underlying data—making multi-level rollups computationally expensive. + +With Timescale, you can use the `percentile_agg` hyperfunction in a continuous aggregate to compute and store a partial aggregation state for each minute. This state efficiently summarizes the distribution of latencies for that time bucket, without storing or sorting all individual values. Later, to produce an hourly or daily percentile, you simply combine the stored partials—no need to reprocess the raw latency values. + +This approach provides a scalable, efficient solution for percentile-based analytics. By combining hyperfunctions with continuous aggregates, Timescale enables real-time systems to deliver fast, resource-efficient insights across high-ingest, high-resolution datasets—without sacrificing accuracy or flexibility. + + +## Cloud-native architecture + +Real-time analytics requires a scalable, high-performance, and cost-efficient database that can handle high-ingest rates and low-latency queries without overprovisioning. Timescale Cloud is designed for elasticity, enabling independent scaling of storage and compute, workload isolation, and intelligent data tiering. + + +### Independent storage and compute scaling + +Real-time applications generate continuous data streams while requiring instant querying of both fresh and historical data. Traditional databases force users to pre-provision fixed storage, leading to unnecessary costs or unexpected limits. Timescale Cloud eliminates this constraint by dynamically scaling storage based on actual usage: + + + +* Storage expands and contracts automatically as data is added or deleted, avoiding manual intervention. +* Usage-based billing ensures costs align with actual storage consumption, eliminating large upfront allocations. +* Compute can be scaled independently to optimize query execution, ensuring fast analytics across both recent and historical data. + +With this architecture, databases grow alongside data streams, enabling seamless access to real-time and historical insights while efficiently managing storage costs. + + +### Workload isolation for real-time performance + +Balancing high-ingest rates and low-latency analytical queries on the same system can create contention, slowing down performance. Timescale Cloud mitigates this by allowing read and write workloads to scale independently: + + + +* The primary database efficiently handles both ingestion and real-time rollups without disruption. +* Read replicas scale query performance separately, ensuring fast analytics even under heavy workloads. + + +
    + +
    + +This separation ensures that frequent queries on fresh data don’t interfere with ingestion, making it easier to support live monitoring, anomaly detection, interactive dashboards, and alerting systems. + + +### Intelligent data tiering for cost-efficient real-time analytics + +Not all real-time data is equally valuable—recent data is queried constantly, while older data is accessed less frequently. Timescale Cloud can be configured to automatically tier data to cheaper bottomless object storage, ensuring that hot data remains instantly accessible, while historical data is still available. + + +
    + +
    + + + +* **Recent, high-velocity data** stays in high-performance storage for ultra-fast queries. +* **Older, less frequently accessed data** is automatically moved to cost-efficient object storage but remains queryable and available for building continuous aggregates. + +While many systems support this concept of data cooling, Timescale ensures that the data can still be queried from the same hypertable regardless of its current location. For real-time analytics, this means applications can analyze live data streams without worrying about storage constraints, while still maintaining access to long-term trends when needed. + + +### Cloud-native database observability + +Real-time analytics doesn’t just require fast queries—it requires the ability to understand why queries are fast or slow, where resources are being used, and how performance changes over time. That’s why Timescale is built with deep observability features, giving developers and operators full visibility into their database workloads. + +At the core of this observability is Insights, Timescale’s built-in query monitoring tool. Insights captures per-query statistics from our whole fleet in real time, showing you exactly how your database is behaving under load. It tracks key metrics like execution time, planning time, number of rows read and returned, I/O usage, and buffer cache hit rates—not just for the database as a whole, but for each individual query. + +Insights lets you do the following: + + + +* Identify slow or resource-intensive queries instantly +* Spot long-term performance regressions or trends +* Understand query patterns and how they evolve over time +* See the impact of schema changes, indexes, or continuous aggregates on workload performance +* Monitor and compare different versions of the same query to optimize execution + +All this is surfaced through an intuitive interface, available directly in Timescale Cloud, with no instrumentation or external monitoring infrastructure required. + +Beyond query-level visibility, Timescale also exposes metrics around service resource consumption, compression, continuous aggregates, and data tiering, allowing you to track how data moves through the system—and how those background processes impact storage and query performance. + +Together, these observability features give you the insight and control needed to operate a real-time analytics database at scale, with confidence, clarity, and performance you can trust**.** + + +## Ensuring reliability and scalability + +Maintaining high availability, efficient resource utilization, and data durability is essential for real-time applications. Timescale provides robust operational features to ensure seamless performance under varying workloads. + +* **High-availability (HA) replicas**: deploy multi-AZ HA replicas to provide fault tolerance and ensure minimal downtime. In the event of a primary node failure, replicas are automatically promoted to maintain service continuity. +* **Connection pooling**: optimize database connections by efficiently managing and reusing them, reducing overhead and improving performance for high-concurrency applications. +* **Backup and recovery**: leverage continuous backups, Point-in-Time Recovery (PITR), and automated snapshotting to protect against data loss. Restore data efficiently to minimize downtime in case of failures or accidental deletions. + +These operational capabilities ensure Timescale remains reliable, scalable, and resilient, even under demanding real-time workloads. + + +## Conclusion + +Real-time analytics is critical for modern applications, but traditional databases struggle to balance high-ingest performance, low-latency queries, and flexible data mutability. Timescale extends PostgreSQL to solve this challenge, combining automatic partitioning, hybrid row-columnar storage, and intelligent compression to optimize both transactional and analytical workloads. + +With continuous aggregates, hyperfunctions, and advanced query optimizations, Timescale ensures sub-second queries even on massive datasets that combine current and historic data. Its cloud-native architecture further enhances scalability with independent compute and storage scaling, workload isolation, and cost-efficient data tiering—allowing applications to handle real-time and historical queries seamlessly. + +For developers, this means building high-performance, real-time analytics applications without sacrificing SQL compatibility, transactional guarantees, or operational simplicity. + +Timescale delivers the best of PostgreSQL, optimized for real-time analytics. + From c16db3c6b81cd76dbf7b952109ba1bb6e0981c70 Mon Sep 17 00:00:00 2001 From: Iain Cox Date: Fri, 21 Mar 2025 17:04:59 +0100 Subject: [PATCH 36/90] chore: move download. (#3957) --- about/whitepaper.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/about/whitepaper.md b/about/whitepaper.md index 0854567eea..7fa97e0956 100644 --- a/about/whitepaper.md +++ b/about/whitepaper.md @@ -14,7 +14,12 @@ Timescale combines TimescaleDB, an open-source PostgreSQL extension, and Timesca This document outlines the architectural choices and optimizations that power Timescale’s performance and scalability while preserving PostgreSQL’s reliability and transactional guarantees. -Want to read this whitepaper from the comfort of your own computer? Download [Timescale architecture for real-time analytics (PDF)](https://assets.timescale.com/docs/downloads/Timescale_Architecture_for_Real-time_Analytics.pdf). +Want to read this whitepaper from the comfort of your own computer? + +
    + [Timescale architecture for real-time analytics (PDF)](https://assets.timescale.com/docs/downloads/Timescale_Architecture_for_Real-time_Analytics.pdf) +
    + ## Introduction From b8b65617f18c865bee7573039e9e47d23d81a4ce Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Mon, 24 Mar 2025 11:59:25 +0200 Subject: [PATCH 37/90] TEST VARIABLE (#3959) --- api/alter_job.md | 28 ++++++++++++++-------------- api/jobs.md | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/api/alter_job.md b/api/alter_job.md index 374c26e741..a67f555b64 100644 --- a/api/alter_job.md +++ b/api/alter_job.md @@ -30,20 +30,20 @@ other useful statistics for deciding what the new schedule should be. ### Optional arguments -|Name|Type| Description | -|-|-|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -|`schedule_interval`|`INTERVAL`| The interval at which the job runs. Defaults to 24 hours. | -|`max_runtime`|`INTERVAL`| The maximum amount of time the job is allowed to run by the background worker scheduler before it is stopped. | -|`max_retries`|`INTEGER`| The number of times the job is retried if it fails. | -|`retry_period`|`INTERVAL`| The amount of time the scheduler waits between retries of the job on failure. | -|`scheduled`|`BOOLEAN`| Set to `FALSE` to exclude this job from being run as background job. | -|`config`|`JSONB`| Job-specific configuration, passed to the function when it runs. This includes:
  • verbose_log: boolean, defaults to false. Enable verbose logging output when running the compression policy.
  • maxchunks_to_compress: integer, defaults to 0 (no limit). The maximum number of chunks to compress during a policy run.
  • recompress: boolean, defaults to true. Recompress partially compressed chunks.
  • compress_after: see [add_compression_policy][add-policy].
  • compress_created_before: see [add_compression_policy][add-policy].
  • hypercore_use_access_method: boolean, defaults to false. Use hypercore TAM to compress chunks.
    • | -|`next_start`|`TIMESTAMPTZ`| The next time at which to run the job. The job can be paused by setting this value to `infinity`, and restarted with a value of `now()`. | -|`if_exists`|`BOOLEAN`| Set to `true`to issue a notice instead of an error if the job does not exist. Defaults to false. | -|`check_config`|`REGPROC`| A function that takes a single argument, the `JSONB` `config` structure. The function is expected to raise an error if the configuration is not valid, and return nothing otherwise. Can be used to validate the configuration when updating a job. Only functions, not procedures, are allowed as values for `check_config`. | -|`fixed_schedule`|`BOOLEAN`| To enable fixed scheduled job runs, set to `TRUE`. | -|`initial_start`|`TIMESTAMPTZ`| Set the time when the `fixed_schedule` job run starts. For example, `19:10:25-07`. | -|`timezone`|`TEXT`| Address the 1-hour shift in start time when clocks change from [Daylight Saving Time to Standard Time](https://en.wikipedia.org/wiki/Daylight_saving_time). For example, `America/Sao_Paulo`. | +|Name|Type| Description | +|-|-|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +|`schedule_interval`|`INTERVAL`| The interval at which the job runs. Defaults to 24 hours. | +|`max_runtime`|`INTERVAL`| The maximum amount of time the job is allowed to run by the background worker scheduler before it is stopped. | +|`max_retries`|`INTEGER`| The number of times the job is retried if it fails. | +|`retry_period`|`INTERVAL`| The amount of time the scheduler waits between retries of the job on failure. | +|`scheduled`|`BOOLEAN`| Set to `FALSE` to exclude this job from being run as background job. | +|`config`|`JSONB`| $JOB_CAP-specific configuration, passed to the function when it runs. This includes:
  • verbose_log: boolean, defaults to false. Enable verbose logging output when running the compression policy.
  • maxchunks_to_compress: integer, defaults to 0 (no limit). The maximum number of chunks to compress during a policy run.
  • recompress: boolean, defaults to true. Recompress partially compressed chunks.
  • compress_after: see [add_compression_policy][add-policy].
  • compress_created_before: see [add_compression_policy][add-policy].
  • hypercore_use_access_method: boolean, defaults to false. Use hypercore TAM to compress chunks.
    • | +|`next_start`|`TIMESTAMPTZ`| The next time at which to run the job. The job can be paused by setting this value to `infinity`, and restarted with a value of `now()`. | +|`if_exists`|`BOOLEAN`| Set to `true`to issue a notice instead of an error if the job does not exist. Defaults to false. | +|`check_config`|`REGPROC`| A function that takes a single argument, the `JSONB` `config` structure. The function is expected to raise an error if the configuration is not valid, and return nothing otherwise. Can be used to validate the configuration when updating a job. Only functions, not procedures, are allowed as values for `check_config`. | +|`fixed_schedule`|`BOOLEAN`| To enable fixed scheduled job runs, set to `TRUE`. | +|`initial_start`|`TIMESTAMPTZ`| Set the time when the `fixed_schedule` job run starts. For example, `19:10:25-07`. | +|`timezone`|`TEXT`| Address the 1-hour shift in start time when clocks change from [Daylight Saving Time to Standard Time](https://en.wikipedia.org/wiki/Daylight_saving_time). For example, `America/Sao_Paulo`. | When a job begins, the `next_start` parameter is set to `infinity`. This diff --git a/api/jobs.md b/api/jobs.md index cc3342db9c..900b241bb3 100644 --- a/api/jobs.md +++ b/api/jobs.md @@ -11,7 +11,7 @@ api: # timescaledb_information.jobs -Shows information about all jobs registered with the automation framework. +Shows information about all $JOBs registered with the automation framework. ### Arguments From 678ac828b76459dc040c3cc8cf92481a9f0b316c Mon Sep 17 00:00:00 2001 From: Daniel Levy <145054291+dannydanlev@users.noreply.github.com> Date: Mon, 24 Mar 2025 08:39:16 -0400 Subject: [PATCH 38/90] Update changelog.md (#3953) * Update changelog.md Signed-off-by: Daniel Levy <145054291+dannydanlev@users.noreply.github.com> Co-authored-by: atovpeko --- about/changelog.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/about/changelog.md b/about/changelog.md index df1a069533..5445cc0ba1 100644 --- a/about/changelog.md +++ b/about/changelog.md @@ -8,6 +8,19 @@ keywords: [changelog, upgrades, updates, releases] All the latest features and updates to Timescale products. +## 🤩 In-Console Livesync for PostgreSQL + + +You can now set up an active data ingestion pipeline with Livesync for PostgreSQL in Timescale Console. This tool enables you to replicate your source database tables into Timescale's hypertables indefinitely. Yes, you heard that right—keep Livesync running for as long as you need, ensuring that your existing source PostgreSQL tables stay in sync with Timescale Cloud. Read more about setting up and using [Livesync for PostgreSQL](https://docs.timescale.com/migrate/latest/livesync/). + +![Livesync in Timescale Console](https://assets.timescale.com/docs/images/timescale-cloud-livesync-tile.png) + +![Set up Timescale Livesync](https://assets.timescale.com/docs/images/set-up-timescale-cloud-livesync.png) + +![Select tables for Livesync](https://assets.timescale.com/docs/images/select-tables-for-timescale-cloud-livesync.png) + +![Timescale Livesync running](https://assets.timescale.com/docs/images/livesync-view-status.png) + ## 💾 16K dimensions on pgvectorscale plus new pgai Vectorizer support From e6bb1d421c041476a5909572cb570c6d95c4eecc Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Tue, 25 Mar 2025 15:49:04 +0200 Subject: [PATCH 39/90] Rename user-defined actions to jobs (#3942) --- .github/styles/Google/WordList.yml | 1 - _code-samples/toolkit/two-step_aggregation.md | 2 +- .../background-worker-failed-start.md | 2 +- about/timescaledb-editions.md | 2 +- api/actions.md | 21 --- api/add_job.md | 66 ------- api/compression/compress_chunk.md | 2 +- api/compression/recompress_chunk.md | 2 +- api/delete_job.md | 31 --- api/hypercore/add_columnstore_policy.md | 2 +- api/hypercore/alter_table.md | 2 +- api/hypercore/chunk_columnstore_stats.md | 2 +- api/hypercore/convert_to_columnstore.md | 2 +- api/hypercore/convert_to_rowstore.md | 4 +- api/hypercore/index.md | 2 +- .../chunk_compression_settings.md | 0 api/{ => informational-views}/chunks.md | 0 .../compression_settings.md | 0 .../continuous_aggregates.md | 0 api/{ => informational-views}/data_nodes.md | 0 api/{ => informational-views}/dimensions.md | 0 .../hypertable_compression_settings.md | 0 api/{ => informational-views}/hypertables.md | 0 .../index.md} | 2 +- api/{ => informational-views}/job_errors.md | 10 +- api/{ => informational-views}/job_history.md | 10 +- api/{ => informational-views}/job_stats.md | 2 +- api/{ => informational-views}/jobs.md | 48 ++--- api/{ => informational-views}/policies.md | 2 +- api/jobs-automation/add_job.md | 66 +++++++ api/{ => jobs-automation}/alter_job.md | 73 ++++---- api/jobs-automation/delete_job.md | 31 +++ api/jobs-automation/index.md | 21 +++ api/{ => jobs-automation}/run_job.md | 16 +- api/page-index/page-index.js | 9 +- api/remove_all_policies.md | 2 +- lambda/redirects.js | 60 +++++- migrate/troubleshooting.md | 2 +- .../about-distributed-hypertables.md | 2 +- self-hosted/manage-storage.md | 4 +- .../multinode-maintenance.md | 4 +- use-timescale/jobs/create-and-manage-jobs.md | 176 ++++++++++++++++++ .../example-downsample-and-compress.md | 28 ++- .../example-generic-retention.md | 28 +-- .../example-tiered-storage.md | 31 ++- use-timescale/jobs/index.md | 36 ++++ use-timescale/page-index/page-index.js | 37 ++-- .../about-user-defined-actions.md | 121 ------------ .../user-defined-actions/alter-and-delete.md | 63 ------- .../create-and-register.md | 72 ------- use-timescale/user-defined-actions/index.md | 37 ---- .../user-defined-actions/test-and-debug.md | 46 ----- 52 files changed, 544 insertions(+), 638 deletions(-) delete mode 100644 api/actions.md delete mode 100644 api/add_job.md delete mode 100644 api/delete_job.md rename api/{ => informational-views}/chunk_compression_settings.md (100%) rename api/{ => informational-views}/chunks.md (100%) rename api/{ => informational-views}/compression_settings.md (100%) rename api/{ => informational-views}/continuous_aggregates.md (100%) rename api/{ => informational-views}/data_nodes.md (100%) rename api/{ => informational-views}/dimensions.md (100%) rename api/{ => informational-views}/hypertable_compression_settings.md (100%) rename api/{ => informational-views}/hypertables.md (100%) rename api/{informational-views.md => informational-views/index.md} (92%) rename api/{ => informational-views}/job_errors.md (92%) rename api/{ => informational-views}/job_history.md (93%) rename api/{ => informational-views}/job_stats.md (98%) rename api/{ => informational-views}/jobs.md (50%) rename api/{ => informational-views}/policies.md (95%) create mode 100644 api/jobs-automation/add_job.md rename api/{ => jobs-automation}/alter_job.md (80%) create mode 100644 api/jobs-automation/delete_job.md create mode 100644 api/jobs-automation/index.md rename api/{ => jobs-automation}/run_job.md (62%) create mode 100644 use-timescale/jobs/create-and-manage-jobs.md rename use-timescale/{user-defined-actions => jobs}/example-downsample-and-compress.md (72%) rename use-timescale/{user-defined-actions => jobs}/example-generic-retention.md (64%) rename use-timescale/{user-defined-actions => jobs}/example-tiered-storage.md (68%) create mode 100644 use-timescale/jobs/index.md delete mode 100644 use-timescale/user-defined-actions/about-user-defined-actions.md delete mode 100644 use-timescale/user-defined-actions/alter-and-delete.md delete mode 100644 use-timescale/user-defined-actions/create-and-register.md delete mode 100644 use-timescale/user-defined-actions/index.md delete mode 100644 use-timescale/user-defined-actions/test-and-debug.md diff --git a/.github/styles/Google/WordList.yml b/.github/styles/Google/WordList.yml index fba544db4a..a3026b9ca0 100644 --- a/.github/styles/Google/WordList.yml +++ b/.github/styles/Google/WordList.yml @@ -55,7 +55,6 @@ swap: static IP address: static external IP address time weighted: time-weighted url: URL - user defined actions: user-defined actions vanilla: regular|standard vs\.: versus World Wide Web: web diff --git a/_code-samples/toolkit/two-step_aggregation.md b/_code-samples/toolkit/two-step_aggregation.md index 279efd2bc8..e6aa015457 100644 --- a/_code-samples/toolkit/two-step_aggregation.md +++ b/_code-samples/toolkit/two-step_aggregation.md @@ -151,6 +151,6 @@ GROUP BY id, time_bucket('1 day'::interval, bucket) ##### NB: There are some two-step aggregates like `tdigest` __ADD LINK? and expose and other bits...__ when we document that function where two-step aggregation can lead to more error or different results, because the algorithm is not deterministic in its re-aggregation, but we will note that clearly in the documentation when that happens, it is unusual. ### Retrospective analysis over downsampled data -[Continuous aggregates][continuous-aggregates] (or separate aggregation tables powered by a cron job or [user-defined action]( __LINK__ ) ) aren't just used for speeding up queries, they're also used for [data retention]( __LINK__ ). But this can mean that they are very difficult to modify as your data ages. Unfortunately this is also when you are learning more things about the analysis you want to do on your data. By keeping them in their raw aggregate form, the user has the flexibility to apply different accessors to do retrospective analysis. With a one-step aggregate the user needs to determine, say, which percentiles are important when we create the continous aggregate, with a two-step aggregate the user can simply determine they're going to want an approximate percentile, and then determine when doing the analysis whether they want the median, the 90th, 95th or 1st percentile. No need to modify the aggregate or try to re-calculate from data that may no longer exist in the system. +[Continuous aggregates][continuous-aggregates] (or separate aggregation tables powered by a cron job or a [custom job]( __LINK__ ) ) aren't just used for speeding up queries, they're also used for [data retention]( __LINK__ ). But this can mean that they are very difficult to modify as your data ages. Unfortunately this is also when you are learning more things about the analysis you want to do on your data. By keeping them in their raw aggregate form, the user has the flexibility to apply different accessors to do retrospective analysis. With a one-step aggregate the user needs to determine, say, which percentiles are important when we create the continous aggregate, with a two-step aggregate the user can simply determine they're going to want an approximate percentile, and then determine when doing the analysis whether they want the median, the 90th, 95th or 1st percentile. No need to modify the aggregate or try to re-calculate from data that may no longer exist in the system. [continuous-aggregates]: /use-timescale/:currentVersion:/continuous-aggregates/about-continuous-aggregates/ \ No newline at end of file diff --git a/_troubleshooting/self-hosted/background-worker-failed-start.md b/_troubleshooting/self-hosted/background-worker-failed-start.md index 5191c790d8..c96b81a9fd 100644 --- a/_troubleshooting/self-hosted/background-worker-failed-start.md +++ b/_troubleshooting/self-hosted/background-worker-failed-start.md @@ -16,7 +16,7 @@ apis: - [hypertables, add_compression_policy()] - [hypertables, add_continuous_aggregate_policy()] keywords: [jobs, policies, actions] -tags: [jobs, scheduled jobs, background jobs, background workers, automation framework, policies, user-defined actions] +tags: [jobs, scheduled jobs, background jobs, background workers, automation framework, policies] --- -## timescaledb_experimental.policies +# timescaledb_experimental.policies The `policies` view provides information on all policies set on continuous diff --git a/api/jobs-automation/add_job.md b/api/jobs-automation/add_job.md new file mode 100644 index 0000000000..c5720bd528 --- /dev/null +++ b/api/jobs-automation/add_job.md @@ -0,0 +1,66 @@ +--- +api_name: add_job() +excerpt: Add a job to run a function or procedure automatically +topics: [jobs] +keywords: [jobs] +tags: [scheduled jobs, background jobs, automation framework] +api: + license: community + type: function +--- + +# add_job() Community + +Register a $JOB for scheduling by the automation framework. For more information about scheduling, including example $JOBs, see the [jobs documentation section][using-jobs]. + +## Required arguments + +|Name|Type| Description | +|-|-|---------------------------------------------------------------| +|`proc`|REGPROC| Name of the function or procedure to register as a $JOB. | +|`schedule_interval`|INTERVAL| Interval between executions of this $JOB. Defaults to 24 hours | + +## Optional arguments + +|Name|Type| Description | +|-|-|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +|`config`|JSONB| $JOB_CAPs-specific configuration, passed to the function when it runs | +|`initial_start`|TIMESTAMPTZ| Time the $JOB is first run. In the case of fixed schedules, this also serves as the origin on which $JOB executions are aligned. If omitted, the current time is used as origin in the case of fixed schedules. | +|`scheduled`|BOOLEAN| Set to `FALSE` to exclude this $JOB from scheduling. Defaults to `TRUE`. | +|`check_config`|`REGPROC`| A function that takes a single argument, the `JSONB` `config` structure. The function is expected to raise an error if the configuration is not valid, and return nothing otherwise. Can be used to validate the configuration when adding a $JOB. Only functions, not procedures, are allowed as values for `check_config`. | +|`fixed_schedule`|BOOLEAN| Set to `FALSE` if you want the next start of a $JOB to be determined as its last finish time plus the schedule interval. Set to `TRUE` if you want the next start of a $JOB to begin `schedule_interval` after the last start. Defaults to `TRUE` | +|`timezone`|TEXT| A valid time zone. If fixed_schedule is `TRUE`, subsequent executions of the $JOB are aligned on its initial start. However, daylight savings time (DST) changes may shift this alignment. Set to a valid time zone if you want to mitigate this issue. Defaults to `NULL`. | + +## Returns + +|Column|Type|Description| +|-|-|-| +|`job_id`|INTEGER|TimescaleDB background job ID| + +## Sample use + +Register the `user_defined_action` procedure to run every hour: + +```sql +CREATE OR REPLACE PROCEDURE user_defined_action(job_id int, config jsonb) LANGUAGE PLPGSQL AS +$$ +BEGIN + RAISE NOTICE 'Executing action % with config %', job_id, config; +END +$$; + +SELECT add_job('user_defined_action','1h'); +SELECT add_job('user_defined_action','1h', fixed_schedule => false); +``` + +Register the `user_defined_action` procedure to run at midnight every Sunday. +The `initial_start` provided must satisfy these requirements, so it must be a Sunday midnight: + +```sql +-- December 4, 2022 is a Sunday +SELECT add_job('user_defined_action','1 week', initial_start => '2022-12-04 00:00:00+00'::timestamptz); +-- if subject to DST +SELECT add_job('user_defined_action','1 week', initial_start => '2022-12-04 00:00:00+00'::timestamptz, timezone => 'Europe/Berlin'); +``` + +[using-jobs]: /use-timescale/:currentVersion:/jobs diff --git a/api/alter_job.md b/api/jobs-automation/alter_job.md similarity index 80% rename from api/alter_job.md rename to api/jobs-automation/alter_job.md index a67f555b64..ff233b8d41 100644 --- a/api/alter_job.md +++ b/api/jobs-automation/alter_job.md @@ -3,7 +3,7 @@ api_name: alter_job() excerpt: Alter a job that is scheduled to run automatically topics: [jobs] keywords: [jobs] -tags: [scheduled jobs, user-defined actions, automation framework, background jobs, alter, change] +tags: [scheduled jobs, automation framework, background jobs, alter, change] api: license: community type: function @@ -13,22 +13,22 @@ import EarlyAccess from "versionContent/_partials/_early_access.mdx"; # alter_job() Community -Actions scheduled using the TimescaleDB automation framework run periodically in -a background worker. You can change the schedule of these jobs with the -`alter_job` function. To alter an existing job, refer to it by `job_id`. The -`job_id` runs a given action, and its current schedule can be found in the +$JOB_CAPs scheduled using the TimescaleDB automation framework run periodically in +a background worker. You can change the schedule of these $JOBs with the +`alter_job` function. To alter an existing $JOB, refer to it by `job_id`. The +`job_id` runs a given $JOB, and its current schedule can be found in the `timescaledb_information.jobs` view, which lists information about every -scheduled action, as well as in `timescaledb_information.job_stats`. The -`job_stats` view also gives information about when each job was last run and +scheduled $JOBs, as well as in `timescaledb_information.job_stats`. The +`job_stats` view also gives information about when each $JOB was last run and other useful statistics for deciding what the new schedule should be. -### Required arguments +## Required arguments |Name|Type|Description| |-|-|-| -|`job_id`|`INTEGER`|The ID of the policy job being modified| +|`job_id`|`INTEGER`|The ID of the policy $JOB being modified| -### Optional arguments +## Optional arguments |Name|Type| Description | |-|-|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| @@ -45,33 +45,32 @@ other useful statistics for deciding what the new schedule should be. |`initial_start`|`TIMESTAMPTZ`| Set the time when the `fixed_schedule` job run starts. For example, `19:10:25-07`. | |`timezone`|`TEXT`| Address the 1-hour shift in start time when clocks change from [Daylight Saving Time to Standard Time](https://en.wikipedia.org/wiki/Daylight_saving_time). For example, `America/Sao_Paulo`. | - -When a job begins, the `next_start` parameter is set to `infinity`. This -prevents the job from attempting to be started again while it is running. When -the job completes, whether or not the job is successful, the parameter is +When a $JOB begins, the `next_start` parameter is set to `infinity`. This +prevents the $JOB from attempting to be started again while it is running. When +the $JOB completes, whether or not the job is successful, the parameter is automatically updated to the next computed start time. Note that altering the `next_start` value is only effective for the next -execution of the job in case of fixed schedules. On the next execution, it will +execution of the $JOB in case of fixed schedules. On the next execution, it will automatically return to the schedule. -### Returns +## Returns -|Column|Type|Description| -|-|-|-| -|`job_id`|`INTEGER`|The ID of the job being modified| -|`schedule_interval`|`INTERVAL`|The interval at which the job runs. Defaults to 24 hours| -|`max_runtime`|`INTERVAL`|The maximum amount of time the job is allowed to run by the background worker scheduler before it is stopped| -|`max_retries`|INTEGER|The number of times the job is retried if it fails| -|`retry_period`|`INTERVAL`|The amount of time the scheduler waits between retries of the job on failure| -|`scheduled`|`BOOLEAN`|Returns `true` if the job is executed by the TimescaleDB scheduler| -|`config`|`JSONB`|Job-specific configuration, passed to the function when it runs| -|`next_start`|`TIMESTAMPTZ`|The next time to run the job| -|`check_config`|`TEXT`|The function used to validate updated job configurations| +|Column|Type| Description | +|-|-|---------------------------------------------------------------------------------------------------------------| +|`job_id`|`INTEGER`| The ID of the $JOB being modified | +|`schedule_interval`|`INTERVAL`| The interval at which the $JOB runs. Defaults to 24 hours | +|`max_runtime`|`INTERVAL`| The maximum amount of time the $JOB is allowed to run by the background worker scheduler before it is stopped | +|`max_retries`|INTEGER| The number of times the $JOB is retried if it fails | +|`retry_period`|`INTERVAL`| The amount of time the scheduler waits between retries of the $JOB on failure | +|`scheduled`|`BOOLEAN`| Returns `true` if the $JOB is executed by the TimescaleDB scheduler | +|`config`|`JSONB`| $JOB_CAPs-specific configuration, passed to the function when it runs | +|`next_start`|`TIMESTAMPTZ`| The next time to run the $JOB | +|`check_config`|`TEXT`| The function used to validate updated $JOB configurations | -### Sample usage +## Sample usage -Reschedules job ID `1000` so that it runs every two days: +Reschedules $JOB ID `1000` so that it runs every two days: ```sql SELECT alter_job(1000, schedule_interval => INTERVAL '2 days'); @@ -85,15 +84,15 @@ FROM timescaledb_information.jobs WHERE proc_name = 'policy_compression' AND hypertable_name = 'conditions' ``` -Reschedules continuous aggregate job ID `1000` so that it next runs at 9:00:00 on 15 March, 2020: +Reschedules continuous aggregate $JOB ID `1000` so that it next runs at 9:00:00 on 15 March, 2020: ```sql SELECT alter_job(1000, next_start => '2020-03-15 09:00:00.0+00'); ``` -### Calculation of next start on failure +## Calculation of next start on failure -When a job run results in a runtime failure, the next start of the job is calculated taking into account both its `retry_period` and `schedule_interval`. +When a $JOB run results in a runtime failure, the next start of the $JOB is calculated taking into account both its `retry_period` and `schedule_interval`. The `next_start` time is calculated using the following formula: ``` next_start = finish_time + consecutive_failures * retry_period ± jitter @@ -101,14 +100,16 @@ next_start = finish_time + consecutive_failures * retry_period ± jitter where jitter (± 13%) is added to avoid the "thundering herds" effect. + To ensure that the `next_start` time is not put off indefinitely or produce timestamps so large they end up out of range, it is capped at 5*`schedule_interval`. Also, more than 20 consecutive failures are not considered, so if the number of consecutive failures is higher, then it multiplies by 20. -Additionally, in the case of jobs with fixed schedules, the system ensures that if the next start calculated as specified, surpasses the next scheduled execution, then the job is executed again at the next scheduled slot and not after that. This ensures that the job does not miss scheduled executions. +Additionally, for $JOBs with fixed schedules, the system ensures that if the next start ( calculated as specified), surpasses the next scheduled execution, the $JOB is executed again at the next scheduled slot and not after that. This ensures that the $JOB does not miss scheduled executions. + +There is a distinction between runtime failures that do not cause the $JOB to crash and $JOB crashes. +In the event of a $JOB crash, the next start calculation follows the same formula, +but it is always at least 5 minutes after the $JOB's last finish, to give an operator enough time to disable it before another crash. -Finally, there is a distinction between runtime failures that do not cause the job to crash and job crashes. -In the event of a job crash, the next start calculation still follows the above formula, -but it is always at least 5 minutes after the job's last finish, to give an operator enough time to disable it before another crash. diff --git a/api/jobs-automation/delete_job.md b/api/jobs-automation/delete_job.md new file mode 100644 index 0000000000..e771a53126 --- /dev/null +++ b/api/jobs-automation/delete_job.md @@ -0,0 +1,31 @@ +--- +api_name: delete_job() +excerpt: Delete a job from the automatic scheduler +topics: [jobs] +keywords: [jobs, delete] +tags: [background jobs, scheduled jobs, automation framework] +api: + license: community + type: function +--- + +# delete_job() Community + +Delete a $JOB registered with the automation framework. +This works for $JOBs as well as policies. + +If the $JOB is currently running, the process is terminated. + +## Required arguments + +|Name|Type|Description| +|---|---|---| +|`job_id`| INTEGER | TimescaleDB background $JOB id | + +## Sample usage + +Delete the $JOB with the $JOB id 1000: + +```sql +SELECT delete_job(1000); +``` diff --git a/api/jobs-automation/index.md b/api/jobs-automation/index.md new file mode 100644 index 0000000000..0f9917282b --- /dev/null +++ b/api/jobs-automation/index.md @@ -0,0 +1,21 @@ +--- +title: Jobs +excerpt: Timescale Cloud API reference for jobs. Includes SQL functions for adding, altering, deleting, and running a job +keywords: [jobs] +tags: [background jobs, scheduled jobs, automation framework] +--- + +# $JOB_CAPs Community + +$JOB_CAPs allow you to run functions and procedures implemented in a +language of your choice on a schedule within Timescale. This allows +automatic periodic tasks that are not covered by existing policies and +even enhancing existing policies with additional functionality. + +The following APIs and views allow you to manage the $JOBs that you create and +get details around automatic $JOBs used by other TimescaleDB functions like +continuous aggregation refresh policies and data retention policies. To view the +policies that you set or the policies that already exist, see +[informational views][informational-views]. + +[informational-views]: /api/:currentVersion:/informational-views/jobs/ diff --git a/api/run_job.md b/api/jobs-automation/run_job.md similarity index 62% rename from api/run_job.md rename to api/jobs-automation/run_job.md index d9643f0c9d..a3fcb33b62 100644 --- a/api/run_job.md +++ b/api/jobs-automation/run_job.md @@ -11,25 +11,27 @@ api: # run_job() Community -Run a previously registered job in the current session. -This works for user-defined actions as well as policies. +Run a previously registered $JOB in the current session. +This works for $JOB as well as policies. Since `run_job` is implemented as stored procedure it cannot be executed inside a SELECT query but has to be executed with `CALL`. -Any background worker job can be run in the foreground when executed with + +Any background worker $JOB can be run in the foreground when executed with `run_job`. You can use this with an increased log level to help debug problems. + -#### Required arguments +## Required arguments |Name|Description| |---|---| -|`job_id`| (INTEGER) TimescaleDB background job ID | +|`job_id`| (INTEGER) TimescaleDB background $JOB ID | -#### Sample usage +## Sample usage -Set log level shown to client to `DEBUG1` and run the job with the job ID 1000: +Set log level shown to client to `DEBUG1` and run the $JOB with the $JOB ID 1000: ```sql SET client_min_messages TO DEBUG1; diff --git a/api/page-index/page-index.js b/api/page-index/page-index.js index dd4b6e046c..98998f8668 100644 --- a/api/page-index/page-index.js +++ b/api/page-index/page-index.js @@ -7,7 +7,7 @@ module.exports = [ excerpt: "The API Reference is your go-to resource for all functions, VIEWs and special feature interfaces available with the TimescaleDB extension", description: - "User-defined actions allow you to run functions and procedures implemented in a language of your choice on a schedule within TimescaleDB. This allows...", + "Jobs allow you to run functions and procedures implemented in a language of your choice on a schedule within TimescaleDB. This allows...", children: [ { title: "Hypertables & chunks", @@ -242,7 +242,6 @@ module.exports = [ }, { title: "Data retention", - type: "directory", href: "data-retention", children: [ { @@ -256,9 +255,8 @@ module.exports = [ ], }, { - title: "Actions and automation", - type: "directory", - href: "actions", + title: "Jobs and automation", + href: "jobs-automation", children: [ { title: "add_job", @@ -494,7 +492,6 @@ module.exports = [ }, { title: "Informational views", - type: "directory", href: "informational-views", children: [ { diff --git a/api/remove_all_policies.md b/api/remove_all_policies.md index 00c1cd29a0..a0d2eb484e 100644 --- a/api/remove_all_policies.md +++ b/api/remove_all_policies.md @@ -47,7 +47,7 @@ Returns true if successful. Remove all policies from a continuous aggregate named `example_continuous_aggregate`. This includes refresh policies, compression -policies, and data retention policies. It doesn't include user-defined jobs: +policies, and data retention policies. It doesn't include custom $JOBs: ```sql SELECT timescaledb_experimental.remove_all_policies('example_continuous_aggregate'); diff --git a/lambda/redirects.js b/lambda/redirects.js index 9cf4b25eb9..c6821cae0c 100644 --- a/lambda/redirects.js +++ b/lambda/redirects.js @@ -492,7 +492,7 @@ module.exports = [ }, { from: "/timescaledb/latest/overview/core-concepts/user-defined-actions", - to: "https://docs.timescale.com/use-timescale/latest/user-defined-actions/", + to: "https://docs.timescale.com/use-timescale/latest/jobs/", }, { from: "/timescaledb/latest/overview/deployment-options", @@ -544,7 +544,7 @@ module.exports = [ }, { from: "/timescaledb/latest/using-timescaledb/actions", - to: "https://docs.timescale.com/use-timescale/latest/user-defined-actions/", + to: "https://docs.timescale.com/use-timescale/latest/jobs/", }, { from: "/timescaledb/latest/using-timescaledb/telemetry", @@ -913,4 +913,60 @@ module.exports = [ from: '/use-timescale/latest/integrations/observability-alerting/tableau/', to: 'https://docs.timescale.com/use-timescale/latest/integrations/tableau/', }, + { + from: '/use-timescale/latest/user-defined-actions/troubleshooting/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/troubleshooting/', + }, + { + from: '/use-timescale/latest/user-defined-actions/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/', + }, + { + from: '/use-timescale/latest/user-defined-actions/about-user-defined-actions/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/create-and-manage-jobs/', + }, + { + from: '/use-timescale/latest/user-defined-actions/create-and-register/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/create-and-manage-jobs/', + }, + { + from: '/use-timescale/latest/user-defined-actions/test-and-debug/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/create-and-manage-jobs/', + }, + { + from: '/use-timescale/latest/user-defined-actions/alter-and-delete/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/create-and-manage-jobs/', + }, + { + from: '/use-timescale/latest/user-defined-actions/example-generic-retention/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/example-generic-retention/', + }, + { + from: '/use-timescale/latest/user-defined-actions/example-tiered-storage/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/example-tiered-storage/', + }, + { + from: '/use-timescale/latest/user-defined-actions/example-downsample-and-compress/', + to: 'https://docs.timescale.com/use-timescale/latest/jobs/example-downsample-and-compress/', + }, + { + from: '/api/latest/actions/add_job/', + to: 'https://docs.timescale.com/api/latest/jobs-automation/add_job/', + }, + { + from: '/api/latest/actions/alter_job/', + to: 'https://docs.timescale.com/api/latest/jobs-automation/alter_job/', + }, + { + from: '/api/latest/actions/delete_job/', + to: 'https://docs.timescale.com/api/latest/jobs-automation/delete_job/', + }, + { + from: '/api/latest/actions/run_job/', + to: 'https://docs.timescale.com/api/latest/jobs-automation/run_job/', + }, + { + from: '/api/latest/actions/', + to: 'https://docs.timescale.com/api/latest/jobs-automation/', + }, ]; diff --git a/migrate/troubleshooting.md b/migrate/troubleshooting.md index 372cf9c17f..61b5db8780 100644 --- a/migrate/troubleshooting.md +++ b/migrate/troubleshooting.md @@ -235,7 +235,7 @@ pg_restore -d "$TARGET" \ ## Ownership of background jobs The `_timescaledb_config.bgw_jobs` table is used to manage background jobs. -This includes both user-defined actions, compression policies, retention +This includes custom $JOBs, compression policies, retention policies, and continuous aggregate refresh policies. On Timescale, this table has a trigger which ensures that no database user can create or modify jobs owned by another database user. This trigger can provide an obstacle for migrations. diff --git a/self-hosted/distributed-hypertables/about-distributed-hypertables.md b/self-hosted/distributed-hypertables/about-distributed-hypertables.md index 35c02a1afe..886970e9ee 100644 --- a/self-hosted/distributed-hypertables/about-distributed-hypertables.md +++ b/self-hosted/distributed-hypertables/about-distributed-hypertables.md @@ -205,7 +205,7 @@ for several reasons: it isn't guaranteed to have a consistent result across each node. An example non-immutable function is [`random()`][random-func], which depends on the current seed. -* The query includes a user-defined function. The access node assumes the +* The query includes a $JOB function. The access node assumes the function doesn't exist on the data nodes, and doesn't push it down. Timescale uses several optimizations to avoid these limitations, and push down diff --git a/self-hosted/manage-storage.md b/self-hosted/manage-storage.md index 219b14f027..c4d993dad5 100644 --- a/self-hosted/manage-storage.md +++ b/self-hosted/manage-storage.md @@ -139,8 +139,8 @@ Alternatively, you can set up a third tablespace called `history_indexes`, and move the data to `history` and the indexes to `history_indexes`. In Timescale 2.0 and later, you can use `move_chunk` with the job scheduler -framework. For more information, see the [user-defined actions section][actions]. +framework. For more information, see the [$JOBs section][jobs]. -[actions]: /use-timescale/:currentVersion:/user-defined-actions/ +[jobs]: /use-timescale/:currentVersion:/jobs/ [api-move-chunk]: /api/:currentVersion:/hypertable/move_chunk [api-reorder-chunk]: /api/:currentVersion:/hypertable/reorder_chunk diff --git a/self-hosted/multinode-timescaledb/multinode-maintenance.md b/self-hosted/multinode-timescaledb/multinode-maintenance.md index fb6e946913..b7f884af0f 100644 --- a/self-hosted/multinode-timescaledb/multinode-maintenance.md +++ b/self-hosted/multinode-timescaledb/multinode-maintenance.md @@ -30,9 +30,7 @@ completed their part of the distributed transaction can complete it later when they become available. This transaction log requires regular cleanup to remove transactions that have completed, and complete those that haven't. We highly recommended that you configure the access node to run a maintenance -job that regularly cleans up any unfinished distributed transactions. - -The custom maintenance job can be run as a user-defined action. For example: +job that regularly cleans up any unfinished distributed transactions. For example: diff --git a/use-timescale/jobs/create-and-manage-jobs.md b/use-timescale/jobs/create-and-manage-jobs.md new file mode 100644 index 0000000000..09d66a5911 --- /dev/null +++ b/use-timescale/jobs/create-and-manage-jobs.md @@ -0,0 +1,176 @@ +--- +title: Create and manage jobs +excerpt: Jobs are custom PostgreSQL functions and procedures that you set up to run on a schedule. Create, register, test, alter, and delete jobs in Timescale Cloud +products: [cloud, mst, self_hosted] +keywords: [jobs] +tags: [scheduled jobs, background jobs, automation framework] +--- + +import Prerequisites from "versionContent/_partials/_prereqs-cloud-and-self.mdx"; + +# Create and manage $JOBs + +$JOB_CAPs in Timescale are custom functions or procedures that run on a schedule that you define. This page explains how to create, test, alter, and delete a $JOB. + +## Prerequisites + + + +## Create a $JOB + +To create a $JOB, create a [function][postgres-createfunction] or [procedure][postgres-createprocedure] that you want your $SERVICE_SHORT to execute, then set it up to run on a schedule. + + + +1. **Define a function or procedure in the language of your choice** + + Wrap it in a `CREATE` statement: + + ```sql + CREATE FUNCTION (required arguments) + RETURNS AS $$ + DECLARE + ; + BEGIN + ; + RETURN { | value } + END; LANGUAGE ; + ``` + + For example, to create a function that returns the total row count of a table within a $SERVICE_SHORT: + + ```sql + CREATE FUNCTION totalRecords (job_id INT DEFAULT NULL, config JSONB DEFAULT NULL) + RETURNS integer AS $total$ + declare + total integer; + BEGIN + SELECT count(*) into total FROM fill_measurements; + RETURN total; + END; + $total$ LANGUAGE plpgsql; + ``` + + `job_id` and `config` are required arguments. This returns `CREATE FUNCTION` to indicate that the function has successfully been created. + +1. **Call the function to validate** + + For example: + + ```sql + select totalRecords(); + ``` + + The result looks like this: + + ```sql + totalrecords + -------------- + 48600500 + (1 row) + ``` + +1. **Register your $JOB with [`add_job`][api-add_job]** + + Pass the name of your $JOB, the schedule you want it to run on, and the content of your config. For the `config` value, if you don't need any special configuration parameters, set to `NULL`. For example, to run the `totalRecords` function every hour: + + ```sql + SELECT add_job('totalRecords', '1h', config => '{"hypertable":"metr"}'); + ``` + + The call returns a `job_id` and stores it along with `config` in the Timescale catalog. + + The $JOB runs on the schedule you set. You can also run it manually with [`run_job`][api-run_job] passing `job_id`. When the $JOB runs, `job_id` and `config` are passed as arguments. + +1. **Validate the job** + + List all currently registered $JOBs with [`timescaledb_information.jobs`][api-timescaledb_information-jobs]: + + ```sql + SELECT * FROM timescaledb_information.jobs; + ``` + + The result looks like this: + + ```sql + job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | config | next_start | hypertable_schema | hypertable_name + --------+----------------------------+-------------------+-------------+-------------+--------------+-----------------------+------------------+-----------+-----------+------------------------+-------------------------------+-------------------+----------------- + 1 | Telemetry Reporter [1] | 24:00:00 | 00:01:40 | -1 | 01:00:00 | _timescaledb_internal | policy_telemetry | postgres | t | | 2022-08-18 06:26:39.524065+00 | | + 1000 | User-Defined Action [1000] | 01:00:00 | 00:00:00 | -1 | 00:05:00 | public | totalrecords | tsdbadmin | t | {"hypertable": "metr"} | 2022-08-17 07:17:24.831698+00 | | + (2 rows) + ``` + + + +## Test and debug a $JOB + +To debug a $JOB, increase the log level and run the $JOB manually with [`run_job`][api-run_job] in the foreground. Because `run_job` is a stored procedure and not a function, run it with [`CALL`][postgres-call] instead of `SELECT`. + + + +1. **Set the minimum log level to `DEBUG1`** + + ```sql + SET client_min_messages TO DEBUG1; + ``` + +1. **Run the $JOB** + + Replace `1000` with your `job_id`: + + ```sql + CALL run_job(1000); + ``` + + + +## Alter and delete a $JOB + +Alter an existing $JOB with [`alter_job`][api-alter_job]. You can change both the config and the schedule on which the $JOB runs. + + + +1. **Change a $JOB's config** + + To replace the entire JSON config for a $JOB, call `alter_job` with a new `config` object. For example, replace the JSON config for a $JOB with ID `1000`: + + ```sql + SELECT alter_job(1000, config => '{"hypertable":"metrics"}'); + ``` + +1. **Turn off $JOB scheduling** + + To turn off automatic scheduling of a $JOB, call `alter_job` and set `scheduled`to `false`. You can still run the $JOB manually with `run_job`. For example, turn off the scheduling for a $JOB with ID `1000`: + + ```sql + SELECT alter_job(1000, scheduled => false); + ``` + +1. **Re-enable automatic scheduling of a $JOB** + + To re-enable automatic scheduling of a $JOB, call `alter_job` and set `scheduled` to `true`. For example, re-enable scheduling for a $JOB with ID `1000`: + + ```sql + SELECT alter_job(1000, scheduled => true); + ``` + +1. **Delete a $JOB with [`delete_job`][api-delete_job]** + + For example, to delete a $JOB with ID `1000`: + + ```sql + SELECT delete_job(1000); + ``` + + + +[postgres-call]: https://www.postgresql.org/docs/current/sql-call.html +[api-alter_job]: /api/:currentVersion:/jobs-automation/alter_job +[api-delete_job]: /api/:currentVersion:/jobs-automation/delete_job +[plpgsql]: https://www.postgresql.org/docs/current/plpgsql-overview.html +[api-add_job]: /api/:currentVersion:/jobs-automation/add_job +[api-run_job]: /api/:currentVersion:/jobs-automation/run_job +[api-timescaledb_information-jobs]: /api/:currentVersion:/informational-views/jobs/ +[postgres-createfunction]: https://www.postgresql.org/docs/current/xfunc.html +[postgres-createprocedure]: https://www.postgresql.org/docs/current/xproc.html +[plpgsql]: https://www.postgresql.org/docs/current/plpgsql-overview.html \ No newline at end of file diff --git a/use-timescale/user-defined-actions/example-downsample-and-compress.md b/use-timescale/jobs/example-downsample-and-compress.md similarity index 72% rename from use-timescale/user-defined-actions/example-downsample-and-compress.md rename to use-timescale/jobs/example-downsample-and-compress.md index 55f7a20180..46dd83065d 100644 --- a/use-timescale/user-defined-actions/example-downsample-and-compress.md +++ b/use-timescale/jobs/example-downsample-and-compress.md @@ -1,29 +1,27 @@ --- -title: Use a user-defined action to downsample and compress chunks -excerpt: Downsample and compress your hypertable chunks by combining a continuous aggregate refresh policy with compression. Not quite what you need? Create a custom user-defined action +title: Use a job to downsample and compress chunks +excerpt: Downsample and compress your hypertable chunks by combining a continuous aggregate refresh policy with compression. Not quite what you need? Create a custom job products: [cloud, mst, self_hosted] -keywords: [actions, compression, downsample] +keywords: [jobs, compression, downsample] --- -# Use a user-defined action to downsample and compress chunks +# Use a $JOB to downsample and compress $CHUNKs -Timescale lets you downsample and compress chunks by combining a -[continuous aggregate refresh policy][cagg-refresh] with a +Timescale lets you downsample and compress $CHUNKs by combining a +[$CAGG refresh policy][cagg-refresh] with a [compression policy][compression]. If you want to implement features not supported by those policies, you can write -a user-defined action to downsample and compress chunks instead. The following +a $JOB to downsample and compress $CHUNKs instead. The following example downsamples raw data to an average over hourly data. This is an -illustrative example, which can be done more simply with a continuous aggregate +illustrative example, which can be done more simply with a $CAGG policy. But you can make the query arbitrarily complex. -## Using a user-defined action to downsample and compress chunks - -1. Create a procedure that first queries the chunks of a hypertable to - determine if they are older than the `lag` parameter. The hypertable in this - example is named `metrics`. If the chunk is not already compressed, +1. Create a procedure that first queries the $CHUNKs of a $HYPERTABLE to + determine if they are older than the `lag` parameter. The $HYPERTABLE in this + example is named `metrics`. If the $CHUNK is not already compressed, downsample it by taking the average of the raw data. Then compress it. A temporary table is used to store the data while calculating the average. @@ -80,8 +78,8 @@ policy. But you can make the query arbitrarily complex. $$; ``` -1. Register the job to run daily. In the `config`, set `lag` to 12 months - to drop chunks containing data older than 12 months. +1. Register the $JOB to run daily. In the `config`, set `lag` to 12 months + to drop $CHUNKs containing data older than 12 months. ```sql SELECT add_job('downsample_compress','1d', config => '{"lag":"12 month"}'); diff --git a/use-timescale/user-defined-actions/example-generic-retention.md b/use-timescale/jobs/example-generic-retention.md similarity index 64% rename from use-timescale/user-defined-actions/example-generic-retention.md rename to use-timescale/jobs/example-generic-retention.md index 7875dd19fe..4a7e613db5 100644 --- a/use-timescale/user-defined-actions/example-generic-retention.md +++ b/use-timescale/jobs/example-generic-retention.md @@ -1,23 +1,21 @@ --- -title: Use a user-defined action to create a generic retention policy +title: Use a job to create a generic retention policy excerpt: In Timescale Cloud, you can add a data retention policy to a hypertable, to store data more efficiently. Take it one step further by creating a generic data retention policy for your entire service products: [cloud, mst, self_hosted] -keywords: [actions, data retention] +keywords: [jobs, data retention] --- -# Use a user-defined action to create a generic retention policy +# Use a $JOB to create a generic retention policy Timescale natively supports adding a -[data retention policy][data-retention-policy] to a hypertable. If you want to -add a generic data retention policy to _all_ hypertables, you can write a -user-defined action. +[data retention policy][data-retention-policy] to a $HYPERTABLE. If you want to +add a generic data retention policy to all $HYPERTABLEs, you can create a custom +$JOB. -## Using a user-defined action to create a generic retention policy - -1. Create a procedure that drops chunks from any hypertable if they are older - than the `drop_after` parameter. To get all hypertables, the +1. Create a procedure that drops $CHUNKs from any $HYPERTABLE if they are older + than the `drop_after` parameter. To get all $HYPERTABLEs, the `timescaledb_information.hypertables` table is queried. ```sql @@ -42,17 +40,19 @@ user-defined action. $$; ``` -1. Register the job to run daily. In the `config`, set `drop_after` to 12 months - to drop chunks containing data older than 12 months. +1. Register the $JOB to run daily. In the `config`, set `drop_after` to 12 months + to drop $CHUNKs containing data older than 12 months. ```sql SELECT add_job('generic_retention','1d', config => '{"drop_after":"12 month"}'); ``` + You can further refine this policy by adding filters to your procedure. For -example, add a `WHERE` clause to the `PERFORM` query to only drop chunks from -particular hypertables. +example, add a `WHERE` clause to the `PERFORM` query to only drop $CHUNKs from +particular $HYPERTABLEs. + diff --git a/use-timescale/user-defined-actions/example-tiered-storage.md b/use-timescale/jobs/example-tiered-storage.md similarity index 68% rename from use-timescale/user-defined-actions/example-tiered-storage.md rename to use-timescale/jobs/example-tiered-storage.md index 895e05c8a6..d9ce80c3d0 100644 --- a/use-timescale/user-defined-actions/example-tiered-storage.md +++ b/use-timescale/jobs/example-tiered-storage.md @@ -1,29 +1,28 @@ --- -title: Use a user-defined action to implement automatic tablespace management -excerpt: Save on storage costs for old data. Timescale Cloud enables you to automatically move hypertable chunks between tablespaces with move_chunk. +title: Use a job to implement automatic tablespace management +excerpt: Save on storage costs for old data. Timescale Cloud enables you to automatically move hypertable chunks between tablespaces with move_chunk products: [cloud, mst, self_hosted] -keywords: [actions, tablespaces] +keywords: [jobs, tablespaces] --- -# Use a user-defined action to implement automatic tablespace management +# Use a $JOB to implement automatic tablespace management -[Moving older data to a different tablespace][moving-data] can help you save on +Moving older data to a different tablespace can help you save on storage costs. Timescale supports automatic tablespace management by providing the `move_chunk` function to move chunks between tablespaces. To schedule the -moves automatically, you can write a user-defined action. +moves automatically, you can write a custom $JOB. -On the Timescale cloud platform, use the [Tiered Storage](https://docs.timescale.com/use-timescale/latest/data-tiering/) -offering which handles this by providing a [tiering policy API](https://docs.timescale.com/use-timescale/latest/data-tiering/enabling-data-tiering/#add-a-tiering-policy) -to move data to low-cost object storage backed by Amazon S3. + +On $CLOUD_LONG, use [tiered storage](https://docs.timescale.com/use-timescale/latest/data-tiering/) which handles this by providing a [tiering policy API](https://docs.timescale.com/use-timescale/latest/data-tiering/enabling-data-tiering/#add-a-tiering-policy) to move data to low-cost object storage backed by Amazon S3. - +To implement automatic $CHUNK moving with a $JOB: -## Using a user-defined action to implement automatic chunk moving + -1. Create a procedure that moves chunks to a different tablespace if they +1. Create a procedure that moves $CHUNKs to a different tablespace if they contain data older than the `lag` parameter. ```sql @@ -73,9 +72,9 @@ to move data to low-cost object storage backed by Amazon S3. $$; ``` -1. Register the job to run daily. In the config, set `hypertable` to `metrics` - to implement automatic chunk moves on the `metrics` hypertable. Set `lag` to - 12 months to move chunks containing data older than 12 months. Set +1. Register the $JOB to run daily. In the config, set `hypertable` to `metrics` + to implement automatic $CHUNK moves on the `metrics` $HYPERTABLE. Set `lag` to + 12 months to move $CHUNKs containing data older than 12 months. Set `tablespace` to the destination tablespace. ```sql @@ -88,5 +87,3 @@ to move data to low-cost object storage backed by Amazon S3. - -[moving-data]: /use-timescale/:currentVersion:/user-defined-actions/example-tiered-storage/ diff --git a/use-timescale/jobs/index.md b/use-timescale/jobs/index.md new file mode 100644 index 0000000000..761f1bbcd5 --- /dev/null +++ b/use-timescale/jobs/index.md @@ -0,0 +1,36 @@ +--- +title: Jobs in Timescale Cloud +excerpt: Increase efficiency and reduce overhead by automating routine tasks. Set up custom jobs on top of Timescale Cloud policies. Learn how to create them in your service +products: [cloud, mst, self_hosted] +keywords: [jobs] +tags: [jobs, scheduled jobs, background jobs, automation framework] +--- + +# $JOB_CAPs + +Timescale natively includes some $JOB-scheduling policies, such as: + +* [$CAGG_CAP policies][caggs] to automatically refresh $CAGGs +* [Compression policies][compressing] to compress historical data +* [Retention policies][retention] to drop historical data +* [Reordering policies][reordering] to reorder data within $CHUNKs + +If these don't cover your use case, you can create and schedule custom-defined $JOBs to run within +your $SERVICE_LONG. They help you automate periodic tasks that aren't covered by the native policies. + +In this section you: + +* [Create and manage $JOBs][create-jobs] +* Get over the following $JOB examples: + * Set up a [generic data retention][generic-retention] policy that applies across all $HYPERTABLEs + * Implement [automatic moving of $CHUNKs between tablespaces][manage-storage] + * Automatically [downsample and compress][downsample-compress] older $CHUNKs + +[create-jobs]: /use-timescale/:currentVersion:/jobs/create-and-manage-jobs/ +[downsample-compress]: /use-timescale/:currentVersion:/jobs/example-downsample-and-compress +[generic-retention]: /use-timescale/:currentVersion:/jobs/example-generic-retention +[manage-storage]: /use-timescale/:currentVersion:/jobs/example-tiered-storage/ +[caggs]: /use-timescale/:currentVersion:/continuous-aggregates/refresh-policies/ +[compressing]: /use-timescale/:currentVersion:/compression/ +[reordering]: /api/:currentVersion:/hypertable/add_reorder_policy/ +[retention]: /use-timescale/:currentVersion:/data-retention/create-a-retention-policy/ \ No newline at end of file diff --git a/use-timescale/page-index/page-index.js b/use-timescale/page-index/page-index.js index bf7deac314..a36da9aa29 100644 --- a/use-timescale/page-index/page-index.js +++ b/use-timescale/page-index/page-index.js @@ -694,44 +694,29 @@ module.exports = [ ] }, { - title: "User-defined actions", - href: "user-defined-actions", + title: "Jobs", + href: "jobs", children: [ { - title: "About user-defined actions", - href: "about-user-defined-actions", - excerpt: "Learn about user-defined actions", + title: "Create and manage jobs", + href: "create-and-manage-jobs", + excerpt: "Create, test, alter, and delete jobs", }, { - title: "Create and register a user-defined action", - href: "create-and-register", - excerpt: "Create a user-defined action", - }, - { - title: "Test and debug a user-defined action", - href: "test-and-debug", - excerpt: "Test and debug user-defined actions", - }, - { - title: "Alter and delete a user-defined action", - href: "alter-and-delete", - excerpt: "Edit and delete user-defined actions", - }, - { - title: "Use an action for generic retention", + title: "Use a job for generic retention", href: "example-generic-retention", - excerpt: "Example user-defined action for a retention policy", + excerpt: "A job example for a retention policy", }, { - title: "Use an action for tablespace management", + title: "Use a job for tablespace management", href: "example-tiered-storage", excerpt: - "Example user-defined action for automatically moving chunks between tablespaces", + "A job example for automatically moving chunks between tablespaces", }, { - title: "Use an action for downsampling and compression", + title: "Use a job for downsampling and compression", href: "example-downsample-and-compress", - excerpt: "Example user-defined action for downsample and compress", + excerpt: "A job example for downsampling and compressing data", }, { title: "Troubleshooting", diff --git a/use-timescale/user-defined-actions/about-user-defined-actions.md b/use-timescale/user-defined-actions/about-user-defined-actions.md deleted file mode 100644 index 8361381058..0000000000 --- a/use-timescale/user-defined-actions/about-user-defined-actions.md +++ /dev/null @@ -1,121 +0,0 @@ ---- -title: About user-defined actions -excerpt: User-defined actions are custom PostgreSQL functions and procedures that you set up to run on a schedule. This provides a set-it-and-forget-it approach to maintaining your service and data -products: [cloud, mst, self_hosted] -keywords: [actions] -tags: [user-defined actions, background jobs, scheduled jobs, automation framework] ---- - -# About user-defined actions - -You can create user-defined actions that periodically run custom functions or -procedures on your database. - -## PostgreSQL functions - -You can use PostgreSQL functions to create database operations that would -typically take several queries or steps. Functions can also be used by other -applications that interact with your database to perform tasks without -additional code. - -Your function needs to be wrapped in a `CREATE` statement. You can also use -`CREATE OR REPLACE`, although this is not recommended. This statement also -allows you to define the language of your commands in this statement. -User-defined actions can be written in any language of your choice. This guide uses -the SQL procedural language [PL/pgSQL][plpgsql]. - -The basic syntax of a function suitable for a user-defined action is: - -```sql -CREATE FUNCTION (required arguments) -RETURNS AS $$ - DECLARE - ; - BEGIN - ; - RETURN { | value } - END; LANGUAGE ; -``` - -This is an example of a function that returns the total row count of a table -within a database. - -Use this code to create the function. Note that `job_id` and `config` are required arguments. - -```sql -CREATE FUNCTION totalRecords (job_id INT DEFAULT NULL, config JSONB DEFAULT NULL) -RETURNS integer AS $total$ -declare - total integer; -BEGIN - SELECT count(*) into total FROM fill_measurements; - RETURN total; -END; -$total$ LANGUAGE plpgsql; -``` - -When you execute this code in psql, it returns `CREATE FUNCTION` to indicate -that it has successfully been created. You can then execute a call to the -function, like this: - -```sql -select totalRecords(); -``` - -The result looks like this: - -```sql - totalrecords --------------- - 48600500 -(1 row) -``` - -## The job scheduler - -When you have created your function, you need to register it with the job -scheduler to make the function run regularly. You can do with the `add_job` function. -This example adds the `totalRecords` function, and tells it to run every hour: - -```sql -SELECT add_job('totalRecords', '1h', config => '{"hypertable":"metr"}'); -``` - -For the `config` value, if you don't need any special configuration parameters, -you can use `NULL`. - -You can see a list of all your currently registered jobs by querying the job -scheduler, like this: - -```sql -SELECT * FROM timescaledb_information.jobs; -``` - -The result looks like this: - -```sql - job_id | application_name | schedule_interval | max_runtime | max_retries | retry_period | proc_schema | proc_name | owner | scheduled | config | next_start | hypertable_schema | hypertable_name ---------+----------------------------+-------------------+-------------+-------------+--------------+-----------------------+------------------+-----------+-----------+------------------------+-------------------------------+-------------------+----------------- - 1 | Telemetry Reporter [1] | 24:00:00 | 00:01:40 | -1 | 01:00:00 | _timescaledb_internal | policy_telemetry | postgres | t | | 2022-08-18 06:26:39.524065+00 | | - 1000 | User-Defined Action [1000] | 01:00:00 | 00:00:00 | -1 | 00:05:00 | public | totalrecords | tsdbadmin | t | {"hypertable": "metr"} | 2022-08-17 07:17:24.831698+00 | | -(2 rows) -``` - -## Native job-scheduling policies - -TimescaleDB natively includes some job-scheduling policies, such as: - -* [Continuous aggregate policies][caggs] to automatically refresh continuous - aggregates -* [Compression policies][compressing] to compress historical data -* [Retention policies][retention] to drop historical data -* [Reordering policies][reordering] to reorder data within chunks - -If these don't cover your use case, or if you want to expand upon the native -policy features, you can write a user-defined action. - -[caggs]: /use-timescale/:currentVersion:/continuous-aggregates/refresh-policies/ -[compressing]: /use-timescale/:currentVersion:/compression/ -[reordering]: /api/:currentVersion:/hypertable/add_reorder_policy/ -[retention]: /use-timescale/:currentVersion:/data-retention/create-a-retention-policy/ -[plpgsql]: https://www.postgresql.org/docs/current/plpgsql-overview.html diff --git a/use-timescale/user-defined-actions/alter-and-delete.md b/use-timescale/user-defined-actions/alter-and-delete.md deleted file mode 100644 index ee0726d34b..0000000000 --- a/use-timescale/user-defined-actions/alter-and-delete.md +++ /dev/null @@ -1,63 +0,0 @@ ---- -title: Alter and delete user-defined actions -excerpt: When a user-defined action has run its course, you can update, reschedule, or delete it. See how to manage user-defined actions in Timescale Cloud -products: [cloud, mst, self_hosted] -keywords: [actions] -tags: [user-defined actions, scheduled jobs, background jobs, automation framework] ---- - -# Alter and delete user-defined actions - -Alter an existing job by using [`alter_job`][api-alter_job]. You can change both -the config and the schedule on which the job runs. Delete a job by using -[`delete_job`][api-delete_job]. - - -To alter or delete a job, you need to know its `job_id`. Find the `job_id` by -querying the `timescaledb_information.jobs` table. - -```sql -SELECT * FROM timescaledb_information.jobs; -``` - - - -## Change a job's config - -To replace the entire JSON config for a job, call `alter_job` with a new -`config` object. For example, replace the JSON config for a job with id `1000`: - -```sql -SELECT alter_job(1000, config => '{"hypertable":"metrics"}'); -``` - -## Turn off scheduling of a job - -To turn off automatic scheduling of a job, call `alter_job` and set `scheduled` -to `false`. You can still run the job manually with `run_job`. For example, -turn off the scheduling for a job with id `1000`: - -```sql -SELECT alter_job(1000, scheduled => false); -``` - -## Re-enable automatic scheduling of a job - -To re-enable automatic scheduling of a job, call `alter_job` and set `scheduled` -to `true`. For example, re-enable scheduling for a job with id `1000`: - -```sql -SELECT alter_job(1000, scheduled => true); -``` - -## Delete a job - -Delete a job from the automation framework with [`delete_job`][api-delete_job]. -For example, to delete a job with id `1000`: - -```sql -SELECT delete_job(1000); -``` - -[api-alter_job]: /api/:currentVersion:/actions/alter_job -[api-delete_job]: /api/:currentVersion:/actions/delete_job diff --git a/use-timescale/user-defined-actions/create-and-register.md b/use-timescale/user-defined-actions/create-and-register.md deleted file mode 100644 index b8f11d8608..0000000000 --- a/use-timescale/user-defined-actions/create-and-register.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -title: Create and register user-defined actions -excerpt: Creating a user-defined action in your Timescale Cloud service is a two-step process. First, define a function or procedure, then register it with add_job. See usage examples -products: [cloud, mst, self_hosted] -keywords: [actions] -tags: [user-defined actions, scheduled jobs, background jobs, automation framework] ---- - -# Create and register user-defined actions - -Adding a user-defined action to your database is a 2-step process: - -1. [Define a function or procedure](#define-a-function-or-procedure) -1. [Register the action with the job scheduler](#register-an-action) - -## Define a function or procedure - -To create an action, begin by defining the -[function][postgres-createfunction] or -[procedure][postgres-createprocedure] that you want your database to execute. - -Your function needs to be wrapped in a `CREATE` statement. You can also use -`CREATE OR REPLACE`, although this is not recommended. This statement also -allows you to define the language of your commands in this statement. -User-defined actions can be written in any language you choose. This guide uses -the SQL procedural language [PL/pgSQL][plpgsql]. - -This example defines a simple procedure that raises a notice: - -```sql -CREATE PROCEDURE user_defined_action(job_id INT, config JSONB) - LANGUAGE PLPGSQL AS - $$ - BEGIN - RAISE NOTICE 'Executing job % with config %', job_id, config; - END - $$; -``` - -## Register an action - -To make the job scheduler run your action, you need to register it. Use the -[`add_job`][api-add_job] function. Supply the name of your action, the schedule -you want it to run on, and the content of your config. If your job needs no -parameters, use a NULL config. - -For example: - -```sql -SELECT add_job('user_defined_action', '1h', config => '{"hypertable":"metrics"}'); -``` - -The `add_job` call returns a `job_id`. It stores the `job_id` and `config` in -the TimescaleDB catalog. - -The action runs on the schedule you set. It also runs when you manually start it -by calling [`run_job`][api-run_job] with the `job_id`. When it runs, the -`job_id` and `config` are passed as arguments to the procedure. - -To list all currently registered jobs, query -[`timescaledb_information.jobs`][api-timescaledb_information-jobs]: - -```sql -SELECT * FROM timescaledb_information.jobs; -``` - -[api-add_job]: /api/:currentVersion:/actions/add_job -[api-run_job]: /api/:currentVersion:/actions/run_job -[api-timescaledb_information-jobs]: /api/:currentVersion:/informational-views/jobs/ -[postgres-createfunction]: https://www.postgresql.org/docs/current/xfunc.html -[postgres-createprocedure]: https://www.postgresql.org/docs/current/xproc.html -[plpgsql]: https://www.postgresql.org/docs/current/plpgsql-overview.html diff --git a/use-timescale/user-defined-actions/index.md b/use-timescale/user-defined-actions/index.md deleted file mode 100644 index 6cefc225f4..0000000000 --- a/use-timescale/user-defined-actions/index.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: User-defined actions -excerpt: Increase efficiency and reduce overhead by automating routine tasks. User-defined actions help you set up custom jobs on top of Timescale Cloud policies. Learn how to create them in your service -products: [cloud, mst, self_hosted] -keywords: [actions] -tags: [user-defined actions, scheduled jobs, background jobs, automation framework] ---- - -# User-defined actions - -User-defined actions let you schedule custom-defined procedures to run within -Timescale. They help you automate periodic tasks that aren't covered by -Timescale's policy features. - -In this section: - -* [Learn about user-defined actions][about-user-defined-actions] before you - start using them -* [Create and register][create-and-register] a user-defined action -* [Test and debug][test-and-debug] a user-defined action -* [Alter and delete][alter-and-delete] a user-defined action -* Examples of user-defined actions: - * Use a user-defined action to set up a - [generic data retention][generic-retention] policy that applies across - all hypertables - * Use a user-defined action to implement - [automatic moving of chunks between tablespaces][manage-storage] - * Use a user-defined action to automatically - [downsample and compress][downsample-compress] older chunks - -[about-user-defined-actions]: /use-timescale/:currentVersion:/user-defined-actions/about-user-defined-actions/ -[alter-and-delete]: /use-timescale/:currentVersion:/user-defined-actions/alter-and-delete/ -[create-and-register]: /use-timescale/:currentVersion:/user-defined-actions/create-and-register/ -[downsample-compress]: /use-timescale/:currentVersion:/user-defined-actions/example-downsample-and-compress -[generic-retention]: /use-timescale/:currentVersion:/user-defined-actions/example-generic-retention -[manage-storage]: /use-timescale/:currentVersion:/user-defined-actions/example-tiered-storage/ -[test-and-debug]: /use-timescale/:currentVersion:/user-defined-actions/test-and-debug/ diff --git a/use-timescale/user-defined-actions/test-and-debug.md b/use-timescale/user-defined-actions/test-and-debug.md deleted file mode 100644 index d21f5a664a..0000000000 --- a/use-timescale/user-defined-actions/test-and-debug.md +++ /dev/null @@ -1,46 +0,0 @@ ---- -title: Test and debug a user-defined action -excerpt: Having issues while setting up user-defined actions in Timescale Cloud? Find solutions to the most common ones -products: [cloud, mst, self_hosted] -keywords: [actions, debug] ---- - -# Test and debug a user-defined action - -To debug an action, you can increase the log level and run the job manually in -the foreground. - -Use the [`run_job`][api-run_job] procedure, which takes a `job_id` argument. -Because `run_job` is a stored procedure and not a function, it needs to be -executed with [`CALL`][postgres-call] instead of `SELECT`. - - - -## Testing and debugging a job - -1. Change the minimum log level shown to the client. Set it to `DEBUG1`. - - ```sql - SET client_min_messages TO DEBUG1; - ``` - -1. Run the job. Replace `1000` with your actual `job_id`. - - ```sql - CALL run_job(1000); - ``` - - -To find the `job_id` for your job, query the `timescaledb_information.jobs` -table. - -```sql -SELECT * FROM timescaledb_information.jobs; -``` - - - - - -[api-run_job]: /api/:currentVersion:/actions/run_job -[postgres-call]: https://www.postgresql.org/docs/current/sql-call.html From 8dfd8a10845b1f86e22ee15d0ce4aa3ac01df8df Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Tue, 25 Mar 2025 17:02:33 +0200 Subject: [PATCH 40/90] Update wording in jobs index (#3967) * update --- use-timescale/jobs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/use-timescale/jobs/index.md b/use-timescale/jobs/index.md index 761f1bbcd5..d43f9e88a2 100644 --- a/use-timescale/jobs/index.md +++ b/use-timescale/jobs/index.md @@ -21,7 +21,7 @@ your $SERVICE_LONG. They help you automate periodic tasks that aren't covered by In this section you: * [Create and manage $JOBs][create-jobs] -* Get over the following $JOB examples: +* See the following examples: * Set up a [generic data retention][generic-retention] policy that applies across all $HYPERTABLEs * Implement [automatic moving of $CHUNKs between tablespaces][manage-storage] * Automatically [downsample and compress][downsample-compress] older $CHUNKs From a50328215426cd41899a5bb4279bb4c137906a0a Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Wed, 26 Mar 2025 11:38:09 +0200 Subject: [PATCH 41/90] link fix (#3969) --- api/informational-views/job_errors.md | 2 +- api/informational-views/job_history.md | 2 +- api/page-index/page-index.js | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/api/informational-views/job_errors.md b/api/informational-views/job_errors.md index 5e6b85b64c..601a62ed3d 100644 --- a/api/informational-views/job_errors.md +++ b/api/informational-views/job_errors.md @@ -84,4 +84,4 @@ For example, the owner can change the retention interval like this: SELECT alter_job(id,config:=jsonb_set(config,'{drop_after}', '"2 weeks"')) FROM _timescaledb_config.bgw_job WHERE id = 2; ``` -[jobs]: /api/:currentVersion:/jobs/ +[jobs]: /api/:currentVersion:/jobs-automation/ diff --git a/api/informational-views/job_history.md b/api/informational-views/job_history.md index 2dda21c448..c3ebe7c96c 100644 --- a/api/informational-views/job_history.md +++ b/api/informational-views/job_history.md @@ -89,4 +89,4 @@ For example, the owner can change the retention interval like this: SELECT alter_job(id,config:=jsonb_set(config,'{drop_after}', '"2 weeks"')) FROM _timescaledb_config.bgw_job WHERE id = 3; ``` -[jobs]: /api/:currentVersion:/jobs/ +[jobs]: /api/:currentVersion:/jobs-automation/ diff --git a/api/page-index/page-index.js b/api/page-index/page-index.js index 98998f8668..878f43d68b 100644 --- a/api/page-index/page-index.js +++ b/api/page-index/page-index.js @@ -242,6 +242,7 @@ module.exports = [ }, { title: "Data retention", + type: "directory", href: "data-retention", children: [ { From 0febbcd841170596cd460b60947b27ff7d2bccfe Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Thu, 27 Mar 2025 09:44:43 +0200 Subject: [PATCH 42/90] Update the docs readme (#3930) Co-authored-by: Iain Cox --- CONTRIBUTING.md | 54 ++--- README.md | 350 +++--------------------------- _multi-code-block.md | 37 ---- _partials/_formatting_examples.md | 162 ++++++++++++-- _procedure-block.md | 28 --- about/contribute-to-timescale.md | 13 +- 6 files changed, 197 insertions(+), 447 deletions(-) delete mode 100644 _multi-code-block.md delete mode 100644 _procedure-block.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f65a02313e..5ebd236f9d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,31 +2,7 @@ Timescale documentation is open for contribution from all community members. The current source is in this repository. -This page explains the process and guidelines to follow when contributing to Timescale documentation. - -## Contribution process - -You can contribute to Timescale documentation in the following ways: - -- [Create an issue][docs-issues] in this repository and describe the proposed change. Our doc team will take care of it. -- Update the docs yourself and have your change reviewed and published by our doc team. - -To make the contribution yourself: - -1. Get the documentation source: - - - No write access? [Fork this repository][github-fork]. - - Already have a write access? [Clone this repository][github-clone]. - -2. Create a branch from `latest`, make your changes, and raise a pull request back to `latest`. - -3. Sign a Contributor License Agreement (CLA). - - You have to sign the CLA only the first time you raise a PR. This helps to ensure that the community is free to use your contributions. - -4. Review your changes. - - The documentation site is generated in a separate private repository using [Gatsby][gatsby]. Once you raise a PR for any branch, GitHub will **automatically** generate a preview for your changes and attach the link in the comments. Any new commits will be visible at the same URL. If you don't see the latest changes, try an incognito browser window. Automated builds are not available for PRs from forked repositories. +This page explains the structure and language guidelines for contributing to Timescale documentation. See the [README][readme] for how to contribute. ## Language @@ -58,14 +34,14 @@ Each major doc section has a dedicated directory with `.md` files inside, repres - **Troubleshooting pages** are not written as whole Markdown files, but are programmatically assembled from individual files in the`_troubleshooting` folder. Each entry describes a single troubleshooting case and its solution, and contains the following front matter: - |Key| Type |Required| Description | - |-|-------|-|--------------------------------------------------------------------------| - |`title`| string |✅| The title of the troubleshooting entry, displayed as a heading above it | - |`section`| The literal string `troubleshooting` |✅| Must be `troubleshooting`, used to identify troubleshooting entries during site build | - |`products` or `topics`| array of strings |✅ (can have either or both, but must have at least one)| The products or topics related to the entry. The entry will show up on the troubleshooting pages for the listed products and topics. | + |Key| Type |Required| Description | + |-|-------|-|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| + |`title`| string |✅| The title of the troubleshooting entry, displayed as a heading above it | + |`section`| The literal string `troubleshooting` |✅| Must be `troubleshooting`, used to identify troubleshooting entries during site build | + |`products` or `topics`| array of strings |✅ (can have either or both, but must have at least one)| The products or topics related to the entry. The entry shows up on the troubleshooting pages for the listed products and topics. | |`errors`| object of form `{language: string, message: string}` |❌| The error, if any, related to the troubleshooting entry. Displayed as a code block right underneath the title. `language` is the programming language to use for syntax highlighting. | - |`keywords`| array of strings |❌| These are displayed at the bottom of every troubleshooting page. Each keyword links to a collection of all pages associated with that keyword. | - |`tags`| array of strings |❌| Concepts, actions, or things associated with the troubleshooting entry. These are not displayed in the UI, but they affect the calculation of related pages. | + |`keywords`| array of strings |❌| These are displayed at the bottom of every troubleshooting page. Each keyword links to a collection of all pages associated with that keyword. | + |`tags`| array of strings |❌| Concepts, actions, or things associated with the troubleshooting entry. These are not displayed in the UI, but they affect the calculation of related pages. | Beneath the front matter, describe the error and its solution in regular Markdown. You can also use any other components allowed within the docs site. @@ -114,13 +90,13 @@ To change the structure, add or delete pages in a section, modify the correspond | `children` | Array of page entries | ❌ | Child pages of the current page. For regular pages, the children should be located in a directory with the same name as the parent. The parent is the `index.md` file in that directory. For`directory` pages, the children should be located in the same directory as the parent. | | `pageComponents` | One of `[['featured-cards'], ['content-list']]` | ❌ | Any page that has child pages can list its children in either card or list style at the bottom of the page. Specify the desired style with this key. | | `featuredChildren` | Array of URLs | ❌ | Similar to `pageComponents`, this displays the children of the current page, but only the selected ones. | -| `index` | string | ❌ | If a section landing page needs to be different from the `index.md` file in that directory, this field specifies the corresponding Markdown file name. | +| `index` | string | ❌ | If a section landing page needs to be different from the `index.md` file in that directory, this field specifies the corresponding Markdown file name. | ## Reuse text in multiple pages Partials allow you to reuse snippets of content in multiple places. All partials live in the `_partials` top-level directory. To make a new partial, create a new -`.md` file in this directory. The filename must start with an underscore. Then import it into the target page and reference in the relevant place. See [Formatting examples][formatting]. +`.md` file in this directory. The filename must start with an underscore. Then import it into the target page as an `.mdx` file and reference in the relevant place. See [Formatting examples][formatting]. ## Formatting @@ -155,7 +131,7 @@ See [Formatting examples][formatting] for details. When adding screenshots to the docs, aim for a full-screen view to provide better context. Reduce the size of your browser so there is as little wasted space as possible. -Attach the image to your issue or PR, and the doc team will upload and insert it for you. +Attach the image to your issue or PR, and the doc team uploads and inserts it for you. ## SEO optimization @@ -178,13 +154,11 @@ To make a documentation page more visible and clear for Google: The previous documentation source is in the deprecated repository called [docs.timescale.com-content][legacy-source]. [legacy-source]: https://github.com/timescale/docs.timescale.com-content -[docs-issues]: https://github.com/timescale/docs/issues -[github-fork]: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo -[github-clone]: https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository -[gatsby]: https://www.gatsbyjs.com/ + [google-style]: https://developers.google.com/style [markdown-syntax]: https://www.markdownguide.org/extended-syntax/ [github-docs]: https://github.com/timescale/docs [use-navigation]: use-timescale/page-index/page-index.js [formatting]: _partials/_formatting_examples.md -[variables]: https://docs.timescale.com/variables-for-contributors/ \ No newline at end of file +[variables]: https://docs.timescale.com/variables-for-contributors/ +[readme]: README.md \ No newline at end of file diff --git a/README.md b/README.md index 3fc140ce91..0197c58c5e 100644 --- a/README.md +++ b/README.md @@ -1,335 +1,55 @@ -# README +
    + + + + Timescale logo + +
    -This is the source for content for docs.timescale.com, starting with release 2.0. -All documentation for previous versions is in the deprecated repository called -[docs.timescale.com-content](https://github.com/timescale/docs.timescale.com-content). +
    -The documentation site is statically generated with -[Gatsby](https://www.gatsbyjs.com/). The source code is in a separate -repository, which pulls in content from this repository on each build. +

    Timescale Cloud is the modern PostgreSQL data platform for all your applications. It enhances PostgreSQL to handle time series, events, real-time analytics, and vector search—all in a single database alongside transactional workloads. +

    -**All files are written in standard markdown.** +[![Docs](https://img.shields.io/badge/Read_the_Timescale_docs-black?style=for-the-badge&logo=readthedocs&logoColor=white)](https://docs.timescale.com/) +[![SLACK](https://img.shields.io/badge/Ask_the_Timescale_community-black?style=for-the-badge&logo=slack&logoColor=white)](https://timescaledb.slack.com/archives/C4GT3N90X) +[![Try Timescale Cloud for free](https://img.shields.io/badge/Try_Timescale_for_free-black?style=for-the-badge&logo=timescale&logoColor=white)](https://console.cloud.timescale.com/signup) -## Contributing +
    -We welcome and appreciate any help the community can provide to make -Timescale's documentation better. +This repository contains the current source for Timescale documentation available at https://docs.timescale.com/. -You can help either by opening an -[issue](https://github.com/timescale/docs/issues) with -any suggestions or bug reports, or by forking this repository, making your own -contribution, and submitting a pull request. +We welcome contributions! You can contribute to Timescale documentation in the following ways: -Before we accept any contributions, Timescale contributors need to -sign the [Contributor License Agreement](https://cla-assistant.io/timescale/docs)(CLA). -By signing a CLA, we can ensure that the community is free and confident in its -ability to use your contributions. +- [Create an issue][docs-issues] in this repository and describe the proposed change. Our doc team takes care of it. +- Update the docs yourself and have your change reviewed and published by our doc team. -## Docs review +## Contribute to the Timescale docs -Once a PR has been started for any branch, a GitHub action will attach a unique -URL to preview your changes and allow others to more effectively review your -updates. The URL will be added as a comment to your PR. +To make the contribution yourself: -Any new commits to that PR will rebuild and be visible at the same URL. +1. Get the documentation source: -## Main sections and tree navigation + - No write access? [Fork this repository][github-fork]. + - Already have a write access? [Clone this repository][github-clone]. -Each major section that is incorporated into docs.timescale.com has a navigation -hierarchy governed by the appropriate `page-index.js` file and underlying -directory structure. Depending on the content you are adding or editing, you may -have to modify the `page-index.js` file that exists for that specific sub-project. +2. Create a branch from `latest`, make your changes, and raise a pull request back to `latest`. -For instance, if you are adding a new function to the 'API Reference' section, -then you need to modify the `page-index.js` file inside of `api-reference` folder, -specifically at `api-reference/page-index/page-index.js`. +3. Sign a Contributor License Agreement (CLA). -Most content is then held within the parent folder as defined by the hierarchy. For -instance, the **Overview** section is a parent to a number of other pages and -folders. Therefore, there is a folder in the repo for `overview` with more Markdown -files and folder inside as the hierarchy dictates. + You have to sign the CLA only the first time you raise a PR. This helps to ensure that the community is free to use your contributions. -As a general rule, every folder must have an `index.md` file inside which will -have the content and layout for the "landing page" of that parent folder. +4. Review your changes. -### `page-index.js` layout - -The format of this file is generally self explanatory, but the following -rules should be understood to ensure you get the results you expect. - -|Key|Type|Required|Description| -|-|-|-|-| -|`href`|string|Yes|The URL segment to use for the page. If there is a corresponding Markdown file, `href` must also match the name of the Markdown file, minus the file extension.| -|`title`|string|Yes|The title of the page, used as the page name within the navigation tree| -|`type`|One of `[directory, placeholder, redirect-to-child-page]`|No|If no type is specified, the page is built as a default page, turning the corresponding Markdown file into a webpage. If the type is `directory`, the corresponding file is turned into a webpage, _and_ the page becomes a directory. `directory` pages are the exception to the rule that the page index matches the file directory structure. Child pages of `directory` pages sit on the same level as the `directory` page inside the repository. They only become children during the site build. If the type is `placeholder`, an entry is made in the navigation tree, but a Markdown file is not converted into a webpage. The Markdown file doesn't even need to exist. Rather, the corresponding page is produced programmatically upon site build. If not produced, the link in the navigation tree returns a 404. If the type is `redirect-to-child-page`, an entry is made in the navigation tree, no page is built, and the link in the navigation tree goes directly to the first child.| -|`children`|Array of page entries|No|Child pages of the current page. If the parent page's type is not `directory`, the children should be located in a directory with the same name as the parent. The parent is the `index.md` file in that directory. If the parent page's type is `directory`, the children should be located in the same directory as the parent.| -|`pageComponents`|One of `[['featured-cards'], ['content-list']]`|No|Any page that has child pages can list its children in either card or list style at the bottom of the page. Specify the desired style with this key.| - -**Example** - -```js - href: "overview", - pageComponents: ["featured-cards"], - children: [ - { - title: "What is time-series data?", - href: "what-is-time-series-data", - }, - { - title: "Core concepts", - href: "core-concepts", - children : [ - { - title: "Hypertables & Chunks", - href: "hypertables-and-chunks", - }, - { - title: "Scaling", - href: "scaling", - }, - ... - ], - }, - ] -``` - -## Formatting and content rules - -### Internal page links - -None of the internal page links within these files work on GitHub inside of -the raw Markdown files that are being reviewed. Instead, the review link discussed -above should be utilized for verifying all internal links. - -Internal links do not need to include the domain name, . - -### External links: - -Input as-is. - -### Anchor tags - -By default, H1, H2, H3, and H4 headings have automatically generated anchor -tags. H2 headings also show up in the Table of Contents (on the right side of -the screen on wide windows). - -### Code blocks - -#### Command formatting - -When showing commands being entered from a command line, do not include a -character for the prompt. Do this: - -```bash -some_command -``` - -instead of this: - -```bash -some_command -``` - -or this: - -```bash -> some_command -``` - -Otherwise the code highlighter may be disrupted. - -#### Syntax highlighting - -When using a code block, add the appropriate language identifier after the -initial three backticks to provide the appropriate highlighting on the -rendered documentation site. - -Programming language samples aside, most code blocks are one of: -`bash`, `sql`, `json`. - -#### Line numbers and copy button - -Code blocks have line numbers and copy buttons by default. To remove line -numbers and copy buttons on a case-by-case basis, use the `CodeBlock` component -directly rather than a native Markdown code block. - -To remove the copy button, set `canCopy` to `false`. - -To remove line numbers, set `showLineNumbers` to `false`. - -```markdown - -``` - -### Partials - -Partials allow you to reuse snippets of content in multiple places. All partials -live in the `_partials` top-level directory. To make a new partial, create a new -`.mdx` file. The filename must start with an underscore. - -To insert the partial in another document, put an import statement in the -document. The import statement goes before the content and after any -frontmatter. For example: - -`import PartialName from 'versionContent/_partials/_partial-name.mdx';` - -`PartialName` can be any CamelCased name, but it is recommended to use the -CamelCased filename to prevent name clashes, because partial names are global -across all MDX files. - -Where you need the partial to display, insert it as a self-closing tag: - -`` - -### General formatting conventions - -To maintain consistency, please follow these general rules. - -* Maintain text editor width for paragraphs at 80 characters. We ask you to do - this to assist in reviewing documentation changes. When text is very wide, it - is difficult to visually see where text has changed within a paragraph and keeping - a narrow width on text assists in making PRs easier to review. **Most editors such - as Visual Studio Code have settings to do this visually.** -* Most links should be reference-style links where the link address is at the - bottom of the page. The two exceptions are: - * Links within highlight blocks (Note, Important, or Warning). These must be inline links for now - * Links to anchors on the same page as the link itself. -* All functions, commands and standalone function arguments (ex. `SELECT`, - `time_bucket`) should be set as inline code within backticks ("\`command\`"). -* Functions should not be written with parentheses unless the function is - being written with arguments within the parentheses. -* "PostgreSQL" is the way to write the elephant database name, rather than - "Postgres." "TimescaleDB" refers to the database, "Timescale" refers to the - company. -* Use backticks when referring to the object of a user interface action. - For example: Click `Get started` to proceed with the tutorial. - -### Callout and highlight blocks - -To create a callout around a paragraph of text, wrap it with the following custom -React component tag. **Reminder**, any links within the callout text MUST have -inline link styles. - -The `type` can currently support a value of `"note"`, `"warning"`, -`"important"`, `"deprecation"` or `"cloud"`". `cloud` produces a callout -for content specifically referring to hosted Timescale. - -```html - -Callout text goes here... - -Example link style would [look like this](http://example.com/) - -``` - -### Tags - -You can use tags to indicate links to downloadable files, or to indicate -metadata about functions. Available tags: - -* Download tags: `Markdown link to download` -* Experimental tags: `` or - `` -* Toolkit tag: `` -* Community tag: `` - -By default, tags have a solid background and gray text. There is also a hollow -variant: - -```markdown -Text to display in a tag -``` - -### Procedures - -Procedures are used to indicate a sequence of steps. For syntax, see [the -procedure example](./_procedure-block.md). - -### Optional label - -Used to indicate an optional step within a procedure. Syntax: `` - -### Multi-code blocks - -Multi-code blocks are code blocks with a language or OS selector. For syntax, -see [the multi-code-block example](./_multi-code-block.md). - -### Tabs - -Tabs can be used to display content that differs based on a user selection. The -syntax is: - -```md - - - - -Content goes here - - - - - -Content goes here - - - - -``` - -Note that spacing is important. - -### Editing troubleshooting sections - -Troubleshooting pages are not written as whole Markdown files, but are -programmatically assembled from troubleshooting entries in the -`_troubleshooting` folder. Each entry describes a single troubleshooting case -and its solution, and contains the following frontmatter: - -|Key|Type|Required|Description| -|-|-|-|-| -|`title`|string|Yes|The title of the troubleshooting entry, displayed as a heading above it| -|`section`|the literal string `troubleshooting`|Yes|Must be `troubleshooting`, used to identify troubleshooting entries during site build| -|`products` or `topics`|array of strings|Yes (can have either or both, but must have at least one)|The products or topics related to the entry. The entry will show up on the troubleshooting pages for the listed products and topics.| -|`errors`|object of form `{language: string, message: string}`|No|The error, if any, related to the troubleshooting entry. Displayed as a code block right underneath the title. `language` is the programming language to use for syntax highlighting.| -|`keywords`|array of strings|No|These are displayed at the bottom of every troubleshooting page. Each keyword links to a collection of all pages associated with that keyword.| -|`tags`|array of strings|No|Concepts, actions, or things associated with the troubleshooting entry. These are not displayed in the UI, but they affect the calculation of related pages.| - -Beneath the frontmatter, describe the error and its solution in normal Markdown. -You can also use any other components allowed within the docs site. - -The entry shows up on the troubleshooting pages for its associated products and -topics. If the page doesn't already exist, add an entry for it in the page -index, setting `type` to `placeholder`. For more information, see the section on -[page index layout](#page-indexjs-layout). - -### Editing the API section - -There is a specific format for the API section which consists of: - -* **Function name** with empty parentheses (if function takes arguments). Ex. `add_dimension()` -* A brief, specific description of the function -* Any warnings necessary -* **Required Arguments** - * A table with columns for "Name," "Type," and "Description" -* **Optional Arguments** - * A table with columns for "Name," "Type," and "Description" -* Any specific instructions about the arguments, including valid types -* **Sample Usage** - * One or two literal examples of the function being used to demonstrate argument syntax. - -See the API file to get an idea. + The documentation site is generated in a separate private repository using [Gatsby][gatsby]. Once you raise a PR for any branch, GitHub **automatically** generates a preview for your changes and attach the link in the comments. Any new commits are visible at the same URL. If you don't see the latest changes, try an incognito browser window. Automated builds are not available for PRs from forked repositories. +See the [Contributing guide](CONTRIBUTING.md) for style and language guidance. +## Learn about Timescale +Timescale is PostgreSQL made powerful. To learn more about the company and its products, visit [timescale.com](https://www.timescale.com). +[docs-issues]: https://github.com/timescale/docs/issues +[github-fork]: https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/working-with-forks/fork-a-repo +[github-clone]: https://docs.github.com/en/repositories/creating-and-managing-repositories/cloning-a-repository +[gatsby]: https://www.gatsbyjs.com/ \ No newline at end of file diff --git a/_multi-code-block.md b/_multi-code-block.md deleted file mode 100644 index d638a5123e..0000000000 --- a/_multi-code-block.md +++ /dev/null @@ -1,37 +0,0 @@ - - - - - -```ruby -class AddTimescale < ActiveRecord::Migration[5.2] - def change - enable_extension("timescaledb") unless extensions.include? "timescaledb" - end -end -``` - - - - - -```python -def start: - if this: - return - -``` - - - - - -```python -def different_python: - return - -``` - - - - \ No newline at end of file diff --git a/_partials/_formatting_examples.md b/_partials/_formatting_examples.md index 486b3087af..34e8360369 100644 --- a/_partials/_formatting_examples.md +++ b/_partials/_formatting_examples.md @@ -4,10 +4,38 @@ This page illustrates and provides examples of the formatting available for Time ## Procedure -Use for a logical sequence of steps to achieve a goal. For example, create a hypertable. +Use for a logical sequence of steps to achieve a goal. For example, create a hypertable. ![Procedure example](https://assets.timescale.com/docs/images/procedure-syntax.png) +Syntax: + + + + 1. **Step 1 summary in bold** + + Step 1 explanation in details. + + ``` + step 1 code + ``` + + 1. **Step 2 summary in bold** + + Step 2 explanation in details. + + 1. Sub-step 1. + + ``` + Sub-step 1 code + ``` + + 1. Sub-step 2. + + 1. Sub-step 3. + + + See a [use example][data-tiering] in the docs. ## Highlight blocks @@ -18,44 +46,92 @@ Use sparingly and only if it's essential to attract the reader's attention. ![Note highlight](https://assets.timescale.com/docs/images/highlight-note.png) + Syntax: + + + + Additional relevant information worth highlighting. + + + See a [use example][disable-chunk-skipping] in the docs. - Important ![Important highlight](https://assets.timescale.com/docs/images/highlight-important.png) + Syntax: + + + + Important things to keep in mind. + + + See a [use example][decompress-chunks] in the docs. - Warning ![Caution highlight](https://assets.timescale.com/docs/images/highlight-warning.png) + Syntax: + + + + Caution! + + + See a [use example][alerting] in the docs. - Deprecation ![Deprecated highlight](https://assets.timescale.com/docs/images/highlight-deprecation.png) + Syntax: + + + + A deprecation notice. + + + See a [use example][deprecation] in the docs. - Cloud ![Cloud highlight](https://assets.timescale.com/docs/images/highlight-cloud.png) - Syntax example: + Syntax: - ```text - + - A note dealing specifically with Timescale Cloud. + A note dealing specifically with Timescale Cloud. - - ``` + ## Tabs ![Tabs](https://assets.timescale.com/docs/images/tabs-example.png) +Syntax: + + + + + + First tab content + + + + + + Second tab content + + + + + See a [use example][live-migration] in the docs. ## Code blocks @@ -68,13 +144,25 @@ To remove line numbers and the copy button, use the `CodeBlock` component with ` ![Custom code block](https://assets.timescale.com/docs/images/custom-code-block.png) -See a [use example][aggregation] in the docs. +Syntax: + + + +See a [use example][connection-pooling] in the docs. ## Multi-tab code blocks ![Multi-tab code block](https://assets.timescale.com/docs/images/multi-tab-code.png) -Syntax example: +Syntax: @@ -97,7 +185,7 @@ Syntax example: ```go - different python code + go code ``` @@ -110,43 +198,75 @@ Syntax example: ![Download tag](https://assets.timescale.com/docs/images/tag-download.png) - See a [use example][installation-windows] in the docs. + Syntax: + + Markdown link to download + + See a [use example][installation-windows] in the docs. - Experimental ![Experimental tag](https://assets.timescale.com/docs/images/tag-experimental.png) - See a [use example][time-bucket] in the docs. + Syntax: + + Experimental + + See a [use example][time-bucket] in the docs. - Toolkit ![Tooklit tag](https://assets.timescale.com/docs/images/tag-toolkit.png) - See a [use example][time-weighted-average] in the docs. + Syntax: + + Toolkit + + See a [use example][time-weighted-average] in the docs. - Community ![Community tag](https://assets.timescale.com/docs/images/tag-community.png) - See a [use example][remove-reorder-policy] in the docs. + Syntax: + + Community + + See a [use example][remove-reorder-policy] in the docs. - Hollow ![Hollow tag](https://assets.timescale.com/docs/images/hollow-tag.png) - Syntax example: + Syntax: - ```text - Text to display in a tag - ``` + ```text + Text to display in a tag + ``` ## Partials -Import a partial from the `_partials` directory and then reference it in the relevant part of the page. See a [use example][live-migration] in the docs. +Import a partial from the `_partials` directory and then reference it in the relevant part of the page. + +Syntax: + + import PartialName from "versionContent/_partials/_partial.mdx"; + + + +See a [use example][live-migration] in the docs. ## Links -Links should be [reference-style Markdown links][reference-links]. For example: +Links should be [reference-style Markdown links][reference-links]. + +Syntax: + + [Anchor][link-label] + + [link-label]: absolute or relative URL + +For example: [A link to the data tiering section in docs][data-tiering] @@ -157,7 +277,7 @@ Links should be [reference-style Markdown links][reference-links]. For example: [deprecation]: ../_partials/_deprecated.md [live-migration]: ../migrate/live-migration.md [fenced-code-blocks]: https://www.markdownguide.org/extended-syntax/#fenced-code-blocks -[aggregation]: ../getting-started/test-drive-timescale-features.md +[connection-pooling]: ../use-timescale/services/connection-pooling.md [installation-windows]: ../self-hosted/install/installation-windows.md [time-bucket]: ../api/time_bucket_ng.md [time-weighted-average]: ../api/time-weighted-averages.md diff --git a/_procedure-block.md b/_procedure-block.md deleted file mode 100644 index ef01fc2e8d..0000000000 --- a/_procedure-block.md +++ /dev/null @@ -1,28 +0,0 @@ - - -### Procedure Title - -1. Do this - - ```bash - code code code - ``` - -2. Then do this - - ```ruby - def function - print('hello') - ``` - -3. TimescaleDB is a time-series database, built on top of PostgreSQL. More than that, - however, it's a relational database for time-series. Developers who use TimescaleDB - get the benefit of a purpose-built time-series database, plus a classic relational - database (PostgreSQL), all in one, with full SQL support. - - ```python - def start: - print('start') - ``` - - \ No newline at end of file diff --git a/about/contribute-to-timescale.md b/about/contribute-to-timescale.md index 9be102edd9..0e16623c63 100644 --- a/about/contribute-to-timescale.md +++ b/about/contribute-to-timescale.md @@ -8,7 +8,7 @@ tags: [github] # Contribute to $COMPANY -$TIMESCALE_DB, pgai, pgvectorscale, TimescaleDB Toolkit, and the $COMPANY documentation are all open source. They are available in GitHub for you use, review, and update. This page shows you where you can add to $COMPANY products. +$TIMESCALE_DB, $PGAI_SHORT, $PGVECTORSCALE, $TOOLKIT_LONG, and the $COMPANY documentation are all open source. They are available in GitHub for you to use, review, and update. This page shows you where you can add to $COMPANY products. ## Contribute to the code for $COMPANY products @@ -19,17 +19,17 @@ $COMPANY appreciates any help the community can provide to make its products bet Head over to the $COMPANY source repositories to learn, review, and help improve our products! -* [TimescaleDB][timescaledb]: a PostgreSQL extension for high-performance real-time analytics on time-series and event data. -* [pgai][pgai]: a suite of tools to develop RAG, semantic search, and other AI applications more easily with PostgreSQL. -* [pgvectorscale][pgvectorscale]: a complement to pgvector for higher performance embedding search and cost-efficient storage for AI applications. -* [TimescaleDB Toolkit][toolkit]: all things analytics when using $TIMESCALE_DB, with a particular focus on developer ergonomics and performance. +* [$TIMESCALE_DB][timescaledb]: a $PG extension for high-performance real-time analytics on time-series and event data. +* [$PGAI_SHORT][pgai]: a suite of tools to develop RAG, semantic search, and other AI applications more easily with $PG. +* [$PGVECTORSCALE][pgvectorscale]: a complement to pgvector for higher performance embedding search and cost-efficient storage for AI applications. +* [$TOOLKIT_LONG][toolkit]: all things analytics when using $TIMESCALE_DB, with a particular focus on developer ergonomics and performance. ## Contribute to $COMPANY documentation $COMPANY documentation is hosted in the [docs GitHub repository][github-docs] and open for contribution from all community members. -See the [contribution guide][contribution-guide] for details. +See the [README][readme] and [contribution guide][contribution-guide] for details. [contribution-guide]: https://github.com/timescale/docs/blob/latest/CONTRIBUTING.md [github-docs]: https://github.com/timescale/docs @@ -37,4 +37,5 @@ See the [contribution guide][contribution-guide] for details. [pgai]: https://github.com/timescale/pgai/blob/main/CONTRIBUTING.md [pgvectorscale]: https://github.com/timescale/pgvectorscale/blob/main/CONTRIBUTING.md [toolkit]: https://github.com/timescale/timescaledb-toolkit +[readme]: https://github.com/timescale/docs/blob/latest/README.md From 9706fbdd4110dddeba66d18eb2f3db6713bee0b6 Mon Sep 17 00:00:00 2001 From: Iain Cox Date: Thu, 27 Mar 2025 12:31:19 +0100 Subject: [PATCH 43/90] fix: update link to say columnstore. (#3962) --- api/hypercore/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/hypercore/index.md b/api/hypercore/index.md index 6e961ae557..e8dbc58370 100644 --- a/api/hypercore/index.md +++ b/api/hypercore/index.md @@ -83,7 +83,7 @@ Chunks in the columnstore have the following limitations: * `ROW LEVEL SECURITY` is not supported on chunks in the columnstore. * To add unique constraints on chunks in the columnstore [convert_the chunk to rowstore][convert_to_rowstore], - add the constraints to your data, then [convert the chunk back to the rowstore][convert_to_columnstore]. + add the constraints to your data, then [convert the chunk back to the columnstore][convert_to_columnstore]. * [SkipScan][skipscan] does not currently work on chunks in the columnstore. From cb40327d0f2ce6a8a3f5c3fad742f06850af32be Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Thu, 27 Mar 2025 18:33:12 +0200 Subject: [PATCH 44/90] Get started update (data mode) (#3955) * draft Co-authored-by: Iain --- _partials/_cloud-connect-service.md | 89 +++++++++++++++++++ _partials/_cloud-connect.md | 57 ------------ _partials/_cloud-intro.md | 12 +-- _partials/_service-overview.md | 2 +- _partials/_timescale-intro.md | 2 +- getting-started/index.md | 4 +- getting-started/run-queries-from-console.md | 52 +++++------ getting-started/services.md | 10 +-- .../try-key-features-timescale-products.md | 52 +++++------ 9 files changed, 154 insertions(+), 126 deletions(-) create mode 100644 _partials/_cloud-connect-service.md delete mode 100644 _partials/_cloud-connect.md diff --git a/_partials/_cloud-connect-service.md b/_partials/_cloud-connect-service.md new file mode 100644 index 0000000000..472a3de959 --- /dev/null +++ b/_partials/_cloud-connect-service.md @@ -0,0 +1,89 @@ + + + +1. **Check your $SERVICE_SHORT is running correctly** + + In [$CONSOLE][services-portal], check that your $SERVICE_SHORT is marked as `Running`. + + ![Check $SERVICE_SHORT is running](https://assets.timescale.com/docs/images/console-services-view.png) + +1. **Connect to your $SERVICE_SHORT** + + Connect using SQL editor in $CONSOLE or with psql in the command line: + + + + + + + + 1. In $CONSOLE, select your $SERVICE_SHORT. + + 1. Click `SQL editor`. + + ![SQL editor](https://assets.timescale.com/docs/images/ops-view-sql-editor.png) + + 1. Run a test query: + + ```sql + SELECT CURRENT_DATE; + ``` + + This query gives you the current date, you have successfully connected to your $SERVICE_SHORT. + + + + And that is it, you are up and running. Enjoy developing with $COMPANY. + + + + + + + + 1. Install [psql][psql]. + + 1. Run the following command in the terminal using the service URL from the config file you have saved during service creation: + + ``` + psql "" + ``` + + 1. Run a test query: + + ```sql + SELECT CURRENT_DATE; + ``` + + This query returns the current date. You have successfully connected to your $SERVICE_SHORT. + + + + And that is it, you are up and running. Enjoy developing with $COMPANY. + + + + + + + + +You can also use the powerful [data mode][popsql] to connect and run queries on your service. + +Quick recap. You: +- Manage your $SERVICE_SHORTs in the [$OPS_MODE][portal-ops-mode] in $CONSOLE: add $READ_REPLICAs and enable + high availability, configure compression, change parameters, and so on. +- Analyze your data in the [$DATA_MODE][portal-data-mode] in $CONSOLE: write queries with + autocomplete, save them in folders, share them, create charts/dashboards, and much more. +- Store configuration and security information in your config file. + +[portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services +[portal-data-mode]: https://console.cloud.timescale.com/dashboard/services?popsql +[account-portal]: https://console.cloud.timescale.com/dashboard/account +[services-portal]: https://console.cloud.timescale.com/dashboard/services +[install-psql]: /use-timescale/:currentVersion:/integrations/psql/ +[popsql]: /getting-started/:currentVersion:/run-queries-from-console/#data-mode +[run-sqleditor]: /getting-started/:currentVersion:/run-queries-from-console/#sql-editor +[install-psql]: /use-timescale/:currentVersion:/integrations/psql/ +[hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/#hypertable-partitioning +[psql]: /use-timescale/:currentVersion:/integrations/psql/ diff --git a/_partials/_cloud-connect.md b/_partials/_cloud-connect.md deleted file mode 100644 index 66688b1f43..0000000000 --- a/_partials/_cloud-connect.md +++ /dev/null @@ -1,57 +0,0 @@ - - - -To connect to a $SERVICE_SHORT: - -1. **Check your $SERVICE_SHORT is running correctly** - - In [$CONSOLE][services-portal], check that your $SERVICE_SHORT is marked as `Running`. - - ![Check $SERVICE_SHORT is running](https://assets.timescale.com/docs/images/console-services-view.png) - -1. **Connect to your $SERVICE_SHORT** - - Use either: - - [$DATA_MODE_CAP][popsql] in $CONSOLE: - 1. In the [$DATA_MODE][portal-data-mode] in $CONSOLE, select a $SERVICE_SHORT and enter your password, then click `Connect`. - - You find your password in the config file you just downloaded. - 2. Select a query to edit: - ![Select a query to edit](https://assets.timescale.com/docs/images/data-mode-query-window.png) - - - [$SQL_EDITOR][run-sqleditor] in $CONSOLE: - - In the [$OPS_MODE][portal-ops-mode] in $CONSOLE, select a $SERVICE_SHORT, then click `SQL editor`. - - ![Check $SERVICE_SHORT is running](https://assets.timescale.com/docs/images/ops-view-sql-editor.png) - - - [psql][install-psql] on the command line: - - Connect to your $SERVICE_SHORT with the value of `Service URL` from the config file you - just saved: - - @:/tsdb?sslmode=require" - `} /> - - You can now run queries for this $SERVICE_SHORT. - -Quick recap. You: -- Manage your $SERVICE_SHORTs in the [$OPS_MODE][portal-ops-mode] in $CONSOLE: add $READ_REPLICAs and enable - high availability, configure compression, change parameters, and so on. -- Analyze your data in the [$DATA_MODE][portal-data-mode] in $CONSOLE: write queries with - autocomplete, save them in folders, share them, create charts/dashboards, and much more. -- Store configuration and security information in your config file. - - - - -[portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services -[portal-data-mode]: https://console.cloud.timescale.com/dashboard/services?popsql -[account-portal]: https://console.cloud.timescale.com/dashboard/account -[services-portal]: https://console.cloud.timescale.com/dashboard/services -[install-psql]: /use-timescale/:currentVersion:/integrations/psql/ -[popsql]: /getting-started/:currentVersion:/run-queries-from-console/#data-mode -[run-sqleditor]: /getting-started/:currentVersion:/run-queries-from-console/#sql-editor -[install-psql]: /use-timescale/:currentVersion:/integrations/psql/ -[hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/#hypertable-partitioning diff --git a/_partials/_cloud-intro.md b/_partials/_cloud-intro.md index 08de9fc2be..7a8f9d45fe 100644 --- a/_partials/_cloud-intro.md +++ b/_partials/_cloud-intro.md @@ -1,22 +1,22 @@ -$CLOUD_LONG is the modern PostgreSQL data platform for all your applications. It enhances PostgreSQL to handle time series, events, +$CLOUD_LONG is the modern $PG data platform for all your applications. It enhances $PG to handle time series, events, real-time analytics, and vector search—all in a single database alongside transactional workloads. -You get one system that handles live data ingestion, late and out-of-order updates, and low latency queries, with the performance, reliability, and scalability your app needs. Ideal for IoT, crypto, finance, SaaS, and a myriad other domains, $CLOUD_LONG allows you to build data-heavy, mission-critical apps while retaining the familiarity and reliability of PostgreSQL. +You get one system that handles live data ingestion, late and out-of-order updates, and low latency queries, with the performance, reliability, and scalability your app needs. Ideal for IoT, crypto, finance, SaaS, and a myriad other domains, $CLOUD_LONG allows you to build data-heavy, mission-critical apps while retaining the familiarity and reliability of $PG. -A $SERVICE_LONG is a single optimized 100% PostgreSQL database instance that you +A $SERVICE_LONG is a single optimized 100% $PG database instance that you use as is, or extend with the capabilities specific to your business needs: -- **Time-series and analytics**: PostgreSQL with $TIMESCALE_DB. The PostgreSQL you know and love, +- **Time-series and analytics**: $PG with $TIMESCALE_DB. The $PG you know and love, supercharged with functionality for storing and querying [time-series data][what-is-time-series] at scale for real-time analytics and other use cases. Get faster time-based queries with $HYPERTABLEs, $CAGGs, and columnar storage. Save on storage with native compression, data retention policies, and bottomless data tiering to Amazon S3. -- **AI and vector**: PostgreSQL with vector extensions. Use PostgreSQL as a vector database with +- **AI and vector**: $PG with vector extensions. Use $PG as a vector database with purpose built extensions for building AI applications from start to scale. Get fast and accurate similarity search with the pgvector and pgvectorscale extensions. Create vector embeddings and perform LLM reasoning on your data with the $PGAI_SHORT extension. - **PostgreSQL**: the trusted industry-standard RDBMS. Ideal for applications requiring strong data consistency, complex relationships, and advanced querying capabilities. Get ACID compliance, extensive SQL support, JSON handling, and extensibility through custom functions, data types, and extensions. $CLOUD_LONG continuously - monitors your $SERVICE_SHORTs and prevents common PostgreSQL out-of-memory crashes. + monitors your $SERVICE_SHORTs and prevents common $PG out-of-memory crashes. All $SERVICE_LONGs include the tooling you expect for production and developer environments: [live migration][live-migration], [automatic backups and PITR][automatic-backups], [high availability][high-availability], [$READ_REPLICAs][readreplica], [data forking][operations-forking], [connection pooling][connection-pooling], [tiered storage][data-tiering], diff --git a/_partials/_service-overview.md b/_partials/_service-overview.md index 7457efc781..25d576dd21 100644 --- a/_partials/_service-overview.md +++ b/_partials/_service-overview.md @@ -3,7 +3,7 @@ You manage your $SERVICE_LONGs and interact with your data in $CONSOLE using the | **$OPS_MODE_CAP** | **$DATA_MODE_CAP** | |-------------------|---------------------| | ![$CONSOLE $OPS_MODE][ops-mode] | ![$CONSOLE $DATA_MODE][data-mode] | -| **You use the $OPS_MODE to:**
    • Ensure data security with high availability and $READ_REPLICAs
    • Save money using compressed and tiered storage
    • Enable PostgreSQL extensions to add extra functionality
    • Increase security using $VPCs
    • Perform day-to-day administration
    | **Powered by $POPSQL, you use the $DATA_MODE to:**
    • Write queries with autocomplete
    • Visualize data with charts and dashboards
    • Schedule queries and dashboards for alerts or recurring reports
    • Share queries and dashboards
    | +| **You use the $OPS_MODE to:**
    • Ensure data security with high availability and $READ_REPLICAs
    • Save money using compressed and tiered storage
    • Enable $PG extensions to add extra functionality
    • Increase security using $VPCs
    • Perform day-to-day administration
    | **Powered by $POPSQL, you use the $DATA_MODE to:**
    • Write queries with autocomplete
    • Visualize data with charts and dashboards
    • Schedule queries and dashboards for alerts or recurring reports
    • Share queries and dashboards
    | [ops-mode]: https://assets.timescale.com/docs/images/ops-mode-overview.png [data-mode]: https://assets.timescale.com/docs/images/data-mode-overview.png \ No newline at end of file diff --git a/_partials/_timescale-intro.md b/_partials/_timescale-intro.md index c78b8c69e7..b69b5205e7 100644 --- a/_partials/_timescale-intro.md +++ b/_partials/_timescale-intro.md @@ -1,2 +1,2 @@ -Timescale extends PostgreSQL for all of your resource-intensive production workloads, so you +$COMPANY extends $PG for all of your resource-intensive production workloads, so you can build faster, scale further, and stay under budget. diff --git a/getting-started/index.md b/getting-started/index.md index e486f08d7e..b99ae1c5b2 100644 --- a/getting-started/index.md +++ b/getting-started/index.md @@ -20,9 +20,7 @@ This section shows you how to: 1. [Run queries from $CONSOLE][run-queries-from-console]: securely interact with your data in the $CONSOLE UI. 1. [Try the main features in $COMPANY products][test-drive]: rapidly implement the features in $CLOUD_LONG that enable you to ingest and query data faster while keeping the costs low. -What next? See the [use case tutorials][tutorials], interact with the data in your $SERVICE_LONG using -[your favorite programming language][connect-with-code], integrate your $SERVICE_LONG with a range of -[third-party tools][integrations], plain old [Use Timescale][use-timescale], or dive into [the API][use-the-api]. + [tutorials]: /tutorials/:currentVersion:/ [connect-with-code]: /quick-start/:currentVersion:/ diff --git a/getting-started/run-queries-from-console.md b/getting-started/run-queries-from-console.md index 2e560d87f3..bf334f7254 100644 --- a/getting-started/run-queries-from-console.md +++ b/getting-started/run-queries-from-console.md @@ -18,7 +18,7 @@ In $CONSOLE you can use the following ways to run SQL queries against your $SERV - [$DATA_MODE_CAP][run-popsql]: a rich experience powered by $POPSQL. You can write queries with autocomplete, save them in folders, share them, create charts/dashboards, and much more. -- [SQL assistant in the $DATA_MODE][sql-assistant]: write, fix, and organize SQL faster and more accurately. +- [$SQL_ASSISTANT_SHORT in the $DATA_MODE][sql-assistant]: write, fix, and organize SQL faster and more accurately. - [$SQL_EDITOR in the $OPS_MODE][run-sqleditor]: a simple $SQL_EDITOR in the $OPS_MODE that lets you run ad-hoc ephemeral queries. This is useful for quick one-off tasks like creating an index on a small table or @@ -37,7 +37,7 @@ Available features are: - **Real-time collaboration**: work with your team directly in the $DATA_MODE query editor with live presence and multiple cursors. - **[Schema browser](https://docs.popsql.com/docs/schema)**: understand the structure of your $SERVICE_SHORT and see usage data on tables and columns. -- **[SQL assistant][sql-assistant]**: write, fix, and organize SQL faster and more accurately using AI. +- **[$SQL_ASSISTANT_SHORT][sql-assistant]**: write, fix, and organize SQL faster and more accurately using AI. - **Autocomplete**: get suggestions as you type your queries. - **[Version history](https://docs.popsql.com/docs/version-history)**: access previous versions of a query from the built-in revision history, or connect to a git repo. @@ -46,7 +46,7 @@ Available features are: - **[Query variables](https://docs.popsql.com/docs/query-variables)**: use Liquid to parameterize your queries or use `if` statements. - **Cross-platform**: works from [$CONSOLE][portal-data-mode] or download the [desktop](https://popsql.com/download) app for macOS, Windows, and Linux. -- **Easy connection**: to $CLOUD_LONG, PostgreSQL, Redshift, Snowflake, BigQuery, MySQL, SQL Server, [and more](https://popsql.com/connections). +- **Easy connection**: to $CLOUD_LONG, $PG, Redshift, Snowflake, BigQuery, MySQL, SQL Server, [and more](https://popsql.com/connections). ### Connect to your $SERVICE_LONG in the $DATA_MODE @@ -106,7 +106,7 @@ If your $SERVICE_LONG runs inside a $VPC, do one of the following to enable acce #### What happens if another member of my $COMPANY project uses the $DATA_MODE? -The number of $DATA_MODE seats you are allocated depends on your [pricing plan][pricing-plan-features]. +The number of $DATA_MODE seats you are allocated depends on your [$PRICING_PLAN][pricing-plan-features]. #### Will using the $DATA_MODE affect the performance of my $SERVICE_LONG? @@ -126,9 +126,9 @@ If you'd like to prevent write operations such as insert or update, instead of using the `tsdbadmin` user, create a read-only user for your $SERVICE_SHORT and use that in the $DATA_MODE. -## SQL assistant +## $SQL_ASSISTANT_SHORT -SQL assistant in [$CONSOLE][portal-data-mode] helps you write, fix, and organize SQL faster and more accurately. +$SQL_ASSISTANT_SHORT in [$CONSOLE][portal-data-mode] helps you write, fix, and organize SQL faster and more accurately.
    @@ -138,47 +138,47 @@ SQL assistant in [$CONSOLE][portal-data-mode] helps you write, fix, and organize ### Key capabilities -SQL assistant offers a range of features to improve your SQL workflow, including: +$SQL_ASSISTANT_SHORT offers a range of features to improve your SQL workflow, including: -- **Real-time help**: SQL assistant provides in-context help for writing and understanding SQL. Use it to: +- **Real-time help**: $SQL_ASSISTANT_SHORT provides in-context help for writing and understanding SQL. Use it to: - - **Understand functions**: need to know how functions like `LAG()` or `ROW_NUMBER()` work? SQL assistant explains it with examples. - - **Interpret complex queries**: SQL assistant breaks down dense queries, giving you a clear view of each part. + - **Understand functions**: need to know how functions like `LAG()` or `ROW_NUMBER()` work? $SQL_ASSISTANT_SHORT explains it with examples. + - **Interpret complex queries**: $SQL_ASSISTANT_SHORT breaks down dense queries, giving you a clear view of each part. -- **Error resolution**: SQL assistant diagnoses errors as they happen, you can resolve issues without leaving your editor. Features include: +- **Error resolution**: $SQL_ASSISTANT_SHORT diagnoses errors as they happen, you can resolve issues without leaving your editor. Features include: - - **Error debugging**: if your query fails, SQL assistant identifies the issue and suggests a fix. - - **Performance tuning**: for slow queries, SQL assistant provides optimization suggestions to improve performance immediately. + - **Error debugging**: if your query fails, $SQL_ASSISTANT_SHORT identifies the issue and suggests a fix. + - **Performance tuning**: for slow queries, $SQL_ASSISTANT_SHORT provides optimization suggestions to improve performance immediately. - **Query organization**: to keep your query library organized, and help your team understand the - purpose of each query, SQL assistant automatically adds titles and summaries to your queries. + purpose of each query, $SQL_ASSISTANT_SHORT automatically adds titles and summaries to your queries. ### Limitations to keep in mind -For best results with SQL assistant: +For best results with $SQL_ASSISTANT_SHORT: -* **Schema awareness**: SQL assistant references schema data but may need extra context +* **Schema awareness**: $SQL_ASSISTANT_SHORT references schema data but may need extra context in complex environments. Specify tables, columns, or joins as needed. -* **Business logic**: SQL assistant does not inherently know specific business terms +* **Business logic**: $SQL_ASSISTANT_SHORT does not inherently know specific business terms such as active user. Define these terms clearly to improve results. ### Security, privacy, and data usage Security and privacy is prioritized in $CONSOLE. In [$DATA_MODE][portal-data-mode], project members -manage SQL assistant settings under [`User name` > `Settings` > `SQL Assistant`][sql-editor-settings]. +manage $SQL_ASSISTANT_SHORT settings under [`User name` > `Settings` > `SQL Assistant`][sql-editor-settings]. ![SQL assistant settings](https://assets.timescale.com/docs/images/sql-editor-preferences.png) -SQL assistant settings are: +$SQL_ASSISTANT_SHORT settings are: * **Opt-in features**: all AI features are off by default. Only [members][project-members] of your $CLOUD_LONG project can enable them. -* **Data protection**: your queries and data remain private and are not used for AI training. - SQL assistant operates with strict security protocols. +* **Data protection**: your queries and data remain private and are not used for AI training. + $SQL_ASSISTANT_SHORT operates with strict security protocols. * **Data usage**: to provide AI support, $COMPANY may share the query title, description and body, also the - database connection type such as PostgreSQL, and the schema. -* **Sample data**: to give the LLM more context so you have better SQL suggestions, enable sample data sharing in the SQL assistant preferences. -* **Telemetry**: to improve SQL assistant, $COMPANY collects telemetry and usage data, including prompts, responses, and query metadata. + database connection type such as $PG, and the schema. +* **Sample data**: to give the LLM more context so you have better SQL suggestions, enable sample data sharing in the $SQL_ASSISTANT_SHORT preferences. +* **Telemetry**: to improve $SQL_ASSISTANT_SHORT, $COMPANY collects telemetry and usage data, including prompts, responses, and query metadata. ## $OPS_MODE_CAP $SQL_EDITOR @@ -213,8 +213,8 @@ To use $SQL_EDITOR: ## Cloud $SQL_EDITOR licenses * **$SQL_EDITOR in the $OPS_MODE**: free for anyone with a [$CLOUD_LONG account][create-cloud-account]. -* **Data mode**: the number of seats you are allocated depends on your [pricing plan][pricing-plan-features]. - [SQL assistant][sql-assistant] is currently free for all users. In the future, limits or paid options may be +* **Data mode**: the number of seats you are allocated depends on your [$PRICING_PLAN][pricing-plan-features]. + [$SQL_ASSISTANT_SHORT][sql-assistant] is currently free for all users. In the future, limits or paid options may be introduced as we work to build the best experience. * **$POPSQL standalone**: there is a free plan available to everyone, as well as paid plans. See [$POPSQL Pricing][popsql-pricing] for full details. diff --git a/getting-started/services.md b/getting-started/services.md index ccc0a93db4..6db6c72ef6 100644 --- a/getting-started/services.md +++ b/getting-started/services.md @@ -7,7 +7,7 @@ content_group: Getting started --- import Install from "versionContent/_partials/_cloud-installation.mdx"; -import Connect from "versionContent/_partials/_cloud-connect.mdx"; +import Connect from "versionContent/_partials/_cloud-connect-service.mdx"; import ServiceOverview from "versionContent/_partials/_service-overview.mdx"; import CloudIntro from "versionContent/_partials/_cloud-intro.mdx"; import WhereNext from "versionContent/_partials/_where-to-next.mdx"; @@ -28,7 +28,7 @@ To start using $CLOUD_LONG for your data: ## Create a $SERVICE_LONG -Now that you have an active $CLOUD_LONG account, you create and manage your $SERVICE_SHORTs in $CONSOLE. When you create a $SERVICE_SHORT, you give a structure for your future data, which you then add manually or migrate from other services. All relevant $CLOUD_LONG features under your $PRICING_PLAN are automatically available when you create a $SERVICE_SHORT. +Now that you have an active $CLOUD_LONG account, you create and manage your $SERVICE_SHORTs in $CONSOLE. When you create a $SERVICE_SHORT, you effectively create a blank $PG database with additional $CLOUD_LONG features available under your $PRICING_PLAN. You then add or migrate your data into this database. @@ -36,7 +36,7 @@ Now that you have an active $CLOUD_LONG account, you create and manage your $SER ![Create a $SERVICE_LONG](https://assets.timescale.com/docs/images/create-timescale-service.png) -1. Follow the next steps in `Create a service` to configure the compute size, environment, availability, region, and $SERVICE_SHORT name. Then click `Create service`. +1. Follow the next steps in `Create a service` to configure the region, compute size, environment, availability, connectivity, and $SERVICE_SHORT name. Then click `Create service`. Your $SERVICE_SHORT is constructed and ready to use in a few seconds. @@ -52,12 +52,10 @@ shows you how to connect. ## Connect to your $SERVICE_SHORT -A $SERVICE_LONG comes with access control to its data. To be able to run queries and perform other operations, connect to the $SERVICE_SHORT with credentials generated during its creation. +To run queries and perform other operations, connect to your $SERVICE_SHORT: -And that is it, you are up and running. Enjoy developing with $COMPANY. - diff --git a/getting-started/try-key-features-timescale-products.md b/getting-started/try-key-features-timescale-products.md index af7f6502c4..58bafcd4f9 100644 --- a/getting-started/try-key-features-timescale-products.md +++ b/getting-started/try-key-features-timescale-products.md @@ -10,10 +10,10 @@ import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.md # Try the key $COMPANY features -$CLOUD_LONG scales PostgreSQL to ingest and query vast amounts of live data. $CLOUD_LONG +$CLOUD_LONG scales $PG to ingest and query vast amounts of live data. $CLOUD_LONG provides a range of features and optimizations that supercharge your queries while keeping the costs down. For example: -* The hypercore row-columnar engine makes queries up to 350x faster, ingests 44% faster, and reduces storage by 90%. +* The $HYPERCORE row-columnar engine makes queries up to 350x faster, ingests 44% faster, and reduces storage by 90%. * Tiered storage seamlessly moves your data from high performance storage for frequently accessed data to low cost bottomless storage for rarely accessed data. The following figure shows how $CLOUD_LONG optimizes your data for superfast real-time analytics and reduced @@ -30,7 +30,7 @@ ingest and query data faster while keeping the costs low. ## Optimize time-series data in $HYPERTABLEs -Time-series data represents how a system, process, or behavior changes over time. $HYPERTABLE_CAPs are PostgreSQL tables +Time-series data represents how a system, process, or behavior changes over time. $HYPERTABLE_CAPs are $PG tables that help you improve insert and query performance by automatically partitioning your data by time. Each $HYPERTABLE is made up of child tables called $CHUNKs. Each $CHUNK is assigned a range of time, and only contains data from that range. When you run a query, $CLOUD_LONG identifies the correct $CHUNK and runs the query on it, instead of going through the entire table. You can also tune $HYPERTABLEs to increase performance @@ -38,9 +38,9 @@ even more. ![Main features and tiered data](https://assets.timescale.com/docs/images/hypertable-structure.png ) -$HYPERTABLE_CAPs exist alongside regular PostgreSQL tables. -You use regular PostgreSQL tables for relational data, and interact with $HYPERTABLEs -and regular PostgreSQL tables in the same way. +$HYPERTABLE_CAPs exist alongside regular $PG tables. +You use regular $PG tables for relational data, and interact with $HYPERTABLEs +and regular $PG tables in the same way. This section shows you how to create regular tables and $HYPERTABLEs, and import relational and time-series data from external files. @@ -54,9 +54,9 @@ relational and time-series data from external files. This test dataset contains second-by-second trade data for the most-traded crypto-assets and a regular table of asset symbols and company names. - To import up to 100GB of data directly from your current PostgreSQL-based database, - [migrate with downtime][migrate-with-downtime] using native PostgreSQL tooling. To seamlessly import 100GB-10TB+ - of data, use the [live migration][migrate-live] tooling supplied by $COMPANY. To add data from non-PostgreSQL data sources, see [Import and ingest data][data-ingest]. + To import up to 100GB of data directly from your current $PG-based database, + [migrate with downtime][migrate-with-downtime] using native $PG tooling. To seamlessly import 100GB-10TB+ + of data, use the [live migration][migrate-live] tooling supplied by $COMPANY. To add data from non-$PG data sources, see [Import and ingest data][data-ingest]. 1. Upload data from the CSVs to your $SERVICE_SHORT: @@ -85,13 +85,13 @@ relational and time-series data from external files. ```bash psql -d "postgres://:@:/" ``` - You use your [connection details][connection-info] to fill in this PostgreSQL connection string. + You use your [connection details][connection-info] to fill in this $PG connection string. 2. Create tables for the data to import: - For the time-series data: - 1. In your sql client, create a normal PostgreSQL table: + 1. In your sql client, create a normal $PG table: ```sql CREATE TABLE crypto_ticks ( @@ -110,7 +110,7 @@ relational and time-series data from external files. - For the relational data: - In your sql client, create a normal PostgreSQL table: + In your sql client, create a normal $PG table: ```sql CREATE TABLE crypto_assets ( symbol TEXT NOT NULL, @@ -137,7 +137,7 @@ relational and time-series data from external files. 1. **Have a quick look at your data** - You query $HYPERTABLEs in exactly the same way as you would a relational PostgreSQL table. + You query $HYPERTABLEs in exactly the same way as you would a relational $PG table. Use one of the following SQL editors to run a query and see the data you uploaded: - **Data mode**: write queries, visualize data, and share your results in [$CONSOLE][portal-data-mode] for all your $SERVICE_LONGs. - **SQL editor**: write, fix, and organize SQL faster and more accurately in [$CONSOLE][portal-ops-mode] for a $SERVICE_LONG. @@ -158,7 +158,7 @@ and the $HYPERTABLE behind the $CAGG is automatically updated in the background. ![Reduced data calls with $CAGGs](https://assets.timescale.com/docs/images/continuous-aggregate.png ) You create $CAGGs on uncompressed data in high-performance storage. They continue to work -on [data in the columnstore][test-drive-enable-compression] +on [data in the $COLUMNSTORE][test-drive-enable-compression] and [rarely accessed data in tiered storage][test-drive-tiered-storage]. You can even create [$CAGGs on top of your $CAGGs][hierarchical-caggs]. @@ -181,7 +181,7 @@ $CONSOLE. You can also do this using psql. 1. **Create a $CAGG** For a $CAGG, data grouped using a $TIME_BUCKET is stored in a - PostgreSQL `MATERIALIZED VIEW` in a $HYPERTABLE. `timescaledb.continuous` ensures that this data + $PG `MATERIALIZED VIEW` in a $HYPERTABLE. `timescaledb.continuous` ensures that this data is always up to date. In your SQL editor, use the following code to create a $CAGG on the real-time data in the `crypto_ticks` table: @@ -268,14 +268,14 @@ a $CAGG, run the query part of the $CAGG ## Enhance query performance for analytics -Hypercore is the $CLOUD_LONG hybrid row-columnar storage engine, designed specifically for real-time analytics and -powered by time-series data. The advantage of hypercore is its ability to seamlessly switch between row-oriented and +$HYPERCORE_CAP is the $CLOUD_LONG hybrid row-columnar storage engine, designed specifically for real-time analytics and +powered by time-series data. The advantage of $HYPERCORE is its ability to seamlessly switch between row-oriented and column-oriented storage. This flexibility enables $CLOUD_LONG to deliver the best of both worlds, solving the key challenges in real-time analytics. ![Move from rowstore to columstore in hypercore](https://assets.timescale.com/docs/images/hypercore.png ) -When you convert $CHUNKs from the rowstore to the columnstore, multiple records are grouped into a single row. +When you convert $CHUNKs from the $ROWSTORE to the $COLUMNSTORE, multiple records are grouped into a single row. The columns of this row hold an array-like structure that stores all the data. Because a single row takes up less disk space, you can reduce your $CHUNK size by more than 90%, and can also speed up your queries. This helps you save on storage costs, and keeps your queries operating at lightning speed. @@ -285,9 +285,9 @@ regularly. For example, last week's market data. -1. **Enable hypercore on a $HYPERTABLE** +1. **Enable $HYPERCORE on a $HYPERTABLE** - Create a [job][job] that automatically moves $CHUNKs in a $HYPERTABLE to the columnstore at a specific time interval. + Create a [job][job] that automatically moves $CHUNKs in a $HYPERTABLE to the $COLUMNSTORE at a specific time interval. ```sql ALTER TABLE crypto_ticks SET ( @@ -297,7 +297,7 @@ regularly. For example, last week's market data. You [segmentby][alter-table-arguments] to speed up queries. -1. **Add a policy to convert $CHUNKs to the columnstore at a specific time interval** +1. **Add a policy to convert $CHUNKs to the $COLUMNSTORE at a specific time interval** For example, yesterday's data: ``` sql @@ -307,7 +307,7 @@ regularly. For example, last week's market data. 1. **View your data space saving** - When you convert data to the columnstore, as well as being optimized for analytics, it is compressed by more than + When you convert data to the $COLUMNSTORE, as well as being optimized for analytics, it is compressed by more than 90%. This helps you save on storage costs and keeps your queries operating at lightning speed. To see the amount of space saved: ``` sql @@ -328,8 +328,8 @@ regularly. For example, last week's market data. ## Slash storage charges -In the previous sections, you used $CAGGs to make fast analytical queries, and -hypercore to reduce storage costs on frequently accessed data. To reduce storage costs even more, +In the previous sections, you used $CAGGs to make fast analytical queries, and +$HYPERCORE to reduce storage costs on frequently accessed data. To reduce storage costs even more, you create tiering policies to move rarely accessed data to the object store. The object store is low-cost bottomless data storage built on Amazon S3. However, no matter the tier, you can [query your data when you need][querying-tiered-data]. $CLOUD_LONG seamlessly accesses the correct storage @@ -337,7 +337,7 @@ tier and generates the response. ![Tiered storage](https://assets.timescale.com/docs/images/tiered-storage.png ) -Data tiering is available in the [$SCALE and $ENTERPRISE][pricing-plans] pricing plans for $CLOUD_LONG. +Data tiering is available in the [$SCALE and $ENTERPRISE][pricing-plans] $PRICING_PLANs for $CLOUD_LONG. To set up data tiering: @@ -394,7 +394,7 @@ data loss during failover. ![$SERVICE_LONG replicas](https://assets.timescale.com/docs/images/ha-read-replica.png ) -High availability is available in the [$SCALE and $ENTERPRISE][pricing-plans] pricing plans for $CLOUD_LONG. +High availability is available in the [$SCALE and $ENTERPRISE][pricing-plans] $PRICING_PLANs for $CLOUD_LONG. From 18032961bc8331ca375f886251e4524ec83824f6 Mon Sep 17 00:00:00 2001 From: Iain Cox Date: Fri, 28 Mar 2025 13:38:06 +0100 Subject: [PATCH 45/90] fix: take table out of SQL tags (#3974) --- use-timescale/compression/manual-compression.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/use-timescale/compression/manual-compression.md b/use-timescale/compression/manual-compression.md index a967e1b970..a6832c02ee 100644 --- a/use-timescale/compression/manual-compression.md +++ b/use-timescale/compression/manual-compression.md @@ -29,12 +29,11 @@ use a hypertable called `example`, and compress chunks older than three days. 1. This returns a list of chunks. Take note of the chunks' names: - ```sql ||show_chunks| |---|---| |1|_timescaledb_internal_hyper_1_2_chunk| |2|_timescaledb_internal_hyper_1_3_chunk| - ``` + @@ -61,12 +60,11 @@ manually compress each one. The results show the chunks for the given hypertable, their compression status, and some other statistics: - ```sql |chunk_schema|chunk_name|compression_status|before_compression_table_bytes|before_compression_index_bytes|before_compression_toast_bytes|before_compression_total_bytes|after_compression_table_bytes|after_compression_index_bytes|after_compression_toast_bytes|after_compression_total_bytes|node_name| |---|---|---|---|---|---|---|---|---|---|---|---| |_timescaledb_internal|_hyper_1_1_chunk|Compressed|8192 bytes|16 kB|8192 bytes|32 kB|8192 bytes|16 kB|8192 bytes|32 kB|| |_timescaledb_internal|_hyper_1_20_chunk|Uncompressed|||||||||| - ``` + 1. Repeat for all chunks you want to compress. From cfba4043ca4af95830529a722f4aacdf7d5a057e Mon Sep 17 00:00:00 2001 From: Mats Kindahl Date: Mon, 31 Mar 2025 09:51:45 +0200 Subject: [PATCH 46/90] chore: fix syntax error in job function template (#3976) --- use-timescale/jobs/create-and-manage-jobs.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/use-timescale/jobs/create-and-manage-jobs.md b/use-timescale/jobs/create-and-manage-jobs.md index 09d66a5911..9826c0fe81 100644 --- a/use-timescale/jobs/create-and-manage-jobs.md +++ b/use-timescale/jobs/create-and-manage-jobs.md @@ -29,12 +29,13 @@ To create a $JOB, create a [function][postgres-createfunction] or [procedure][po ```sql CREATE FUNCTION (required arguments) RETURNS AS $$ - DECLARE - ; - BEGIN - ; - RETURN { | value } - END; LANGUAGE ; + DECLARE + ; + BEGIN + ; + RETURN { | value } + END; + $$ LANGUAGE ; ``` For example, to create a function that returns the total row count of a table within a $SERVICE_SHORT: @@ -173,4 +174,4 @@ Alter an existing $JOB with [`alter_job`][api-alter_job]. You can change both th [api-timescaledb_information-jobs]: /api/:currentVersion:/informational-views/jobs/ [postgres-createfunction]: https://www.postgresql.org/docs/current/xfunc.html [postgres-createprocedure]: https://www.postgresql.org/docs/current/xproc.html -[plpgsql]: https://www.postgresql.org/docs/current/plpgsql-overview.html \ No newline at end of file +[plpgsql]: https://www.postgresql.org/docs/current/plpgsql-overview.html From d7ece0e17a6e49fd956c2a0c6be4e5b142936979 Mon Sep 17 00:00:00 2001 From: Philip Krauss <35487337+philkra@users.noreply.github.com> Date: Mon, 31 Mar 2025 13:07:42 +0200 Subject: [PATCH 47/90] Add TSDB version 2.18 and 2.19 to the compat matrix (#3978) --- self-hosted/upgrades/upgrade-pg.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/self-hosted/upgrades/upgrade-pg.md b/self-hosted/upgrades/upgrade-pg.md index beb8282847..8a246e2b98 100644 --- a/self-hosted/upgrades/upgrade-pg.md +++ b/self-hosted/upgrades/upgrade-pg.md @@ -18,6 +18,8 @@ TimescaleDB is a PostgreSQL extension. Ensure that you upgrade to compatible ver ||PostgreSQL 17|PostgreSQL 16|PostgreSQL 15|PostgreSQL 14|PostgreSQL 13|PostgreSQL 12|PostgreSQL 11|PostgreSQL 10|PostgreSQL 9.6| |-|-|-|-|-|-|-|-|-| +|TimescaleDB 2.19 and higher|✅|✅|✅|✅|❌|❌|❌|❌|❌| +|TimescaleDB 2.18 and higher|✅|✅|✅|✅|❌|❌|❌|❌|❌| |TimescaleDB 2.17 and higher|✅|✅|✅|✅|❌|❌|❌|❌|❌| |TimescaleDB 2.16 and higher|❌|✅|✅|✅|❌|❌|❌|❌|❌|❌| |TimescaleDB 2.15 and higher|❌|✅|✅|✅|✅|❌|❌|❌|❌|❌| From 0fa5612af1eb9d50278424d6b9ab498b93a8fc41 Mon Sep 17 00:00:00 2001 From: Iain Cox Date: Tue, 1 Apr 2025 09:42:43 +0200 Subject: [PATCH 48/90] Add the update matrix to the Timescale Cloud maintenance page. (#3979) * chore: open sections and icons * chore: sort out upgrade pages. * chore: sort out upgrade pages. --- ...self_postgres_timescaledb_compatibility.md | 31 ++++++++-------- self-hosted/upgrades/major-upgrade.md | 3 +- self-hosted/upgrades/upgrade-pg.md | 19 ++-------- use-timescale/upgrades.md | 35 ++++++++++--------- 4 files changed, 39 insertions(+), 49 deletions(-) diff --git a/_partials/_migrate_self_postgres_timescaledb_compatibility.md b/_partials/_migrate_self_postgres_timescaledb_compatibility.md index 4e4b9473f2..65d5ef78cc 100644 --- a/_partials/_migrate_self_postgres_timescaledb_compatibility.md +++ b/_partials/_migrate_self_postgres_timescaledb_compatibility.md @@ -1,19 +1,20 @@ -| Version number |PostgreSQL 17|PostgreSQL 16|PostgreSQL 15|PostgreSQL 14|PostgreSQL 13|PostgreSQL 12|PostgreSQL 11|PostgreSQL 10| -|---------------------------|-|-|-|-|-|-|-|-| -| TimescaleDB
    2.18.x |✅|✅|✅|✅|❌|❌|❌|❌|❌| -| TimescaleDB
    2.17.x |✅|✅|✅|✅|❌|❌|❌|❌|❌| -| TimescaleDB
    2.16.x |❌|✅|✅|✅|❌|❌|❌|❌|❌|❌| -| TimescaleDB
    2.15.x |❌|✅|✅|✅|✅|❌|❌|❌|❌|❌| -| TimescaleDB
    2.14.x |❌|✅|✅|✅|✅|❌|❌|❌|❌|❌| -| TimescaleDB
    2.13.x |❌|✅|✅|✅|✅|❌|❌|❌|❌| -| TimescaleDB
    2.12.x |❌|❌|✅|✅|✅|❌|❌|❌|❌| -| TimescaleDB
    2.10.x |❌|❌|✅|✅|✅|✅|❌|❌|❌| -| TimescaleDB
    2.5 - 2.9 |❌|❌|❌|✅|✅|✅|❌|❌|❌| -| TimescaleDB
    2.4 |❌|❌|❌|❌|✅|✅|❌|❌|❌| -| TimescaleDB
    2.1 - 2.3 |❌|❌|❌|❌|✅|✅|✅|❌|❌| -| TimescaleDB
    2.0 |❌|❌|❌|❌|❌|✅|✅|❌|❌ -| TimescaleDB
    1.7 |❌|❌|❌|❌|❌|✅|✅|✅|✅| +| Version number |PostgreSQL 17|PostgreSQL 16|PostgreSQL 15|PostgreSQL 14|PostgreSQL 13|PostgreSQL 12|PostgreSQL 11|PostgreSQL 10| +|----------------------------------|-|-|-|-|-|-|-|-| +| TimescaleDB
    2.19 and higher |✅|✅|✅|✅|❌|❌|❌|❌|❌| +| TimescaleDB
    2.18.x |✅|✅|✅|✅|❌|❌|❌|❌|❌| +| TimescaleDB
    2.17.x |✅|✅|✅|✅|❌|❌|❌|❌|❌| +| TimescaleDB
    2.16.x |❌|✅|✅|✅|❌|❌|❌|❌|❌|❌| +| TimescaleDB
    2.15.x |❌|✅|✅|✅|✅|❌|❌|❌|❌|❌| +| TimescaleDB
    2.14.x |❌|✅|✅|✅|✅|❌|❌|❌|❌|❌| +| TimescaleDB
    2.13.x |❌|✅|✅|✅|✅|❌|❌|❌|❌| +| TimescaleDB
    2.12.x |❌|❌|✅|✅|✅|❌|❌|❌|❌| +| TimescaleDB
    2.10.x |❌|❌|✅|✅|✅|✅|❌|❌|❌| +| TimescaleDB
    2.5 - 2.9 |❌|❌|❌|✅|✅|✅|❌|❌|❌| +| TimescaleDB
    2.4 |❌|❌|❌|❌|✅|✅|❌|❌|❌| +| TimescaleDB
    2.1 - 2.3 |❌|❌|❌|❌|✅|✅|✅|❌|❌| +| TimescaleDB
    2.0 |❌|❌|❌|❌|❌|✅|✅|❌|❌ +| TimescaleDB
    1.7 |❌|❌|❌|❌|❌|✅|✅|✅|✅| We recommend not using TimescaleDB with PostgreSQL 17.1, 16.5, 15.9, 14.14, 13.17, 12.21. These minor versions [introduced a breaking binary interface change][postgres-breaking-change] that, diff --git a/self-hosted/upgrades/major-upgrade.md b/self-hosted/upgrades/major-upgrade.md index b6e2aa2d6e..d3017050d0 100644 --- a/self-hosted/upgrades/major-upgrade.md +++ b/self-hosted/upgrades/major-upgrade.md @@ -127,11 +127,12 @@ notice is shown. - + To upgrade TimescaleDB in a Docker container, see the [Docker container upgrades](/self-hosted/latest/upgrades/upgrade-docker) section. + ## Verify the updated policy settings and jobs diff --git a/self-hosted/upgrades/upgrade-pg.md b/self-hosted/upgrades/upgrade-pg.md index 8a246e2b98..d68f8b9e25 100644 --- a/self-hosted/upgrades/upgrade-pg.md +++ b/self-hosted/upgrades/upgrade-pg.md @@ -6,6 +6,7 @@ keywords: [upgrades, PostgreSQL, versions, compatibility] --- import PlanUpgrade from "versionContent/_partials/_plan_upgrade.mdx"; +import SupportMatrix from "versionContent/_partials/_migrate_self_postgres_timescaledb_compatibility.mdx"; import ConsiderCloud from "versionContent/_partials/_consider-cloud.mdx"; import PlanMigrationPath from "versionContent/_partials/_migrate_self_postgres_plan_migration_path.mdx"; @@ -15,23 +16,7 @@ TimescaleDB is a PostgreSQL extension. Ensure that you upgrade to compatible ver - -||PostgreSQL 17|PostgreSQL 16|PostgreSQL 15|PostgreSQL 14|PostgreSQL 13|PostgreSQL 12|PostgreSQL 11|PostgreSQL 10|PostgreSQL 9.6| -|-|-|-|-|-|-|-|-|-| -|TimescaleDB 2.19 and higher|✅|✅|✅|✅|❌|❌|❌|❌|❌| -|TimescaleDB 2.18 and higher|✅|✅|✅|✅|❌|❌|❌|❌|❌| -|TimescaleDB 2.17 and higher|✅|✅|✅|✅|❌|❌|❌|❌|❌| -|TimescaleDB 2.16 and higher|❌|✅|✅|✅|❌|❌|❌|❌|❌|❌| -|TimescaleDB 2.15 and higher|❌|✅|✅|✅|✅|❌|❌|❌|❌|❌| -|TimescaleDB 2.14 and higher|❌|✅|✅|✅|✅|❌|❌|❌|❌|❌| -|TimescaleDB 2.13 and higher|❌|✅|✅|✅|✅|❌|❌|❌|❌| -|TimescaleDB 2.12 and higher|❌|❌|✅|✅|✅|❌|❌|❌|❌| -|TimescaleDB 2.10 and higher|❌|❌|✅|✅|✅|✅|❌|❌|❌| -|TimescaleDB 2.5 to 2.9|❌|❌|❌|✅|✅|✅|❌|❌|❌| -|TimescaleDB 2.4|❌|❌|❌|❌|✅|✅|❌|❌|❌| -|TimescaleDB 2.1 to 2.3|❌|❌|❌|❌|✅|✅|✅|❌|❌| -|TimescaleDB 2.0|❌|❌|❌|❌|❌|✅|✅|❌|❌ -|TimescaleDB 1.7|❌|❌|❌|❌|❌|✅|✅|✅|✅| +## Prerequisites diff --git a/use-timescale/upgrades.md b/use-timescale/upgrades.md index f97d95a13a..effed6dd80 100644 --- a/use-timescale/upgrades.md +++ b/use-timescale/upgrades.md @@ -8,6 +8,8 @@ cloud_ui: - [services, :serviceId, operations, maintenance] --- +import SupportMatrix from "versionContent/_partials/_migrate_self_postgres_timescaledb_compatibility.mdx"; + # Maintenance and upgrades On Timescale, minor software updates are handled automatically, and @@ -19,8 +21,10 @@ is a critical security vulnerability that affects you, maintenance might need to occur outside of the scheduled maintenance window. + After a maintenance update, the DNS name remains the same, but the IP address it points to often changes. + In most cases, the updates that occur during your maintenance windows do not @@ -120,33 +124,30 @@ can plan accordingly. However, in some cases, we might not be able to do so. ## Upgrade to a new PostgreSQL version -Timescale currently supports PostgreSQL 14, 15, 16, and 17. You can see -your PostgreSQL and Timescale versions from the Timescale service +You can also manually upgrade to the newest supported PostgreSQL version from the service overview page. - - -You can also manually upgrade to the newest supported PostgreSQL version -(PostgreSQL 17) from the service overview page. - Upgrading to a newer version of PostgreSQL allows you to take advantage of new features, enhancements, and security fixes. It also ensures that you are using a version of PostgreSQL that's compatible with the newest version of Timescale, -allowing you to take advantage of everything Timescale has to offer. For more -information about feature changes between versions, see the +allowing you to take advantage of everything Timescale has to offer. + +The following table shows you the compatible versions of $PG and $TIMESCALE_DB. + + + +For more information about feature changes between versions, see the [PostgreSQL release notes][postgres-relnotes] and [Timescale release notes][timescale-relnotes]. + + + Your Timescale service is unavailable until the upgrade is complete. This can take up to 20 minutes. It is recommended to test on a fork first for a better estimate. + ### Recommended practices for upgrading @@ -165,13 +166,14 @@ For a smooth upgrade experience, make sure you: this fork to only pay for storage until you are comfortable deleting it. + Timescale services with replicas cannot be upgraded. To upgrade a service with a replica, you must first delete the replica and then upgrade the service. -### Upgrading to a new PostgreSQL version +### Upgrade to a new PostgreSQL version 1. In the Timescale console, navigate to `Services` and click the service you want to upgrade. @@ -186,6 +188,7 @@ with a replica, you must first delete the replica and then upgrade the service. + [cloud-login]: https://cloud.timescale.com From 1669f49f464ca0a318f32c5b0b88189de45789c0 Mon Sep 17 00:00:00 2001 From: Monae Date: Tue, 1 Apr 2025 07:54:51 -0500 Subject: [PATCH 49/90] fix: add compression link in caggs intro (#3981) Co-authored-by: Iain Cox --- _partials/_caggs-intro.md | 1 + 1 file changed, 1 insertion(+) diff --git a/_partials/_caggs-intro.md b/_partials/_caggs-intro.md index 1215a8f69f..b56fb0f0ef 100644 --- a/_partials/_caggs-intro.md +++ b/_partials/_caggs-intro.md @@ -30,6 +30,7 @@ By default, querying continuous aggregates provides you with real-time data. Pre-aggregated data from the materialized view is combined with recent data that hasn't been aggregated yet. This gives you up-to-date results on every query. +[compression]: /use-timescale/:currentVersion:/compression/about-compression [data-tiering]: /use-timescale/:currentVersion:/data-tiering/ [hypercore]: /use-timescale/:currentVersion:/hypercore/ [hierarchical-caggs]: /use-timescale/:currentVersion:/continuous-aggregates/hierarchical-continuous-aggregates/ From 29b2797a6e97a3b540d928794b5be4d0bc7cad27 Mon Sep 17 00:00:00 2001 From: Lucas Coelho Date: Wed, 2 Apr 2025 11:46:44 +0100 Subject: [PATCH 50/90] chore: set default open accordions (#3975) * chore: set default open accordions * trigger workflow * chore: make the most useful pages visible when you land on the docs. --------- Co-authored-by: Iain Cox --- about/page-index/page-index.js | 26 +- getting-started/page-index/page-index.js | 4 +- use-timescale/page-index/page-index.js | 328 ++++++++++++----------- 3 files changed, 183 insertions(+), 175 deletions(-) diff --git a/about/page-index/page-index.js b/about/page-index/page-index.js index 40af9ab0ef..6b827dbc43 100644 --- a/about/page-index/page-index.js +++ b/about/page-index/page-index.js @@ -2,6 +2,7 @@ module.exports = [ { title: "About Timescale products", href: "about", + defaultOpen: true, filePath: "index.md", pageComponents: ["featured-cards"], excerpt: @@ -10,23 +11,19 @@ module.exports = [ { title: "Timescale architecture for real-time analytics", href: "whitepaper", - excerpt: "A whitepaper detailing the architectural choices and optimizations for real-time analytics that power Timescale" - }, - { - title: "Changelog", - href: "changelog", - excerpt: "A summary of the latest changes to all Timescale products.", - }, - { - title: "Release notes", - href: "release-notes", - excerpt: "Release information for TimescaleDB v2.0.0 - v2.15.1", + excerpt: + "A whitepaper detailing the architectural choices and optimizations for real-time analytics that power Timescale", }, { title: "Pricing plans and account management", href: "pricing-and-account-management", excerpt: "Pricing plans for Timescale Cloud services", }, + { + title: "Changelog", + href: "changelog", + excerpt: "A summary of the latest changes to all Timescale products.", + }, { title: "TimescaleDB editions", href: "timescaledb-editions", @@ -36,7 +33,12 @@ module.exports = [ title: "Contribute to Timescale", href: "contribute-to-timescale", excerpt: "Contribute to the Timescale codebase", - } + }, + { + title: "Release notes", + href: "release-notes", + excerpt: "Release information for TimescaleDB v2.0.0 - v2.15.1", + }, ], }, ]; diff --git a/getting-started/page-index/page-index.js b/getting-started/page-index/page-index.js index 01cc595287..3f46a8a488 100644 --- a/getting-started/page-index/page-index.js +++ b/getting-started/page-index/page-index.js @@ -2,6 +2,7 @@ module.exports = [ { title: "Get started", href: "getting-started", + defaultOpen: true, excerpt: "Get started with Timescale", children: [ { @@ -12,7 +13,8 @@ module.exports = [ { title: "Try the key Timescale features", href: "try-key-features-timescale-products", - excerpt: "Improve database performance with Hypertables, time bucketing, continuous aggregates, compression, data tiering, and high availability", + excerpt: + "Improve database performance with Hypertables, time bucketing, continuous aggregates, compression, data tiering, and high availability", }, { title: "Run your queries from Timescale Console", diff --git a/use-timescale/page-index/page-index.js b/use-timescale/page-index/page-index.js index a36da9aa29..1f0bcd01eb 100644 --- a/use-timescale/page-index/page-index.js +++ b/use-timescale/page-index/page-index.js @@ -2,11 +2,150 @@ module.exports = [ { title: "Use Timescale", href: "use-timescale", + defaultOpen: true, filePath: "index.md", pageComponents: ["content-list"], excerpt: "How to connect to Timescale, administer, and configure the database.", children: [ + { + title: "Hypertables", + href: "hypertables", + children: [ + { + title: "About hypertables", + href: "about-hypertables", + excerpt: "Learn about hypertables in Timescale", + }, + { + title: "Create hypertables", + href: "create", + excerpt: "Create hypertables", + }, + { + title: "Change hypertable chunk intervals", + href: "change-chunk-intervals", + excerpt: "Change and view chunk time intervals for a hypertable", + }, + { + title: "Alter hypertables", + href: "alter", + excerpt: "Alter hypertables", + }, + { + title: "Create unique indexes on hypertables", + href: "hypertables-and-unique-indexes", + excerpt: "Create hypertables with unique indexes", + }, + { + title: "Improve query performance", + href: "improve-query-performance", + excerpt: "Skip chunks", + }, + { + title: "Drop hypertables", + href: "drop", + excerpt: "Drop hypertables", + }, + { + title: "Troubleshoot hypertables", + href: "troubleshooting", + type: "placeholder", + excerpt: "Troubleshooting and error fixes for hypertables", + }, + ], + }, + { + title: "Hypercore", + href: "hypercore", + excerpt: + "Seamlessly switch between row-oriented and column-oriented storage", + children: [ + { + title: "Optimize your data for real-time analytics", + href: "real-time-analytics-in-hypercore", + excerpt: "Automate", + }, + { + title: "Modify data in Hypercore", + href: "modify-data-in-hypercore", + excerpt: "Update data stored in the columnstore", + }, + { + title: "Improve query and upsert performance", + href: "secondary-indexes", + excerpt: "Automate", + }, + ], + }, + { + title: "Continuous aggregates", + href: "continuous-aggregates", + children: [ + { + title: "About continuous aggregates", + href: "about-continuous-aggregates", + excerpt: "About continuous aggregates", + }, + { + title: "Create a continuous aggregate", + href: "create-a-continuous-aggregate", + excerpt: "Create continuous aggregates", + }, + { + title: "Hierarchical continuous aggregates", + href: "hierarchical-continuous-aggregates", + }, + { + title: "Refresh policies for continuous aggregates", + href: "refresh-policies", + excerpt: "Manage refresh policies for continuous aggregates", + }, + { + title: "Create an index on a continuous aggregate", + href: "create-index", + excerpt: + "Manage automatic index creation and manually create additional indexes", + }, + { + title: "Time in continuous aggregates", + href: "time", + excerpt: "Manage time in continuous aggregates", + }, + { + title: "Drop data from continuous aggregates", + href: "drop-data", + excerpt: "Drop data from continuous aggregates", + }, + { + title: "Manage materialized hypertables", + href: "materialized-hypertables", + excerpt: "Manage materialized hypertables in continuous aggregates", + }, + { + title: "Real time aggregates", + href: "real-time-aggregates", + excerpt: "Manage real time aggregates in continuous aggregates", + }, + { + title: "Compress continuous aggregates", + href: "compression-on-continuous-aggregates", + excerpt: "Compress continuous aggregates", + }, + { + title: "Migrate a continuous aggregate to the new form", + href: "migrate", + excerpt: + "Migrate old continuous aggregates to the new form introduced in Timescale 2.7", + }, + { + title: "Troubleshoot continuous aggregates", + href: "troubleshooting", + type: "placeholder", + excerpt: "Troubleshoot continuous aggregates", + }, + ], + }, { title: "Timescale Cloud regions", href: "regions", @@ -30,8 +169,7 @@ module.exports = [ { title: "Service management", href: "service-management", - excerpt: - "Timescale services operations, Service management tab", + excerpt: "Timescale services operations, Service management tab", }, { title: "Manually change resources", @@ -41,8 +179,7 @@ module.exports = [ { title: "Connection pooling", href: "connection-pooling", - excerpt: - "Using a connection pool with your Timescale services", + excerpt: "Using a connection pool with your Timescale services", }, { title: "I/O boost", @@ -145,76 +282,6 @@ module.exports = [ }, ], }, - { - title: "Hypertables", - href: "hypertables", - children: [ - { - title: "About hypertables", - href: "about-hypertables", - excerpt: "Learn about hypertables in Timescale", - }, - { - title: "Create hypertables", - href: "create", - excerpt: "Create hypertables", - }, - { - title: "Change hypertable chunk intervals", - href: "change-chunk-intervals", - excerpt: "Change and view chunk time intervals for a hypertable", - }, - { - title: "Alter hypertables", - href: "alter", - excerpt: "Alter hypertables", - }, - { - title: "Create unique indexes on hypertables", - href: "hypertables-and-unique-indexes", - excerpt: "Create hypertables with unique indexes", - }, - { - title: "Improve query performance", - href: "improve-query-performance", - excerpt: "Skip chunks", - }, - { - title: "Drop hypertables", - href: "drop", - excerpt: "Drop hypertables", - }, - { - title: "Troubleshoot hypertables", - href: "troubleshooting", - type: "placeholder", - excerpt: "Troubleshooting and error fixes for hypertables", - }, - ], - }, - { - title: "Hypercore", - href: "hypercore", - excerpt: - "Seamlessly switch between row-oriented and column-oriented storage", - children: [ - { - title: "Optimize your data for real-time analytics", - href: "real-time-analytics-in-hypercore", - excerpt: "Automate", - }, - { - title: "Modify data in Hypercore", - href: "modify-data-in-hypercore", - excerpt: "Update data stored in the columnstore", - }, - { - title: "Improve query and upsert performance", - href: "secondary-indexes", - excerpt: "Automate", - }, - ], - }, { title: "Schema management", href: "schema-management", @@ -302,7 +369,8 @@ module.exports = [ { title: "Import and ingest data", href: "ingest-data", - excerpt: "Ingest data into a Timescale Cloud service from third-party sources", + excerpt: + "Ingest data into a Timescale Cloud service from third-party sources", children: [ { title: "Import data from CSV", @@ -325,80 +393,14 @@ module.exports = [ { title: "Ingest data with Kafka", href: "ingest-kafka", - excerpt: "Import data into a Timescale Cloud service using the PostgreSQL Kafka connector", + excerpt: + "Import data into a Timescale Cloud service using the PostgreSQL Kafka connector", }, { title: "Ingest metrics with Telegraf", href: "ingest-telegraf", - excerpt: "Ingest metrics into a Timescale Cloud service using the Telegraf plugin", - }, - ], - }, - { - title: "Continuous aggregates", - href: "continuous-aggregates", - children: [ - { - title: "About continuous aggregates", - href: "about-continuous-aggregates", - excerpt: "About continuous aggregates", - }, - { - title: "Create a continuous aggregate", - href: "create-a-continuous-aggregate", - excerpt: "Create continuous aggregates", - }, - { - title: "Hierarchical continuous aggregates", - href: "hierarchical-continuous-aggregates", - }, - { - title: "Refresh policies for continuous aggregates", - href: "refresh-policies", - excerpt: "Manage refresh policies for continuous aggregates", - }, - { - title: "Create an index on a continuous aggregate", - href: "create-index", excerpt: - "Manage automatic index creation and manually create additional indexes", - }, - { - title: "Time in continuous aggregates", - href: "time", - excerpt: "Manage time in continuous aggregates", - }, - { - title: "Drop data from continuous aggregates", - href: "drop-data", - excerpt: "Drop data from continuous aggregates", - }, - { - title: "Manage materialized hypertables", - href: "materialized-hypertables", - excerpt: "Manage materialized hypertables in continuous aggregates", - }, - { - title: "Real time aggregates", - href: "real-time-aggregates", - excerpt: "Manage real time aggregates in continuous aggregates", - }, - { - title: "Compress continuous aggregates", - href: "compression-on-continuous-aggregates", - excerpt: "Compress continuous aggregates", - }, - { - title: "Migrate a continuous aggregate to the new form", - href: "migrate", - excerpt: - "Migrate old continuous aggregates to the new form introduced in Timescale 2.7", - }, - { - title: "Troubleshoot continuous aggregates", - href: "troubleshooting", - type: "placeholder", - excerpt: "Troubleshoot continuous aggregates", + "Ingest metrics into a Timescale Cloud service using the Telegraf plugin", }, ], }, @@ -443,7 +445,8 @@ module.exports = [ { title: "Tiered storage", href: "data-tiering", - excerpt: "Save on storage costs by tiering older data to a low-cost bottomless object storage tier", + excerpt: + "Save on storage costs by tiering older data to a low-cost bottomless object storage tier", children: [ { title: "About the object storage tier", @@ -454,20 +457,17 @@ module.exports = [ { title: "Manage tiering", href: "enabling-data-tiering", - excerpt: - "How to enable the object storage tier", + excerpt: "How to enable the object storage tier", }, { title: "Querying tiered data", href: "querying-tiered-data", - excerpt: - "How to query tiered data", + excerpt: "How to query tiered data", }, { title: "Replicas and forks with tiered data", href: "tiered-data-replicas-forks", - excerpt: - "How tiered data works on replicas and forks", + excerpt: "How tiered data works on replicas and forks", }, { title: "Troubleshooting", @@ -628,8 +628,7 @@ module.exports = [ { title: "Export to Prometheus", href: "metrics-to-prometheus", - excerpt: - "Export telemetry data to Prometheus", + excerpt: "Export telemetry data to Prometheus", }, ], }, @@ -675,7 +674,7 @@ module.exports = [ href: "postgis", excerpt: "Using the postgis PostgreSQL extension", }, - ] + ], }, { title: "Backup, restore, and PITR", @@ -689,9 +688,9 @@ module.exports = [ { title: "Point-in-time recovery", href: "point-in-time-recovery", - excerpt: "PITR on Timescale services" - } - ] + excerpt: "PITR on Timescale services", + }, + ], }, { title: "Jobs", @@ -733,7 +732,8 @@ module.exports = [ { title: "Find your connection details", href: "find-connection-details", - excerpt: "Find connection information for your Timescale Cloud service", + excerpt: + "Find connection information for your Timescale Cloud service", }, { title: "Amazon CloudWatch", @@ -778,7 +778,8 @@ module.exports = [ { title: "Corporate data center", href: "corporate-data-center", - excerpt: "Integrate your corporate data center with Timescale Cloud", + excerpt: + "Integrate your corporate data center with Timescale Cloud", }, { title: "Datadog", @@ -891,7 +892,8 @@ module.exports = [ { title: "Client credentials", href: "client-credentials", - excerpt: "Client credentials to programmatically access your Timescale account", + excerpt: + "Client credentials to programmatically access your Timescale account", }, { title: "Read only role", @@ -907,12 +909,14 @@ module.exports = [ { title: "Connect securely from any cloud", href: "transit-gateway", - excerpt: "Peer your Timescale Cloud service with AWS Transit Gateway", + excerpt: + "Peer your Timescale Cloud service with AWS Transit Gateway", }, { title: "VPC peering and AWS PrivateLink", href: "vpc", - excerpt: "Secure your Timescale Service with VPC peering and AWS PrivateLink", + excerpt: + "Secure your Timescale Service with VPC peering and AWS PrivateLink", }, { title: "IP allow list", From ba5e2dbaf4b163838fef8a194c434ef9fb51657e Mon Sep 17 00:00:00 2001 From: Lucas Coelho Date: Thu, 3 Apr 2025 12:03:24 +0200 Subject: [PATCH 51/90] chore: add workflow to delete preview link (#3945) Co-authored-by: Robson Coelho --- .github/workflows/delete-preview.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .github/workflows/delete-preview.yml diff --git a/.github/workflows/delete-preview.yml b/.github/workflows/delete-preview.yml new file mode 100644 index 0000000000..cfb31012e1 --- /dev/null +++ b/.github/workflows/delete-preview.yml @@ -0,0 +1,17 @@ +name: PR preview cleanup + +on: + pull_request: + types: [closed] + +jobs: + remove-rpeview: + runs-on: ubuntu-latest + name: Remove preview + steps: + - name: Repository Dispatch + uses: peter-evans/repository-dispatch@26b39ed245ab8f31526069329e112ab2fb224588 + with: + token: ${{ secrets.ORG_AUTOMATION_TOKEN }} + repository: timescale/web-documentation + event-type: remove-docs-content From c5ec22e13ffe7d4edf7f321290131a34c3a6c9af Mon Sep 17 00:00:00 2001 From: Lucas Coelho Date: Thu, 3 Apr 2025 12:59:19 +0200 Subject: [PATCH 52/90] fix: add missing branch name payload (#3986) --- .github/workflows/delete-preview.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/delete-preview.yml b/.github/workflows/delete-preview.yml index cfb31012e1..8a3e230ccc 100644 --- a/.github/workflows/delete-preview.yml +++ b/.github/workflows/delete-preview.yml @@ -9,9 +9,14 @@ jobs: runs-on: ubuntu-latest name: Remove preview steps: + - name: Set variables + id: timescale + run: | + echo "DEV_FOLDER=$(echo ${GITHUB_HEAD_REF})" >> $GITHUB_OUTPUT - name: Repository Dispatch uses: peter-evans/repository-dispatch@26b39ed245ab8f31526069329e112ab2fb224588 with: token: ${{ secrets.ORG_AUTOMATION_TOKEN }} repository: timescale/web-documentation event-type: remove-docs-content + client-payload: '{"branch": "${{ steps.timescale.outputs.DEV_FOLDER }}", "pr_number": "${{ env.PR_NUMBER }}"}' From 6c74a92774af285e816ad6ec0995e2fd3f43db16 Mon Sep 17 00:00:00 2001 From: Iain Cox Date: Fri, 4 Apr 2025 12:07:15 +0200 Subject: [PATCH 53/90] chore: setup livesync for console and terminal. (#3925) * chore: setup livesync for console and terminal. Co-authored-by: Arunprasad Rajkumar --- _partials/_cloud_self_configuration.md | 2 +- _partials/_early_access.md | 2 +- _partials/_early_access_2_18_0.md | 1 + _partials/_hypercore_manual_workflow.md | 2 +- .../_livesync-configure-source-database.md | 43 +++ _partials/_livesync-console.md | 103 +++++++ _partials/_livesync-limitations.md | 10 + _partials/_livesync-terminal.md | 242 +++++++++++++++ ...igrate_live_tune_source_database_awsrds.md | 2 +- _partials/_prereqs-cloud-and-self.md | 2 +- _partials/_prereqs-cloud-only.md | 2 +- api/hypercore/alter_materialized_view.md | 2 +- api/hypercore/alter_table.md | 2 +- migrate/livesync.md | 284 ++---------------- migrate/page-index/page-index.js | 2 +- use-timescale/OLD-cloud-multi-node.md | 2 +- .../real-time-analytics-in-hypercore.md | 2 +- .../metrics-logging/service-metrics.md | 2 +- .../services/OLD_create-a-pg-service.md | 2 +- 19 files changed, 444 insertions(+), 265 deletions(-) create mode 100644 _partials/_early_access_2_18_0.md create mode 100644 _partials/_livesync-configure-source-database.md create mode 100644 _partials/_livesync-console.md create mode 100644 _partials/_livesync-limitations.md create mode 100644 _partials/_livesync-terminal.md diff --git a/_partials/_cloud_self_configuration.md b/_partials/_cloud_self_configuration.md index 4a5ab0b425..b478a63f1e 100644 --- a/_partials/_cloud_self_configuration.md +++ b/_partials/_cloud_self_configuration.md @@ -1,4 +1,4 @@ -import EarlyAccess from "versionContent/_partials/_early_access.mdx"; +import EarlyAccess from "versionContent/_partials/_early_access_2_18_0.mdx"; ## Policies diff --git a/_partials/_early_access.md b/_partials/_early_access.md index 89ad944fba..66072e60a1 100644 --- a/_partials/_early_access.md +++ b/_partials/_early_access.md @@ -1 +1 @@ -Early access: TimescaleDB v2.18.0 +Early access diff --git a/_partials/_early_access_2_18_0.md b/_partials/_early_access_2_18_0.md new file mode 100644 index 0000000000..89ad944fba --- /dev/null +++ b/_partials/_early_access_2_18_0.md @@ -0,0 +1 @@ +Early access: TimescaleDB v2.18.0 diff --git a/_partials/_hypercore_manual_workflow.md b/_partials/_hypercore_manual_workflow.md index 1598c3e6b5..a5d57b81c0 100644 --- a/_partials/_hypercore_manual_workflow.md +++ b/_partials/_hypercore_manual_workflow.md @@ -1,4 +1,4 @@ -import EarlyAccess from "versionContent/_partials/_early_access.mdx"; +import EarlyAccess from "versionContent/_partials/_early_access_2_18_0.mdx"; 1. **Stop the jobs that are automatically adding chunks to the columnstore** diff --git a/_partials/_livesync-configure-source-database.md b/_partials/_livesync-configure-source-database.md new file mode 100644 index 0000000000..67ad8799b2 --- /dev/null +++ b/_partials/_livesync-configure-source-database.md @@ -0,0 +1,43 @@ +1. **Tune the Write Ahead Log (WAL) on the PostgreSQL source database** + + ```sql + psql $SOURCE -c "ALTER SYSTEM SET wal_level=’logical’;" + psql $SOURCE -c "ALTER SYSTEM SET max_wal_sender=10;" + psql $SOURCE -c "ALTER SYSTEM SET wal_sender_timeout=0;" + ``` + * [ GUC “wal_level” as “logical”](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) + * [GUC “max_wal_senders” as 10](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-MAX-WAL-SENDERS) + * [GUC “wal_sender_timeout” as 0](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT) + +1. **Enable update and delete replication on the source database** + + Replica identity assists data replication by identifying the rows being modified. + By default each table and hypertable in the source database defaults to the primary key of the table being replicated. + However, you can also have: + + - **A viable unique index**: each table has a unique, non-partial, non-deferrable index that includes only columns + marked as `NOT NULL`. If a `UNIQUE` index does not exists, create one to assist the migration. You can delete it after + live sync. For each table, set `REPLICA IDENTITY` to the viable unique index: + + ```sql + psql -X -d $SOURCE -c 'ALTER TABLE REPLICA IDENTITY USING INDEX <_index_name>' + ``` + + - **No primary key or viable unique index**: use brute force. For each table, set `REPLICA IDENTITY` to `FULL`: + + ```sql + psql -X -d $SOURCE -c 'ALTER TABLE
    REPLICA IDENTITY FULL' + ``` + For each `UPDATE` or `DELETE` statement, PostgreSQL reads the whole table to find all matching rows. + This results in significantly slower replication. If you are expecting a large number of `UPDATE` or `DELETE` + operations on the table, best practice is to not use `FULL` + + To capture only `INSERT` and ignore `UPDATE`s and `DELETE`s, use a + [publish config](https://www.postgresql.org/docs/current/sql-createpublication.html#SQL-CREATEPUBLICATION-PARAMS-WITH-PUBLISH) + while [creating the publication][lives-sync-specify-tables]. + + +1. **Restart your source database** + + +[lives-sync-specify-tables]: /migrate/:currentVersion:/livesync/#specify-the-tables-to-synchronize diff --git a/_partials/_livesync-console.md b/_partials/_livesync-console.md new file mode 100644 index 0000000000..0cd9d8968c --- /dev/null +++ b/_partials/_livesync-console.md @@ -0,0 +1,103 @@ +import PrereqCloud from "versionContent/_partials/_prereqs-cloud-only.mdx"; +import LivesyncLimitations from "versionContent/_partials/_livesync-limitations.mdx"; +import LivesyncConfigureSourceDatabase from "versionContent/_partials/_livesync-configure-source-database.mdx"; +import TuneSourceDatabaseAWSRDS from "versionContent/_partials/_migrate_live_tune_source_database_awsrds.mdx"; + +## Prerequisites + + + +- Install the [PostgreSQL client tools][install-psql] on your sync machine. + +- Ensure that the source $PG instance and the target $SERVICE_LONG have the same extensions installed. + + LiveSync does not create extensions on the target. If the table uses column types from an extension, + first create the extension on the target $SERVICE_LONG before syncing the table. + +## Limitations + + + +## Set your connection string + +This variable holds the connection information for the source database. In Terminal on your migration machine, +set the following: + +```bash +export SOURCE="postgres://:@:/" +``` + + +Avoid using connection strings that route through connection poolers like PgBouncer or similar tools. This tool +requires a direct connection to the database to function properly. + + + +## Tune your source database + + + + + + + + + + + + + + + + + + + + + + +## Synchronize data to your $SERVICE_LONG + +To sync data from your PostgreSQL database to your $SERVICE_LONG using $CONSOLE: + + + +1. **Connect to your $SERVICE_LONG** + + In [$CONSOLE][portal-ops-mode], select the service to sync live data to. +1. **Start livesync** + 1. Click `Actions` > `livesync for PostgreSQL`. + +1. **Connect the source database and target $SERVICE_SHORT** + + ![Livesync wizard](https://assets.timescale.com/docs/images/livesync-wizard.png) + + In `livesync for PostgreSQL`: + 1. Set the `Livesync Name`. + 2. Set the` PostgreSQL Connection String` to point to the source database you want to sync to Timescale. + 3. Press `Continue`. + $CONSOLE connects to the source database and retrieves the schema information. + +1. **Optimize the data to syncronize in hypertables** + + ![livesync start](https://assets.timescale.com/docs/images/livesync-start.png) + 1. Select the table to sync, and press `+`. + $CONSOLE checks the table schema and, if possible suggests the column to use as the time dimension in a hypertable. + 1. Repeat this step for each table you want to sync. + 1. Press `Start Livesync`. + + $CONSOLE starts livesync between the source database and the target $SERVICE_SHORT and displays the progress. + +1. **Monitor syncronization** + 1. To view the progress of the livesync, click the name of the livesync process: + ![livesync view status](https://assets.timescale.com/docs/images/livesync-view-status.png) + 1. To pause and restart livesync, click the buttons on the right of the livesync process and select an action: + ![livesync start stop](https://assets.timescale.com/docs/images/livesync-start-stop.png) + + + +And that is it, you are using Livesync to synchronize all the data, or specific tables, from a PostgreSQL database +instance to your $SERVICE_LONG in real-time. + +[install-psql]: /use-timescale/:currentVersion:/integrations/psql/ +[portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services diff --git a/_partials/_livesync-limitations.md b/_partials/_livesync-limitations.md new file mode 100644 index 0000000000..1ad3d2d108 --- /dev/null +++ b/_partials/_livesync-limitations.md @@ -0,0 +1,10 @@ +* Schema changes must be co-ordinated. + + Make compatible changes to the schema in your $SERVICE_LONG first, then make + the same changes to the source PostgreSQL instance. +* Ensure that the source $PG instance and the target $SERVICE_LONG have the same extensions installed. + + LiveSync does not create extensions on the target. If the table uses column types from an extension, + first create the extension on the target $SERVICE_LONG before syncing the table. +* There is WAL volume growth on the source PostgreSQL instance during large table copy. +* This works for PostgreSQL databases only as source. Timescaledb is not yet supported. diff --git a/_partials/_livesync-terminal.md b/_partials/_livesync-terminal.md new file mode 100644 index 0000000000..8b4965bd22 --- /dev/null +++ b/_partials/_livesync-terminal.md @@ -0,0 +1,242 @@ +import MigrationPrerequisites from "versionContent/_partials/_migrate_prerequisites.mdx"; +import SetupConnectionStrings from "versionContent/_partials/_migrate_live_setup_connection_strings.mdx"; +import LivesyncLimitations from "versionContent/_partials/_livesync-limitations.mdx"; +import LivesyncConfigureSourceDatabase from "versionContent/_partials/_livesync-configure-source-database.mdx"; +import TuneSourceDatabaseAWSRDS from "versionContent/_partials/_migrate_live_tune_source_database_awsrds.mdx"; + +## Prerequisites + + + +- Ensure that the source $PG instance and the target $SERVICE_LONG have the same extensions installed. + + LiveSync does not create extensions on the target. If the table uses column types from an extension, + first create the extension on the target $SERVICE_LONG before syncing the table. + +- [Install Docker][install-docker] on your sync machine. + + You need a minimum of a 4 CPU/16GB EC2 instance to run Livesync. + +- Install the [PostgreSQL client tools][install-psql] on your sync machine. + + This includes `psql`, `pg_dump`, and `pg_dumpall`. + + +## Limitations + + + +- The Schema is not migrated by Livesync, you use pg_dump/restore to migrate schema + +## Set your connection strings + +The `` in the `SOURCE` connection must have the replication role granted in order to create a replication slot. + + + + +## Tune your source database + + + + + + + + + + + + + + + + + + + + + + +## Migrate the table schema to the $SERVICE_LONG + +Use pg_dump to: + + + +1. **Download the schema from the source database** + + ```shell + pg_dump $SOURCE \ + --no-privileges \ + --no-owner \ + --no-publications \ + --no-subscriptions \ + --no-table-access-method \ + --no-tablespaces \ + --schema-only \ + --file=schema.sql + ``` + +1. **Apply the schema on the target $SERVICE_SHORT** + ```shell + psql $TARGET -f schema.sql + ``` + + + +## Convert partitions and tables with time-series data into hypertables + +For efficient querying and analysis, you can convert tables which contain time-series or +events data, and tables that are already partitioned using PostgreSQL declarative partition into +[hypertables][about-hypertables]. + + + +1. **Convert tables to hyperatables** + + Run the following on each table in the target $SERVICE_LONG to convert it to a hypertable: + + ```shell + psql -X -d $TARGET -c "SELECT create_hypertable('
    ', by_range('', ''::interval));" + ``` + + For example, to convert the *metrics* table into a hypertable with *time* as a partition column and + *1 day* as a partition interval: + + ```shell + psql -X -d $TARGET -c "SELECT create_hypertable('public.metrics', by_range('time', '1 day'::interval));" + ``` + +1. **Convert PostgreSQL partitions to hyperatables** + + Rename the partition and create a new normal table with the same name as the partitioned table, then + convert to a hypertable: + + ```shell + psql $TARGET -f - < + + +## Synchronize data to your $SERVICE_LONG + +You use the Livesync docker image to synchronize changes in real-time from a PostgreSQL database +instance to a $SERVICE_LONG: + + + +1. **Start Livesync** + + As you run Livesync continuously, best practice is to run it as a background process. + + ```shell + docker run -d --rm --name livesync timescale/live-sync:v0.0.0-alpha.7 run --publication analytics --subscription livesync --source $SOURCE --target $TARGET + ``` + +1. **Trace progress** + + Once Livesync is running as a docker daemon, you can also capture the logs: + ```shell + docker logs -f livesync + ``` + +1. **View the tables being synchronized** + + ```bash + psql $TARGET -c "SELECT * FROM _ts_live_sync.subscription_rel" + + subname | schemaname | tablename | rrelid | state | lsn + ----------+------------+-----------+--------+-------+----- + livesync | public | metrics | 17261 | d | + ``` + Possible values for `state` are: + + - d: initial table data sync + + - f: initial table data sync completed + + - s: catching up with the latest change + + - r: table is ready, synching live changes + +1. **Stop Livesync** + + ```shell + docker stop live-sync + ``` + +1. **Cleanup** + + You need to manually execute a SQL snippet to cleanup replication slots created by the live-migration. + + ```shell + psql $SOURCE -f - < + + +## Specify the tables to synchronize + +After the Livesync docker is up and running, you [`CREATE PUBLICATION`][create-publication] on the SOURCE database to +specify the list of tables which you intend to synchronize. Once you create a PUBLICATION, it is +automatically picked by Livesync, which starts synching the tables expressed as part of it. + +For example: + + + +1. **Create a publication named `analytics` which publishes `metrics` and `tags` tables** + + `PUBLICATION` enables you to add all the tables in the schema or even all the tables in the database. However, it + requires superuser privileges on most of the managed PostgreSQL offerings. + + ```sql + CREATE PUBLICATION analytics FOR TABLE metrics, tags; + ``` + +1. **Add tables after to an existing publication with a call to [ALTER PUBLICATION][alter-publication]** + + ```sql + ALTER PUBLICATION analytics ADD TABLE events; + ``` + +1. **Publish PostgreSQL declarative partitioned table** + + To publish declaratively partitioned table changes to your $SERVICE_LONG, set the `publish_via_partition_root` + special `PUBLICATION` config to `true`: + + ```sql + ALTER PUBLICATION analytics SET(publish_via_partition_root=true); + ``` + +1. **Stop synching a table in the `PUBLICATION` with a call to `DROP TABLE`** + + ```sql + ALTER PUBLICATION analytics DROP TABLE tags; + ``` + + + + +[create-publication]: https://www.postgresql.org/docs/current/sql-createpublication.html +[alter-publication]: https://www.postgresql.org/docs/current/sql-alterpublication.html +[install-docker]: https://docs.docker.com/engine/install/ +[about-hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/ +[lives-sync-specify-tables]: /migrate/:currentVersion:/livesync/#specify-the-tables-to-synchronize +[compression]: /use-timescale/:currentVersion:/compression/about-compression +[caggs]: /use-timescale/:currentVersion:/continuous-aggregates/about-continuous-aggregates/ +[join-livesync-on-slack]: https://app.slack.com/client/T4GT3N2JK/C086NU9EZ88 +[install-psql]: /use-timescale/:currentVersion:/integrations/psql/ diff --git a/_partials/_migrate_live_tune_source_database_awsrds.md b/_partials/_migrate_live_tune_source_database_awsrds.md index 6bfe993060..118885fe04 100644 --- a/_partials/_migrate_live_tune_source_database_awsrds.md +++ b/_partials/_migrate_live_tune_source_database_awsrds.md @@ -31,7 +31,7 @@ Updating parameters on a PostgreSQL instance will cause an outage. Choose a time Changing parameters will cause an outage. Wait for the database instance to reboot before continuing. 1. Verify that the settings are live in your database. -1. **Enable live-migration to replicate `DELETE` and`UPDATE` operations** +1. **Enable replication `DELETE` and`UPDATE` operations** diff --git a/_partials/_prereqs-cloud-and-self.md b/_partials/_prereqs-cloud-and-self.md index 1a6d3a6619..e55dacb480 100644 --- a/_partials/_prereqs-cloud-and-self.md +++ b/_partials/_prereqs-cloud-and-self.md @@ -1,6 +1,6 @@ To follow the procedure on this page you need to: -* Create a [target $SERVICE_LONG][create-service] +* Create a [target $SERVICE_LONG][create-service]. This procedure also works for [self-hosted $TIMESCALE_DB][enable-timescaledb]. diff --git a/_partials/_prereqs-cloud-only.md b/_partials/_prereqs-cloud-only.md index 3c9a7d93bb..6d1f87a964 100644 --- a/_partials/_prereqs-cloud-only.md +++ b/_partials/_prereqs-cloud-only.md @@ -1,5 +1,5 @@ To follow the procedure on this page you need to: -* Create a [target $SERVICE_LONG][create-service] +* Create a [target $SERVICE_LONG][create-service]. [create-service]: /getting-started/:currentVersion:/services/ diff --git a/api/hypercore/alter_materialized_view.md b/api/hypercore/alter_materialized_view.md index f4dc10cfe5..a5fffe35ab 100644 --- a/api/hypercore/alter_materialized_view.md +++ b/api/hypercore/alter_materialized_view.md @@ -10,7 +10,7 @@ api: --- import Since2180 from "versionContent/_partials/_since_2_18_0.mdx"; -import EarlyAccess from "versionContent/_partials/_early_access.mdx"; +import EarlyAccess from "versionContent/_partials/_early_access_2_18_0.mdx"; # ALTER MATERIALIZED VIEW (Hypercore) Community diff --git a/api/hypercore/alter_table.md b/api/hypercore/alter_table.md index cc5c1da77b..ba1ce93a00 100644 --- a/api/hypercore/alter_table.md +++ b/api/hypercore/alter_table.md @@ -11,7 +11,7 @@ products: [cloud, self_hosted] --- import Since2180 from "versionContent/_partials/_since_2_18_0.mdx"; -import EarlyAccess from "versionContent/_partials/_early_access.mdx"; +import EarlyAccess from "versionContent/_partials/_early_access_2_18_0.mdx"; # ALTER TABLE (Hypercore) diff --git a/migrate/livesync.md b/migrate/livesync.md index d89d0180ce..dda5b78c87 100644 --- a/migrate/livesync.md +++ b/migrate/livesync.md @@ -8,277 +8,57 @@ tags: [recovery, logical backup, replication] import MigrationPrerequisites from "versionContent/_partials/_migrate_prerequisites.mdx"; import SetupConnectionStrings from "versionContent/_partials/_migrate_live_setup_connection_strings.mdx"; - +import LivesyncConsole from "versionContent/_partials/_livesync-console.mdx" +import LivesyncTerminal from "versionContent/_partials/_livesync-terminal.mdx" +import EarlyAccessNoRelease from "versionContent/_partials/_early_access.mdx"; # Livesync from PostgreSQL to Timescale Cloud -You use the Livesync Docker image to synchronize all data, or specific tables, from a PostgreSQL database -instance to a $SERVICE_LONG in real-time. You run Livesync continuously, turning PostgreSQL into a primary database -with a $SERVICE_LONG as a logical replica. This enables you to leverage $CLOUD_LONG’s real-time analytics capabilities -on your replica data. +You use Livesync to synchronize all the data, or specific tables, from a PostgreSQL database instance to your +$SERVICE_LONG in real-time. You run Livesync continuously, turning PostgreSQL into a primary database with your +$SERVICE_LONG as a logical replica. This enables you to leverage $CLOUD_LONG’s real-time analytics capabilities on +your replica data. - -You use Livesync for for data synchronization, rather than migration. It is in alpha and is not recommended for -production use. - +![livesync view status](https://assets.timescale.com/docs/images/livesync-view-status.png) -Livesync leverages the PostgreSQL logical replication protocol, a well-established and widely -understood feature in the PostgreSQL ecosystem. By relying on this protocol, Livesync ensures -compatibility, familiarity, and a broader knowledge base, making it easier for you to adopt and -integrate. +Livesync leverages the a well-established PostgreSQL logical replication protocol. By relying on this protocol, +Livesync ensures compatibility, familiarity, and a broader knowledge base. Making it easier for you to adopt Livesync +and integrate your data. -You use Livesync to: +You use Livesync for data synchronization, rather than migration. Livesync can: * Copy existing data from a PostgreSQL instance to a $SERVICE_LONG: - - Copy data at up to 150 GB/hr. You need at least a 4 CPU/16GB source database, and a 4 CPU/16GB target $SERVICE_SHORT. - - Copy the publication tables in parallel. However, large tables are still copied using a single connection. - Parallel copying is in the backlog. - - Forget foreign key relationships. Livesync disables foreign key validation during the sync. For example, if a - `metrics` table refers to the `id` column on the `tags` table, you can still sync only the `metrics` table - without worrying about their foreign key relationships. + - Copy data at up to 150 GB/hr. + + You need at least a 4 CPU/16GB source database, and a 4 CPU/16GB target $SERVICE_SHORT. + - Copy the publication tables in parallel. + + Large tables are still copied using a single connection. Parallel copying is in the backlog. + - Forget foreign key relationships. + + Livesync disables foreign key validation during the sync. For example, if a `metrics` table refers to + the `id` column on the `tags` table, you can still sync only the `metrics` table without worrying about their + foreign key relationships. - Track progress. PostgreSQL expose `COPY` progress under in `pg_stat_progress_copy`. * Synchronize real-time changes from a PostgreSQL instance to a $SERVICE_LONG. * Add and remove tables on demand using the [PostgreSQL PUBLICATION interface](https://www.postgresql.org/docs/current/sql-createpublication.html). * Enable features such as [hypertables][about-hypertables], [columnstore][compression], and [continuous aggregates][caggs] on your logical replica. -If you have an questions or feedback, talk to us in [#livesync in Timescale Community][join-livesync-on-slack]. - -# Prerequisites - - - -- [Install Docker][install-docker] on your sync machine. - You need a minimum of a 4 CPU/16GB EC2 instance to run Livesync - -- Install the PostgreSQL client tools on your sync machine. - - This includes `psql`, `pg_dump`, and `pg_dumpall`. - -## Limitations - -* The Schema is not migrated by Livesync, you use pg_dump/restore to migrate schema -* Schema changes must be co-ordinated. Make compatible changes to the schema in your $SERVICE_LONG first, then make - the same changes to the source PostgreSQL instance. -* There is WAL volume growth on the source PostgreSQL instance during large table copy. -* This works for PostgreSQL databases only as source. Timescaledb is not yet supported. - -## Set your connection strings - -The `` in the `SOURCE` connection must have the replication role granted in order to create a replication slot. - - - -## Configure the source database - -You need to tune the Write Ahead Log (WAL) on the PostgreSQL source database: -* PostgreSQL[ GUC “wal_level” as “logical”](https://www.postgresql.org/docs/current/runtime-config-wal.html#GUC-WAL-LEVEL) -* PostgreSQL[GUC “max_wal_senders” as 10](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-MAX-WAL-SENDERS) - -To do this: - -```sql -psql $SOURCE -c "SET wal_level=’logical’;" -psql $SOURCE -c "SET max_wal_sender=10;" -``` - -## Enable update and delete replication on the source database - -Replica identity assists data replication by identifying the rows being modified. -By default each table and hypertable in the source database defaults to the primary key of the table being replicated. -However, you can also have: - -- **A viable unique index**: each table has a unique, non-partial, non-deferrable index that includes only columns - marked as `NOT NULL`. If a `UNIQUE` index does not exists, create one to assist the migration. You can delete it after - live sync. For each table, set `REPLICA IDENTITY` to the viable unique index: - - ```sql - psql -X -d $SOURCE -c 'ALTER TABLE
    REPLICA IDENTITY USING INDEX <_index_name>' - - -- **No primary key or viable unique index**: use brute force. For each table, set `REPLICA IDENTITY` to `FULL`: - - ```sql - psql -X -d $SOURCE -c 'ALTER TABLE
    REPLICA IDENTITY FULL' - ``` - For each `UPDATE` or `DELETE` statement, PostgreSQL reads the whole table to find all matching rows. - This results in significantly slower replication. If you are expecting a large number of `UPDATE` or `DELETE` - operations on the table, best practice is to not use `FULL` - -To capture only `INSERT` and ignore `UPDATE`s and `DELETE`s, use a -[publish config](https://www.postgresql.org/docs/current/sql-createpublication.html#SQL-CREATEPUBLICATION-PARAMS-WITH-PUBLISH) -while [creating the publication][lives-sync-specify-tables]. - -## Migrate the table schema to the $SERVICE_LONG - -Use pg_dump to: - - - -1. **Download the schema from the source database** - - ```shell - pg_dump $SOURCE \ - --no-privileges \ - --no-owner \ - --no-publications \ - --no-subscriptions \ - --no-table-access-method \ - --no-tablespaces \ - --schema-only \ - --file=schema.sql - ``` - -1. **Apply the schema on the target $SERVICE_SHORT** - ```shell - psql $TARGET -f schema.sql - ``` - - - -## Convert partitions and tables with time-series data into hypertables - -For efficient querying and analysis, you can convert tables which contain time-series or -events data, and tables that are already partitioned using PostgreSQL declarative partition into -[hypertables][about-hypertables]. - - - -1. **Convert tables to hyperatables** +: Livesync is not supported for production use. If you have an questions or feedback, talk to us in #livesync in Timescale Community. - Run the following on each table in the target $SERVICE_LONG to convert it to a hypertable: + - ```shell - psql -X -d $TARGET -c "SELECT create_hypertable('
    ', by_range('', ''::interval));" - ``` + - For example, to convert the *metrics* table into a hypertable with *time* as a partition column and - *1 day* as a partition interval: - - ```shell - psql -X -d $TARGET -c "SELECT create_hypertable('public.metrics', by_range('time', '1 day'::interval));" - ``` - -1. **Convert PostgreSQL partitions to hyperatables** - - Rename the partition and create a new normal table with the same name as the partitioned table, then - convert to a hypertable: - - ```shell - psql $TARGET -f - < - - -## Syncronize data from your source database to the $SERVICE_LONG - -You use the Livesync docker image to synchronize changes in real-time from a PostgreSQL database -instance to a $SERVICE_LONG: - - - -1. **Start Livesync** - - As you run Livesync continuously, best practice is to run it as a background process. - - ```shell - docker run -d --rm --name livesync timescale/live-sync:v0.0.0-alpha.7 run --publication analytics --subscription livesync --source $SOURCE --target $TARGET - ``` - -1. **Trace progress** - - Once Livesync is running as a docker daemon, you can also capture the logs: - ```shell - docker logs -f livesync - ``` - -1. **View the tables being synchronized** - - ```bash - psql $TARGET -c "SELECT * FROM _ts_live_sync.subscription_rel" - - subname | schemaname | tablename | rrelid | state | lsn - ----------+------------+-----------+--------+-------+----- - livesync | public | metrics | 17261 | d | - ``` - Possible values for `state` are: - - - d: initial table data sync - - - f: initial table data sync completed - - - s: catching up with the latest change - - - r: table is ready, synching live changes - -1. **Stop Livesync** - - ```shell - docker stop live-sync - ``` - -1. **Cleanup** - - You need to manually execute a SQL snippet to cleanup replication slots created by the live-migration. - - ```shell - psql $SOURCE -f - < - - -## Specify the tables to synchronize - -After the Livesync docker is up and running, you [`CREATE PUBLICATION`][create-publication] on the SOURCE database to -specify the list of tables which you intend to synchronize. Once you create a PUBLICATION, it is -automatically picked by Livesync, which starts synching the tables expressed as part of it. - -For example: - - - -1. **Create a publication named `analytics` which publishes `metrics` and `tags` tables** - - `PUBLICATION` enables you to add all the tables in the schema or even all the tables in the database. However, it - requires superuser privileges on most of the managed PostgreSQL offerings. - - ```sql - CREATE PUBLICATION analytics FOR TABLE metrics, tags; - ``` - -1. **Add tables after to an existing publication with a call to [ALTER PUBLICATION][alter-publication]** - - ```sql - ALTER PUBLICATION analytics ADD TABLE events; - ``` - -1. **Publish PostgreSQL declarative partitioned table** - - To publish declaratively partitioned table changes to your $SERVICE_LONG, set the `publish_via_partition_root` - special `PUBLICATION` config to `true`: - - ```sql - ALTER PUBLICATION analytics SET(publish_via_partition_root=true); - ``` + -1. **Stop synching a table in the `PUBLICATION` with a call to `DROP TABLE`** + + - ```sql - ALTER PUBLICATION analytics DROP TABLE tags; - ``` + - + + [create-publication]: https://www.postgresql.org/docs/current/sql-createpublication.html diff --git a/migrate/page-index/page-index.js b/migrate/page-index/page-index.js index 29dc6c3d96..6d35ca8667 100644 --- a/migrate/page-index/page-index.js +++ b/migrate/page-index/page-index.js @@ -16,7 +16,7 @@ module.exports = [ excerpt: "Migrate a large database with low downtime", }, { - title: "Livesync from Postgres to Timescale Cloud", + title: "Livesync from PostgreSQL to Timescale Cloud", href: "livesync", excerpt: "Synchronize updates to a primary postgres database instance to Timescale Cloud service in real-time", }, diff --git a/use-timescale/OLD-cloud-multi-node.md b/use-timescale/OLD-cloud-multi-node.md index c6dd61f207..5aba352f99 100644 --- a/use-timescale/OLD-cloud-multi-node.md +++ b/use-timescale/OLD-cloud-multi-node.md @@ -6,7 +6,7 @@ keywords: [multi-node, scaling] tags: [cluster, distributed hypertables] --- -import EarlyAccess from "versionContent/_partials/_early_access.mdx"; +import EarlyAccess from "versionContent/_partials/_early_access_2_18_0.mdx"; # Multi-node diff --git a/use-timescale/hypercore/real-time-analytics-in-hypercore.md b/use-timescale/hypercore/real-time-analytics-in-hypercore.md index 70119a97e3..d7b0c3296c 100644 --- a/use-timescale/hypercore/real-time-analytics-in-hypercore.md +++ b/use-timescale/hypercore/real-time-analytics-in-hypercore.md @@ -9,7 +9,7 @@ import PrereqCloud from "versionContent/_partials/_prereqs-cloud-only.mdx"; import HCConversionOverview from "versionContent/_partials/_hypercore-conversion-overview.mdx"; import HCPolicyWorkflow from "versionContent/_partials/_hypercore_policy_workflow.mdx"; import UsageBasedStorage from "versionContent/_partials/_usage-based-storage-intro.mdx"; -import EarlyAccess from "versionContent/_partials/_early_access.mdx"; +import EarlyAccess from "versionContent/_partials/_early_access_2_18_0.mdx"; # Optimize your data for real-time analytics diff --git a/use-timescale/metrics-logging/service-metrics.md b/use-timescale/metrics-logging/service-metrics.md index 8d5e1aac84..21862ff28f 100644 --- a/use-timescale/metrics-logging/service-metrics.md +++ b/use-timescale/metrics-logging/service-metrics.md @@ -9,7 +9,7 @@ cloud_ui: - [services, :serviceId, metrics] --- -import EarlyAccess from "versionContent/_partials/_early_access.mdx"; +import EarlyAccess from "versionContent/_partials/_early_access_2_18_0.mdx"; # Service metrics diff --git a/use-timescale/services/OLD_create-a-pg-service.md b/use-timescale/services/OLD_create-a-pg-service.md index 7448a2c3d0..0578c9679d 100644 --- a/use-timescale/services/OLD_create-a-pg-service.md +++ b/use-timescale/services/OLD_create-a-pg-service.md @@ -5,7 +5,7 @@ products: [cloud] keywords: [services, PostgreSQL, create, install] --- -import EarlyAccess from "versionContent/_partials/_early_access.mdx"; +import EarlyAccess from "versionContent/_partials/_early_access_2_18_0.mdx"; # Create a PostgreSQL service From 0ab0e8ca8d3a64c91ce3b2cd97ae1989364e466a Mon Sep 17 00:00:00 2001 From: Iain Cox Date: Fri, 4 Apr 2025 13:25:44 +0200 Subject: [PATCH 54/90] Changelog April 4 --- about/changelog.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/about/changelog.md b/about/changelog.md index 5445cc0ba1..5c3566700a 100644 --- a/about/changelog.md +++ b/about/changelog.md @@ -8,6 +8,35 @@ keywords: [changelog, upgrades, updates, releases] All the latest features and updates to Timescale products. +## 📒Faster vector search and improved job information + + +### pgvectorscale 0.7.0: faster filtered filtered vector search with filtered indexes + +This pgvectorscale release adds label-based filtered vector search to the StremingDiskANN index. +This enables you to return more precise and efficient results by combining vector +similarity search with label filtering while still uitilizing the ANN index. This is a common need for large-scale RAG and Agentic applications +that rely on vector searches with metadata filters to return relevant results. Filtered indexes add +even more capabilities for filtered search at scale, complementing the high accuracy streaming filtering already +present in pgvectorscale. The implementation is inspired by Microsoft's Filtered DiskANN research. +For more information, see the [pgvectorscale release notes][log-28032025-pgvectorscale-rn] and a +[usage example][log-28032025-pgvectorscale-example]. + +### Job errors and individual job pages + +Each job now has an individual page in $CONSOLE, and displays additional details about job errors. You use +this information to debug failing jobs. + +To see the job information page, in [$CONSOLE][console], select the $SERVICE_SHORT to check, then click `Jobs` > job ID to investigate. + +- Successful jobs: + + ![Log success in Timescale Console](https://assets.timescale.com/docs/images/changelog-job-success-page.png) + +- Unsuccessful jobs with errors: + + ![Log errors in Timescale Console](https://assets.timescale.com/docs/images/changelog-job-error-page.png) + ## 🤩 In-Console Livesync for PostgreSQL @@ -868,3 +897,5 @@ To learn more, see the [postgresql-unit documentation](https://github.com/df7cb/ [popsql-web]: https://app.popsql.com/login [popsql-desktop]: https://popsql.com/download [console]: https://console.cloud.timescale.com/dashboard/services +[log-28032025-pgvectorscale-rn]: https://github.com/timescale/pgvectorscale/releases/tag/0.7.0 +[log-28032025-pgvectorscale-example]: https://github.com/timescale/pgvectorscale?tab=readme-ov-file#label-based-filtering-with-diskann From 0123437f05acf58e46e0eecd0d95bff89d85dca2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fabr=C3=ADzio=20de=20Royes=20Mello?= Date: Fri, 4 Apr 2025 10:54:58 -0300 Subject: [PATCH 55/90] fix: Remove deprecated old-format CAggs information (#3989) --- .../about-continuous-aggregates.md | 29 +++++-------------- .../create-a-continuous-aggregate.md | 8 ----- .../materialized-hypertables.md | 2 +- 3 files changed, 8 insertions(+), 31 deletions(-) diff --git a/use-timescale/continuous-aggregates/about-continuous-aggregates.md b/use-timescale/continuous-aggregates/about-continuous-aggregates.md index 85dab2c3d9..f6182ed439 100644 --- a/use-timescale/continuous-aggregates/about-continuous-aggregates.md +++ b/use-timescale/continuous-aggregates/about-continuous-aggregates.md @@ -212,30 +212,22 @@ Continuous aggregates consist of: ### Materialization hypertable Continuous aggregates take raw data from the original hypertable, aggregate it, -and store the intermediate state in a materialization hypertable. When you query -the continuous aggregate view, the state is returned to you as needed. +and store the aggregated data in a materialization hypertable. When you query +the continuous aggregate view, the aggregated data is returned to you as needed. Using the same temperature example, the materialization table looks like this: -|day|location|chunk|avg temperature partial| +|day|location|chunk|avg temperature| |-|-|-|-| -|2021/01/01|New York|1|{3, 219}| -|2021/01/01|Stockholm|1|{4, 280}| +|2021/01/01|New York|1|73| +|2021/01/01|Stockholm|1|70| |2021/01/02|New York|2|| -|2021/01/02|Stockholm|2|{5, 345}| +|2021/01/02|Stockholm|2|69| The materialization table is stored as a Timescale hypertable, to take advantage of the scaling and query optimizations that hypertables offer. Materialization tables contain a column for each group-by clause in the query, -a `chunk` column identifying which chunk in the raw data this entry came from, -and a `partial aggregate` column for each aggregate in the query. - -The partial column is used internally to calculate the output. In this example, -because the query looks for an average, the partial column contains the number -of rows seen, and the sum of all their values. The most important thing to know -about partials is that they can be combined to create new partials spanning all -of the old partials' rows. This is important if you combine groups that span -multiple chunks. +and an `aggregate` column for each aggregate in the query. For more information, see [materialization hypertables][cagg-mat-hypertables]. @@ -248,13 +240,6 @@ unblocks other transactions, and materializes the aggregates. The first transaction is very quick, and most of the work happens during the second transaction, to ensure that the work does not interfere with other operations. -When you query the continuous aggregate view, the materialization engine -combines the aggregate partials into a single partial for each time range, and -calculates the value that is returned. For example, to compute an average, each -partial sum is added up to a total sum, and each partial count is added up to a -total count, then the average is computed as the total sum divided by the total -count. - ### Invalidation engine Any change to the data in a hypertable could potentially invalidate some diff --git a/use-timescale/continuous-aggregates/create-a-continuous-aggregate.md b/use-timescale/continuous-aggregates/create-a-continuous-aggregate.md index 6b1228d27e..85d3787136 100644 --- a/use-timescale/continuous-aggregates/create-a-continuous-aggregate.md +++ b/use-timescale/continuous-aggregates/create-a-continuous-aggregate.md @@ -75,14 +75,6 @@ the hypertable. The time bucket allows you to define a time interval, instead of having to use specific timestamps. For example, you can define a time bucket as five minutes, or one day. -When the continuous aggregate is materialized, the materialization table stores -partials, which are then used to calculate the result of the query. This means a -certain amount of processing capacity is required for any query, and the amount -required becomes greater as the interval gets smaller. Because of this, if you -have very small intervals, it can be more efficient to run the aggregate query -on the raw data in the hypertable. You should test both methods to determine -what is best for your dataset and desired bucket interval. - You can't use [time_bucket_gapfill][api-time-bucket-gapfill] directly in a continuous aggregate. This is because you need access to previous data to determine the gapfill content, which isn't yet available when you create the diff --git a/use-timescale/continuous-aggregates/materialized-hypertables.md b/use-timescale/continuous-aggregates/materialized-hypertables.md index 9a87388511..999d41b398 100644 --- a/use-timescale/continuous-aggregates/materialized-hypertables.md +++ b/use-timescale/continuous-aggregates/materialized-hypertables.md @@ -9,7 +9,7 @@ tags: [materialized views] # Materialized hypertables Continuous aggregates take raw data from the original hypertable, aggregate it, -and store the intermediate state in a materialization hypertable. You can modify +and store the aggregated data in a materialization hypertable. You can modify this materialized hypertable in the same way as any other hypertable. ## Discover the name of a materialized hypertable From 3db2d7df131af4e63c57dff20af8ab46b113b333 Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Mon, 7 Apr 2025 10:23:06 +0300 Subject: [PATCH 56/90] Rearrange internal API structure Co-authored-by: Iain Cox --- _partials/_formatting_examples.md | 4 ++-- .../_hypertable-detailed-size-api.md | 19 ++++--------------- .../_hypertable-size-api.md | 19 ++++--------------- api/compression/chunk_compression_stats.md | 6 +++--- api/compression/compress_chunk.md | 8 ++++---- api/compression/decompress_chunk.md | 8 ++++---- .../hypertable_compression_stats.md | 6 +++--- api/compression/remove_compression_policy.md | 6 +++--- .../add_continuous_aggregate_policy.md | 8 ++++---- .../add_policies.md | 0 .../alter_materialized_view.md | 6 +++--- .../alter_policies.md | 0 .../cagg_migrate.md | 0 .../create_materialized_view.md | 0 .../drop_materialized_view.md | 4 ++-- .../hypertable_detailed_size.md | 14 ++++++++++++++ api/continuous-aggregates/hypertable_size.md | 15 +++++++++++++++ api/continuous-aggregates/index.md | 15 +++++++++++++++ .../refresh_continuous_aggregate.md | 6 +++--- .../remove_all_policies.md | 0 .../remove_continuous_aggregate_policy.md | 0 .../remove_policies.md | 0 .../show_policies.md | 0 .../add_retention_policy.md | 8 ++++---- .../index.md} | 0 .../remove_retention_policy.md | 6 +++--- .../add_data_node.md | 12 ++++++------ .../alter_data_node.md | 12 ++++++------ .../attach_data_node.md | 8 ++++---- ...eanup_copy_chunk_operation_experimental.md | 6 +++--- .../copy_chunk_experimental.md | 10 +++++----- .../create_distributed_hypertable.md | 10 +++++----- .../create_distributed_restore_point.md | 0 .../delete_data_node.md | 10 +++++----- .../detach_data_node.md | 10 +++++----- .../distributed_exec.md | 6 +++--- .../index.md} | 0 .../move_chunk_experimental.md | 10 +++++----- .../set_number_partitions.md | 6 +++--- .../set_replication_factor.md | 6 +++--- api/{ => hypertable}/add_dimension.md | 0 api/{ => hypertable}/add_dimension_old.md | 8 ++++---- api/{ => hypertable}/add_reorder_policy.md | 0 api/{ => hypertable}/attach_tablespace.md | 6 +++--- api/{ => hypertable}/chunks_detailed_size.md | 6 +++--- api/{ => hypertable}/create_hypertable.md | 0 api/{ => hypertable}/create_hypertable_old.md | 0 api/{ => hypertable}/create_index.md | 2 +- api/{ => hypertable}/detach_tablespace.md | 6 +++--- api/{ => hypertable}/detach_tablespaces.md | 4 ++-- .../disable_chunk_skipping.md | 8 ++++---- api/{ => hypertable}/drop_chunks.md | 0 api/{ => hypertable}/enable_chunk_skipping.md | 0 .../hypertable_approximate_detailed_size.md | 6 +++--- .../hypertable_approximate_size.md | 6 +++--- api/hypertable/hypertable_detailed_size.md | 14 ++++++++++++++ api/{ => hypertable}/hypertable_index_size.md | 6 +++--- api/hypertable/hypertable_size.md | 14 ++++++++++++++ api/{hypertable.md => hypertable/index.md} | 0 api/{ => hypertable}/merge_chunks.md | 0 api/{ => hypertable}/move_chunk.md | 6 +++--- api/{ => hypertable}/remove_reorder_policy.md | 6 +++--- api/{ => hypertable}/reorder_chunk.md | 8 ++++---- .../set_chunk_time_interval.md | 0 api/{ => hypertable}/set_integer_now_func.md | 0 api/{ => hypertable}/show_chunks.md | 0 api/{ => hypertable}/show_tablespaces.md | 4 ++-- .../chunk_compression_settings.md | 4 ++-- api/informational-views/chunks.md | 4 ++-- .../compression_settings.md | 4 ++-- .../continuous_aggregates.md | 4 ++-- .../hypertable_compression_settings.md | 4 ++-- api/informational-views/hypertables.md | 4 ++-- api/informational-views/job_errors.md | 6 +++--- api/informational-views/job_history.md | 6 +++--- api/informational-views/job_stats.md | 4 ++-- api/informational-views/jobs.md | 4 ++-- api/informational-views/policies.md | 4 ++-- api/page-index/page-index.js | 4 ---- 79 files changed, 236 insertions(+), 190 deletions(-) rename api/hypertable_detailed_size.md => _partials/_hypertable-detailed-size-api.md (85%) rename api/hypertable_size.md => _partials/_hypertable-size-api.md (82%) rename api/{ => continuous-aggregates}/add_continuous_aggregate_policy.md (98%) rename api/{ => continuous-aggregates}/add_policies.md (100%) rename api/{ => continuous-aggregates}/alter_materialized_view.md (98%) rename api/{ => continuous-aggregates}/alter_policies.md (100%) rename api/{ => continuous-aggregates}/cagg_migrate.md (100%) rename api/{ => continuous-aggregates}/create_materialized_view.md (100%) rename api/{ => continuous-aggregates}/drop_materialized_view.md (97%) create mode 100644 api/continuous-aggregates/hypertable_detailed_size.md create mode 100644 api/continuous-aggregates/hypertable_size.md create mode 100644 api/continuous-aggregates/index.md rename api/{ => continuous-aggregates}/refresh_continuous_aggregate.md (98%) rename api/{ => continuous-aggregates}/remove_all_policies.md (100%) rename api/{ => continuous-aggregates}/remove_continuous_aggregate_policy.md (100%) rename api/{ => continuous-aggregates}/remove_policies.md (100%) rename api/{ => continuous-aggregates}/show_policies.md (100%) rename api/{ => data-retention}/add_retention_policy.md (97%) rename api/{data-retention.md => data-retention/index.md} (100%) rename api/{ => data-retention}/remove_retention_policy.md (92%) rename api/{ => distributed-hypertables}/add_data_node.md (98%) rename api/{ => distributed-hypertables}/alter_data_node.md (96%) rename api/{ => distributed-hypertables}/attach_data_node.md (97%) rename api/{ => distributed-hypertables}/cleanup_copy_chunk_operation_experimental.md (90%) rename api/{ => distributed-hypertables}/copy_chunk_experimental.md (92%) rename api/{ => distributed-hypertables}/create_distributed_hypertable.md (98%) rename api/{ => distributed-hypertables}/create_distributed_restore_point.md (100%) rename api/{ => distributed-hypertables}/delete_data_node.md (96%) rename api/{ => distributed-hypertables}/detach_data_node.md (96%) rename api/{ => distributed-hypertables}/distributed_exec.md (97%) rename api/{distributed-hypertables.md => distributed-hypertables/index.md} (100%) rename api/{ => distributed-hypertables}/move_chunk_experimental.md (91%) rename api/{ => distributed-hypertables}/set_number_partitions.md (95%) rename api/{ => distributed-hypertables}/set_replication_factor.md (97%) rename api/{ => hypertable}/add_dimension.md (100%) rename api/{ => hypertable}/add_dimension_old.md (99%) rename api/{ => hypertable}/add_reorder_policy.md (100%) rename api/{ => hypertable}/attach_tablespace.md (97%) rename api/{ => hypertable}/chunks_detailed_size.md (97%) rename api/{ => hypertable}/create_hypertable.md (100%) rename api/{ => hypertable}/create_hypertable_old.md (100%) rename api/{ => hypertable}/create_index.md (99%) rename api/{ => hypertable}/detach_tablespace.md (97%) rename api/{ => hypertable}/detach_tablespaces.md (94%) rename api/{ => hypertable}/disable_chunk_skipping.md (96%) rename api/{ => hypertable}/drop_chunks.md (100%) rename api/{ => hypertable}/enable_chunk_skipping.md (100%) rename api/{ => hypertable}/hypertable_approximate_detailed_size.md (98%) rename api/{ => hypertable}/hypertable_approximate_size.md (98%) create mode 100644 api/hypertable/hypertable_detailed_size.md rename api/{ => hypertable}/hypertable_index_size.md (97%) create mode 100644 api/hypertable/hypertable_size.md rename api/{hypertable.md => hypertable/index.md} (100%) rename api/{ => hypertable}/merge_chunks.md (100%) rename api/{ => hypertable}/move_chunk.md (97%) rename api/{ => hypertable}/remove_reorder_policy.md (92%) rename api/{ => hypertable}/reorder_chunk.md (96%) rename api/{ => hypertable}/set_chunk_time_interval.md (100%) rename api/{ => hypertable}/set_integer_now_func.md (100%) rename api/{ => hypertable}/show_chunks.md (100%) rename api/{ => hypertable}/show_tablespaces.md (92%) diff --git a/_partials/_formatting_examples.md b/_partials/_formatting_examples.md index 34e8360369..885e4d4330 100644 --- a/_partials/_formatting_examples.md +++ b/_partials/_formatting_examples.md @@ -271,7 +271,7 @@ For example: [A link to the data tiering section in docs][data-tiering] [data-tiering]: ../use-timescale/data-tiering/enabling-data-tiering.md -[disable-chunk-skipping]: ../api/disable_chunk_skipping.md +[disable-chunk-skipping]: ../api/hypertable/disable_chunk_skipping.md [decompress-chunks]: ../use-timescale/compression/decompress-chunks.md [alerting]: ../use-timescale/alerting.md [deprecation]: ../_partials/_deprecated.md @@ -281,7 +281,7 @@ For example: [installation-windows]: ../self-hosted/install/installation-windows.md [time-bucket]: ../api/time_bucket_ng.md [time-weighted-average]: ../api/time-weighted-averages.md -[remove-reorder-policy]: ../api/remove_reorder_policy.md +[remove-reorder-policy]: ../api/hypertable/remove_reorder_policy.md [reference-links]: https://www.markdownguide.org/basic-syntax/#reference-style-links diff --git a/api/hypertable_detailed_size.md b/_partials/_hypertable-detailed-size-api.md similarity index 85% rename from api/hypertable_detailed_size.md rename to _partials/_hypertable-detailed-size-api.md index 6643eface4..275e29eb88 100644 --- a/api/hypertable_detailed_size.md +++ b/_partials/_hypertable-detailed-size-api.md @@ -1,14 +1,3 @@ ---- -api_name: hypertable_detailed_size() -excerpt: Get detailed information about disk space used by a hypertable -topics: [hypertables] -keywords: [hypertables, information] -tags: [statistics, size, disk space] -api: - license: apache - type: function ---- - # hypertable_detailed_size() Get detailed information about disk space used by a hypertable or @@ -27,13 +16,13 @@ instead. For more information about using hypertables, including chunk size partitioning, see the [hypertable section][hypertable-docs]. -### Required arguments +## Required arguments |Name|Type|Description| |---|---|---| | `hypertable` | REGCLASS | Hypertable or continuous aggregate to show detailed size of. | -### Returns +## Returns |Column|Type|Description| |-|-|-| @@ -48,7 +37,7 @@ If executed on a relation that is not a hypertable, the function returns `NULL`. -### Sample usage +## Sample usage Get the size information for a hypertable. @@ -68,4 +57,4 @@ The access node is listed without a user-given node name. Normally, the access node holds no data, but still maintains, for example, index information that occupies a small amount of disk space. -[hypertable-docs]: /use-timescale/:currentVersion:/hypertables/ +[hypertable-docs]: /use-timescale/:currentVersion:/hypertables/ \ No newline at end of file diff --git a/api/hypertable_size.md b/_partials/_hypertable-size-api.md similarity index 82% rename from api/hypertable_size.md rename to _partials/_hypertable-size-api.md index 83959b59c2..b5379b8675 100644 --- a/api/hypertable_size.md +++ b/_partials/_hypertable-size-api.md @@ -1,14 +1,3 @@ ---- -api_name: hypertable_size() -excerpt: Get the total disk space used by a hypertable -topics: [hypertables] -keywords: [hypertables, information] -tags: [disk space, size] -api: - license: apache - type: function ---- - # hypertable_size() Get the total disk space used by a hypertable or continuous aggregate, @@ -26,13 +15,13 @@ instead. For more information about using hypertables, including chunk size partitioning, see the [hypertable section][hypertable-docs]. -### Required arguments +## Required arguments |Name|Type|Description| |-|-|-| |`hypertable`|REGCLASS|Hypertable or continuous aggregate to show size of.| -### Returns +## Returns |Name|Type|Description| |-|-|-| @@ -42,7 +31,7 @@ see the [hypertable section][hypertable-docs]. `NULL` is returned if the function is executed on a non-hypertable relation. -### Sample usage +## Sample usage Get the size information for a hypertable. @@ -71,4 +60,4 @@ SELECT hypertable_size('device_stats_15m'); 73728 ``` -[hypertable-docs]: /use-timescale/:currentVersion:/hypertables/ +[hypertable-docs]: /use-timescale/:currentVersion:/hypertables/ \ No newline at end of file diff --git a/api/compression/chunk_compression_stats.md b/api/compression/chunk_compression_stats.md index 6b7014bf3b..2c3feb6ff4 100644 --- a/api/compression/chunk_compression_stats.md +++ b/api/compression/chunk_compression_stats.md @@ -24,13 +24,13 @@ the chunk. An insert into a compressed chunk does not update the compressed sizes. For more information about how to compute chunk sizes, see the `chunks_detailed_size` section. -### Required arguments +## Required arguments |Name|Type|Description| |-|-|-| |`hypertable`|REGCLASS|Name of the hypertable| -### Returns +## Returns |Column|Type|Description| |-|-|-| @@ -47,7 +47,7 @@ sizes. For more information about how to compute chunk sizes, see the |`after_compression_total_bytes`|BIGINT|Size of the entire chunk table (table+indexes+toast) after compression (NULL if currently uncompressed)| |`node_name`|TEXT|nodes on which the chunk is located, applicable only to distributed hypertables| -### Sample usage +## Sample usage ```sql SELECT * FROM chunk_compression_stats('conditions') diff --git a/api/compression/compress_chunk.md b/api/compression/compress_chunk.md index 1bad13976d..82d781ec78 100644 --- a/api/compression/compress_chunk.md +++ b/api/compression/compress_chunk.md @@ -31,26 +31,26 @@ You can get a list of chunks belonging to a hypertable using the [`show_chunks` function](/api/latest/hypertable/show_chunks/). -### Required arguments +## Required arguments |Name|Type|Description| |---|---|---| | `chunk_name` | REGCLASS | Name of the chunk to be compressed| -### Optional arguments +## Optional arguments |Name|Type|Description| |---|---|---| | `if_not_compressed` | BOOLEAN | Disabling this will make the function error out on chunks that are already compressed. Defaults to true.| | `hypercore_use_access_method` | BOOLEAN | `NULL` |✖| Set to `true` to use hypercore table access metod. If set to `NULL` it will use the value from `timescaledb.default_hypercore_use_access_method`. | -### Returns +## Returns |Column|Type|Description| |---|---|---| | `compress_chunk` | REGCLASS | Name of the chunk that was compressed| -### Sample usage +## Sample usage Compress a single chunk. diff --git a/api/compression/decompress_chunk.md b/api/compression/decompress_chunk.md index 3ec7ac6fdf..e503cf2e05 100644 --- a/api/compression/decompress_chunk.md +++ b/api/compression/decompress_chunk.md @@ -28,25 +28,25 @@ the next scheduled job. -### Required arguments +## Required arguments |Name|Type|Description| |---|---|---| |`chunk_name`|`REGCLASS`|Name of the chunk to be decompressed.| -### Optional arguments +## Optional arguments |Name|Type|Description| |---|---|---| |`if_compressed`|`BOOLEAN`|Disabling this will make the function error out on chunks that are not compressed. Defaults to true.| -### Returns +## Returns |Column|Type|Description| |---|---|---| |`decompress_chunk`|`REGCLASS`|Name of the chunk that was decompressed.| -### Sample usage +## Sample usage Decompress a single chunk: diff --git a/api/compression/hypertable_compression_stats.md b/api/compression/hypertable_compression_stats.md index 39f141ff76..8c515ff3a3 100644 --- a/api/compression/hypertable_compression_stats.md +++ b/api/compression/hypertable_compression_stats.md @@ -23,13 +23,13 @@ see the [hypertable section][hypertable-docs]. For more information about compression, see the [compression section][compression-docs]. -### Required arguments +## Required arguments |Name|Type|Description| |-|-|-| |`hypertable`|REGCLASS|Hypertable to show statistics for| -### Returns +## Returns |Column|Type|Description| |-|-|-| @@ -51,7 +51,7 @@ Returns show `NULL` if the data is currently uncompressed. -### Sample usage +## Sample usage ```sql SELECT * FROM hypertable_compression_stats('conditions'); diff --git a/api/compression/remove_compression_policy.md b/api/compression/remove_compression_policy.md index 3b9c72372c..3af46e54ea 100644 --- a/api/compression/remove_compression_policy.md +++ b/api/compression/remove_compression_policy.md @@ -20,19 +20,19 @@ If you need to remove the compression policy. To restart policy-based compression you need to add the policy again. To view the policies that already exist, see [informational views][informational-views]. -### Required arguments +## Required arguments |Name|Type|Description| |-|-|-| |`hypertable`|REGCLASS|Name of the hypertable or continuous aggregate the policy should be removed from| -### Optional arguments +## Optional arguments |Name|Type|Description| |---|---|---| | `if_exists` | BOOLEAN | Setting to true causes the command to fail with a notice instead of an error if a compression policy does not exist on the hypertable. Defaults to false.| -### Sample usage +## Sample usage Remove the compression policy from the 'cpu' table: diff --git a/api/add_continuous_aggregate_policy.md b/api/continuous-aggregates/add_continuous_aggregate_policy.md similarity index 98% rename from api/add_continuous_aggregate_policy.md rename to api/continuous-aggregates/add_continuous_aggregate_policy.md index 6a36cffdf5..919e3b7c2e 100644 --- a/api/add_continuous_aggregate_policy.md +++ b/api/continuous-aggregates/add_continuous_aggregate_policy.md @@ -15,7 +15,7 @@ Create a policy that automatically refreshes a continuous aggregate. To view the policies that you set or the policies that already exist, see [informational views][informational-views]. -### Required arguments +## Required arguments |Name|Type|Description| |-|-|-| @@ -43,7 +43,7 @@ about how continuous aggregates use real-time aggregation, see the [real-time aggregation section](/use-timescale/:currentVersion:/continuous-aggregates/real-time-aggregates/). -### Optional arguments +## Optional arguments |Name|Type|Description| |-|-|-| @@ -59,13 +59,13 @@ Setting `buckets_per_batch` greater than zero means that the refresh window is s Because each `batch` is an individual transaction, executing a policy in batches make the data visible for the users before the entire job is executed. Batches are processed from the most recent data to the oldest. -### Returns +## Returns |Column|Type|Description| |-|-|-| |`job_id`|INTEGER|TimescaleDB background job ID created to implement this policy| -### Sample use +## Sample use Add a policy that refreshes the last month once an hour, excluding the latest hour from the aggregate. For performance reasons, we recommend that you diff --git a/api/add_policies.md b/api/continuous-aggregates/add_policies.md similarity index 100% rename from api/add_policies.md rename to api/continuous-aggregates/add_policies.md diff --git a/api/alter_materialized_view.md b/api/continuous-aggregates/alter_materialized_view.md similarity index 98% rename from api/alter_materialized_view.md rename to api/continuous-aggregates/alter_materialized_view.md index 57eb5de81f..2471768b39 100644 --- a/api/alter_materialized_view.md +++ b/api/continuous-aggregates/alter_materialized_view.md @@ -28,13 +28,13 @@ continuous aggregate view: ALTER MATERIALIZED VIEW SET ( timescaledb.
    REPLICA IDENTITY USING INDEX <_index_name>' ``` - **No primary key or viable unique index**: use brute force. For each table, set `REPLICA IDENTITY` to `FULL`: - + ```sql - psql -X -d $SOURCE -c 'ALTER TABLE
    REPLICA IDENTITY FULL' + psql $SOURCE -c 'ALTER TABLE
    REPLICA IDENTITY FULL' ``` - For each `UPDATE` or `DELETE` statement, PostgreSQL reads the whole table to find all matching rows. - This results in significantly slower replication. If you are expecting a large number of `UPDATE` or `DELETE` - operations on the table, best practice is to not use `FULL` + For each `UPDATE` or `DELETE` statement, PostgreSQL reads the whole table to find all matching rows. + This results in significantly slower replication. If you are expecting a large number of `UPDATE` or `DELETE` + operations on the table, best practice is to not use `FULL` + + To capture only `INSERT` and ignore `UPDATE`s and `DELETE`s, use a + [publish config](https://www.postgresql.org/docs/current/sql-createpublication.html#SQL-CREATEPUBLICATION-PARAMS-WITH-PUBLISH) + while [creating the publication][lives-sync-specify-tables]. + + +1. **Create a user for livesync and assign permissions** + + 1. Create ``: + + ```sql + psql $SOURCE -c "CREATE USER PASSWORD ''" + ``` + + You can use an existing user. However, you must ensure that the user has the following permissions. - To capture only `INSERT` and ignore `UPDATE`s and `DELETE`s, use a - [publish config](https://www.postgresql.org/docs/current/sql-createpublication.html#SQL-CREATEPUBLICATION-PARAMS-WITH-PUBLISH) - while [creating the publication][lives-sync-specify-tables]. + 1. Assign the user permissions on the source database: + + ```sql + psql $SOURCE <; + GRANT SELECT ON ALL TABLES IN SCHEMA "public" TO ; + ALTER DEFAULT PRIVILEGES IN SCHEMA "public" GRANT SELECT ON TABLES TO ; + GRANT CREATE ON DATABASE to ; + EOF + ``` + + If you are sycing from AWS RDS and Aurora to $CLOUD_LONG, run the following command: + ```sql + psql $SOURCE -d "GRANT rds_replication TO ;" + + 1. On each table you want to sync, make `` the owner: + ```sql + psql $SOURCE -c 'ALTER TABLE
    OWNER TO ;' + ``` 1. **Restart your source database** diff --git a/_partials/_livesync-console.md b/_partials/_livesync-console.md index cb338dab3a..5791e7fe0a 100644 --- a/_partials/_livesync-console.md +++ b/_partials/_livesync-console.md @@ -74,8 +74,10 @@ To sync data from your PostgreSQL database to your $SERVICE_LONG using $CONSOLE: In `livesync for PostgreSQL`: 1. Set the `Livesync Name`. - 2. Set the` PostgreSQL Connection String` to point to the source database you want to sync to Timescale. - 3. Press `Continue`. + 1. Set the` PostgreSQL Connection String` to point to the source database you want to sync to Timescale. + + This is the connection string for [``][livesync-tune-source-db]. + 1. Press `Continue`. $CONSOLE connects to the source database and retrieves the schema information. 1. **Optimize the data to syncronize in hypertables** @@ -101,3 +103,5 @@ instance to your $SERVICE_LONG in real-time. [install-psql]: /integrations/:currentVersion:/psql/ [portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services +[livesync-tune-source-db]: /migrate/:currentVersion:/livesync/#tune-your-source-database + From fafbcea1d405fe7149bf22f78ea6dab2eccdeeaf Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Fri, 11 Apr 2025 11:27:53 +0300 Subject: [PATCH 65/90] Add a note about conflicting packages for redhat installations Co-authored-by: Iain Cox --- _partials/_install-self-hosted-redhat-based.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/_partials/_install-self-hosted-redhat-based.md b/_partials/_install-self-hosted-redhat-based.md index 2f06e54f3c..cdc3f53773 100644 --- a/_partials/_install-self-hosted-redhat-based.md +++ b/_partials/_install-self-hosted-redhat-based.md @@ -72,6 +72,8 @@ 1. **Install TimescaleDB** + To avoid errors, **do not** install $TDB_APACHE and $TDB_COMMUNITY at the same time. + ```bash sudo yum install timescaledb-2-postgresql-17 postgresql17 ``` @@ -87,7 +89,7 @@ - 1. Initialize the PostgreSQL instance: + 1. **Initialize the PostgreSQL instance** ```bash sudo /usr/pgsql-17/bin/postgresql-17-setup initdb From a13b5d4ed9ae6007218b3fe197691594c0d77438 Mon Sep 17 00:00:00 2001 From: Iain Cox Date: Fri, 11 Apr 2025 12:05:49 +0200 Subject: [PATCH 66/90] chore: Move start coding to get started. (#4000) * chore: Move start coding to get started. Signed-off-by: Iain Cox --- .../_livesync-configure-source-database.md | 25 ++++++ _partials/_quickstart-intro.md | 5 +- .../_start-coding-golang.md | 82 ++++--------------- .../_start-coding-java.md | 77 +++-------------- .../_start-coding-node.md | 52 ++---------- .../_start-coding-python.md | 67 +++------------ .../_start-coding-ruby.md | 4 - _partials/_where-to-next.md | 2 +- getting-started/index.md | 15 ++-- getting-started/page-index/page-index.js | 6 ++ getting-started/run-queries-from-console.md | 3 +- getting-started/services.md | 3 +- .../start-coding-with-timescale.md | 56 +++++++++++++ .../try-key-features-timescale-products.md | 4 +- lambda/regexRedirects.js | 4 + mst/ingest-data.md | 2 +- page-index/page-index.js | 2 - quick-start/index.md | 17 ---- quick-start/page-index/page-index.js | 34 -------- 19 files changed, 154 insertions(+), 306 deletions(-) rename quick-start/golang.md => _partials/_start-coding-golang.md (92%) rename quick-start/java.md => _partials/_start-coding-java.md (91%) rename quick-start/node.md => _partials/_start-coding-node.md (88%) rename quick-start/python.md => _partials/_start-coding-python.md (90%) rename quick-start/ruby.md => _partials/_start-coding-ruby.md (99%) create mode 100644 getting-started/start-coding-with-timescale.md delete mode 100644 quick-start/index.md delete mode 100644 quick-start/page-index/page-index.js diff --git a/_partials/_livesync-configure-source-database.md b/_partials/_livesync-configure-source-database.md index 306718d19a..82039bedcc 100644 --- a/_partials/_livesync-configure-source-database.md +++ b/_partials/_livesync-configure-source-database.md @@ -70,6 +70,31 @@ psql $SOURCE -c 'ALTER TABLE
    OWNER TO ;' ``` +1. **Create the livesync user and assign permissions** + + 1. Create the livesync user: + + ```sql + psql $SOURCE -c "CREATE USER timescale_livesync PASSWORD 'livesync1234'" + ``` + 1. Assign the user permissions on the source database: + + ```sql + psql $SOURCE < OWNER TO timescale_livesync;' + ``` + 1. **Restart your source database** diff --git a/_partials/_quickstart-intro.md b/_partials/_quickstart-intro.md index a788977d18..d9c564c75f 100644 --- a/_partials/_quickstart-intro.md +++ b/_partials/_quickstart-intro.md @@ -1,3 +1,2 @@ -This code quick start enables you to easily integrate your app with $CLOUD_LONG. -Use your favorite programming language to connect to your $SERVICE_LONG, create and manage -hypertables, then ingest and query data. +Easily integrate your app with $CLOUD_LONG. Use your favorite programming language to connect to your +$SERVICE_LONG, create and manage hypertables, then ingest and query data. diff --git a/quick-start/golang.md b/_partials/_start-coding-golang.md similarity index 92% rename from quick-start/golang.md rename to _partials/_start-coding-golang.md index dcc08e5e92..59b6579582 100644 --- a/quick-start/golang.md +++ b/_partials/_start-coding-golang.md @@ -1,41 +1,21 @@ ---- -title: "Quick Start: Go and TimescaleDB" -excerpt: Get started with Timescale Cloud or TimescaleDB using Go -keywords: [Golang, Go] ---- -import Intro from "versionContent/_partials/_quickstart-intro.mdx"; - -# Go quick start - - - -This quick start guide walks you through: - -* [Connecting to TimescaleDB][connect] -* [Creating a relational table][create-table] -* [Creating a hypertable][create-a-hypertable] -* [Inserting data][insert] -* [Executing a query][query] +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; ## Prerequisites -Before you start, make sure you have: + -* Installed [TimescaleDB][install]. -* Installed [Go][golang-install]. -* Installed the [PGX driver for Go][pgx-driver-github]. +- Install [Go][golang-install]. +- Install the [PGX driver for Go][pgx-driver-github]. -## Connect to TimescaleDB +## Connect to your $SERVICE_SHORT -In this section, you create a connection to TimescaleDB using the PGX driver. +In this section, you create a connection to $CLOUD_LONG using the PGX driver. PGX is a toolkit designed to help Go developers work directly with PostgreSQL. You can use it to help your Go application interact directly with TimescaleDB. - - 1. Locate your TimescaleDB credentials and use them to compose a connection string for PGX. @@ -107,9 +87,7 @@ You can use it to help your Go application interact directly with TimescaleDB. ```go os.Getenv("DATABASE_CONNECTION_STRING") ``` - - - + Alternatively, you can connect to TimescaleDB using a connection pool. @@ -118,8 +96,6 @@ result in faster database queries: - - 1. To create a connection pool that can be used for concurrent connections to your database, use the `pgxpool.New()` function instead of `pgx.Connect()`. Also note that this script imports @@ -159,8 +135,6 @@ result in faster database queries: } ``` - - ## Create a relational table @@ -173,8 +147,6 @@ percentage of the sensors. - - 1. Compose a string that contains the SQL statement to create a relational table. This example creates a table called `sensors`, with columns for ID, type, and location: @@ -223,8 +195,6 @@ percentage of the sensors. } ``` - - ## Generate a hypertable @@ -235,8 +205,6 @@ and most other tasks are executed on the hypertable. - - 1. Create a variable for the `CREATE TABLE SQL` statement for your hypertable. Notice how the hypertable has the compulsory time column: @@ -313,8 +281,6 @@ and most other tasks are executed on the hypertable. } ``` - - ## Insert rows of data @@ -329,8 +295,6 @@ the process. - - 1. Open a connection pool to the database, then use the prepared statements to formulate an `INSERT` SQL statement, and execute it: @@ -381,8 +345,6 @@ the process. } ``` - - Instead of inserting a single row of data at a time, you can use this procedure @@ -390,8 +352,6 @@ to insert multiple rows of data, instead: - - 1. This example uses PostgreSQL to generate some sample time-series to insert into the `sensor_data` hypertable. Define the SQL statement to generate the data, called `queryDataGeneration`. Then use the `.Query()` function to @@ -603,8 +563,6 @@ to insert multiple rows of data, instead: } ``` - - Inserting multiple rows of data using this method executes as many `insert` @@ -616,8 +574,6 @@ the previous procedure. It uses the pgx `Batch` object: - - 1. This example batch inserts data into the database: ```go @@ -742,8 +698,6 @@ the previous procedure. It uses the pgx `Batch` object: } ``` - - ## Execute a query @@ -752,8 +706,6 @@ This section covers how to execute queries against your database. - - 1. Define the SQL query you'd like to run on the database. This example uses a SQL query that combines time-series and relational data. It returns the average CPU values for every 5 minute interval, for sensors located on @@ -904,9 +856,7 @@ This section covers how to execute queries against your database. } } ``` - - - + ## Next steps @@ -928,12 +878,12 @@ your Go application, be sure to check out these advanced TimescaleDB tutorials: [pgx-docs]: https://pkg.go.dev/github.com/jackc/pgx [pgx-driver-github]: https://github.com/jackc/pgx [install]: /getting-started/:currentVersion:/ -[connect]: /quick-start/:currentVersion:/golang/#connect-to-timescaledb -[create-table]: /quick-start/:currentVersion:/golang/#create-a-relational-table -[create-a-hypertable]: /quick-start/:currentVersion:/golang/#generate-a-hypertable -[insert]: /quick-start/:currentVersion:/golang/#insert-rows-of-data -[query]: /quick-start/:currentVersion:/golang/#execute-a-query +[connect]: /getting-started/:currentVersion:/start-coding-with-timescale/#connect-to-timescaledb +[create-table]: /getting-started/:currentVersion:/start-coding-with-timescale/#create-a-relational-table +[create-a-hypertable]: /getting-started/:currentVersion:/start-coding-with-timescale/#generate-a-hypertable +[insert]: /getting-started/:currentVersion:/start-coding-with-timescale/#insert-rows-of-data +[query]: /getting-started/:currentVersion:/start-coding-with-timescale/#execute-a-query [create-hypertable-docs]: /use-timescale/:currentVersion:/hypertables/create/ -[insert]: /quick-start/:currentVersion:/golang/#insert-a-row-into-your-timescale-database -[query]: /quick-start/:currentVersion:/golang/#execute-a-query-on-your-timescale-database -[create-hypertable]: /quick-start/:currentVersion:/python/#generate-a-hypertable +[insert]: /getting-started/:currentVersion:/start-coding-with-timescale/#insert-a-row-into-your-timescale-database +[query]: /getting-started/:currentVersion:/start-coding-with-timescale/#execute-a-query-on-your-timescale-database +[create-hypertable]: /getting-started/:currentVersion:/start-coding-with-timescale/#generate-a-hypertable diff --git a/quick-start/java.md b/_partials/_start-coding-java.md similarity index 91% rename from quick-start/java.md rename to _partials/_start-coding-java.md index f3b9389477..2066fef8b7 100644 --- a/quick-start/java.md +++ b/_partials/_start-coding-java.md @@ -1,45 +1,23 @@ ---- -title: "Quick Start: Java and TimescaleDB" -excerpt: Get started with Timescale Cloud or TimescaleDB using Java -keywords: [Java] ---- - -import Intro from "versionContent/_partials/_quickstart-intro.mdx"; - -# Java quick start - - - -This quick start guide walks you through: - -* [Connecting to TimescaleDB][connect] -* [Creating a relational table][create-table] -* [Creating a hypertable][create-a-hypertable] -* [Inserting data][insert] -* [Executing a query][query] +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; ## Prerequisites -Before you start, make sure you have: + -* Installed TimescaleDB. For more information, see the - [installation documentation][install]. -* Installed the [Java Development Kit (JDK)][jdk]. -* Installed the [PostgreSQL JDBC driver][pg-jdbc-driver]. +* Install the [Java Development Kit (JDK)][jdk]. +* Install the [PostgreSQL JDBC driver][pg-jdbc-driver]. -All code in this quick start is for Java 16 and later. If you are working +All code in this quick start is for Java 16 and later. If you are working with older JDK versions, use legacy coding techniques. -## Connect to TimescaleDB +## Connect to your $SERVICE_SHORT -In this section, you create a connection to TimescaleDB using an application in +In this section, you create a connection to your $SERVICE_SHORT using an application in a single file. You can use any of your favorite build tools, including `gradle` or `maven`. - - 1. Create a directory containing a text file called `Main.java`, with this content: ```java @@ -91,9 +69,9 @@ or `maven`. java -cp *.jar Main.java ``` -If the command is successful, a string similar to -`org.postgresql.Driver@7f77e91b` is printed to your console. This means that you -are ready to connect to TimescaleDB from Java. + If the command is successful, a string similar to + `org.postgresql.Driver@7f77e91b` is printed to your console. This means that you + are ready to connect to TimescaleDB from Java. 1. Locate your TimescaleDB credentials and use them to compose a connection string for JDBC. @@ -145,8 +123,6 @@ are ready to connect to TimescaleDB from Java. If the command is successful, a string similar to `{ApplicationName=PostgreSQL JDBC Driver}` is printed to your console. - - ## Create a relational table @@ -159,8 +135,6 @@ percentage of the sensors. - - 1. Compose a string which contains the SQL statement to create a relational table. This example creates a table called `sensors`, with columns `id`, `type` and `location`: @@ -210,9 +184,7 @@ percentage of the sensors. } } ``` - - - + ## Create a hypertable @@ -223,8 +195,6 @@ and most other tasks are executed on the hypertable. - - 1. Create a `CREATE TABLE` SQL statement for your hypertable. Notice how the hypertable has the compulsory time column: @@ -297,8 +267,6 @@ and most other tasks are executed on the hypertable. } ``` - - ## Insert data @@ -308,8 +276,6 @@ section, you can insert single rows, or insert by batches of rows. - - 1. Open a connection to the database, use prepared statements to formulate the `INSERT` SQL statement, then execute the statement: @@ -329,8 +295,6 @@ section, you can insert single rows, or insert by batches of rows. } ``` - - If you want to insert a batch of rows by using a batching mechanism. In this @@ -339,11 +303,6 @@ example, you generate some sample time-series data to insert into the - - 1. Insert batches of rows: ```java @@ -367,8 +326,6 @@ defaultExpanded={false}> } ``` - - ## Execute a query @@ -377,8 +334,6 @@ This section covers how to execute queries against your database. - - ## Execute queries on TimescaleDB 1. Define the SQL query you'd like to run on the database. This example @@ -428,8 +383,6 @@ This section covers how to execute queries against your database. ... ``` - - ## Next steps @@ -446,7 +399,7 @@ advanced TimescaleDB tutorials: This section contains complete code samples. - +### Complete code sample ```java package com.timescale.java; @@ -534,9 +487,7 @@ public class Main { } ``` - - - +### Execute more complex queries ```java package com.timescale.java; @@ -645,8 +596,6 @@ public class Main { } ``` - - [jdk]: https://openjdk.java.net [pg-jdbc-driver-artifact]: https://jdbc.postgresql.org/download/ [pg-jdbc-driver-conn-docs]: https://jdbc.postgresql.org/documentation/datasource/ diff --git a/quick-start/node.md b/_partials/_start-coding-node.md similarity index 88% rename from quick-start/node.md rename to _partials/_start-coding-node.md index a0160a19ec..0520c53c25 100644 --- a/quick-start/node.md +++ b/_partials/_start-coding-node.md @@ -1,31 +1,11 @@ ---- -title: "Quick Start: Node.js and TimescaleDB" -excerpt: Get started with Timescale Cloud or TimescaleDB using Node.js -keywords: [Node, node.js] ---- - -import Intro from "versionContent/_partials/_quickstart-intro.mdx"; - -# Node.js quick start - - - -This quick start guide walks you through: - -* [Connecting to TimescaleDB][connect] -* [Creating a relational table][create-table] -* [Creating a hypertable][create-a-hypertable] -* [Inserting data][insert] -* [Executing a query][query] +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; ## Prerequisites -Before you start, make sure you have: + -* Installed TimescaleDB. For more information, see the - [installation documentation][install]. -* Installed [Node.js][node-install]. -* Installed the Node.js package manager [npm][npm-install]. +* Install [Node.js][node-install]. +* Install the Node.js package manager [npm][npm-install]. ## Connect to TimescaleDB @@ -34,8 +14,6 @@ ORM (object relational mapper) called [Sequelize][sequelize-info]. - - 1. At the command prompt, initialize a new Node.js app: ```bash @@ -157,8 +135,6 @@ ORM (object relational mapper) called [Sequelize][sequelize-info]. Connection has been established successfully. ``` - - ## Create a relational table @@ -167,8 +143,6 @@ In this section, you create a relational table called `page_loads`. - - 1. Use the Sequelize command line tool to create a table and model called `page_loads`: ```bash @@ -237,8 +211,6 @@ In this section, you create a relational table called `page_loads`. 1. Instantiate a `PageLoads` object and save it to the database. - - ## Create a hypertable @@ -249,8 +221,6 @@ and most other tasks are executed on the hypertable. - - 1. Create a migration to modify the `page_loads` relational table, and change it to a hypertable by first running the following command: @@ -304,8 +274,6 @@ and most other tasks are executed on the hypertable. == 20200601202912-add_hypertable: migrated (0.426s) ``` - - ## Insert rows of data @@ -314,10 +282,6 @@ This section covers how to insert data into your hypertables. - - -## Insert rows into TimescaleDB - 1. In the `index.js` file, modify the `/` route to get the `user-agent` from the request object (`req`) and the current timestamp. Then, call the `create` method on `PageLoads` model, supplying the user agent and timestamp @@ -342,9 +306,7 @@ This section covers how to insert data into your hypertables. } }) ``` - - - + ## Execute a query @@ -355,8 +317,6 @@ is displayed. - - 1. Modify the `/` route in the `index.js` file to call the Sequelize `findAll` function and retrieve all data from the `page_loads` table using the `PageLoads` model: @@ -385,8 +345,6 @@ is displayed. Now, when you reload the page, you should see all of the rows currently in the `page_loads` table. - - [node-install]: https://nodejs.org diff --git a/quick-start/python.md b/_partials/_start-coding-python.md similarity index 90% rename from quick-start/python.md rename to _partials/_start-coding-python.md index 13b39c3590..77d3d8bf2d 100644 --- a/quick-start/python.md +++ b/_partials/_start-coding-python.md @@ -1,32 +1,13 @@ ---- -title: "Quick Start: Python and TimescaleDB" -excerpt: Get started with Timescale Cloud or TimescaleDB using Python -keywords: [Python] ---- - -import Intro from "versionContent/_partials/_quickstart-intro.mdx"; - -# Python quick start - - - -This quick start guide walks you through: - -* [Connecting to TimescaleDB][connect] -* [Creating a relational table][create-table] -* [Creating a hypertable][create-a-hypertable] -* [Inserting data][insert] -* [Executing a query][query] +import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; ## Prerequisites -Before you start, make sure you have: + + +* Install the `psycopg2` library. -* Installed TimescaleDB. For more information, see the - [installation documentation][install]. -* Installed the `psycopg2` library. For more information, see the - [psycopg2 documentation][psycopg2-docs]. -* [](#)A [Python virtual environment][virtual-env]. + For more information, see the [psycopg2 documentation][psycopg2-docs]. +* Create a [Python virtual environment][virtual-env]. [](#) ## Connect to TimescaleDB @@ -37,8 +18,6 @@ prevents common attacks such as SQL injection. - - 1. Import the psycogpg2 library: ```python @@ -108,8 +87,6 @@ prevents common attacks such as SQL injection. print(cursor.fetchone()) ``` - - ## Create a relational table @@ -122,8 +99,6 @@ percentage of the sensors. - - 1. Compose a string which contains the SQL statement to create a relational table. This example creates a table called `sensors`, with columns `id`, `type` and `location`: @@ -149,8 +124,6 @@ percentage of the sensors. cursor.close() ``` - - ## Create a hypertable @@ -161,8 +134,6 @@ and most other tasks are executed on the hypertable. - - 1. Create a string variable that contains the `CREATE TABLE` SQL statement for your hypertable. Notice how the hypertable has the compulsory time column: @@ -202,9 +173,7 @@ and most other tasks are executed on the hypertable. conn.commit() cursor.close() ``` - - - + ## Insert rows of data @@ -215,8 +184,6 @@ section, you can use `psycopg2` with prepared statements, or you can use - - 1. This example inserts a list of tuples, or relational data, called `sensors`, into the relational table named `sensors`. Open a cursor with a connection to the database, use prepared statements to formulate the `INSERT` SQL @@ -251,8 +218,6 @@ section, you can use `psycopg2` with prepared statements, or you can use conn.commit() ``` - - If you choose to use `pgcopy` instead, install the `pgcopy` package @@ -265,8 +230,6 @@ from pgcopy import CopyManager - - 1. Generate some random sensor data using the `generate_series` function provided by PostgreSQL. This example inserts a total of 480 rows of data (4 readings, every 5 minutes, for 24 hours). In your application, this would be @@ -350,8 +313,6 @@ from pgcopy import CopyManager print(cursor.fetchall()) ``` - - ## Execute a query @@ -367,9 +328,9 @@ For more information about properly using placeholders in `psycopg2`, see the For more information about how to execute more complex queries in `psycopg2`, see the [psycopg2 documentation][psycopg2-docs-basics]. - +### Execute a query - + 1. Define the SQL query you'd like to run on the database. This example is a simple `SELECT` statement querying each row from the previously created @@ -412,16 +373,14 @@ see the [psycopg2 documentation][psycopg2-docs-basics]. Using this cursor, `cursor.fetchall()` returns a list of dictionary-like objects. - - For more complex queries, you can use prepared statements to ensure queries are executed safely against the database. - +### Execute queries using prepared statements - + 1. Write the query using prepared statements: @@ -442,9 +401,7 @@ executed safely against the database. cursor.execute(query, data) results = cursor.fetchall() ``` - - - + [install]: /getting-started/latest/ diff --git a/quick-start/ruby.md b/_partials/_start-coding-ruby.md similarity index 99% rename from quick-start/ruby.md rename to _partials/_start-coding-ruby.md index 857e6e2a2c..145952a2a8 100644 --- a/quick-start/ruby.md +++ b/_partials/_start-coding-ruby.md @@ -7,10 +7,6 @@ keywords: [Ruby] import QuickstartIntro from "versionContent/_partials/_quickstart-intro.mdx"; import IntegrationPrereqs from "versionContent/_partials/_integration-prereqs.mdx"; -# Ruby quick start - - - ## Prerequisites diff --git a/_partials/_where-to-next.md b/_partials/_where-to-next.md index 1a3344859e..6c4aa2ac8e 100644 --- a/_partials/_where-to-next.md +++ b/_partials/_where-to-next.md @@ -4,8 +4,8 @@ your $SERVICE_LONG with a range of [third-party tools][integrations], plain old into [the API][use-the-api]. [tutorials]: /tutorials/:currentVersion:/ -[connect-with-code]: /quick-start/:currentVersion:/ [integrations]: /integrations/:currentVersion:/ +[connect-with-code]: /getting-started/:currentVersion:/start-coding-with-timescale/ [use-the-api]: /api/:currentVersion:/ [use-timescale]: /use-timescale/:currentVersion:/ [try-timescale-features]: /getting-started/:currentVersion:/try-key-features-timescale-products/ diff --git a/getting-started/index.md b/getting-started/index.md index e202b2225f..c828b0fa76 100644 --- a/getting-started/index.md +++ b/getting-started/index.md @@ -1,8 +1,7 @@ --- title: Get started with Timescale excerpt: Supercharge your real-time analytics on time-series data with Timescale Cloud. Create a free account, launch your first service, and use some of the advanced features -products: [cloud] -layout_components: [next_prev_large] +products: [cloud, mst, self_hosted] content_group: Getting started --- @@ -16,14 +15,17 @@ import WhereNext from "versionContent/_partials/_where-to-next.mdx"; This section shows you how to: -1. [Create and connect to a $SERVICE_LONG][services-create]: choose the capabilities that match your business and engineering needs on $COMPANY's cloud-based $PG platform. -1. [Run queries from $CONSOLE][run-queries-from-console]: securely interact with your data in the $CONSOLE UI. -1. [Try the main features in $COMPANY products][test-drive]: rapidly implement the features in $CLOUD_LONG that enable you to ingest and query data faster while keeping the costs low. +- [Create and connect to a $SERVICE_LONG][services-create]: choose the capabilities that match your business and + engineering needs on $COMPANY's cloud-based $PG platform. +- [Try the main features in $COMPANY products][test-drive]: rapidly implement the features in $CLOUD_LONG that + enable you to ingest and query data faster while keeping the costs low. +- [Start coding with Timescale][start-coding]: quickly integrate $CLOUD_LONG and $TIMESCALE_DB into your apps using your favorite programming language. +- [Run queries from $CONSOLE][run-queries-from-console]: securely interact with your data in the $CONSOLE UI. [tutorials]: /tutorials/:currentVersion:/ -[connect-with-code]: /quick-start/:currentVersion:/ +[connect-with-code]: /getting-started/:currentVersion:/start-coding-with-timescale/ [integrations]: /integrations/:currentVersion:/ [use-the-api]: /api/:currentVersion:/ [use-timescale]: /use-timescale/:currentVersion:/ @@ -31,5 +33,6 @@ This section shows you how to: [services-connect]: /getting-started/:currentVersion:/services/#connect-to-your-service [test-drive]: /getting-started/:currentVersion:/try-key-features-timescale-products/ [run-queries-from-console]: /getting-started/:currentVersion:/run-queries-from-console/ +[start-coding]: /getting-started/:currentVersion:/start-coding-with-timescale/ [ingest-data]: /getting-started/:currentVersion:/try-key-features-timescale-products/#optimize-time-series-data-in-hypertables diff --git a/getting-started/page-index/page-index.js b/getting-started/page-index/page-index.js index 3f46a8a488..4d51fd1ea8 100644 --- a/getting-started/page-index/page-index.js +++ b/getting-started/page-index/page-index.js @@ -16,6 +16,12 @@ module.exports = [ excerpt: "Improve database performance with Hypertables, time bucketing, continuous aggregates, compression, data tiering, and high availability", }, + { + title: "Start coding with Timescale", + href: "start-coding-with-timescale", + excerpt: + "Integrate Timescale Cloud with your app using your preferred programming language", + }, { title: "Run your queries from Timescale Console", href: "run-queries-from-console", diff --git a/getting-started/run-queries-from-console.md b/getting-started/run-queries-from-console.md index 44fdb6568e..51b72b886e 100644 --- a/getting-started/run-queries-from-console.md +++ b/getting-started/run-queries-from-console.md @@ -1,8 +1,7 @@ --- title: Run your queries from Timescale Console excerpt: Choose the right tool to manage your data. Timescale Cloud offers the data mode, the SQL editor, and the SQL Assistant to better address your needs -keywords: [popsql, sql editor, chart, dashboard] -layout_components: [next_prev_large] +products: [cloud, mst, self_hosted] content_group: Getting started --- diff --git a/getting-started/services.md b/getting-started/services.md index 96dcd59b87..c11362ae31 100644 --- a/getting-started/services.md +++ b/getting-started/services.md @@ -1,8 +1,7 @@ --- title: Create your first Timescale service excerpt: Timescale Cloud offers a range of capabilities to accommodate your real-time analytics and AI and vector workloads. Learn more about each of them and create your first service in Timescale Console -products: [cloud] -layout_components: [next_prev_large] +products: [cloud, mst, self_hosted] content_group: Getting started --- diff --git a/getting-started/start-coding-with-timescale.md b/getting-started/start-coding-with-timescale.md new file mode 100644 index 0000000000..252bcae7ee --- /dev/null +++ b/getting-started/start-coding-with-timescale.md @@ -0,0 +1,56 @@ +--- +title: Start coding with Timescale +excerpt: Integrate Timescale Cloud with your app using your preferred programming language. Connect to a service, + create and manage hypertables, then and ingest and query data +products: [cloud, mst, self_hosted] +content_group: Getting started +--- + +import StartCodingRuby from "versionContent/_partials/_start-coding-ruby.mdx"; +import StartCodingPython from "versionContent/_partials/_start-coding-python.mdx"; +import StartCodingNode from "versionContent/_partials/_start-coding-node.mdx"; +import StartCodingGoLang from "versionContent/_partials/_start-coding-golang.mdx"; +import StartCodingJava from "versionContent/_partials/_start-coding-java.mdx"; + +# Start coding with Timescale + +Easily integrate your app with $CLOUD_LONG. Use your favorite programming language to connect to your +$SERVICE_LONG, create and manage hypertables, then ingest and query data. + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +You are not limited to these languages. $CLOUD_LONG is based on $PG, you can interface +with $TIMESCALE_DB and $CLOUD_LONG using any [$PG client driver][postgres-drivers]. + + +[postgres-drivers]: https://wiki.postgresql.org/wiki/List_of_drivers diff --git a/getting-started/try-key-features-timescale-products.md b/getting-started/try-key-features-timescale-products.md index f02c24b639..0441471283 100644 --- a/getting-started/try-key-features-timescale-products.md +++ b/getting-started/try-key-features-timescale-products.md @@ -1,7 +1,7 @@ --- title: Try the key Timescale features excerpt: Improve database performance with hypertables, time bucketing, compression and continuous aggregates. -products: [cloud] +products: [cloud, mst, self_hosted] content_group: Getting started --- @@ -405,7 +405,7 @@ What next? See the [use case tutorials][tutorials], interact with the data in yo [third-party tools][integrations], plain old [Use Timescale][use-timescale], or dive into [the API][use-the-api]. [tutorials]: /tutorials/:currentVersion:/ -[connect-with-code]: /quick-start/:currentVersion:/ +[connect-with-code]: /getting-started/:currentVersion:/start-coding-with-timescale/ [integrations]: /integrations/:currentVersion:/ [use-the-api]: /api/:currentVersion:/ [use-timescale]: /use-timescale/:currentVersion:/ diff --git a/lambda/regexRedirects.js b/lambda/regexRedirects.js index ba56a10a0b..07aec09cb8 100644 --- a/lambda/regexRedirects.js +++ b/lambda/regexRedirects.js @@ -12,6 +12,10 @@ module.exports = [ to: (match) => `https://docs.timescale.com/integrations/latest${match[1] || ""}` }, + { + from: /^\/quick-start\/latest\/(\/.*)?$/, + to: `https://docs.timescale.com/getting-started/latest/start-coding-with-timescale/` + }, { from: /^\/v[0-1]\.[0-9]\/tutorials/, to: "https://docs.timescale.com/tutorials/latest/" diff --git a/mst/ingest-data.md b/mst/ingest-data.md index 279cec75f3..e64a29ebe5 100644 --- a/mst/ingest-data.md +++ b/mst/ingest-data.md @@ -154,7 +154,7 @@ recommend that you use the connector with Kafka and Kafka Connect. See [these instructions][gh-kafkaconnector] for using the Kafka connector. -[code-qs]: /quick-start/:currentVersion:/ +[code-qs]: /getting-started/:currentVersion:/start-coding-with-timescale/ [gh-kafkaconnector]: https://github.com/debezium/debezium/tree/master/debezium-connector-postgres [github-parallel-copy]: https://github.com/timescale/timescaledb-parallel-copy [postgres-odbc]: https://odbc.postgresql.org/ diff --git a/page-index/page-index.js b/page-index/page-index.js index 7512fff364..f01047c462 100644 --- a/page-index/page-index.js +++ b/page-index/page-index.js @@ -5,7 +5,6 @@ const gsgPageIndex = require("../getting-started/page-index/page-index"); const timescaleUsingPageIndex = require("../use-timescale/page-index/page-index"); const navigationPageIndex = require("../navigation/page-index/page-index"); const tutorialsPageIndex = require("../tutorials/page-index/page-index.js"); -const codeQuickStartsPageIndex = require("../quick-start/page-index/page-index.js"); const migrationPageIndex = require("../migrate/page-index/page-index.js"); const timescaleAboutPageIndex = require("../about/page-index/page-index"); const integrationsIndex = require("../integrations/page-index/page-index"); @@ -15,7 +14,6 @@ module.exports = [ ...gsgPageIndex, ...timescaleUsingPageIndex, ...tutorialsPageIndex, - ...codeQuickStartsPageIndex, ...integrationsIndex, ...apiReferencePageIndex, ...selfHostedPageIndex, diff --git a/quick-start/index.md b/quick-start/index.md deleted file mode 100644 index e1e25ed346..0000000000 --- a/quick-start/index.md +++ /dev/null @@ -1,17 +0,0 @@ ---- -title: Code quick starts -excerpt: Get started with Timescale Cloud or TimescaleDB in your preferred programming language. Learn to connect to a database, create and manage hypertables, and ingest and query data ---- - -# Start coding with Timescale - -These pages help you to quickly integrate Timescale Cloud and TimescaleDB into your apps using code: - -* [Node.js](/quick-start/:currentVersion:/node/) -* [Python](/quick-start/:currentVersion:/python/) -* [Ruby](/quick-start/:currentVersion:/ruby/) -* [Golang](/quick-start/:currentVersion:/golang/) -* [Java](/quick-start/:currentVersion:/java/) - -You are not limited to these languages. Timescale Cloud is based on PostgreSQL, you can interface -with TimescaleDB and Timescale Cloud using any [PostgreSQL client driver](https://wiki.postgresql.org/wiki/List_of_drivers). diff --git a/quick-start/page-index/page-index.js b/quick-start/page-index/page-index.js deleted file mode 100644 index 92661c627a..0000000000 --- a/quick-start/page-index/page-index.js +++ /dev/null @@ -1,34 +0,0 @@ -module.exports = [ - { - title: "Start coding with Timescale", - href: "quick-start", - excerpt: "A collection of quick starts for common programming languages.", - children: [ - { - title: "Node.js", - href: "node", - excerpt: "Learn about using TimescaleDB with node.js", - }, - { - title: "Python", - href: "python", - excerpt: "Learn about using TimescaleDB with Python", - }, - { - title: "Ruby", - href: "ruby", - excerpt: "Learn about using TimescaleDB with Ruby", - }, - { - title: "Golang", - href: "golang", - excerpt: "Learn about using TimescaleDB with Golang", - }, - { - title: "Java", - href: "java", - excerpt: "Learn about using TimescaleDB with Java", - }, - ], - }, -]; From 385a0f373bd3e8f122a332e40db367462f93f30f Mon Sep 17 00:00:00 2001 From: Rahil Sondhi Date: Fri, 11 Apr 2025 11:27:08 -0500 Subject: [PATCH 67/90] [changelog] TimescaleDB v2.19, new Service Overview page, and log improvements (#4014) --- about/changelog.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/about/changelog.md b/about/changelog.md index 6eaa3f420a..b417ba75dd 100644 --- a/about/changelog.md +++ b/about/changelog.md @@ -8,6 +8,35 @@ keywords: [changelog, upgrades, updates, releases] All the latest features and updates to Timescale products. +## 🪵 TimescaleDB v2.19, new service overview page, and log improvements + + +### TimescaleDB v2.19—query performance and concurrency improvements + +Starting this week, all new services created on Timescale Cloud use [TimescaleDB v2.19](https://github.com/timescale/timescaledb/releases/tag/2.19.0). Existing services will be upgraded gradually during their maintenance window. + +Highlighted features in TimescaleDB v2.19 include: + +- Improved concurrency of `INSERT`, `UPDATE`, and `DELETE` operations on the columnstore by no longer blocking DML statements during the recompression of a chunk. +- Improved system performance during continuous aggregate refreshes by breaking them into smaller batches. This reduces systems pressure and minimizes the risk of spilling to disk. +- Faster and more up-to-date results for queries against continuous aggregates by materializing the most recent data first, as opposed to old data first in prior versions. +- Faster analytical queries with SIMD vectorization of aggregations over text columns and `GROUP BY` over multiple columns. +- Enable chunk size optimization for better query performance in the columnstore by merging them with `merge_chunk`. + +### New service overview page + +The service overview page in Timescale Console has been overhauled to make it simpler and easier to use. Navigate to the `Overview` tab for any of your services and you will find an architecture diagram and general information pertaining to it. You may also see recommendations at the top, for how to optimize your service. + +![New Service Overview page](https://assets.timescale.com/docs/images/new-timescale-service-overview.png) + +To leave the product team your feedback, open `Help & Support` on the left and select `Send feedback to the product team`. + +### Find logs faster + +Finding logs just got easier! We've added a date, time, and timezone picker, so you can jump straight to the exact moment you're interested in—no more endless scrolling. + +![Find logs faster](https://assets.timescale.com/docs/images/find-logs-faster-timescale-console.png) + ## 📒Faster vector search and improved job information From 5c9a2af5cf8b52e385c1b23256df642a6516e250 Mon Sep 17 00:00:00 2001 From: BenR Date: Sat, 12 Apr 2025 02:47:02 -0600 Subject: [PATCH 68/90] Fix small typo in _livesync-configure-source-database.md (#4016) Small but important typo for anyone copying the sql command: ALTER SYSTEM SET max_wal_sender=10; to ALTER SYSTEM SET max_wal_senders=10; Signed-off-by: BenR Co-authored-by: Iain Cox --- _partials/_livesync-configure-source-database.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_partials/_livesync-configure-source-database.md b/_partials/_livesync-configure-source-database.md index 82039bedcc..133b53978b 100644 --- a/_partials/_livesync-configure-source-database.md +++ b/_partials/_livesync-configure-source-database.md @@ -3,7 +3,7 @@ ```sql psql $SOURCE < Date: Mon, 14 Apr 2025 10:14:57 +0300 Subject: [PATCH 69/90] add a note about unique indexes (#4007) --- api/hypertable/create_index.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/api/hypertable/create_index.md b/api/hypertable/create_index.md index 89b8556ef6..f4f587625f 100644 --- a/api/hypertable/create_index.md +++ b/api/hypertable/create_index.md @@ -20,25 +20,25 @@ using a single transaction for the entire hypertable. This allows `INSERT`s, and other operations to be performed concurrently during most of the duration of the `CREATE INDEX` command. While the index is being created on an individual chunk, it functions as if a regular `CREATE INDEX` were called on that chunk, however -other chunks are completely un-blocked. +other chunks are completely unblocked. - This version of `CREATE INDEX` can be used as an alternative to `CREATE INDEX CONCURRENTLY`, which is not currently supported on hypertables. - -If the operation fails partway through, indexes might not be created on all + +- Not supported for `CREATE UNIQUE INDEX`. +- If the operation fails partway through, indexes might not be created on all hypertable chunks. If this occurs, the index on the root table of the hypertable is marked as invalid. You can check this by running `\d+` on the hypertable. The index still works, and is created on new chunks, but if you want to ensure all chunks have a copy of the index, drop and recreate it. -You can also use the following query to find all invalid indexes: + You can also use the following query to find all invalid indexes: -```SQL -SELECT * FROM pg_index i WHERE i.indisvalid IS FALSE; -``` + ```SQL + SELECT * FROM pg_index i WHERE i.indisvalid IS FALSE; + ``` From aee1f07ae8315f6dc298c42761c12bf82243b922 Mon Sep 17 00:00:00 2001 From: Arunprasad Rajkumar Date: Mon, 14 Apr 2025 15:49:14 +0530 Subject: [PATCH 70/90] Fix livesync user creation steps (#4020) * livesync: Streamline user creation instructions Signed-off-by: Arunprasad Rajkumar * Fix order Signed-off-by: Arunprasad Rajkumar --------- Signed-off-by: Arunprasad Rajkumar Co-authored-by: Iain Cox --- ...vesync-configure-source-database-awsrds.md | 74 +++++++++++++++++++ .../_livesync-configure-source-database.md | 71 ++++-------------- _partials/_livesync-console.md | 11 +-- 3 files changed, 93 insertions(+), 63 deletions(-) create mode 100644 _partials/_livesync-configure-source-database-awsrds.md diff --git a/_partials/_livesync-configure-source-database-awsrds.md b/_partials/_livesync-configure-source-database-awsrds.md new file mode 100644 index 0000000000..2e571d5f7b --- /dev/null +++ b/_partials/_livesync-configure-source-database-awsrds.md @@ -0,0 +1,74 @@ +import EnableReplication from "versionContent/_partials/_migrate_live_setup_enable_replication.mdx"; + +Updating parameters on a PostgreSQL instance will cause an outage. Choose a time that will cause the least issues to tune this database. + +1. **Tune the Write Ahead Log (WAL) on the RDS/Aurora PostgreSQL source database** + + 1. In [https://console.aws.amazon.com/rds/home#databases:][databases], + select the RDS instance to migrate. + + 1. Click `Configuration`, scroll down and note the `DB instance parameter group`, then click `Parameter Groups` + + Create security rule to enable RDS EC2 connection + + 1. Click `Create parameter group`, fill in the form with the following values, then click `Create`. + - **Parameter group name** - whatever suits your fancy. + - **Description** - knock yourself out with this one. + - **Engine type** - `PostgreSQL` + - **Parameter group family** - the same as `DB instance parameter group` in your `Configuration`. + 1. In `Parameter groups`, select the parameter group you created, then click `Edit`. + 1. Update the following parameters, then click `Save changes`. + - `rds.logical_replication` set to `1`: record the information needed for logical decoding. + - `wal_sender_timeout` set to `0`: disable the timeout for the sender process. + + 1. In RDS, navigate back to your [databases][databases], select the RDS instance to migrate and click `Modify`. + + 1. Scroll down to `Database options` select your new parameter group and click `Continue`. + 1. Either `Apply immediately` or choose a maintence window, then click `Modify DB instance`. + + Changing parameters will cause an outage. Wait for the database instance to reboot before continuing. + 1. Verify that the settings are live in your database. + +1. **Create a user for livesync and assign permissions** + + 1. Create ``: + + ```sql + psql $SOURCE -c "CREATE USER PASSWORD ''" + ``` + + You can use an existing user. However, you must ensure that the user has the following permissions. + + 1. Assign the user permissions on the source database: + + ```sql + psql $SOURCE <; + GRANT SELECT ON ALL TABLES IN SCHEMA "public" TO ; + ALTER DEFAULT PRIVILEGES IN SCHEMA "public" GRANT SELECT ON TABLES TO ; + GRANT CREATE ON DATABASE to ; + GRANT rds_replication TO ; + EOF + ``` + + If the tables you are syncing are not in the `public` schema, grant the user permissions on the schema you are syncing.: + ```sql + psql $SOURCE < TO ; + EOF + ``` + + 1. On each table you want to sync, make `` the owner: + + ```sql + psql $SOURCE -c 'ALTER TABLE
    OWNER TO ;' + ``` + You can skip this step if the replicating user is already the owner of the tables. + +1. **Enable replication `DELETE` and`UPDATE` operations** + + + +[databases]: https://console.aws.amazon.com/rds/home#databases: diff --git a/_partials/_livesync-configure-source-database.md b/_partials/_livesync-configure-source-database.md index 133b53978b..bb94d8146e 100644 --- a/_partials/_livesync-configure-source-database.md +++ b/_partials/_livesync-configure-source-database.md @@ -1,3 +1,5 @@ +import EnableReplication from "versionContent/_partials/_migrate_live_setup_enable_replication.mdx"; + 1. **Tune the Write Ahead Log (WAL) on the PostgreSQL source database** ```sql @@ -11,33 +13,7 @@ * [GUC “max_wal_senders” as 10](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-MAX-WAL-SENDERS) * [GUC “wal_sender_timeout” as 0](https://www.postgresql.org/docs/current/runtime-config-replication.html#GUC-WAL-SENDER-TIMEOUT) -1. **Enable update and delete replication on the source database** - - Replica identity assists data replication by identifying the rows being modified. - By default each table and hypertable in the source database defaults to the primary key of the table being replicated. - However, you can also have: - - - **A viable unique index**: each table has a unique, non-partial, non-deferrable index that includes only columns - marked as `NOT NULL`. If a `UNIQUE` index does not exists, create one to assist the migration. You can delete it after - live sync. For each table, set `REPLICA IDENTITY` to the viable unique index: - - ```sql - psql $SOURCE -c 'ALTER TABLE
    REPLICA IDENTITY USING INDEX <_index_name>' - ``` - - - **No primary key or viable unique index**: use brute force. For each table, set `REPLICA IDENTITY` to `FULL`: - - ```sql - psql $SOURCE -c 'ALTER TABLE
    REPLICA IDENTITY FULL' - ``` - For each `UPDATE` or `DELETE` statement, PostgreSQL reads the whole table to find all matching rows. - This results in significantly slower replication. If you are expecting a large number of `UPDATE` or `DELETE` - operations on the table, best practice is to not use `FULL` - - To capture only `INSERT` and ignore `UPDATE`s and `DELETE`s, use a - [publish config](https://www.postgresql.org/docs/current/sql-createpublication.html#SQL-CREATEPUBLICATION-PARAMS-WITH-PUBLISH) - while [creating the publication][lives-sync-specify-tables]. - + This will require a restart of the PostgreSQL source database. 1. **Create a user for livesync and assign permissions** @@ -46,9 +22,9 @@ ```sql psql $SOURCE -c "CREATE USER PASSWORD ''" ``` - + You can use an existing user. However, you must ensure that the user has the following permissions. - + 1. Assign the user permissions on the source database: ```sql @@ -57,45 +33,24 @@ GRANT SELECT ON ALL TABLES IN SCHEMA "public" TO ; ALTER DEFAULT PRIVILEGES IN SCHEMA "public" GRANT SELECT ON TABLES TO ; GRANT CREATE ON DATABASE to ; + GRANT replication TO ; EOF ``` - - If you are sycing from AWS RDS and Aurora to $CLOUD_LONG, run the following command: - ```sql - psql $SOURCE -d "GRANT rds_replication TO ;" - - 1. On each table you want to sync, make `` the owner: - - ```sql - psql $SOURCE -c 'ALTER TABLE
    OWNER TO ;' - ``` - -1. **Create the livesync user and assign permissions** - - 1. Create the livesync user: - - ```sql - psql $SOURCE -c "CREATE USER timescale_livesync PASSWORD 'livesync1234'" - ``` - 1. Assign the user permissions on the source database: + If the tables you are syncing are not in the `public` schema, grant the user permissions on the schema you are syncing.: ```sql psql $SOURCE < TO ; EOF ``` - 1. On each table you want to sync, make `livesync` the owner: + 1. On each table you want to sync, make `` the owner: ```sql - psql $SOURCE -c 'ALTER TABLE
    OWNER TO timescale_livesync;' + psql $SOURCE -c 'ALTER TABLE
    OWNER TO ;' ``` + You can skip this step if the replicating user is already the owner of the tables. -1. **Restart your source database** - +1. **Enable replication `DELETE` and`UPDATE` operations** -[lives-sync-specify-tables]: /migrate/:currentVersion:/livesync/#specify-the-tables-to-synchronize + diff --git a/_partials/_livesync-console.md b/_partials/_livesync-console.md index 5791e7fe0a..3e283e0238 100644 --- a/_partials/_livesync-console.md +++ b/_partials/_livesync-console.md @@ -1,7 +1,7 @@ import PrereqCloud from "versionContent/_partials/_prereqs-cloud-only.mdx"; import LivesyncLimitations from "versionContent/_partials/_livesync-limitations.mdx"; import LivesyncConfigureSourceDatabase from "versionContent/_partials/_livesync-configure-source-database.mdx"; -import TuneSourceDatabaseAWSRDS from "versionContent/_partials/_migrate_live_tune_source_database_awsrds.mdx"; +import TuneSourceDatabaseAWSRDS from "versionContent/_partials/_livesync-configure-source-database-awsrds.mdx"; ## Prerequisites @@ -37,19 +37,20 @@ requires a direct connection to the database to function properly. - + + - + - + - + From cd9f96c19d2cd43853134b3f7079d33f78772d35 Mon Sep 17 00:00:00 2001 From: Arunprasad Rajkumar Date: Mon, 14 Apr 2025 19:27:48 +0530 Subject: [PATCH 71/90] Refer latest live-sync docker version (#4022) Signed-off-by: Arunprasad Rajkumar --- _partials/_livesync-terminal.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_partials/_livesync-terminal.md b/_partials/_livesync-terminal.md index a5c291b1cd..506e1b68bc 100644 --- a/_partials/_livesync-terminal.md +++ b/_partials/_livesync-terminal.md @@ -138,7 +138,7 @@ instance to a $SERVICE_LONG: As you run Livesync continuously, best practice is to run it as a background process. ```shell - docker run -d --rm --name livesync timescale/live-sync:v0.0.0-alpha.7 run --publication analytics --subscription livesync --source $SOURCE --target $TARGET + docker run -d --rm --name livesync timescale/live-sync:v0.1.11 run --publication analytics --subscription livesync --source $SOURCE --target $TARGET ``` 1. **Trace progress** From 9148d059c6bb749d44b4bd85e819c22214206416 Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Tue, 15 Apr 2025 11:01:01 +0300 Subject: [PATCH 72/90] Update the CAGGS docs with the end_offset within the current bucket nuance (#3991) --- .../continuous-aggregates/refresh-policies.md | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/use-timescale/continuous-aggregates/refresh-policies.md b/use-timescale/continuous-aggregates/refresh-policies.md index f29d213490..b9be7e2a2e 100644 --- a/use-timescale/continuous-aggregates/refresh-policies.md +++ b/use-timescale/continuous-aggregates/refresh-policies.md @@ -31,18 +31,16 @@ Among others, `add_continuous_aggregate_policy` takes the following arguments: 24 hours. If you set the `start_offset` or `end_offset` to `NULL`, the range is open-ended -and extends to the beginning or end of time. However, it's recommended to set -the `end_offset` so that at least the most recent time bucket is excluded. For -time-series data that mostly contains writes that occur in time stamp order, the -time buckets that see lots of writes quickly have out-of-date aggregates. You -get better performance by excluding the time buckets that are getting a lot of -writes. - -In addition, materializing the most recent bucket might interfere with -[real-time aggregation][future-watermark]. - -See the [API reference][api-reference] -for the full list of required and optional arguments and use examples. +and extends to the beginning or end of time. + +If you set `end_offset` within the current time bucket, this bucket is excluded. This is done for the following reasons: + +- The current bucket is incomplete and can't be refreshed. +- The current bucket gets lots of writes in the time-stamp order and its aggregate becomes outdated very quickly. Excluding it improves performance. + +To include the current time bucket, enable [real-time aggregation][future-watermark]. In Timescale 2.13 and later, it is disabled by default. + +See the [API reference][api-reference] for the full list of required and optional arguments and use examples. From 30fecbeea665845886ed971bbb4848197cd323db Mon Sep 17 00:00:00 2001 From: Victor Perez Date: Tue, 15 Apr 2025 11:18:25 +0200 Subject: [PATCH 73/90] Update timescaledb-parallel-copy references (#4019) * outdated instructions to run timescaledb-parallel-copy --- _partials/_add-data-energy.md | 2 -- _partials/_add-data-nyctaxis.md | 2 -- _partials/_migrate_using_parallel_copy.md | 5 +++-- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/_partials/_add-data-energy.md b/_partials/_add-data-energy.md index c42420ae13..f9d5e32c4c 100644 --- a/_partials/_add-data-energy.md +++ b/_partials/_add-data-energy.md @@ -46,5 +46,3 @@ connection. ``` - -[parallel-copy]: https://github.com/timescale/timescaledb-parallel-copy diff --git a/_partials/_add-data-nyctaxis.md b/_partials/_add-data-nyctaxis.md index 8f5d1313e6..e8aafd1537 100644 --- a/_partials/_add-data-nyctaxis.md +++ b/_partials/_add-data-nyctaxis.md @@ -58,5 +58,3 @@ tolls_amount | 0 improvement_surcharge | 0.3 total_amount | 10.3 ``` - -[parallel-copy]: https://github.com/timescale/timescaledb-parallel-copy diff --git a/_partials/_migrate_using_parallel_copy.md b/_partials/_migrate_using_parallel_copy.md index d8b6db3fc9..c01b6046e5 100644 --- a/_partials/_migrate_using_parallel_copy.md +++ b/_partials/_migrate_using_parallel_copy.md @@ -17,8 +17,9 @@ --connection "host= \ user=tsdbadmin password= \ port= \ - sslmode=require" \ - --db-name tsdb \ + dbname=tsdb \ + sslmode=require + " \ --table \ --file .csv \ --workers \ From d23aabc2d4ab7ffe662480ba8227e23486eeb73b Mon Sep 17 00:00:00 2001 From: Victor Perez Date: Tue, 15 Apr 2025 11:28:39 +0200 Subject: [PATCH 74/90] fix: align migration instructions with smoke test configuration (#4018) * fix: align migration instructions with smoke test configuration --- .../_migrate_dump_roles_schema_data_mst.md | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/_partials/_migrate_dump_roles_schema_data_mst.md b/_partials/_migrate_dump_roles_schema_data_mst.md index dd293cdc30..b928040c45 100644 --- a/_partials/_migrate_dump_roles_schema_data_mst.md +++ b/_partials/_migrate_dump_roles_schema_data_mst.md @@ -23,15 +23,29 @@ ```bash sed -i -E \ - -e '/CREATE ROLE "postgres";/d' \ - -e '/ALTER ROLE "postgres"/d' \ - -e '/CREATE ROLE "tsdbadmin";/d' \ - -e '/ALTER ROLE "tsdbadmin"/d' \ - -e '/GRANT "pg_read_all_stats" TO "tsdbadmin"/d' \ - -e 's/(NO)*SUPERUSER//g' \ - -e 's/(NO)*REPLICATION//g' \ - -e 's/(NO)*BYPASSRLS//g' \ - -e 's/GRANTED BY "[^"]*"//g' \ + -e '/DROP ROLE IF EXISTS "postgres";/d' \ + -e '/DROP ROLE IF EXISTS "tsdbadmin";/d' \ + -e '/CREATE ROLE "postgres";/d' \ + -e '/ALTER ROLE "postgres"/d' \ + -e '/CREATE ROLE "rds/d' \ + -e '/ALTER ROLE "rds/d' \ + -e '/TO "rds/d' \ + -e '/GRANT "rds/d' \ + -e '/GRANT "pg_read_all_stats" TO "tsdbadmin"/d' \ + -e 's/(NO)*SUPERUSER//g' \ + -e 's/(NO)*REPLICATION//g' \ + -e 's/(NO)*BYPASSRLS//g' \ + -e 's/GRANTED BY "[^"]*"//g' \ + -e '/CREATE ROLE "tsdbadmin";/d' \ + -e '/ALTER ROLE "tsdbadmin"/d' \ + -e 's/WITH ADMIN OPTION,/WITH /g' \ + -e 's/WITH ADMIN OPTION//g' \ + -e 's/GRANTED BY ".*"//g' \ + -e '/GRANT "pg_.*" TO/d' \ + -e '/CREATE ROLE "_aiven";/d' \ + -e '/ALTER ROLE "_aiven"/d' \ + -e '/GRANT SET ON PARAMETER "pgaudit.log_max_string_length" TO "_tsdbadmin_auditing"/d' \ + -e '/GRANT SET ON PARAMETER "pgaudit.log_nested_statements" TO "_tsdbadmin_auditing"/d' \ roles.sql ``` From 46d9215e97101c0e4f6dba051fdcc0f4175c6e71 Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Tue, 15 Apr 2025 12:45:02 +0300 Subject: [PATCH 75/90] Updates to docs about attaching VPC to a service Co-authored-by: Iain Cox --- _partials/_vpc-limitations.md | 12 +++ use-timescale/security/vpc.md | 172 +++++++++++++++------------------- 2 files changed, 90 insertions(+), 94 deletions(-) create mode 100644 _partials/_vpc-limitations.md diff --git a/_partials/_vpc-limitations.md b/_partials/_vpc-limitations.md new file mode 100644 index 0000000000..29a314a828 --- /dev/null +++ b/_partials/_vpc-limitations.md @@ -0,0 +1,12 @@ +* You **can attach**: + * Up to 50 Customer $VPCs to a Peering $VPC. + * A $SERVICE_LONG to a single Peering $VPC at a time. + The $SERVICE_SHORT and the Peering $VPC must be in the same AWS region. However, you can peer a Customer $VPC and a Peering $VPC that are in different regions. + * Multiple $SERVICE_LONGs to the same Peering $VPC. +* You **cannot attach** a $SERVICE_LONG to multiple Peering $VPCs at the same time. + + The number of Peering $VPCs you can create in your project depends on your [$PRICING_PLAN][pricing-plans]. + If you need another Peering $VPC, either contact [support@timescale.com](mailto:support@timescale.com) or change your $PRICING_PLAN in [$CONSOLE][console-login]. + +[console-login]: https://console.cloud.timescale.com/ +[pricing-plans]: /about/:currentVersion:/pricing-and-account-management/ \ No newline at end of file diff --git a/use-timescale/security/vpc.md b/use-timescale/security/vpc.md index 770fd70d22..302e011f07 100644 --- a/use-timescale/security/vpc.md +++ b/use-timescale/security/vpc.md @@ -9,7 +9,9 @@ cloud_ui: - [services, :serviceId, operations, vpc] --- -# Secure your $CLOUD_LONG services with VPC Peering and AWS PrivateLink +import VpcLimitations from "versionContent/_partials/_vpc-limitations.mdx"; + +# Secure your $CLOUD_LONG services with $VPC Peering and AWS PrivateLink You use Virtual Private Cloud ($VPC) peering to ensure that your $SERVICE_LONGs are only accessible through your secured AWS infrastructure. This reduces the potential @@ -20,211 +22,197 @@ $CLOUD_LONG is: ![$CLOUD_LONG isolation architecture](https://assets.timescale.com/docs/images/tsc-vpc-architecture.svg) -Your customer apps run inside your AWS Customer VPC, your $CLOUD_LONG services always run +Your customer apps run inside your AWS Customer $VPC, your $SERVICE_LONGs always run inside the secure $CLOUD_LONG $VPC. You control secure communication between apps in -your VPC and your $SERVICE_SHORTs using a dedicated Peering $VPC. The AWS PrivateLink connecting +your $VPC and your $SERVICE_SHORTs using a dedicated Peering $VPC. The AWS PrivateLink connecting $CLOUD_LONG $VPC to the dedicated Peering $VPC gives the same level of protection as using a direct AWS PrivateLink connection. It only enables communication to be initiated from your Customer $VPC -to services running in the $CLOUD_LONG $VPC. $CLOUD_LONG cannot initiate communication with your $VPC. +to $SERVICE_SHORTs running in the $CLOUD_LONG $VPC. $CLOUD_LONG cannot initiate communication with your Customer $VPC. -To configure this secure connection, you first create the $CLOUD_LONG Peering $VPC with +To configure this secure connection, you first create a Peering $VPC with AWS PrivateLink in $CONSOLE. After you have accepted and configured the peering connection to your Customer $VPC, you use AWS Security Groups to -restrict the services in your Customer $VPC that are visible to the Peering $VPC. -The last step is to attach individual services to the Peering $VPC. - -* You define each $CLOUD_LONG $VPC on a [$CLOUD_LONG project level][project-members]. -* You can attach: - * Up to 50 Customer $VPCs to a $CLOUD_LONG $VPC. - * A $SERVICE_LONG to a single $CLOUD_LONG $VPC at a time. - - The $SERVICE_SHORT and $VPC must be in the same AWS region. However, you can peer a Customer $VPC and a - $CLOUD_LONG $VPC that are in different regions. - * Multiple $SERVICE_LONGs to the same $CLOUD_LONG $VPC. -* You cannot attach a $SERVICE_LONG to multiple $CLOUD_LONG $VPCs at the same time. - -The number of $CLOUD_LONG $VPCs you can create in your project depends on your [pricing plan][pricing-plans]. -If you need another $CLOUD_LONG $VPC, either contact [support@timescale.com](mailto:support@timescale.com) or change your -pricing plan in [$CONSOLE][console-login]. +restrict the apps in your Customer $VPC that are visible to the Peering $VPC. +The last step is to attach individual $SERVICE_SHORTs to the Peering $VPC in $CONSOLE. + +* You create each Peering $VPC on a [$CLOUD_LONG project level][project-members]. + + ## Prerequisites -In order to set up VPC peering you need the following permissions in your AWS account: +To set up $VPC peering, you need the following permissions in your AWS account: -* Accept VPC peering requests +* Accept $VPC peering requests * Configure route table rules * Configure security group and firewall rules ## Set up a secured connection between $CLOUD_LONG and AWS -To connect to a $SERVICE_LONG using VPC peering, your apps and infrastructure must be already -running in an Amazon Web Services (AWS) VPC. You can peer your VPC from any AWS region. -However, your $CLOUD_LONG $VPC must be within one of the [Cloud-supported regions][tsc-regions]. +To connect to a $SERVICE_LONG using $VPC peering, your apps and infrastructure must be already +running in an Amazon Web Services (AWS) $VPC. You can peer your $VPC from any AWS region. +However, your Peering $VPC must be within one of the [Cloud-supported regions][tsc-regions]. -The stages to create a secured connection between $CLOUD_LONG services and your AWS infrastructure are: +The stages to create a secured connection between $SERVICE_LONGs and your AWS infrastructure are: -1. [Create a Peering VPC in $CONSOLE][aws-vpc-setup-vpc] -1. [Complete the VPC connection in your AWS][aws-vpc-complete] +1. [Create a Peering $VPC in $CONSOLE][aws-vpc-setup-vpc] +1. [Complete the $VPC connection in your AWS][aws-vpc-complete] 1. [Set up security groups in your AWS][aws-vpc-security-groups] -1. [Attach a Timescale Service to the Peering VPC][aws-vpc-connect-vpcs] +1. [Attach a $SERVICE_LONG to the Peering $VPC][aws-vpc-connect-vpcs] -### Create a Peering VPC in $CONSOLE +### Create a Peering $VPC in $CONSOLE -Create the VPC and the peering connection that enables you to securely route traffic -between $CLOUD_LONG and your own VPC in a logically isolated virtual network. +Create the $VPC and the peering connection that enables you to securely route traffic +between $CLOUD_LONG and your Customer $VPC in a logically isolated virtual network. -1. In [$CONSOLE > Security > VPC][console-vpc], click `Create a VPC`. +1. **In [$CONSOLE > Security > VPC][console-vpc], click `Create a VPC`** ![$CLOUD_LONG new $VPC](https://assets.timescale.com/docs/images/add-peering-vpc.png) - * You can attach: - * Up to 50 Customer $VPCs to a $CLOUD_LONG $VPC. - * A $SERVICE_LONG to a single $CLOUD_LONG $VPC at a time. - The $SERVICE_SHORT and $VPC must be in the same AWS region. However, you can peer a Customer $VPC and a $CLOUD_LONG $VPC that are in different regions. - * Multiple $SERVICE_LONGs to the same $CLOUD_LONG $VPC. - * You cannot attach a $SERVICE_LONG to multiple $CLOUD_LONG $VPCs at the same time. - - The number of $CLOUD_LONG $VPCs you can create in your project depends on your [pricing plan][pricing-plans]. - If you need another $CLOUD_LONG $VPC, either contact [support@timescale.com](mailto:support@timescale.com) or change your pricing plan in [$CONSOLE][console-login]. - -1. Choose your region and IP range, name your VPC, then click `Create VPC`. +1. **Choose your region and IP range, name your $VPC, then click `Create VPC`** ![Create a new VPC in $CLOUD_LONG](https://assets.timescale.com/docs/images/configure-peering-vpc.png) - The IP ranges of the Peering VPC and Customer VPC should not overlap. + The IP ranges of the Peering $VPC and Customer VPC should not overlap. -1. For as many peering connections as you need: +1. **For as many peering connections as you need**: 1. In the `VPC Peering` column, click `Add`. - 2. Enter information about your existing AWS VPC, then click `Add Connection`. + 2. Enter information about your existing Customer VPC, then click `Add Connection`. ![Add peering](https://assets.timescale.com/docs/images/add-peering.png) + + $CLOUD_LONG sends a peering request to your AWS account so you can [complete the VPC connection in AWS][aws-vpc-complete]. + -### Complete the VPC connection in AWS +### Complete the $VPC connection in AWS When you receive the $CLOUD_LONG peering request in AWS, edit your routing table to match -the `IP Range` and `CIDR block` between your AWS and $CLOUD_LONG VPCs. +the `IP Range` and `CIDR block` between your Customer and Peering VPCs. -When you peer a VPC with multiple CIDRs, all CIDRs are added to the $CLOUD_LONG rules automatically. -After you have finished peering, further changes in your VPC's CIDRs are not detected automatically. -If you need to refresh the CIDRs, please recreate the peering connection. +When you peer a $VPC with multiple CIDRs, all CIDRs are added to the $CLOUD_LONG rules automatically. +After you have finished peering, further changes in your $VPC's CIDRs are not detected automatically. +If you need to refresh the CIDRs, recreate the peering connection. The request acceptance process is an important safety mechanism. Do not accept a peering request from an unknown account. -1. In [AWS > VPC Dashboard > Peering connections][aws-dashboard], select the peering connection - request from $CLOUD_LONG. +1. **In [AWS > VPC Dashboard > Peering connections][aws-dashboard], select the peering connection + request from $CLOUD_LONG** Copy the peering connection ID to the clipboard. The connection request starts with `pcx-`. -1. In the peering connection, click `Route Tables`, then select the `Route Table ID` - that corresponds to your VPC. -1. In `Routes`, click `Edit routes`. You see the list of existing Destinations. +1. **In the peering connection, click `Route Tables`, then select the `Route Table ID` + that corresponds to your $VPC** + +1. **In `Routes`, click `Edit routes`** + + You see the list of existing destinations. ![Create a new VPC route](https://assets.timescale.com/docs/images/tsc-vpc-add-route.png). If you do not already have a destination that corresponds to the `IP range / CIDR block` of - your Timescale VPC: + your Peering $VPC: 1. Click `Add route`, and set: - * `Destination`: the CIDR block of your Timescale VPC. For example: `10.0.0.7/17`. + * `Destination`: the CIDR block of your Peering $VPC. For example: `10.0.0.7/17`. * `Target`: the peering connection ID you copied to your clipboard. 2. Click `Save changes`. Network traffic is secured between your AWS account and $CLOUD_LONG for this project. + ### Set up security groups in AWS Security groups allow specific inbound and outbound traffic at the resource level. -You can associate a VPC with one or more security groups and each instance in your -VPC may belong to a different set of security groups. The security group choices -for your VPC are: +You can associate a $VPC with one or more security groups, and each instance in your +$VPC may belong to a different set of security groups. The security group choices +for your $VPC are: -* Create a security group to use for your $CLOUD_LONG VPC only. -* Associate your VPC with an existing security group. -* Do nothing, your VPC is automatically associated with the default one. +* Create a security group to use for your $CLOUD_LONG $VPC only. +* Associate your $VPC with an existing security group. +* Do nothing, your $VPC is automatically associated with the default one. -To create a security group specific to your $CLOUD_LONG VPC: +To create a security group specific to your $CLOUD_LONG Peering VPC: -1. [AWS > VPC Dashboard > Security Groups][aws-security-groups], click `Create security group`. +1. **[AWS > VPC Dashboard > Security Groups][aws-security-groups], click `Create security group`** -1. Enter the rules for this security group: +1. **Enter the rules for this security group**: The AWS Security Groups dashboard - * `VPC`: select the VPC that is peered with Timescale. + * `VPC`: select the $VPC that is peered with $CLOUD_LONG. * `Inbound rules`: leave empty. * `Outbound rules`: * `Type`: `Custom TCP` * `Protocol`: `TCP` * `Port range`: `5432` * `Destination`: `Custom` - * `Info`: the CIDR block of your $CLOUD_LONG VPC. -1. Click `Add rule`, then click `Create security group`. + * `Info`: the CIDR block of your $CLOUD_LONG Peering $VPC. +1. **Click `Add rule`, then click `Create security group`** -### Attach a $CLOUD_LONG service to the Peering VPC +### Attach a $SERVICE_LONG to the Peering $VPC Now that $CLOUD_LONG is communicating securely with your AWS infrastructure, you can attach -one or more services to the VPC. - -After you attach a $SERVICE_SHORT to a VPC, you can only access it through the peered -AWS VPC. It is no longer accessible using the public internet. +one or more $SERVICE_SHORTs to the Peering $VPC. +After you attach a $SERVICE_SHORT to a Peering $VPC, you can only access it through the peered +AWS $VPC. It is no longer accessible using the public internet. -1. In [$CONSOLE > Services][console-services] select the $SERVICE_SHORT you want to - connect to the VPC. -1. Click `Security` > `VPC`. -1. Select the VPC, then click `Attach VPC`. +1. **In [$CONSOLE > Services][console-services] select the $SERVICE_SHORT you want to + connect to the Peering $VPC** +1. **Click `Operations` > `Security` > `VPC`** +1. **Select the $VPC, then click `Attach VPC`** And that is it, your $SERVICE_SHORT is now securely communicating with your AWS -account inside a VPC. +account inside a $VPC. ## Migrate a $CLOUD_LONG $SERVICE_SHORT between VPCs To ensure that your applications continue to run without interruption, you keep -$SERVICE_SHORT attached to the VPC. However, you can change the VPC your -$SERVICE_SHORT is attached to, or disconnect from a VPC and enable access to the +$SERVICE_SHORT attached to the Peering $VPC. However, you can change the Peering $VPC your +$SERVICE_SHORT is attached to, or disconnect from the Peering $VPC and enable access to the $SERVICE_SHORT from the public internet. -$CLOUD_LONG uses a different DNS for services that are attached to a VPC. -When you migrate a $SERVICE_SHORT between public access and a VPC, you need +$CLOUD_LONG uses a different DNS for $SERVICE_SHORTs that are attached to a Peering $VPC. +When you migrate a $SERVICE_SHORT between public access and a Peering $VPC, you need to update your connection string. -1. In [$CONSOLE > Services][console-services] select the $SERVICE_SHORT to migrate. +1. **In [$CONSOLE > Services][console-services] select the $SERVICE_SHORT to migrate** If you don't have a $SERVICE_SHORT, [create a new one][create-service]. -1. Click `Security` > `VPC`. -1. Select the VPC, then click `Attach VPC`. +1. **Click `Operations` > `Security` > `VPC`** +1. **Select the $VPC, then click `Attach VPC`** Migration takes a few minutes to complete and requires a change to DNS settings for the -Service. The $SERVICE_SHORT is not accessible during this time. If you receive a DNS error, allow +$SERVICE_SHORT. The $SERVICE_SHORT is not accessible during this time. If you receive a DNS error, allow some time for DNS propagation. [aws-dashboard]: https://console.aws.amazon.com/vpc/home#PeeringConnections: @@ -234,14 +222,10 @@ some time for DNS propagation. [console-services]: https://console.cloud.timescale.com/dashboard/services [timescale-support]: https://www.timescale.com/contact/ [tsc-regions]: /use-timescale/:currentVersion:/regions/ - - [aws-vpc-setup-vpc]: /use-timescale/:currentVersion:/security/vpc/#create-a-peering-vpc-in-timescale-console [aws-vpc-complete]: /use-timescale/:currentVersion:/security/vpc/#complete-the-vpc-connection-in-aws [aws-vpc-security-groups]: /use-timescale/:currentVersion:/security/vpc/#set-up-security-groups-in-aws [aws-vpc-connect-vpcs]: /use-timescale/:currentVersion:/security/vpc/#attach-a-timescale-service-to-the-peering-vpc - - [create-service]: /getting-started/:currentVersion:/services/#create-a-timescale-cloud-service [pricing-plans]: /about/:currentVersion:/pricing-and-account-management/ [project-members]: /use-timescale/:currentVersion:/members/ From 150eb56ce429bcdf4593bc6782a2640f71e881b3 Mon Sep 17 00:00:00 2001 From: Arunprasad Rajkumar Date: Tue, 15 Apr 2025 18:47:31 +0530 Subject: [PATCH 76/90] Update live-sync console limitation (#4023) Let user know that the indexes are not migrated. Signed-off-by: Arunprasad Rajkumar --- _partials/_livesync-console.md | 4 ++++ _partials/_livesync-limitations.md | 2 +- _partials/_livesync-terminal.md | 9 +++++---- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/_partials/_livesync-console.md b/_partials/_livesync-console.md index 3e283e0238..1cb23420b3 100644 --- a/_partials/_livesync-console.md +++ b/_partials/_livesync-console.md @@ -16,6 +16,10 @@ import TuneSourceDatabaseAWSRDS from "versionContent/_partials/_livesync-configu ## Limitations +- Indexes(including Primary Key and Unique constraints) are not migrated by $SERVICE_LONG. + +We recommend that you create only necessary indexes on the target $SERVICE_LONG depending on your query patterns. + ## Set your connection string diff --git a/_partials/_livesync-limitations.md b/_partials/_livesync-limitations.md index 1ad3d2d108..69949ec0c0 100644 --- a/_partials/_livesync-limitations.md +++ b/_partials/_livesync-limitations.md @@ -7,4 +7,4 @@ LiveSync does not create extensions on the target. If the table uses column types from an extension, first create the extension on the target $SERVICE_LONG before syncing the table. * There is WAL volume growth on the source PostgreSQL instance during large table copy. -* This works for PostgreSQL databases only as source. Timescaledb is not yet supported. +* This works for PostgreSQL databases only as source. TimescaleDB is not yet supported. diff --git a/_partials/_livesync-terminal.md b/_partials/_livesync-terminal.md index 506e1b68bc..412b4d3361 100644 --- a/_partials/_livesync-terminal.md +++ b/_partials/_livesync-terminal.md @@ -39,19 +39,20 @@ The `` in the `SOURCE` connection must have the replication role granted i - + + - + - + - + From acba297ed07d69e6517acb9208b8dc6f284ab4b2 Mon Sep 17 00:00:00 2001 From: Ramon Guiu Date: Thu, 17 Apr 2025 17:04:31 +0200 Subject: [PATCH 77/90] Changelog April 18 (#4033) * Changelog April 18 Signed-off-by: Ramon Guiu * Update changelog.md Signed-off-by: Ramon Guiu * review --------- Signed-off-by: Ramon Guiu Co-authored-by: atovpeko --- about/changelog.md | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/about/changelog.md b/about/changelog.md index b417ba75dd..6129d6c58f 100644 --- a/about/changelog.md +++ b/about/changelog.md @@ -8,6 +8,30 @@ keywords: [changelog, upgrades, updates, releases] All the latest features and updates to Timescale products. +## ☑️ Embeddings spot checks, TimescaleDB v2.19.3, and new models in SQL Assistant + + +### Embeddings spot checks + +In Timescale Cloud, you can now quickly check the quality of the embeddings from the vectorizers' outputs. Construct a similarity search query with additional filters on source metadata using a simple UI. Run the query right away, or copy it to the SQL editor or data mode and further customize it to your needs. Run the check in Timescale Console > `Services` > `AI`: + +![Embedding Quality Inspection](https://assets.timescale.com/docs/images/ai-spot-checks.png) + +### TimescaleDB v2.19.3 + +New services created in Timescale Cloud now use TimescaleDB v2.19.3. Existing services are in the process of being automatically upgraded to this version. + +This release adds a number of bug fixes including: + +- Fix segfault when running a query against columnstore chunks that group by multiple columns, including UUID segmentby columns. +- Fix hypercore table access method segfault on DELETE operations using a segmentby column. + +### New OpenAI, Llama, and Gemini models in SQL Assistant + +The data mode's SQL Assistant now includes support for the latest models from OpenAI and LLama: GPT-4.1 (including mini and nano) and Llama 4 (Scout and Maverick). Additionally, we've added support for Gemini models, in particular Gemini 2.0 Nano and 2.5 Pro (experimental and preview). With the new additions, SQL Assistant supports more than 20 language models so you can select the one best suited to your needs. + +![SQL Assistant - New Models](https:///assets.timescale.com/docs/images/sql-assistant-new-models.png) + ## 🪵 TimescaleDB v2.19, new service overview page, and log improvements From 2b4f3615e7b2df55cc5813bdc3894aa7a979e24d Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Thu, 17 Apr 2025 18:29:00 +0300 Subject: [PATCH 78/90] review (#4034) --- about/changelog.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/about/changelog.md b/about/changelog.md index 6129d6c58f..0664b96056 100644 --- a/about/changelog.md +++ b/about/changelog.md @@ -28,7 +28,7 @@ This release adds a number of bug fixes including: ### New OpenAI, Llama, and Gemini models in SQL Assistant -The data mode's SQL Assistant now includes support for the latest models from OpenAI and LLama: GPT-4.1 (including mini and nano) and Llama 4 (Scout and Maverick). Additionally, we've added support for Gemini models, in particular Gemini 2.0 Nano and 2.5 Pro (experimental and preview). With the new additions, SQL Assistant supports more than 20 language models so you can select the one best suited to your needs. +The data mode's SQL Assistant now includes support for the latest models from OpenAI and Llama: GPT-4.1 (including mini and nano) and Llama 4 (Scout and Maverick). Additionally, we've added support for Gemini models, in particular Gemini 2.0 Nano and 2.5 Pro (experimental and preview). With the new additions, SQL Assistant supports more than 20 language models so you can select the one best suited to your needs. ![SQL Assistant - New Models](https:///assets.timescale.com/docs/images/sql-assistant-new-models.png) @@ -66,7 +66,7 @@ Finding logs just got easier! We've added a date, time, and timezone picker, so ### pgvectorscale 0.7.0: faster filtered filtered vector search with filtered indexes -This pgvectorscale release adds label-based filtered vector search to the StremingDiskANN index. +This pgvectorscale release adds label-based filtered vector search to the StreamingDiskANN index. This enables you to return more precise and efficient results by combining vector similarity search with label filtering while still uitilizing the ANN index. This is a common need for large-scale RAG and Agentic applications that rely on vector searches with metadata filters to return relevant results. Filtered indexes add From bf859262d035b8eaba604efa63d43962ebd1913c Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Mon, 21 Apr 2025 10:08:29 +0300 Subject: [PATCH 79/90] Fixed a link to Ollama in changelog --- about/changelog.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/about/changelog.md b/about/changelog.md index 0664b96056..57d510e6aa 100644 --- a/about/changelog.md +++ b/about/changelog.md @@ -845,7 +845,7 @@ select ollama_generate ; ``` -To learn more, see the [pgai Ollama documentation](https://github.com/timescale/pgai/blob/main/docs/model_calling/ollama.md). +To learn more, see the [pgai Ollama documentation](https://github.com/timescale/pgai/blob/main/docs/vectorizer/quick-start.md). ## 🧙 Compression Wizard From 64e2c0d3d1044cf89fc1310cc541ddee13033f92 Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Tue, 22 Apr 2025 11:42:44 +0300 Subject: [PATCH 80/90] link fix (#4044) --- mlc_config.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mlc_config.json b/mlc_config.json index e6d39aa44c..9eca629fe9 100644 --- a/mlc_config.json +++ b/mlc_config.json @@ -78,6 +78,9 @@ }, { "pattern": "^https://www.tableau.com/support/releases/server/2024.2.6#esdalt" + }, + { + "pattern": "^https://azure.microsoft.com/en-us/products/data-studio" } ] } From 409fcddd145d1129fcd7e0b83cc5e52e3acc0445 Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Tue, 22 Apr 2025 13:46:48 +0300 Subject: [PATCH 81/90] Added h3 to the list of extensions --- use-timescale/extensions/index.md | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/use-timescale/extensions/index.md b/use-timescale/extensions/index.md index ccfbb11fee..22d0e0ae0c 100644 --- a/use-timescale/extensions/index.md +++ b/use-timescale/extensions/index.md @@ -75,17 +75,18 @@ supported extensions: ## Third-party extensions -| Extension | Description | Enabled by default | -|--------------------------------------------------|------------------------------------------------------------|-----------------------| -| [pgaudit][pgaudit] | Detailed session and/or object audit logging | - | -| [pgpcre][pgpcre] | Perl-compatible RegEx | - | -| [pgrouting][pgrouting] | Geospatial routing functionality | - | -| [postgis][postgis] | PostGIS geometry and geography spatial types and functions | - | -| [postgis_raster][postgis-raster] | PostGIS raster types and functions | - | -| [postgis_sfcgal][postgis-sfcgal] | PostGIS SFCGAL functions | - | -| [postgis_tiger_geocoder][postgis-tiger-geocoder] | PostGIS tiger geocoder and reverse geocoder | - | -| [postgis_topology][postgis-topology] | PostGIS topology spatial types and functions | - | -| [unit][unit] | SI units for PostgreSQL | - | +| Extension | Description | Enabled by default | +|--------------------------------------------------|------------------------------------------------------------|--------------------| +| [h3][h3] | H3 bindings for PostgreSQL | - | +| [pgaudit][pgaudit] | Detailed session and/or object audit logging | - | +| [pgpcre][pgpcre] | Perl-compatible RegEx | - | +| [pgrouting][pgrouting] | Geospatial routing functionality | - | +| [postgis][postgis] | PostGIS geometry and geography spatial types and functions | - | +| [postgis_raster][postgis-raster] | PostGIS raster types and functions | - | +| [postgis_sfcgal][postgis-sfcgal] | PostGIS SFCGAL functions | - | +| [postgis_tiger_geocoder][postgis-tiger-geocoder] | PostGIS tiger geocoder and reverse geocoder | - | +| [postgis_topology][postgis-topology] | PostGIS topology spatial types and functions | - | +| [unit][unit] | SI units for PostgreSQL | - | [amcheck]: https://www.postgresql.org/docs/current/amcheck.html [autoinc]: https://www.postgresql.org/docs/current/contrib-spi.html#CONTRIB-SPI-AUTOINC @@ -100,6 +101,7 @@ supported extensions: [earthdistance]: https://www.postgresql.org/docs/current/earthdistance.html [fuzzystrmatch]: https://www.postgresql.org/docs/current/fuzzystrmatch.html [hstore]: https://www.postgresql.org/docs/current/hstore.html +[h3]: https://pgxn.org/dist/h3/ [insert-username]: https://www.postgresql.org/docs/current/contrib-spi.html#CONTRIB-SPI-INSERT-USERNAME [intagg]: https://www.postgresql.org/docs/current/intagg.html [intarray]: https://www.postgresql.org/docs/current/intarray.html From 7a46f718d33257b5caa41f2f50cf9f708147759d Mon Sep 17 00:00:00 2001 From: Arunprasad Rajkumar Date: Tue, 22 Apr 2025 18:05:09 +0530 Subject: [PATCH 82/90] Improve privilege grant steps for livesync user creation section (#4036) * Improve priviledge grant steps for livesync user creation section Signed-off-by: Arunprasad Rajkumar * Fix review comments Signed-off-by: Arunprasad Rajkumar --------- Signed-off-by: Arunprasad Rajkumar --- ...vesync-configure-source-database-awsrds.md | 18 +++++++++++++++--- .../_livesync-configure-source-database.md | 19 ++++++++++++++++--- _partials/_livesync-console.md | 8 ++++++-- 3 files changed, 37 insertions(+), 8 deletions(-) diff --git a/_partials/_livesync-configure-source-database-awsrds.md b/_partials/_livesync-configure-source-database-awsrds.md index 2e571d5f7b..0cb2d35809 100644 --- a/_partials/_livesync-configure-source-database-awsrds.md +++ b/_partials/_livesync-configure-source-database-awsrds.md @@ -41,6 +41,18 @@ Updating parameters on a PostgreSQL instance will cause an outage. Choose a time You can use an existing user. However, you must ensure that the user has the following permissions. + 1. Grant permissions to create a replication slot: + + ```sql + psql $SOURCE -c "GRANT rds_replication TO " + ``` + + 1. Grant permissions to create a publication: + + ```sql + psql $SOURCE -c "GRANT CREATE ON DATABASE TO " + ``` + 1. Assign the user permissions on the source database: ```sql @@ -48,15 +60,15 @@ Updating parameters on a PostgreSQL instance will cause an outage. Choose a time GRANT USAGE ON SCHEMA "public" TO ; GRANT SELECT ON ALL TABLES IN SCHEMA "public" TO ; ALTER DEFAULT PRIVILEGES IN SCHEMA "public" GRANT SELECT ON TABLES TO ; - GRANT CREATE ON DATABASE to ; - GRANT rds_replication TO ; EOF ``` - If the tables you are syncing are not in the `public` schema, grant the user permissions on the schema you are syncing.: + If the tables you are syncing are not in the `public` schema, grant the user permissions for each schema you are syncing.: ```sql psql $SOURCE < TO ; GRANT SELECT ON ALL TABLES IN SCHEMA TO ; + ALTER DEFAULT PRIVILEGES IN SCHEMA GRANT SELECT ON TABLES TO ; EOF ``` diff --git a/_partials/_livesync-configure-source-database.md b/_partials/_livesync-configure-source-database.md index bb94d8146e..3ce73d7a4e 100644 --- a/_partials/_livesync-configure-source-database.md +++ b/_partials/_livesync-configure-source-database.md @@ -25,6 +25,18 @@ import EnableReplication from "versionContent/_partials/_migrate_live_setup_enab You can use an existing user. However, you must ensure that the user has the following permissions. + 1. Grant permissions to create a replication slot: + + ```sql + psql $SOURCE -c "ALTER ROLE REPLICATION" + ``` + + 1. Grant permissions to create a publication: + + ```sql + psql $SOURCE -c "GRANT CREATE ON DATABASE TO " + ``` + 1. Assign the user permissions on the source database: ```sql @@ -32,15 +44,15 @@ import EnableReplication from "versionContent/_partials/_migrate_live_setup_enab GRANT USAGE ON SCHEMA "public" TO ; GRANT SELECT ON ALL TABLES IN SCHEMA "public" TO ; ALTER DEFAULT PRIVILEGES IN SCHEMA "public" GRANT SELECT ON TABLES TO ; - GRANT CREATE ON DATABASE to ; - GRANT replication TO ; EOF ``` - If the tables you are syncing are not in the `public` schema, grant the user permissions on the schema you are syncing.: + If the tables you are syncing are not in the `public` schema, grant the user permissions for each schema you are syncing.: ```sql psql $SOURCE < TO ; GRANT SELECT ON ALL TABLES IN SCHEMA TO ; + ALTER DEFAULT PRIVILEGES IN SCHEMA GRANT SELECT ON TABLES TO ; EOF ``` @@ -51,6 +63,7 @@ import EnableReplication from "versionContent/_partials/_migrate_live_setup_enab ``` You can skip this step if the replicating user is already the owner of the tables. + 1. **Enable replication `DELETE` and`UPDATE` operations** diff --git a/_partials/_livesync-console.md b/_partials/_livesync-console.md index 1cb23420b3..1c011cba14 100644 --- a/_partials/_livesync-console.md +++ b/_partials/_livesync-console.md @@ -16,9 +16,13 @@ import TuneSourceDatabaseAWSRDS from "versionContent/_partials/_livesync-configu ## Limitations -- Indexes(including Primary Key and Unique constraints) are not migrated by $SERVICE_LONG. +* Indexes(including Primary Key and Unique constraints) are not migrated by $SERVICE_LONG. -We recommend that you create only necessary indexes on the target $SERVICE_LONG depending on your query patterns. + We recommend that you create only necessary indexes on the target $SERVICE_LONG depending on your query patterns. + +* Tables with user defined types are not migrated by $SERVICE_LONG. + + You need to create the user defined types on the target $SERVICE_LONG before syncing the table. From 11a3255dd716f31cc18587dca73d6c08c21b8a48 Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Thu, 24 Apr 2025 11:24:15 +0300 Subject: [PATCH 83/90] Mention the agent mode in SQL assistant docs Co-authored-by: Iain Cox --- getting-started/run-queries-from-console.md | 31 ++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/getting-started/run-queries-from-console.md b/getting-started/run-queries-from-console.md index 51b72b886e..77eb3b1585 100644 --- a/getting-started/run-queries-from-console.md +++ b/getting-started/run-queries-from-console.md @@ -127,7 +127,7 @@ use that in the $DATA_MODE. ## $SQL_ASSISTANT_SHORT -$SQL_ASSISTANT_SHORT in [$CONSOLE][portal-data-mode] helps you write, fix, and organize SQL faster and more accurately. +$SQL_ASSISTANT_SHORT in [$CONSOLE][portal-data-mode] uses the power of AI to help you write, fix, and organize SQL faster and more accurately.
    @@ -152,6 +152,35 @@ $SQL_ASSISTANT_SHORT offers a range of features to improve your SQL workflow, in - **Query organization**: to keep your query library organized, and help your team understand the purpose of each query, $SQL_ASSISTANT_SHORT automatically adds titles and summaries to your queries. +- **Agent mode**: to get results with minimal involvement from you, $SQL_ASSISTANT_SHORT autopilots through complex tasks and troubleshoots its own problems. No need to go step by step, analyze errors, and try out solutions. Simply turn on the agent mode in the LLM picker and watch $SQL_ASSISTANT_SHORT do all the work for you. + +### Supported LLMs + +$SQL_ASSISTANT_SHORT supports a large number of LLMs, including: + +- GPT-4o mini +- GPT-4o +- GPT-4.1 nano +- GPT-4.1 mini +- GPT-4.1 +- o4-mini (low) +- o4-mini +- o4-mini (high) +- o3 (low) +- o3 +- o3 (high) +- Claude 3.5 Haiku +- Claud 3.7 Sonnet +- Claud 3.7 Sonnet (extended thinking) +- Llama 3.3 70B Versatile +- Llama 3.3 70B Instruct +- Llama 3.1 405B Instruct +- Llama 4 Scout +- Llama 4 Maverick +- DeepSeek R1 Distill - Llama 3.3. 70B +- DeepSeek R1 +- Gemini 2.0 Flash + ### Limitations to keep in mind For best results with $SQL_ASSISTANT_SHORT: From 4fc5bfa65593d4c3ca37097ae6d2d9804efc073e Mon Sep 17 00:00:00 2001 From: Lucas Coelho Date: Thu, 24 Apr 2025 09:12:15 -0300 Subject: [PATCH 84/90] Datadog agent integration (#4025) * Add Datadog Agent instructions Signed-off-by: Ramon Guiu * chore: setup structure. * chore: updates * chore: update on review. * chore: update on review. * chore: try a build fix. * ci: trigger build * chore: attempt file rename to generate properly * chore: update import path --------- Signed-off-by: Ramon Guiu Signed-off-by: Iain Cox Co-authored-by: Ramon Guiu Co-authored-by: Iain Co-authored-by: Robson Coelho --- ...exporter.md => _manage-a-data-exporter.md} | 10 +- integrations/cloudwatch.md | 2 +- integrations/datadog.md | 123 ++++++++++++++++-- integrations/find-connection-details.md | 20 +++ use-timescale/metrics-logging/integrations.md | 6 +- 5 files changed, 146 insertions(+), 15 deletions(-) rename _partials/{_manage-data-exporter.md => _manage-a-data-exporter.md} (97%) diff --git a/_partials/_manage-data-exporter.md b/_partials/_manage-a-data-exporter.md similarity index 97% rename from _partials/_manage-data-exporter.md rename to _partials/_manage-a-data-exporter.md index cfc23cabef..64cc7c2bc7 100644 --- a/_partials/_manage-data-exporter.md +++ b/_partials/_manage-a-data-exporter.md @@ -1,4 +1,4 @@ -## Attach a data exporter to a $SERVICE_LONG +### Attach a data exporter to a $SERVICE_LONG To send telemetry data to an external monitoring tool, you attach a data exporter to your $SERVICE_LONG. You can attach only one exporter to a $SERVICE_SHORT. @@ -14,7 +14,7 @@ To attach an exporter: -## Monitor $SERVICE_LONG metrics +### Monitor $SERVICE_LONG metrics You can now monitor your $SERVICE_SHORT metrics. Use the following metrics to check the service is running correctly: @@ -36,7 +36,7 @@ Additionally, use the following tags to filter your results. |`role`|`replica` or `primary`| For $SERVICE_SHORT with replicas | |`node-id`|| For multi-node services | -## Edit a data exporter +### Edit a data exporter To update a data exporter: @@ -50,7 +50,7 @@ You cannot change fields such as the provider or the AWS region. -## Delete a data exporter +### Delete a data exporter To remove a data exporter that you no longer need: @@ -73,7 +73,7 @@ To remove a data exporter that you no longer need: -## Reference +### Reference When you create the IAM OIDC provider, the URL must match the region you create the exporter in. It must be one of the following: diff --git a/integrations/cloudwatch.md b/integrations/cloudwatch.md index a3848874d0..40b6fb36d4 100644 --- a/integrations/cloudwatch.md +++ b/integrations/cloudwatch.md @@ -7,7 +7,7 @@ keywords: [integrate] import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; import CloudWatchExporter from "versionContent/_partials/_cloudwatch-data-exporter.mdx"; -import ManageDataExporter from "versionContent/_partials/_manage-data-exporter.mdx"; +import ManageDataExporter from "versionContent/_partials/_manage-a-data-exporter.mdx"; # Integrate Amazon CloudWatch with $CLOUD_LONG diff --git a/integrations/datadog.md b/integrations/datadog.md index 3dc934a47c..45ac3d197a 100644 --- a/integrations/datadog.md +++ b/integrations/datadog.md @@ -7,16 +7,23 @@ keywords: [integrate] import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; import DataDogExporter from "versionContent/_partials/_datadog-data-exporter.mdx"; -import ManageDataExporter from "versionContent/_partials/_manage-data-exporter.mdx"; +import ManageDataExporter from "versionContent/_partials/_manage-a-data-exporter.mdx"; # Integrate Datadog with $CLOUD_LONG -[Datadog][datadog] is a cloud-based monitoring and analytics platform that provides comprehensive visibility into applications, infrastructure, and systems through real-time monitoring, logging, and analytics. +[Datadog][datadog] is a cloud-based monitoring and analytics platform that provides comprehensive visibility into +applications, infrastructure, and systems through real-time monitoring, logging, and analytics. -You can export telemetry data from your $SERVICE_LONGs with the time-series and analytics capability enabled to Datadog. The available metrics include CPU usage, RAM usage, and storage. This integration is available for [Scale and Enterprise][pricing-plan-features] -pricing tiers. +This page explains how to: + +- [Monitor $SERVICE_LONG metrics with Datadog][datadog-monitor-cloud] + + This integration is available for [Scale and Enterprise][pricing-plan-features] pricing plans. + +- Configure Datadog Agent to collect metrics for your $SERVICE_LONG + + This integration is available for all pricing plans. -This pages explains how to export telemetry data from your $SERVICE_LONG into Datadog by creating a $CLOUD_LONG data exporter, then attaching it to the $SERVICE_SHORT. ## Prerequisites @@ -24,19 +31,119 @@ This pages explains how to export telemetry data from your $SERVICE_LONG into Da - Sign up for [Datadog][datadog-signup]. - You need your API key to follow this procedure. [Create one][datadog-api-key] if you haven't yet. + You need your [Datadog API key][datadog-api-key] to follow this procedure. -## Create a data exporter +- Install [Datadog Agent][datadog-agent-install]. + +## Monitor Timescale Cloud service metrics with Datadog + +Export telemetry data from your $SERVICE_LONGs with the time-series and analytics capability enabled to +Datadog using a $CLOUD_LONG data exporter. The available metrics include CPU usage, RAM usage, and storage. + +### Create a data exporter A $CLOUD_LONG data exporter sends telemetry data from a $SERVICE_LONG to a third-party monitoring -tool. You create an exporter on the [project level][projects], in the same AWS region as your $SERVICE_SHORT: +tool. You create an exporter on the [project level][projects], in the same AWS region as your $SERVICE_SHORT: +### Manage a data exporter + +This section shows you how to attach, monitor, edit, and delete a data exporter. + +## Configure Datadog Agent to collect metrics for your $SERVICE_LONGs + +Datadog Agent includes a [$PG integration][datadog-postgres] that you use to collect detailed $PG database +metrics about your $SERVICE_LONGs. + +1. **Connect to your $SERVICE_LONG** + + For $CLOUD_LONG, open an [SQL editor][run-queries] in [$CONSOLE][open-console]. For self-hosted, use [`psql`][psql]. + +1. **Add the `datadog` user to your $SERVICE_LONG** + + ```sql + create user datadog with password ''; + ``` + + ```sql + grant pg_monitor to datadog; + ``` + + ```sql + grant SELECT ON pg_stat_database to datadog; + ``` + +1. **Test the connection and rights for the datadog user** + + Update the following command with your [connection details][connection-info], then run it from the command line: + + ```bash + psql "postgres://datadog:@:/tsdb?sslmode=require" -c \ + "select * from pg_stat_database LIMIT(1);" \ + && echo -e "\e[0;32mPostgres connection - OK\e[0m" || echo -e "\e[0;31mCannot connect to Postgres\e[0m" + ``` + You see the output from the `pg_stat_database` table, which means you have given the correct rights to `datadog`. + +1. **Connect Datadog to your $SERVICE_LONG** + + 1. Open the datadog agent $PG configuration file, usually located at: + - **Linux**: `/etc/datadog-agent/conf.d/postgres.d/conf.yaml` + - **MacOS**: `/opt/datadog-agent/etc/conf.d/postgres.d/conf.yaml` + - **Windows**: `C:\ProgramData\Datadog\conf.d\postgres.d\conf.yaml` + + 1. Integrate Datadog Agent with your $SERVICE_LONG + + Use your [connection details][connection-info] to update the following and add it to the datadog agent $PG + configuration file: + + ```yaml + init_config: + + instances: + - host: + port: + username: datadog + password: > + dbname: tsdb + disable_generic_tags: true + ``` + +1. **Add $CLOUD_LONG metrics** + + Tags to make it easier for build Datadog dashboards that combine metrics from the $CLOUD_LONG data exporter and + Datadog Agent. Use your [connection details][connection-info] to update the following and add it to + `/datadog.yaml`: + + ```yaml + tags: + - project-id: + - service-id: + - region: + ``` + +1. **Restart Datadog Agent** + + See how to [Start, stop, and restart Datadog Agent][datadog-agent-restart]. + +Metrics for your $SERVICE_LONG are now visible in Datadog. Check the Datadog $PG integration documentation for a +comprehensive list of [metrics][datadog-postgres-metrics] collected. + [datadog]: https://www.datadoghq.com/ +[datadog-agent-install]: https://docs.datadoghq.com/getting_started/agent/#installation +[datadog-postgres]: https://docs.datadoghq.com/integrations/postgres/ +[datadog-postgres-metrics]:https://docs.datadoghq.com/integrations/postgres/?tab=host#metrics +[datadog-postgres-setup]: https://docs.datadoghq.com/integrations/postgres/?tab=host#configuration [datadog-signup]: https://www.datadoghq.com/ +[datadog-monitor-cloud]: /use-timescale/:currentVersion:/integrations/datadog/#monitor-timescale-cloud-service-metrics-with-datadog +[datadog-agent]: /use-timescale/:currentVersion:/integrations/datadog/#configure-datadog-agent-to-collect-metrics-for-your-timescale-cloud-services +[datadog-agent-restart]: https://docs.datadoghq.com/agent/configuration/agent-commands/#start-stop-and-restart-the-agent [projects]: /use-timescale/:currentVersion:/members/ [datadog-api-key]: https://docs.datadoghq.com/account_management/api-app-keys/#add-an-api-key-or-client-token [pricing-plan-features]: /about/:currentVersion:/pricing-and-account-management/#features-included-in-each-plan +[run-queries]: /getting-started/:currentVersion:/run-queries-from-console/ +[open-console]: https://console.cloud.timescale.com/dashboard/services +[psql]: /use-timescale/:currentVersion:/integrations/psql/ +[connection-info]: /use-timescale/:currentVersion:/integrations/find-connection-details/ diff --git a/integrations/find-connection-details.md b/integrations/find-connection-details.md index 16c9d83a68..9871a4ab90 100644 --- a/integrations/find-connection-details.md +++ b/integrations/find-connection-details.md @@ -21,6 +21,8 @@ Find the connection details based on your deployment type: +## Connect to your service + Retrieve the connection details for your $SERVICE_LONG: - **In `-credentials.txt`**: @@ -33,6 +35,24 @@ Retrieve the connection details for your $SERVICE_LONG: ![Timescale service connection details](https://assets.timescale.com/docs/images/timescale-service-connection-details.png) +## Find your project and service ID + +To retrieve the connection details for your $CLOUD_LONG project and $SERVICE_LONG: + + + +1. **Retreive your project ID**: + + In [$CONSOLE][console-services], click `Timescale Project`, then click `Copy` next to the project ID. + ![Retrive the project id in console](https://assets.timescale.com/docs/images/console-project-id.png) + +1. **Retrieve your service ID**: + + Click the dots next to the service, then click `Copy` next to the service ID. + ![Retrive the service id in console](https://assets.timescale.com/docs/images/console-service-id.png) + + + diff --git a/use-timescale/metrics-logging/integrations.md b/use-timescale/metrics-logging/integrations.md index dfc2dfbb48..b54eb1a2f1 100644 --- a/use-timescale/metrics-logging/integrations.md +++ b/use-timescale/metrics-logging/integrations.md @@ -11,7 +11,7 @@ cloud_ui: --- import DataDogExporter from "versionContent/_partials/_datadog-data-exporter.mdx"; -import ManageDataExporter from "versionContent/_partials/_manage-data-exporter.mdx"; +import ManageDataExporter from "versionContent/_partials/_manage-a-data-exporter.mdx"; import IntegrationPrereqsCloud from "versionContent/_partials/_integration-prereqs-cloud-only.mdx"; import CloudWatchExporter from "versionContent/_partials/_cloudwatch-data-exporter.mdx"; @@ -50,6 +50,10 @@ tool. You create an exporter on the [project level][projects], in the same AWS r +## Manage a data exporter + +This section shows you how to attach, monitor, edit, and delete a data exporter. + [attach-exporter]: /use-timescale/:currentVersion:/metrics-logging/integrations/#attach-a-data-exporter-to-a-timescale-cloud-service From e3ec0e778a5b50bc58625939884810dfc334f0ff Mon Sep 17 00:00:00 2001 From: Vineeth Pothulapati Date: Thu, 24 Apr 2025 18:11:19 +0530 Subject: [PATCH 85/90] Add Livesync for S3 docs (#4047) * Add Livesync for S3 docs Co-authored-by: Iain Co-authored-by: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> --- _partials/_livesync-console.md | 4 +- _partials/_livesync-terminal.md | 2 +- _partials/_prereqs-cloud-only.md | 8 +- about/changelog.md | 4 +- lambda/redirects.js | 4 + migrate/index.md | 24 ++- ...livesync.md => livesync-for-postgresql.md} | 22 +-- migrate/livesync-for-s3.md | 165 ++++++++++++++++++ migrate/page-index/page-index.js | 11 +- 9 files changed, 220 insertions(+), 24 deletions(-) rename migrate/{livesync.md => livesync-for-postgresql.md} (71%) create mode 100644 migrate/livesync-for-s3.md diff --git a/_partials/_livesync-console.md b/_partials/_livesync-console.md index 1c011cba14..d9ca0192ef 100644 --- a/_partials/_livesync-console.md +++ b/_partials/_livesync-console.md @@ -89,7 +89,7 @@ To sync data from your PostgreSQL database to your $SERVICE_LONG using $CONSOLE: 1. Press `Continue`. $CONSOLE connects to the source database and retrieves the schema information. -1. **Optimize the data to syncronize in hypertables** +1. **Optimize the data to synchronize in hypertables** ![livesync start](https://assets.timescale.com/docs/images/livesync-start.png) 1. Select the table to sync, and press `+`. @@ -112,5 +112,5 @@ instance to your $SERVICE_LONG in real-time. [install-psql]: /integrations/:currentVersion:/psql/ [portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services -[livesync-tune-source-db]: /migrate/:currentVersion:/livesync/#tune-your-source-database +[livesync-tune-source-db]: /migrate/:currentVersion:/livesync-for-postgresql/#tune-your-source-database diff --git a/_partials/_livesync-terminal.md b/_partials/_livesync-terminal.md index 412b4d3361..d133cd86f1 100644 --- a/_partials/_livesync-terminal.md +++ b/_partials/_livesync-terminal.md @@ -236,7 +236,7 @@ For example: [alter-publication]: https://www.postgresql.org/docs/current/sql-alterpublication.html [install-docker]: https://docs.docker.com/engine/install/ [about-hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/ -[lives-sync-specify-tables]: /migrate/:currentVersion:/livesync/#specify-the-tables-to-synchronize +[lives-sync-specify-tables]: /migrate/:currentVersion:/livesync-for-postgresql/#specify-the-tables-to-synchronize [compression]: /use-timescale/:currentVersion:/compression/about-compression [caggs]: /use-timescale/:currentVersion:/continuous-aggregates/about-continuous-aggregates/ [join-livesync-on-slack]: https://app.slack.com/client/T4GT3N2JK/C086NU9EZ88 diff --git a/_partials/_prereqs-cloud-only.md b/_partials/_prereqs-cloud-only.md index 6d1f87a964..a176783bf9 100644 --- a/_partials/_prereqs-cloud-only.md +++ b/_partials/_prereqs-cloud-only.md @@ -1,5 +1,9 @@ -To follow the procedure on this page you need to: +To follow the steps on this page: + +* Create a target [$SERVICE_LONG][create-service] with time-series and analytics enabled. + + You need your [connection details][connection-info]. -* Create a [target $SERVICE_LONG][create-service]. [create-service]: /getting-started/:currentVersion:/services/ +[connection-info]: /integrations/:currentVersion:/find-connection-details/ diff --git a/about/changelog.md b/about/changelog.md index 57d510e6aa..d8f9a732f9 100644 --- a/about/changelog.md +++ b/about/changelog.md @@ -93,7 +93,7 @@ To see the job information page, in [$CONSOLE][console], select the $SERVICE_SHO ## 🤩 In-Console Livesync for PostgreSQL -You can now set up an active data ingestion pipeline with Livesync for PostgreSQL in Timescale Console. This tool enables you to replicate your source database tables into Timescale's hypertables indefinitely. Yes, you heard that right—keep Livesync running for as long as you need, ensuring that your existing source PostgreSQL tables stay in sync with Timescale Cloud. Read more about setting up and using [Livesync for PostgreSQL](https://docs.timescale.com/migrate/latest/livesync/). +You can now set up an active data ingestion pipeline with Livesync for PostgreSQL in Timescale Console. This tool enables you to replicate your source database tables into Timescale's hypertables indefinitely. Yes, you heard that right—keep Livesync running for as long as you need, ensuring that your existing source PostgreSQL tables stay in sync with Timescale Cloud. Read more about setting up and using [Livesync for PostgreSQL](https://docs.timescale.com/migrate/latest/livesync-for-postgresql/). ![Livesync in Timescale Console](https://assets.timescale.com/docs/images/timescale-cloud-livesync-tile.png) @@ -308,7 +308,7 @@ All feature requests are automatically published to the [Timescale Forum](https: ### Postgres Livesync (Alpha release) We have built a new solution that helps you continuously replicate all or some of your Postgres tables directly into Timescale Cloud. -[Livesync](https://docs.timescale.com/migrate/latest/livesync/) allows you to keep a current Postgres instance such as RDS as your primary database, and easily offload your real-time analytical queries to Timescale Cloud to boost their performance. If you have any questions or feedback, talk to us in [#livesync in Timescale Community](https://app.slack.com/client/T4GT3N2JK/C086NU9EZ88). +[Livesync](https://docs.timescale.com/migrate/latest/livesync-for-postgresql/) allows you to keep a current Postgres instance such as RDS as your primary database, and easily offload your real-time analytical queries to Timescale Cloud to boost their performance. If you have any questions or feedback, talk to us in [#livesync in Timescale Community](https://app.slack.com/client/T4GT3N2JK/C086NU9EZ88). This is just the beginning—you'll see more from Livesync in 2025! diff --git a/lambda/redirects.js b/lambda/redirects.js index d8d5124e8a..14e3183e9e 100644 --- a/lambda/redirects.js +++ b/lambda/redirects.js @@ -343,6 +343,10 @@ module.exports = [ { from: "/latest/tutorials/tutorial-setup-timescale-prometheus/", to: "https://docs.timescale.com/tutorials/latest/", + + { + from: "/migrate/latest/livesync", + to: "https://docs.timescale.com/migrate/latest/livesync-for-postgresql/", }, { from: "/mst/latest/create-a-service", diff --git a/migrate/index.md b/migrate/index.md index ef8ffbae1a..dfe76d523e 100644 --- a/migrate/index.md +++ b/migrate/index.md @@ -8,11 +8,14 @@ tags: [ingest, migrate, RDS] import OpenSupportRequest from "versionContent/_partials/_migrate_open_support_request.mdx" -# Migrate your data to Timescale Cloud +# Migrate and sync data to Timescale Cloud -You have chosen to migrate your data to Timescale Cloud, thank you. Depending on the amount of data -you need to migrate, and the amount of downtime you can afford, we offer the following ways to migrate -your data: +You have chosen to migrate or sync your data to $CLOUD_LONG, thank you. + +## Migrate your data + +Depending on the amount of data you need to migrate, and the amount of downtime you can afford. $COMPANY offers +the following ways to migrate your data: | Migration strategy | Use when | Downtime requirements | |--------------------------------------------|-----------------------------------------------------------------------------------------------------------------------------|-----------------------| @@ -31,12 +34,25 @@ If you're migrating your data from another source database type, best practice i a `.csv` file, then import to your Timescale Cloud Service using [timescaledb-parallel-copy][parallel-copy]. For other ingestion methods, see [Ingest data from other sources][data-ingest]. +## Livesync your data + +You use $LIVESYNC to synchronize all or some of your data to your $SERVICE_LONG in real-time. You run $LIVESYNC +continuously, using your data as a primary database and your $SERVICE_LONG as a logical replica. This enables you +to leverage $CLOUD_LONG’s real-time analytics capabilities on your replica data. + +| $LIVESYNC options | Downtime requirements | +|----------------------------------------|-----------------------| +| [$LIVESYNC for $PG][livesync-postgres] | None | +| [$LIVESYNC for S3][livesync-s3] | None | + [data-ingest]: /use-timescale/:currentVersion:/ingest-data/ [dual-write]: /migrate/:currentVersion:/dual-write-and-backfill/ [pg-dump-restore]: /migrate/:currentVersion:/pg-dump-and-restore/ [parallel-copy]: /use-timescale/:currentVersion:/ingest-data/import-csv/ +[livesync-postgres]: /migrate/:currentVersion:/livesync-for-postgresql/ +[livesync-s3]: /migrate/:currentVersion:/livesync-for-s3/ [troubleshooting]: /migrate/:currentVersion:/troubleshooting/ [live-migration]: /migrate/:currentVersion:/live-migration/ [pgcopydb]: https://github.com/dimitri/pgcopydb diff --git a/migrate/livesync.md b/migrate/livesync-for-postgresql.md similarity index 71% rename from migrate/livesync.md rename to migrate/livesync-for-postgresql.md index dda5b78c87..4135b81ab0 100644 --- a/migrate/livesync.md +++ b/migrate/livesync-for-postgresql.md @@ -14,18 +14,18 @@ import EarlyAccessNoRelease from "versionContent/_partials/_early_access.mdx"; # Livesync from PostgreSQL to Timescale Cloud -You use Livesync to synchronize all the data, or specific tables, from a PostgreSQL database instance to your -$SERVICE_LONG in real-time. You run Livesync continuously, turning PostgreSQL into a primary database with your +You use $LIVESYNC to synchronize all the data, or specific tables, from a PostgreSQL database instance to your +$SERVICE_LONG in real-time. You run $LIVESYNC continuously, turning PostgreSQL into a primary database with your $SERVICE_LONG as a logical replica. This enables you to leverage $CLOUD_LONG’s real-time analytics capabilities on your replica data. ![livesync view status](https://assets.timescale.com/docs/images/livesync-view-status.png) -Livesync leverages the a well-established PostgreSQL logical replication protocol. By relying on this protocol, -Livesync ensures compatibility, familiarity, and a broader knowledge base. Making it easier for you to adopt Livesync +$LIVESYNC_CAP leverages the well-established PostgreSQL logical replication protocol. By relying on this protocol, +$LIVESYNC ensures compatibility, familiarity, and a broader knowledge base. Making it easier for you to adopt $LIVESYNC and integrate your data. -You use Livesync for data synchronization, rather than migration. Livesync can: +You use $LIVESYNC for data synchronization, rather than migration. Livesync can: * Copy existing data from a PostgreSQL instance to a $SERVICE_LONG: - Copy data at up to 150 GB/hr. @@ -35,16 +35,17 @@ You use Livesync for data synchronization, rather than migration. Livesync can: Large tables are still copied using a single connection. Parallel copying is in the backlog. - Forget foreign key relationships. - Livesync disables foreign key validation during the sync. For example, if a `metrics` table refers to + $LIVESYNC_CAP disables foreign key validation during the sync. For example, if a `metrics` table refers to the `id` column on the `tags` table, you can still sync only the `metrics` table without worrying about their foreign key relationships. - - Track progress. PostgreSQL expose `COPY` progress under in `pg_stat_progress_copy`. + - Track progress. PostgreSQL exposes `COPY` progress under `pg_stat_progress_copy`. + * Synchronize real-time changes from a PostgreSQL instance to a $SERVICE_LONG. -* Add and remove tables on demand using the [PostgreSQL PUBLICATION interface](https://www.postgresql.org/docs/current/sql-createpublication.html). +* Add and remove tables on demand using the [PostgreSQL PUBLICATION interface][postgres-publication-interface]. * Enable features such as [hypertables][about-hypertables], [columnstore][compression], and [continuous aggregates][caggs] on your logical replica. -: Livesync is not supported for production use. If you have an questions or feedback, talk to us in #livesync in Timescale Community. +: livesync is not supported for production use. If you have an questions or feedback, talk to us in #livesync in Timescale Community. @@ -65,7 +66,8 @@ You use Livesync for data synchronization, rather than migration. Livesync can: [alter-publication]: https://www.postgresql.org/docs/current/sql-alterpublication.html [install-docker]: https://docs.docker.com/engine/install/ [about-hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/ -[lives-sync-specify-tables]: /migrate/:currentVersion:/livesync/#specify-the-tables-to-synchronize +[lives-sync-specify-tables]: /migrate/:currentVersion:/livesync-for-postgresql/#specify-the-tables-to-synchronize [compression]: /use-timescale/:currentVersion:/compression/about-compression [caggs]: /use-timescale/:currentVersion:/continuous-aggregates/about-continuous-aggregates/ [join-livesync-on-slack]: https://app.slack.com/client/T4GT3N2JK/C086NU9EZ88 +[postgres-publication-interface]: https://www.postgresql.org/docs/current/sql-createpublication.html diff --git a/migrate/livesync-for-s3.md b/migrate/livesync-for-s3.md new file mode 100644 index 0000000000..81a39d2e74 --- /dev/null +++ b/migrate/livesync-for-s3.md @@ -0,0 +1,165 @@ +--- +title: Livesync S3 to Timescale Cloud +excerpt: Synchronize data from S3 to Timescale Cloud service in real time +products: [cloud] +keywords: [migration, low-downtime, backup] +tags: [recovery, logical backup, replication] +--- + +import PrereqCloud from "versionContent/_partials/_prereqs-cloud-only.mdx"; +import EarlyAccessNoRelease from "versionContent/_partials/_early_access.mdx"; + +# Livesync from S3 to Timescale Cloud + +You use $LIVESYNC to synchronize tabular data, from an S3 bucket to your +$SERVICE_LONG in real time. You run $LIVESYNC continuously, turning S3 into a primary database with your +$SERVICE_LONG as a logical replica. This enables you to leverage $CLOUD_LONG’s real-time analytics capabilities on +your replica data. + +![$LIVESYNC_CAP view status](https://assets.timescale.com/docs/images/livesync-s3-view-status.png) + +You use $LIVESYNC for data synchronization, rather than migration. Livesync can: + +* Sync data from an S3 bucket instance to a $SERVICE_LONG: + - $LIVESYNC uses Glob patterns to identify the objects to sync. + - $LIVESYNC uses the objects returned for subsequent queries. This efficient approach means files are synced in + [lexicographical order][lex-order]. + - $LIVESYNC watches an S3 bucket for new files and imports them automatically. $LIVESYNC runs on a configurable + schedule and tracks processed files. + - For large backlogs, $LIVESYNC checks every minute until caught up. + +* Sync data from multiple file formats: + + * CSV: checked for compression in `.gz` and `.zip` format, then processing using [timescaledb-parallel-copy][parallel-copy] + + * Parquet: converted to CSV, then processed using [timescaledb-parallel-copy][parallel-copy] + +* Enable features such as [hypertables][about-hypertables], [columnstore][compression], and + [continuous aggregates][caggs] on your logical replica. + +$LIVESYNC for S3 continuously imports data from an Amazon S3 bucket into your database. It monitors your S3 bucket for new +files matching a specified pattern and automatically imports them into your designated database table. + +: livesync is not supported for production use. If you have any questions or feedback, talk to us in #livesync in Timescale Community. + +## Prerequisites + + + +- Access to a standard Amazon S3 bucket containing your data files. + Directory buckets are not supported. +- Access credentials for the S3 bucket. + - The following credentials are supported: + - [IAM Role][credentials-iam]. + + - Configure the trust policy. Set the: + + - `Principal`: `arn:aws:iam::142548018081:role/timescale-s3-connections`. + - `ExternalID`: set to the [$CLOUD_LONG project and $SERVICE_LONG ID][connection-project-service-id] of the + $SERVICE_SHORT you are syncing to in the format `/`. + + This is to avoid the [confused deputy problem][confused-deputy-problem]. + - Give the following access permissions: + + - `s3:GetObject`. + - `s3:ListBucket`. + + - [Public anonymous user][credentials-public]. + +## Limitations + +- **CSV**: + - Maximum file size: 1GB + To increase these limits, contact sales@timescale.com + - Maximum row size: 2MB + - Supported compressed formats: + - `.gz` + - `.zip` + - Advanced settings: + - Delimiter: the default character is `,`, you can choose a different delimiter + - Skip Header: skip the first row if your file has headers +- **Parquet**: + - Maximum file size: 1GB + - Maximum row group uncompressed size: 200MB + - Maximum row size: 2MB +- **Sync iteration**: + To prevent system overload, $LIVESYNC tracks up to 100 files for each sync iteration. Additional checks only fill + empty queue slots. + +## Synchronize data to your $SERVICE_LONG + +To sync data from your S3 bucket to your $SERVICE_LONG using $CONSOLE: + + + +1. **Connect to your $SERVICE_LONG** + + In [$CONSOLE][portal-ops-mode], select the service to sync live data to. +1. **Start livesync** + 1. Click `Actions` > `livesync for S3`. + 2. Click `New Livesync for S3` + +1. **Connect the source S3 bucket to the target $SERVICE_SHORT** + + ![Livesync connect to bucket](https://assets.timescale.com/docs/images/livesync-s3-wizard.png) + + 1. In `Livesync for S3`, set the `Bucket name` and `Authentication method`, then click `Continue`. + + For instruction on creating the IAM role you need to connect your S3 bucket, click `Learn how`: + ![Livesync connect to bucket](https://assets.timescale.com/docs/images/livesync-s3-create-credentials.png) + $CONSOLE connects to the source bucket. + 1. In `Define files to sync`, choose the `File type` and set the `Glob pattern`. + + Use the following patterns: + - `/*`: match all files in a folder. Also, any pattern ending with `/` is treated as `/*`. + - `/**`: match all recursively. + - `/**/*.csv`: match a specific file type. + + $LIVESYNC uses prefix filters where possible, place patterns carefully at the end of your glob expression. + AWS S3 doesn't support complex filtering. If your expression filters too many files, the list operation may timeout. + + 1. Click the search icon, you see files to sync. Click `Continue`. + +1. **Optimize the data to synchronize in hypertables** + + $CONSOLE checks the file schema and, if possible, suggests the column to use as the time dimension in a + [hypertable][about-hypertables]. + + ![Livesync choose table](https://assets.timescale.com/docs/images/livesync-s3-create-tables.png) + + 1. Choose the `Data type` for each column, then click `Continue`. + 1. Choose the interval. This can be a minute, an hour or use a [cron expression][cron-expression]. + 1. Repeat this step for each table you want to sync. + 1. Press `Start Livesync`. + + $CONSOLE starts $LIVESYNC between the source database and the target $SERVICE_SHORT and displays the progress. + +1. **Monitor syncronization** + 1. To view the progress of the livesync, click the name of the $LIVESYNC process: + You see the status of the file being synced. Only one file runs at a time. + ![livesync view status](https://assets.timescale.com/docs/images/livesync-s3-view-status.png) + 1. To pause and restart livesync, click the buttons on the right of the $LIVESYNC process and select an action: + During pauses, you can edit the configuration before resuming. + ![livesync start stop](https://assets.timescale.com/docs/images/livesync-s3-start-stop.png) + + + +And that is it, you are using $LIVESYNC to synchronize all the data, or specific files, from an S3 bucket to your +$SERVICE_LONG in real time. + + +[about-hypertables]: /use-timescale/:currentVersion:/hypertables/about-hypertables/ +[lives-sync-specify-tables]: /migrate/:currentVersion:/livesync-for-postgresql/#specify-the-tables-to-synchronize +[compression]: /use-timescale/:currentVersion:/compression/about-compression +[caggs]: /use-timescale/:currentVersion:/continuous-aggregates/about-continuous-aggregates/ +[join-livesync-on-slack]: https://app.slack.com/client/T4GT3N2JK/C086NU9EZ88 +[parallel-copy]: https://github.com/timescale/timescaledb-parallel-copy +[deputy-problem]: https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html +[lex-order]:https://en.wikipedia.org/wiki/Lexicographic_order +[credentials-iam]: https://docs.aws.amazon.com/IAM/latest/UserGuide/id_roles_create_for-user.html#roles-creatingrole-user-console +[credentials-public]: https://docs.aws.amazon.com/AmazonS3/latest/userguide/example-bucket-policies.html#example-bucket-policies-anonymous-user +[portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services +[hypertable-docs]: /use-timescale/:currentVersion:/hypertables/ +[cron-expression]: https://en.wikipedia.org/wiki/Cron#Cron_expression +[confused-deputy-problem]: https://docs.aws.amazon.com/IAM/latest/UserGuide/confused-deputy.html +[connection-project-service-id]: /integrations/:currentVersion:/find-connection-details/#find-your-project-and-service-id diff --git a/migrate/page-index/page-index.js b/migrate/page-index/page-index.js index 6d35ca8667..eb8abfdd35 100644 --- a/migrate/page-index/page-index.js +++ b/migrate/page-index/page-index.js @@ -1,6 +1,6 @@ module.exports = [ { - title: "Migrate your data to Timescale Cloud", + title: "Migrate and sync data to Timescale Cloud", href: "migrate", excerpt: "Migrating your data to Timescale", children: [ @@ -16,10 +16,15 @@ module.exports = [ excerpt: "Migrate a large database with low downtime", }, { - title: "Livesync from PostgreSQL to Timescale Cloud", - href: "livesync", + title: "Livesync from PostgreSQL", + href: "livesync-for-postgresql", excerpt: "Synchronize updates to a primary postgres database instance to Timescale Cloud service in real-time", }, + { + title: "Livesync from S3", + href: "livesync-for-s3", + excerpt: "Synchronize data from S3 to Timescale Cloud service in real time", + }, { title: "Dual-write and backfill", href: "dual-write-and-backfill", From 50594d0719adb78126c5174db532d8d5305f7920 Mon Sep 17 00:00:00 2001 From: Iain Cox Date: Thu, 24 Apr 2025 15:06:58 +0200 Subject: [PATCH 86/90] fix: redirects structure. (#4050) --- lambda/redirects.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lambda/redirects.js b/lambda/redirects.js index 14e3183e9e..d3fca3fa2a 100644 --- a/lambda/redirects.js +++ b/lambda/redirects.js @@ -343,7 +343,11 @@ module.exports = [ { from: "/latest/tutorials/tutorial-setup-timescale-prometheus/", to: "https://docs.timescale.com/tutorials/latest/", - + }, + { + from: "/migrate/latest/livesync", + to: "https://docs.timescale.com/migrate/latest/livesync-for-postgresql/", + }, { from: "/migrate/latest/livesync", to: "https://docs.timescale.com/migrate/latest/livesync-for-postgresql/", From 198d1d601347efed1d2b7055ddcb80a49efc7c3b Mon Sep 17 00:00:00 2001 From: Anastasiia Tovpeko <114177030+atovpeko@users.noreply.github.com> Date: Fri, 25 Apr 2025 12:27:11 +0300 Subject: [PATCH 87/90] Passwordless data mode (#4049) * added passwordless data mode + cleanup --- _partials/_cloud-connect-service.md | 31 +++++- _partials/_cloud-intro.md | 2 +- getting-started/run-queries-from-console.md | 103 +++++++++--------- .../try-key-features-timescale-products.md | 26 +++-- 4 files changed, 92 insertions(+), 70 deletions(-) diff --git a/_partials/_cloud-connect-service.md b/_partials/_cloud-connect-service.md index 5647de4f77..cb02abbf8b 100644 --- a/_partials/_cloud-connect-service.md +++ b/_partials/_cloud-connect-service.md @@ -9,9 +9,33 @@ 1. **Connect to your $SERVICE_SHORT** - Connect using SQL editor in $CONSOLE or with psql in the command line: + Connect using data mode or SQL editor in $CONSOLE, or psql in the command line: + + + + + + 1. In $CONSOLE, toggle `Data`. + + 1. Select your $SERVICE_SHORT in the connection drop-down in the top right. + + ![Select a connection](https://assets.timescale.com/docs/images/data-mode-connection-dropdown.png) + + 1. Run a test query: + + ```sql + SELECT CURRENT_DATE; + ``` + + This query gives you the current date, you have successfully connected to your $SERVICE_SHORT. + + + + And that is it, you are up and running. Enjoy developing with $COMPANY. + + @@ -55,7 +79,7 @@ SELECT CURRENT_DATE; ``` - This query returns the current date. You have successfully connected to your $SERVICE_SHORT. + This query returns the current date. You have successfully connected to your $SERVICE_SHORT. @@ -66,9 +90,6 @@ - - -You can also use the powerful [data mode][popsql] to connect and run queries on your service. Quick recap. You: - Manage your $SERVICE_SHORTs in the [$OPS_MODE][portal-ops-mode] in $CONSOLE: add $READ_REPLICAs and enable diff --git a/_partials/_cloud-intro.md b/_partials/_cloud-intro.md index 7a8f9d45fe..f8341c0779 100644 --- a/_partials/_cloud-intro.md +++ b/_partials/_cloud-intro.md @@ -12,7 +12,7 @@ use as is, or extend with the capabilities specific to your business needs: - **AI and vector**: $PG with vector extensions. Use $PG as a vector database with purpose built extensions for building AI applications from start to scale. Get fast and accurate similarity search with the pgvector and pgvectorscale extensions. Create vector embeddings and perform LLM reasoning on your data with - the $PGAI_SHORT extension. + the pgai extension. - **PostgreSQL**: the trusted industry-standard RDBMS. Ideal for applications requiring strong data consistency, complex relationships, and advanced querying capabilities. Get ACID compliance, extensive SQL support, JSON handling, and extensibility through custom functions, data types, and extensions. $CLOUD_LONG continuously diff --git a/getting-started/run-queries-from-console.md b/getting-started/run-queries-from-console.md index 77eb3b1585..b872dc0de5 100644 --- a/getting-started/run-queries-from-console.md +++ b/getting-started/run-queries-from-console.md @@ -9,7 +9,7 @@ import WhereNext from "versionContent/_partials/_where-to-next.mdx"; # Run your queries from $CONSOLE -As $CLOUD_LONG is based on PostgreSQL, you can use lots of [different tools][integrations] to +As $CLOUD_LONG is based on $PG, you can use lots of [different tools][integrations] to connect to your $SERVICE_SHORT and interact with your data. In $CONSOLE you can use the following ways to run SQL queries against your $SERVICE_SHORT: @@ -20,8 +20,7 @@ In $CONSOLE you can use the following ways to run SQL queries against your $SERV - [$SQL_ASSISTANT_SHORT in the $DATA_MODE][sql-assistant]: write, fix, and organize SQL faster and more accurately. - [$SQL_EDITOR in the $OPS_MODE][run-sqleditor]: a simple $SQL_EDITOR in the $OPS_MODE that lets you run ad-hoc ephemeral - queries. This is useful for quick one-off tasks like creating an index on a small table or - inspecting `pg_stat_statements`. + queries. This is useful for quick one-off tasks like creating an index on a small table or inspecting `pg_stat_statements`. If you prefer the command line to the $OPS_MODE $SQL_EDITOR in $CONSOLE, use [psql][install-psql]. @@ -35,17 +34,15 @@ Available features are: - **Real-time collaboration**: work with your team directly in the $DATA_MODE query editor with live presence and multiple cursors. -- **[Schema browser](https://docs.popsql.com/docs/schema)**: understand the structure of your $SERVICE_SHORT and see usage data on tables and columns. +- **[Schema browser][schema-browser]**: understand the structure of your $SERVICE_SHORT and see usage data on tables and columns. - **[$SQL_ASSISTANT_SHORT][sql-assistant]**: write, fix, and organize SQL faster and more accurately using AI. - **Autocomplete**: get suggestions as you type your queries. -- **[Version history](https://docs.popsql.com/docs/version-history)**: access previous versions of a query from the built-in revision history, or connect to a - git repo. -- **[Charts](https://docs.popsql.com/docs/creating-charts)**: visualize data from inside the UI rather than switch to Sheets or Excel. -- **[Schedules](https://docs.popsql.com/docs/scheduled-queries)**: automatically refresh queries and dashboards to create push alerts. -- **[Query variables](https://docs.popsql.com/docs/query-variables)**: use Liquid to parameterize your queries or use `if` statements. -- **Cross-platform**: works from [$CONSOLE][portal-data-mode] or download the - [desktop](https://popsql.com/download) app for macOS, Windows, and Linux. -- **Easy connection**: to $CLOUD_LONG, $PG, Redshift, Snowflake, BigQuery, MySQL, SQL Server, [and more](https://popsql.com/connections). +- **[Version history][version-history]**: access previous versions of a query from the built-in revision history, or connect to a git repo. +- **[Charts][charts]**: visualize data from inside the UI rather than switch to Sheets or Excel. +- **[Schedules][schedules]**: automatically refresh queries and dashboards to create push alerts. +- **[Query variables][query-variables]**: use Liquid to parameterize your queries or use `if` statements. +- **Cross-platform support**: work from [$CONSOLE][portal-data-mode] or download the [desktop app][popsql-desktop] for macOS, Windows, and Linux. +- **Easy connection**: connect to $CLOUD_LONG, $PG, Redshift, Snowflake, BigQuery, MySQL, SQL Server, [and more][popsql-connections]. ### Connect to your $SERVICE_LONG in the $DATA_MODE @@ -55,24 +52,21 @@ To connect to a $SERVICE_SHORT: 1. **Check your $SERVICE_SHORT is running correctly** - In [$CONSOLE][services-portal], check that your $SERVICE_SHORT is marked as `Running`. + In [$CONSOLE][services-portal], check that your $SERVICE_SHORT is marked as `Running`: - ![Check $SERVICE_LONG is running](https://assets.timescale.com/docs/images/console-services-view.png) + ![Check $SERVICE_LONG is running](https://assets.timescale.com/docs/images/timescale-console-services-view.png) 1. **Connect to your $SERVICE_SHORT** - 1. In the [$DATA_MODE][portal-data-mode] in $CONSOLE, select a $SERVICE_SHORT in the connection drop-down. - 1. Click the pencil icon next to the $SERVICE_SHORT name. + In the [$DATA_MODE][portal-data-mode] in $CONSOLE, select a $SERVICE_SHORT in the connection drop-down: - ![Select a connection](https://assets.timescale.com/docs/images/data-mode-connections.png) - - 1. Click `Edit` next to `Username/Password` and enter [your connection details][connection-info] for this $SERVICE_SHORT, then click `Connect`. + ![Select a connection](https://assets.timescale.com/docs/images/data-mode-connection-dropdown.png) 1. **Run a test query** - Type `SELECT CURRENT_DATE;` in `Scratchpad` and press `Run`: + Type `SELECT CURRENT_DATE;` in `Scratchpad` and click `Run`: - ![Run a simple query](https://assets.timescale.com/docs/images/data-mode-scratchpad.png) + ![Run a simple query](https://assets.timescale.com/docs/images/run-query-in-scratchpad.png) Quick recap. You: - Manage your $SERVICE_SHORTs in the [$OPS_MODE in $CONSOLE][portal-ops-mode] @@ -83,15 +77,11 @@ Quick recap. You: Now you have used the $DATA_MODE in $CONSOLE, see how to easily do the following: -- [Write a query](https://docs.popsql.com/docs/writing-a-query) -- [Share a query with your - teammates](https://docs.popsql.com/docs/sharing-a-link-to-your-query-and-results) -- [Create a chart from your - data](https://docs.popsql.com/docs/creating-charts) -- [Create a dashboard of multiple query - results](https://docs.popsql.com/docs/creating-dashboards) -- [Create schedules for your - queries](https://docs.popsql.com/docs/scheduled-queries) +- [Write a query][write-query] +- [Share a query with your teammates][share-query] +- [Create a chart from your data][create-chart] +- [Create a dashboard of multiple query results][create-dashboard] +- [Create schedules for your queries][create-schedule] ### $DATA_MODE_CAP FAQ @@ -99,7 +89,7 @@ Now you have used the $DATA_MODE in $CONSOLE, see how to easily do the following If your $SERVICE_LONG runs inside a $VPC, do one of the following to enable access for the $POPSQL desktop app: -- Use $POPSQL's [bridge connector](https://docs.popsql.com/docs/bridge-connector). +- Use $POPSQL's [bridge connector][bridge-connector]. - Use an SSH tunnel: when you configure the connection in $POPSQL, under `Advanced Options` enable `Connect over SSH`. - Add $POPSQL's static IPs (`23.20.131.72, 54.211.234.135`) to your allowlist. @@ -119,7 +109,7 @@ If you have a small number of users running performant SQL queries against a $SERVICE_SHORT with sufficient resources, then there should be no degradation to performance. However, if you have a large number of users running queries, or if the queries are computationally expensive, best practice is to create -a [$READ_REPLICA][readreplica] and send analytical queries there. +a [$READ_REPLICA][read-replica] and send analytical queries there. If you'd like to prevent write operations such as insert or update, instead of using the `tsdbadmin` user, create a read-only user for your $SERVICE_SHORT and @@ -190,7 +180,6 @@ For best results with $SQL_ASSISTANT_SHORT: * **Business logic**: $SQL_ASSISTANT_SHORT does not inherently know specific business terms such as active user. Define these terms clearly to improve results. - ### Security, privacy, and data usage Security and privacy is prioritized in $CONSOLE. In [$DATA_MODE][portal-data-mode], project members @@ -208,7 +197,6 @@ $SQL_ASSISTANT_SHORT settings are: * **Sample data**: to give the LLM more context so you have better SQL suggestions, enable sample data sharing in the $SQL_ASSISTANT_SHORT preferences. * **Telemetry**: to improve $SQL_ASSISTANT_SHORT, $COMPANY collects telemetry and usage data, including prompts, responses, and query metadata. - ## $OPS_MODE_CAP $SQL_EDITOR $SQL_EDITOR is an integrated secure UI that you use to run queries and see the results @@ -233,10 +221,11 @@ To use $SQL_EDITOR: 1. **Run a test query** - Type your query in the UI, then click `Run`. The results appear in the lower window. + Type `SELECT CURRENT_DATE;` in the UI and click `Run`. The results appear in the lower window: - + ![Run a simple query](https://assets.timescale.com/docs/images/run-query-in-sql-editor.png) + ## Cloud $SQL_EDITOR licenses @@ -244,27 +233,37 @@ To use $SQL_EDITOR: * **Data mode**: the number of seats you are allocated depends on your [$PRICING_PLAN][pricing-plan-features]. [$SQL_ASSISTANT_SHORT][sql-assistant] is currently free for all users. In the future, limits or paid options may be introduced as we work to build the best experience. -* **$POPSQL standalone**: there is a free plan available to everyone, as well as paid plans. See [$POPSQL Pricing][popsql-pricing] for full - details. - +* **$POPSQL standalone**: there is a free plan available to everyone, as well as paid plans. See [$POPSQL Pricing][popsql-pricing] for full details. -[readreplica]: /use-timescale/:currentVersion:/ha-replicas/read-scaling/ -[run-popsql]: /getting-started/:currentVersion:/run-queries-from-console/#data-mode -[run-sqleditor]: /getting-started/:currentVersion:/run-queries-from-console/#sql-editor +[bridge-connector]: https://docs.popsql.com/docs/bridge-connector +[charts]: https://docs.popsql.com/docs/creating-charts +[connection-info]: /integrations/:currentVersion:/find-connection-details/ +[console]: https://console.cloud.timescale.com +[create-chart]: https://docs.popsql.com/docs/creating-charts +[create-cloud-account]: /getting-started/:currentVersion:/services/#create-a-timescale-cloud-account +[create-dashboard]: https://docs.popsql.com/docs/creating-dashboards +[create-schedule]: https://docs.popsql.com/docs/scheduled-queries +[install-psql]: /integrations/:currentVersion:/psql/ [integrations]: /integrations/:currentVersion:/ -[timescale-console]: https://console.cloud.timescale.com/ +[popsql-connections]: https://popsql.com/connections +[popsql-desktop]: https://popsql.com/download +[popsql-pricing]: https://popsql.com/pricing +[portal-data-mode]: https://console.cloud.timescale.com/dashboard/services?popsql [portal-ops-mode]: https://console.cloud.timescale.com/dashboard/services [pricing-plan-features]: https://www.timescale.com/pricing#features -[install-psql]: /integrations/:currentVersion:/psql/ -[create-cloud-account]: /getting-started/:currentVersion:/services/#create-a-timescale-cloud-account -[portal-data-mode]: https://console.cloud.timescale.com/dashboard/services?popsql -[services-portal]: https://console.cloud.timescale.com/dashboard/services [project-members]: /use-timescale/:currentVersion:/members/ -[console]: https://console.cloud.timescale.com -[sql-editor-settings]: https://console.cloud.timescale.com/dashboard/settings?popsql=%2Fpreferences%2Fai -[portal-data-mode]: https://console.cloud.timescale.com/dashboard/services?popsql +[query-variables]: https://docs.popsql.com/docs/query-variables +[read-replica]: /use-timescale/:currentVersion:/ha-replicas/read-scaling/ +[run-popsql]: /getting-started/:currentVersion:/run-queries-from-console/#data-mode +[run-sqleditor]: /getting-started/:currentVersion:/run-queries-from-console/#ops-mode-sql-editor +[schedules]: https://docs.popsql.com/docs/scheduled-queries +[schema-browser]: https://docs.popsql.com/docs/schema +[services-portal]: https://console.cloud.timescale.com/dashboard/services +[share-query]: https://docs.popsql.com/docs/sharing-a-link-to-your-query-and-results [sql-assistant]: /getting-started/:currentVersion:/run-queries-from-console/#sql-assistant -[connection-info]: /integrations/:currentVersion:/find-connection-details/ -[popsql-pricing]: https://popsql.com/pricing +[sql-editor-settings]: https://console.cloud.timescale.com/dashboard/settings?popsql=%2Fpreferences%2Fai +[timescale-console]: https://console.cloud.timescale.com/ +[version-history]: https://docs.popsql.com/docs/version-history +[write-query]: https://docs.popsql.com/docs/writing-a-query \ No newline at end of file diff --git a/getting-started/try-key-features-timescale-products.md b/getting-started/try-key-features-timescale-products.md index 0441471283..2e4986dc4b 100644 --- a/getting-started/try-key-features-timescale-products.md +++ b/getting-started/try-key-features-timescale-products.md @@ -36,7 +36,7 @@ is made up of child tables called $CHUNKs. Each $CHUNK is assigned a range of ti contains data from that range. When you run a query, $CLOUD_LONG identifies the correct $CHUNK and runs the query on it, instead of going through the entire table. You can also tune $HYPERTABLEs to increase performance even more. -![Main features and tiered data](https://assets.timescale.com/docs/images/hypertable-structure.png ) +![Main features and tiered data](https://assets.timescale.com/docs/images/hypertable-structure.png) $HYPERTABLE_CAPs exist alongside regular $PG tables. You use regular $PG tables for relational data, and interact with $HYPERTABLEs @@ -106,7 +106,7 @@ relational and time-series data from external files. SELECT create_hypertable('crypto_ticks', by_range('time')); ``` To more fully understand how $HYPERTABLEs work, and how to optimize them for performance by - tuning $CHUNK intervals and enabling $CHUNK_SKIPPING, see [the $HYPERTABLEs documentation][hypertables-section]. + tuning $CHUNK intervals and enabling chunk skipping, see [the $HYPERTABLEs documentation][hypertables-section]. - For the relational data: @@ -133,7 +133,7 @@ relational and time-series data from external files. To more fully understand how $HYPERTABLEs work, and how to optimize them for performance by - tuning $CHUNK intervals and enabling $CHUNK_SKIPPING, see [the $HYPERTABLEs documentation][hypertables-section]. + tuning $CHUNK intervals and enabling chunk skipping, see [the $HYPERTABLEs documentation][hypertables-section]. 1. **Have a quick look at your data** @@ -172,18 +172,20 @@ $CONSOLE. You can also do this using psql. - + -1. **In [$CONSOLE][portal-ops-mode], select the $SERVICE_SHORT you uploaded data to, then click `SQL Editor`** +1. **Connect to your $SERVICE_SHORT** + + In [$CONSOLE][portal-data-mode], select your $SERVICE_SHORT in the connection drop-down in the top right. 1. **Create a $CAGG** For a $CAGG, data grouped using a $TIME_BUCKET is stored in a $PG `MATERIALIZED VIEW` in a $HYPERTABLE. `timescaledb.continuous` ensures that this data is always up to date. - In your SQL editor, use the following code to create a $CAGG on the real-time data in + In data mode, use the following code to create a $CAGG on the real-time data in the `crypto_ticks` table: ```sql @@ -228,10 +230,10 @@ $CONSOLE. You can also do this using psql. -1. **In [$CONSOLE][portal-ops-mode], select the $SERVICE_SHORT you uploaded data to**. -1. **Click `Operations` > `Continuous aggregates`, select `crypto_ticks`, then click `Create a Continuous Aggregate`**. +1. **In [$CONSOLE][portal-ops-mode], select the $SERVICE_SHORT you uploaded data to** +1. **Click `Operations` > `Continuous aggregates`, select `crypto_ticks`, then click `Create a Continuous Aggregate`** ![$CAGG wizard](https://assets.timescale.com/docs/images/continuous-aggregate-wizard.png ) -1. **Create a view called `assets_candlestick_daily` on the `time` column with an interval of `1 day`, then click `Next step`**. +1. **Create a view called `assets_candlestick_daily` on the `time` column with an interval of `1 day`, then click `Next step`** 1. **Update the view SQL with the following functions, then click `Run`** ```sql CREATE MATERIALIZED VIEW assets_candlestick_daily @@ -357,7 +359,7 @@ To set up data tiering: 1. **Set the time interval when data is tiered** - In $CONSOLE, click `SQL Editor`, then enable data tiering on a $HYPERTABLE with the following query: + In $CONSOLE, click `Data` to switch to the data mode, then enable data tiering on a $HYPERTABLE with the following query: ```sql SELECT add_tiering_policy('assets_candlestick_daily', INTERVAL '3 weeks'); ``` @@ -386,9 +388,9 @@ To set up data tiering: ## Reduce the risk of downtime and data loss By default, all $SERVICE_LONGs have rapid recovery enabled. However, if your app has very low tolerance -for downtime, $CLOUD_LONG offers $HA_REPLICAs. $HA_REPLICA_SHORTs are exact, up-to-date copies +for downtime, $CLOUD_LONG offers $HA_REPLICAs. HA replicas are exact, up-to-date copies of your database hosted in multiple AWS availability zones (AZ) within the same region as your primary node. -$HA_REPLICA_SHORTa automatically take over operations if the original primary data node becomes unavailable. +HA replicas automatically take over operations if the original primary data node becomes unavailable. The primary node streams its write-ahead log (WAL) to the replicas to minimize the chances of data loss during failover. From 76bc5346eff9b6a07e319dee9594c54a5eeb9019 Mon Sep 17 00:00:00 2001 From: Matthew Peveler Date: Fri, 25 Apr 2025 04:00:53 -0600 Subject: [PATCH 88/90] Update composite hypertable example (#4042) Signed-off-by: Matthew Peveler Co-authored-by: Iain Cox --- api/hypertable/create_hypertable.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/api/hypertable/create_hypertable.md b/api/hypertable/create_hypertable.md index 206da45be5..3d378a30c6 100644 --- a/api/hypertable/create_hypertable.md +++ b/api/hypertable/create_hypertable.md @@ -87,8 +87,6 @@ column type using a range partitioning function. LANGUAGE SQL IMMUTABLE AS 'SELECT $1.reported'; - - SELECT create_hypertable('measurements', by_range('report', partition_func => 'report_reported')); ``` 1. Create the hypertable using the immutable function: From afb524ea005f0dd1fbc5c623c1b1a686af8e2328 Mon Sep 17 00:00:00 2001 From: Iain Date: Mon, 28 Apr 2025 14:28:47 +0200 Subject: [PATCH 89/90] chore: updates on review --- integrations/page-index/page-index.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/integrations/page-index/page-index.js b/integrations/page-index/page-index.js index c005d8d907..e86988f8ea 100644 --- a/integrations/page-index/page-index.js +++ b/integrations/page-index/page-index.js @@ -11,6 +11,11 @@ module.exports = [ href: "find-connection-details", excerpt: "Find connection information for your Timescale Cloud service", }, + { + title: "Airbyte", + href: "airbyte", + excerpt: "Integrate Airbyte with Timescale Cloud", + }, { title: "Amazon CloudWatch", href: "cloudwatch", From 8e26a2d79b69c06fb836b95b5f86e262d47f6cd7 Mon Sep 17 00:00:00 2001 From: Iain Date: Tue, 6 May 2025 14:49:42 +0200 Subject: [PATCH 90/90] chore: merge. --- use-timescale/page-index/page-index.js | 155 ------------------------- 1 file changed, 155 deletions(-) diff --git a/use-timescale/page-index/page-index.js b/use-timescale/page-index/page-index.js index 6f725e5d1a..fe8fa933b5 100644 --- a/use-timescale/page-index/page-index.js +++ b/use-timescale/page-index/page-index.js @@ -729,161 +729,6 @@ module.exports = [ }, ], }, - { - title: "Integrations", - href: "integrations", - excerpt: "Integrate third-party solutions with Timescale Cloud", - children: [ - { - title: "Find your connection details", - href: "find-connection-details", - excerpt: - "Find connection information for your Timescale Cloud service", - }, - { - title: "Amazon CloudWatch", - href: "cloudwatch", - excerpt: "Integrate Amazon Cloudwatch with Timescale Cloud", - }, - { - title: "Amazon SageMaker", - href: "amazon-sagemaker", - excerpt: "Integrate Amazon SageMaker with Timescale Cloud", - }, - { - title: "Amazon Web Services", - href: "aws", - excerpt: "Integrate AWS with Timescale Cloud", - }, - { - title: "Apache Airflow", - href: "apache-airflow", - excerpt: "Integrate Apache Airflow with Timescale products", - }, - { - title: "Apache Kafka", - href: "apache-kafka", - excerpt: "Integrate Apache Kafka with Timescale Cloud", - }, - { - title: "AWS Lambda", - href: "aws-lambda", - excerpt: "Integrate AWS Lambda with Timescale Cloud", - }, - { - title: "Apache Airflow", - href: "apache-airflow", - excerpt: "Integrate Apache Airflow with Timescale Cloud", - }, - { - title: "Azure Data Studio", - href: "azure-data-studio", - excerpt: "Integrate Azure Data Studio with Timescale Cloud", - }, - { - title: "Corporate data center", - href: "corporate-data-center", - excerpt: - "Integrate your corporate data center with Timescale Cloud", - }, - { - title: "Datadog", - href: "datadog", - excerpt: "Integrate Datadog with Timescale Cloud", - }, - { - title: "DBeaver", - href: "dbeaver", - excerpt: "Integrate DBeaver with Timescale Cloud", - }, - { - title: "Decodable", - href: "decodable", - excerpt: "Integrate Decodable with Timescale Cloud", - }, - { - title: "Debezium", - href: "debezium", - excerpt: "Integrate Debezium with Timescale products", - }, - { - title: "Fivetran", - href: "fivetran", - excerpt: "Integrate Fivetran with Timescale Cloud", - }, - { - title: "Google Cloud", - href: "google-cloud", - excerpt: "Integrate Google Cloud with Timescale Cloud", - }, - { - title: "Grafana", - href: "grafana", - excerpt: "Integrate Grafana with Timescale products", - }, - { - title: "Kubernetes", - href: "kubernetes", - excerpt: "Integrate Kubernetes with Timescale products", - }, - { - title: "Microsoft Azure", - href: "microsoft-azure", - excerpt: "Integrate Microsoft Azure with Timescale Cloud", - }, - { - title: "pgAdmin", - href: "pgadmin", - excerpt: "Integrate pgAdmin with Timescale Cloud", - }, - { - title: "PostgreSQL", - href: "postgresql", - excerpt: "Integrate PostgreSQL with Timescale products", - }, - { - title: "Prometheus", - href: "prometheus", - excerpt: "Integrate Prometheus with Timescale Cloud", - }, - { - title: "Power BI", - href: "power-bi", - excerpt: "Integrate Power BI with Timescale Cloud", - }, - { - title: "psql", - href: "psql", - excerpt: "Connect to Timescale Cloud with psql", - }, - { - title: "qStudio", - href: "qstudio", - excerpt: "Integrate qstudio with Timescale Cloud", - }, - { - title: "Supabase", - href: "supabase", - excerpt: "Integrate Supabase with Timescale products", - }, - { - title: "Tableau", - href: "tableau", - excerpt: "Integrate Tableau with Timescale Cloud", - }, - { - title: "Terraform", - href: "terraform", - excerpt: "Integrate Terraform with Timescale Cloud", - }, - { - title: "Troubleshooting Timescale integrations", - href: "troubleshooting", - type: "placeholder", - excerpt: "Resolve integration issues", - }, - ], - }, { title: "Security", href: "security",