From c5d9258488b4a83ef76ab33d0accfb255e92a62d Mon Sep 17 00:00:00 2001 From: annasclaw Date: Thu, 2 Jul 2026 12:37:55 +0100 Subject: [PATCH] fix(docs): remove duplicate content & fix sidebar header link Two fixes for docs duplication introduced by PR #4028: 1. Remove duplicate 'dlt's internal tables' section from destination-tables.md. This section was a verbatim copy of 'Internal dlt tables' in dataset-access/dataset.md (the canonical source). Removes 64 lines of duplicate content. 2. Fix 'Configuration & secrets' sidebar category header linking to its own page (a top-level header with a doc link). Removed the property and added the same doc as an 'Overview' item inside the category, so clicking the header expands the category instead of navigating to a page. Closes #4125 --- .../docs/general-usage/destination-tables.md | 64 ------------------- docs/website/sidebars.js | 9 +-- 2 files changed, 5 insertions(+), 68 deletions(-) diff --git a/docs/website/docs/general-usage/destination-tables.md b/docs/website/docs/general-usage/destination-tables.md index 5cb16e0918..f9a0d14a49 100644 --- a/docs/website/docs/general-usage/destination-tables.md +++ b/docs/website/docs/general-usage/destination-tables.md @@ -275,70 +275,6 @@ load_info = pipeline.run(data, table_name="users") Every time you run this pipeline, a new schema will be created in the destination database with a datetime-based suffix. The data will be loaded into tables in this schema. For example, the first time you run the pipeline, the schema will be named `mydata_20230912064403`, the second time it will be named `mydata_20230912064407`, and so on. -## dlt’s internal tables - -dlt automatically creates internal tables in the destination schema to track pipeline runs, support incremental loading, and manage schema versions. These tables use the `_dlt_` prefix. - -### `_dlt_loads`: Load history tracking -This table records each pipeline run. Every time you execute a pipeline, a new row is added to this table with a unique `load_id`. This table tracks which loads have been completed and supports chaining of transformations. - - -| Column name | Type | Description | -|----------------------|-----------|-------------------------------------------| -| `load_id` | STRING | Unique identifier for the load job | -| `schema_name` | STRING | Name of the schema used during the load | -| `schema_version_hash`| STRING | Hash of the schema version | -| `status` | INTEGER | Load status. Value `0` means completed | -| `inserted_at` | TIMESTAMP | When the load was recorded | - -Only rows with `status = 0` are considered complete. Other values represent incomplete or interrupted loads. The status column can also be used to coordinate multi-step transformations. - -### `_dlt_pipeline_state`: Pipeline state and checkpoints -This table stores the internal state of the pipeline for each run. This state enables incremental loading and allows the pipeline to resume from where it left off if a previous run was interrupted. - - -| Column name | Type | Description | -|-------------------|------------------|------------------------------------------------------| -| `version` | INTEGER | Version of this state entry | -| `engine_version` | INTEGER | Version of the dlt engine used | -| `pipeline_name` | STRING | Name of the pipeline | -| `state` | STRING or BLOB | Serialized Python dictionary of pipeline state | -| `created_at` | TIMESTAMP | When this state entry was created | -| `version_hash` | STRING | Hash to detect changes in the state | -| `_dlt_load_id` | STRING | Reference to related load in `_dlt_loads` | -| `_dlt_id` | STRING | Unique identifier for the pipeline state row | - - -The state column contains a serialized Python dictionary that includes: - - - Incremental progress (e.g. last item or timestamp processed). - - Checkpoints for transformations. - - Source-specific metadata and settings. - -This allows dlt to resume interrupted pipelines, avoid reloading already processed data, and ensure pipelines are idempotent and efficient. - -The `version_hash` is recalculated on each update. dlt uses this table to implement last-value incremental loading. If a run fails or stops, this table ensures the next run picks up from the correct checkpoint. - -### `_dlt_version`: Schema version tracking -This table tracks the history of all schema versions used by the pipeline. Every time dlt updates the schema. For example, when new columns or tables are added, a new entry is written to this table. - -| Column name | Type | Description | -|------------------|------------------|--------------------------------------------------| -| `version` | INTEGER | Numeric version of the schema | -| `engine_version` | INTEGER | Version of the dlt engine used | -| `inserted_at` | TIMESTAMP | Time the schema version entry was created | -| `schema_name` | STRING | Name of the schema | -| `version_hash` | STRING | Unique hash representing the schema content | -| `schema` | STRING or JSON | Full schema in JSON format | - -By keeping previous schema definitions, `_dlt_version` ensures that: - -- Older data remains readable -- New data uses updated schema rules -- Backward compatibility is maintained - -This table also supports troubleshooting and compatibility checks. It lets you track which schema and engine version were used for any load. This helps with debugging and ensures safe evolution of your data model. - ## Loading data into existing tables not created by dlt diff --git a/docs/website/sidebars.js b/docs/website/sidebars.js index bb4327ebae..fdfcf35dbe 100644 --- a/docs/website/sidebars.js +++ b/docs/website/sidebars.js @@ -245,11 +245,12 @@ const sidebars = { { type: "category", label: "Configuration & secrets", - link: { - type: "doc", - id: "general-usage/credentials/index", - }, items: [ + { + type: "doc", + id: "general-usage/credentials/index", + label: "Overview", + }, "general-usage/credentials/setup", "general-usage/credentials/advanced", "general-usage/credentials/vaults",