From c44e85d4deaeb5db0e6cd878ac1b2230ede3047e Mon Sep 17 00:00:00 2001 From: niebayes Date: Mon, 8 Jun 2026 10:19:48 +0000 Subject: [PATCH 1/3] add docs for hybrid cache hint --- dir.yaml | 3 + .../query-optimization/hints/hybrid-cache.md | 59 +++++++++++++++++++ .../query-optimization/hints/hybrid-cache.md | 52 ++++++++++++++++ .../query-optimization/hints/overview.md | 6 +- 4 files changed, 118 insertions(+), 2 deletions(-) create mode 100644 en_US/development-guide/query-optimization/hints/hybrid-cache.md create mode 100644 zh_CN/development-guide/query-optimization/hints/hybrid-cache.md diff --git a/dir.yaml b/dir.yaml index 4782887e..c7a11310 100644 --- a/dir.yaml +++ b/dir.yaml @@ -117,6 +117,9 @@ - title_en: Query Parallelism Control title_cn: 查询并行度控制 path: development-guide/query-optimization/hints/parallel-degree + - title_en: Hybrid Cache Control + title_cn: 混合缓存控制 + path: development-guide/query-optimization/hints/hybrid-cache - title_en: Index Hints title_cn: Index Hints path: development-guide/query-optimization/hints/index diff --git a/en_US/development-guide/query-optimization/hints/hybrid-cache.md b/en_US/development-guide/query-optimization/hints/hybrid-cache.md new file mode 100644 index 00000000..756abe76 --- /dev/null +++ b/en_US/development-guide/query-optimization/hints/hybrid-cache.md @@ -0,0 +1,59 @@ +--- +title: "Datalayers Hybrid Cache Hint Guide" +description: "Control object store cache behavior per query using the hybrid_cache hint." +--- +# Hybrid Cache Hint Guide + +## Overview + +Datalayers uses Hybrid Cache (a foyer-based memory + disk cache) to cache Parquet file metadata and content from object stores, accelerating queries. In some scenarios, bypassing the cache for direct storage reads is preferable. The `hybrid_cache` hint enables per-query control over cache behavior. + +## Syntax + +```sql +-- Disable cache for the current query +SELECT /*+ SET_VAR(hybrid_cache=off) */ * FROM t; + +-- Explicitly enable cache (default behavior) +SELECT /*+ SET_VAR(hybrid_cache=on) */ * FROM t; +``` + +Supported values: + +| Value | Meaning | +|-------|---------| +| `on`, `1`, `true` | Enable cache (default) | +| `off`, `0`, `false` | Disable cache | + +Case insensitive. Can be combined with other hints: + +```sql +SELECT /*+ SET_VAR(parallel_degree=4), SET_VAR(hybrid_cache=off) */ * FROM t; +``` + +## Use Cases + +- **Benchmarking after cache warm-up**: compare cached vs. non-cached query performance +- **Debugging cache issues**: disable cache to verify data consistency when suspecting stale cache entries +- **Large scan queries**: when a single query would evict hot data from the cache, temporarily disable it + +## Verification + +Use `EXPLAIN ANALYZE VERBOSE` to inspect the physical plan output: + +```sql +-- Default: shows hybrid_cache=on +EXPLAIN ANALYZE VERBOSE SELECT * FROM t; + +-- Cache disabled: shows hybrid_cache=off +EXPLAIN ANALYZE VERBOSE SELECT /*+ SET_VAR(hybrid_cache=off) */ * FROM t; +``` + +Look for `hybrid_cache=on` or `hybrid_cache=off` in the `PartitionScanExec` node of the physical plan. + +> **Note**: If the instance has no Hybrid Cache configured (`object_store.file_cache` and `metadata_cache` are both empty), the output always shows `hybrid_cache=off` regardless of the hint value. + +## See Also + +- For SQL Hints overview, see [SQL Hints Guide](./overview.md) +- For query parallelism control, see [Parallel Degree Guide](./parallel-degree.md) diff --git a/zh_CN/development-guide/query-optimization/hints/hybrid-cache.md b/zh_CN/development-guide/query-optimization/hints/hybrid-cache.md new file mode 100644 index 00000000..85e1326f --- /dev/null +++ b/zh_CN/development-guide/query-optimization/hints/hybrid-cache.md @@ -0,0 +1,52 @@ +--- +title: "Datalayers Hybrid Cache Hint 使用指南" +description: "介绍如何通过 hybrid_cache Hint 按查询控制对象存储缓存的启用与禁用。" +--- +# Hybrid Cache Hint 使用指南 + +## 概述 + +Datalayers 默认会使用 Hybrid Cache（内存 + 磁盘混合缓存）缓存对象存储中 Parquet 文件的数据和对象元信息，以加速查询。但某些场景下，用户可能需要绕过缓存。`hybrid_cache` Hint 允许为单个查询临时禁用混合缓存。 + +## 语法 + +```sql +-- 禁用缓存（当前查询不走 Hybrid Cache） +SELECT /*+ SET_VAR(hybrid_cache=off) */ * FROM t; + +-- 显式启用缓存（默认行为，无需设置）。注意，需要用户在配置文件中配置 Hybrid Cache 后才能真正启用 +SELECT /*+ SET_VAR(hybrid_cache=on) */ * FROM t; +``` + +支持的值（大小写不敏感）： + +| 值 | 含义 | +|----|------| +| `on` 或 `1` | 启用缓存（默认） | +| `off` 或 `0` | 禁用缓存 | + +可与其他 Hint 组合使用： + +```sql +SELECT /*+ SET_VAR(parallel_degree=4), SET_VAR(hybrid_cache=off) */ * FROM t; +``` + +## 适用场景 + +- **缓存预热后的基准测试**：对比缓存命中与绕过缓存的性能差异 +- **排查缓存相关问题**：怀疑缓存数据不一致时，禁用缓存以验证 +- **特定大查询**：某条查询扫描数据量远超缓存容量，频繁 evict 其他热点数据，可临时禁用 + +## 如何验证 + +使用 `EXPLAIN` 或 `EXPLAIN ANALYZE` 查看物理计划输出，确认 hybrid cache 的状态： + +```sql +-- 如果用户配置了 Hybrid Cache，PartitionScanExec 算子会显示 hybrid_cache=on +EXPLAIN SELECT * FROM t; + +-- 如果用户没有配置 Hybrid Cache，或通过 hint 禁用缓存，PartitionScanExec 算子会显示 hybrid_cache=off +EXPLAIN SELECT /*+ SET_VAR(hybrid_cache=off) */ * FROM t; +``` + +> **注意**：若用户在启动 Datalayers 时未配置 Hybrid Cache（`object_store.file_cache` 和 `object_store.metadata_cache` 均为空），无论 Hint 设置为何，输出均显示 `hybrid_cache=off`。 diff --git a/zh_CN/development-guide/query-optimization/hints/overview.md b/zh_CN/development-guide/query-optimization/hints/overview.md index ba077295..7dd6754d 100644 --- a/zh_CN/development-guide/query-optimization/hints/overview.md +++ b/zh_CN/development-guide/query-optimization/hints/overview.md @@ -10,10 +10,12 @@ SQL Hints 是嵌入在 SQL 中的特殊注释指令，用于向优化器提供 目前 Datalayers 支持以下 SQL Hints： -- 查询并行度(parallel_degree)控制 +- 查询并行度（parallel_degree）控制 +- 混合缓存控制（hybrid_cache） - Index Hints ## 相关文档 - 了解并行度控制，请参考 [查询并行度优化指南](./parallel-degree.md) -- 了解索引提示，请参考 [Index Hints 使用指南](./index.md) +- 了解混合缓存控制，请参考 [混合缓存控制使用指南](./hybrid-cache.md) +- 了解 Index Hints，请参考 [Index Hints 使用指南](./index.md) From 4ecfeb893aac7916d812be20b666eb9559b6987a Mon Sep 17 00:00:00 2001 From: niebayes Date: Mon, 8 Jun 2026 10:20:06 +0000 Subject: [PATCH 2/3] update en doc --- .../query-optimization/hints/hybrid-cache.md | 31 +++++++------------ 1 file changed, 12 insertions(+), 19 deletions(-) diff --git a/en_US/development-guide/query-optimization/hints/hybrid-cache.md b/en_US/development-guide/query-optimization/hints/hybrid-cache.md index 756abe76..bc8085b0 100644 --- a/en_US/development-guide/query-optimization/hints/hybrid-cache.md +++ b/en_US/development-guide/query-optimization/hints/hybrid-cache.md @@ -6,7 +6,7 @@ description: "Control object store cache behavior per query using the hybrid_cac ## Overview -Datalayers uses Hybrid Cache (a foyer-based memory + disk cache) to cache Parquet file metadata and content from object stores, accelerating queries. In some scenarios, bypassing the cache for direct storage reads is preferable. The `hybrid_cache` hint enables per-query control over cache behavior. +Datalayers uses a Hybrid Cache (memory + disk cache) to cache Parquet file data and object metadata from object stores, accelerating queries. However, in some scenarios, bypassing the cache is desirable. The `hybrid_cache` hint temporarily disables the hybrid cache for an individual query. ## Syntax @@ -14,18 +14,18 @@ Datalayers uses Hybrid Cache (a foyer-based memory + disk cache) to cache Parque -- Disable cache for the current query SELECT /*+ SET_VAR(hybrid_cache=off) */ * FROM t; --- Explicitly enable cache (default behavior) +-- Explicitly enable cache (default behavior, no need to set). Note: Hybrid Cache must be configured in the config file for it to actually take effect SELECT /*+ SET_VAR(hybrid_cache=on) */ * FROM t; ``` -Supported values: +Supported values (case insensitive): | Value | Meaning | |-------|---------| -| `on`, `1`, `true` | Enable cache (default) | -| `off`, `0`, `false` | Disable cache | +| `on` or `1` | Enable cache (default) | +| `off` or `0` | Disable cache | -Case insensitive. Can be combined with other hints: +Can be combined with other hints: ```sql SELECT /*+ SET_VAR(parallel_degree=4), SET_VAR(hybrid_cache=off) */ * FROM t; @@ -39,21 +39,14 @@ SELECT /*+ SET_VAR(parallel_degree=4), SET_VAR(hybrid_cache=off) */ * FROM t; ## Verification -Use `EXPLAIN ANALYZE VERBOSE` to inspect the physical plan output: +Use `EXPLAIN` or `EXPLAIN ANALYZE` to inspect the physical plan output for the hybrid cache state: ```sql --- Default: shows hybrid_cache=on -EXPLAIN ANALYZE VERBOSE SELECT * FROM t; +-- If Hybrid Cache is configured, the PartitionScanExec operator shows hybrid_cache=on +EXPLAIN SELECT * FROM t; --- Cache disabled: shows hybrid_cache=off -EXPLAIN ANALYZE VERBOSE SELECT /*+ SET_VAR(hybrid_cache=off) */ * FROM t; +-- If Hybrid Cache is not configured, or disabled by hint, the PartitionScanExec operator shows hybrid_cache=off +EXPLAIN SELECT /*+ SET_VAR(hybrid_cache=off) */ * FROM t; ``` -Look for `hybrid_cache=on` or `hybrid_cache=off` in the `PartitionScanExec` node of the physical plan. - -> **Note**: If the instance has no Hybrid Cache configured (`object_store.file_cache` and `metadata_cache` are both empty), the output always shows `hybrid_cache=off` regardless of the hint value. - -## See Also - -- For SQL Hints overview, see [SQL Hints Guide](./overview.md) -- For query parallelism control, see [Parallel Degree Guide](./parallel-degree.md) +> **Note**: If Hybrid Cache is not configured at startup (`object_store.file_cache` and `object_store.metadata_cache` are both empty), the output always shows `hybrid_cache=off` regardless of the hint value. From 4c0b5e060b377113dec3d5caad07178bcaf3acc4 Mon Sep 17 00:00:00 2001 From: YinBo Date: Wed, 10 Jun 2026 16:24:25 +0800 Subject: [PATCH 3/3] chore --- .../query-optimization/hints/hybrid-cache.md | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/zh_CN/development-guide/query-optimization/hints/hybrid-cache.md b/zh_CN/development-guide/query-optimization/hints/hybrid-cache.md index 85e1326f..3562df7d 100644 --- a/zh_CN/development-guide/query-optimization/hints/hybrid-cache.md +++ b/zh_CN/development-guide/query-optimization/hints/hybrid-cache.md @@ -1,12 +1,12 @@ --- title: "Datalayers Hybrid Cache Hint 使用指南" -description: "介绍如何通过 hybrid_cache Hint 按查询控制对象存储缓存的启用与禁用。" +description: "介绍如何使用 hybrid_cache Hint 按查询控制是否使用混合缓存。" --- # Hybrid Cache Hint 使用指南 ## 概述 -Datalayers 默认会使用 Hybrid Cache（内存 + 磁盘混合缓存）缓存对象存储中 Parquet 文件的数据和对象元信息，以加速查询。但某些场景下，用户可能需要绕过缓存。`hybrid_cache` Hint 允许为单个查询临时禁用混合缓存。 +Datalayers 在配置文件中打开混合缓存后，在查询时默认使用 Hybrid Cache（内存加磁盘的混合缓存）来加速查询。在某些场景下，绕过缓存会更合适（如对非热点数据进行一次性的分析查询）。`hybrid_cache` Hint 可用于控制单个查询时选择是否使用混合缓存。 ## 语法 @@ -14,16 +14,17 @@ Datalayers 默认会使用 Hybrid Cache（内存 + 磁盘混合缓存）缓存 -- 禁用缓存（当前查询不走 Hybrid Cache） SELECT /*+ SET_VAR(hybrid_cache=off) */ * FROM t; --- 显式启用缓存（默认行为，无需设置）。注意，需要用户在配置文件中配置 Hybrid Cache 后才能真正启用 +-- 显式启用缓存（默认行为） +-- 注意：只有在配置文件中启用了 Hybrid Cache，该设置才会生效 SELECT /*+ SET_VAR(hybrid_cache=on) */ * FROM t; ``` 支持的值（大小写不敏感）： -| 值 | 含义 | -|----|------| -| `on` 或 `1` | 启用缓存（默认） | -| `off` 或 `0` | 禁用缓存 | +| 值 | 含义 | +| :----------- | :--------------- | +| `on` 或 `1` | 启用缓存（默认） | +| `off` 或 `0` | 禁用缓存 | 可与其他 Hint 组合使用： @@ -34,19 +35,18 @@ SELECT /*+ SET_VAR(parallel_degree=4), SET_VAR(hybrid_cache=off) */ * FROM t; ## 适用场景 - **缓存预热后的基准测试**：对比缓存命中与绕过缓存的性能差异 -- **排查缓存相关问题**：怀疑缓存数据不一致时，禁用缓存以验证 -- **特定大查询**：某条查询扫描数据量远超缓存容量，频繁 evict 其他热点数据，可临时禁用 +- **大范围扫描查询**：单条查询扫描的数据量远超缓存容量、可能频繁淘汰热点数据时，可临时禁用缓存 ## 如何验证 -使用 `EXPLAIN` 或 `EXPLAIN ANALYZE` 查看物理计划输出，确认 hybrid cache 的状态： +使用 `EXPLAIN` 或 `EXPLAIN ANALYZE` 查看物理执行计划，确认 Hybrid Cache 的状态： ```sql --- 如果用户配置了 Hybrid Cache，PartitionScanExec 算子会显示 hybrid_cache=on +-- 如果已配置 Hybrid Cache，PartitionScanExec 算子会显示 hybrid_cache=on EXPLAIN SELECT * FROM t; --- 如果用户没有配置 Hybrid Cache，或通过 hint 禁用缓存，PartitionScanExec 算子会显示 hybrid_cache=off +-- 如果未配置 Hybrid Cache，或通过 hint 禁用了缓存，PartitionScanExec 算子会显示 hybrid_cache=off EXPLAIN SELECT /*+ SET_VAR(hybrid_cache=off) */ * FROM t; ``` -> **注意**：若用户在启动 Datalayers 时未配置 Hybrid Cache（`object_store.file_cache` 和 `object_store.metadata_cache` 均为空），无论 Hint 设置为何，输出均显示 `hybrid_cache=off`。 +> **注意**：如果在启动 Datalayers 时未配置 Hybrid Cache（`object_store.file_cache` 和 `object_store.metadata_cache` 均为空），那么无论 Hint 如何设置，输出都会显示 `hybrid_cache=off`。