From 5fb041d52a6130519b4e625fdeb874750822625c Mon Sep 17 00:00:00 2001 From: Elastic Machine Date: Wed, 22 Jul 2026 16:33:10 +0000 Subject: [PATCH 1/2] chore: sync public skills (v0.3.1) --- .github/.release-manifest.json | 6 +- CHANGELOG.md | 11 + README.md | 2 +- .../skills/elasticsearch-esql/SKILL.md | 4 +- .../references/dsl-to-esql-migration.md | 32 +- .../references/environment-setup.md | 4 +- .../references/esql-reference.md | 265 ++++++++++------ .../references/esql-version-history.md | 292 +++++++++--------- .../references/generation-tips.md | 32 +- .../references/query-patterns.md | 6 +- .../references/time-series-queries.md | 34 +- .../elasticsearch-onboarding-playbook.md | 92 ++---- .../use-case-library/use-case-library.md | 4 +- .../vector-hybrid-search.md | 47 +-- .../elasticsearch/elasticsearch-esql/SKILL.md | 4 +- .../references/dsl-to-esql-migration.md | 32 +- .../references/environment-setup.md | 4 +- .../references/esql-reference.md | 265 ++++++++++------ .../references/esql-version-history.md | 292 +++++++++--------- .../references/generation-tips.md | 32 +- .../references/query-patterns.md | 6 +- .../references/time-series-queries.md | 34 +- .../elasticsearch-onboarding-playbook.md | 92 ++---- .../use-case-library/use-case-library.md | 4 +- .../vector-hybrid-search.md | 47 +-- 25 files changed, 873 insertions(+), 770 deletions(-) diff --git a/.github/.release-manifest.json b/.github/.release-manifest.json index 2e73de4..187d325 100644 --- a/.github/.release-manifest.json +++ b/.github/.release-manifest.json @@ -1,7 +1,7 @@ { - "version": "0.3.0", + "version": "0.3.1", "source_repo": "elastic/agent-skills", - "sandbox_sha": "092aaa6a6c68a4f0553145c90c86ea157721ae2e", + "sandbox_sha": "9b3fa5cb2cb8a643d1faf3d2b650bb80e4d7c68e", "skills": { "skills/cloud/access-management": { "name": "cloud-access-management", @@ -37,7 +37,7 @@ }, "skills/elasticsearch/elasticsearch-esql": { "name": "elasticsearch-esql", - "version": "0.3.0" + "version": "0.4.0" }, "skills/elasticsearch/elasticsearch-file-ingest": { "name": "elasticsearch-file-ingest", diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f4ea2a..a738a3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,16 @@ # Changelog +## v0.3.1 + +### Updated Skills + +- `skills/elasticsearch/elasticsearch-esql` (v0.3.0 → v0.4.0) +- `skills/elasticsearch/elasticsearch-onboarding` (v0.1.0) + +### Generated Artifacts + +- Regenerated README skill table + ## v0.3.0 ### New Skills diff --git a/README.md b/README.md index 824e3fe..d5f3c0d 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ Skills in this repository focus on: | [elasticsearch-audit](skills/elasticsearch/elasticsearch-audit/SKILL.md) | Enable, configure, and query Elasticsearch security audit logs. Use when the task involves audit logging setup, event filtering, or investigating security incidents like failed logins. | 0.1.0 | elastic | | [elasticsearch-authn](skills/elasticsearch/elasticsearch-authn/SKILL.md) | Authenticate to Elasticsearch using native, file-based, LDAP/AD, SAML, OIDC, Kerberos, JWT, or certificate realms. Use when connecting with credentials, choosing a realm, or managing API keys. Assumes the target realms are already configured. | 0.1.0 | elastic | | [elasticsearch-authz](skills/elasticsearch/elasticsearch-authz/SKILL.md) | Manage Elasticsearch RBAC: native users, roles, role mappings, document- and field-level security. Use when creating users or roles, assigning privileges, or mapping external realms like LDAP/SAML. | 0.1.1 | elastic | -| [elasticsearch-esql](skills/elasticsearch/elasticsearch-esql/SKILL.md) | Execute ES\|QL (Elasticsearch Query Language) queries, use when the user wants to query Elasticsearch data, analyze logs, aggregate metrics, explore data, or create charts and dashboards from ES\|QL results. | 0.3.0 | elastic | +| [elasticsearch-esql](skills/elasticsearch/elasticsearch-esql/SKILL.md) | Execute ES\|QL (Elasticsearch Query Language) queries, use when the user wants to query Elasticsearch data, analyze logs, aggregate metrics, explore data, or create charts and dashboards from ES\|QL results. | 0.4.0 | elastic | | [elasticsearch-file-ingest](skills/elasticsearch/elasticsearch-file-ingest/SKILL.md) | Ingest and transform data files (CSV/JSON/Parquet/Arrow IPC) into Elasticsearch with stream processing and custom transforms. Use when loading files or batch importing data — not for reindexing, general ingest pipeline design, or bulk API patterns. | 0.2.0 | elastic | | [elasticsearch-onboarding](skills/elasticsearch/elasticsearch-onboarding/SKILL.md) | Help developers new to Elasticsearch get from zero to a working search experience. Guide them through understanding their intent, mapping their data, and building a search experience with best practices baked in. Use this when the user shows intent to build search-related functionality, asks about Elasticsearch-related concepts for their use case, or expresses the need for help getting started with Elasticsearch. | 0.1.0 | elastic | | [elasticsearch-security-troubleshooting](skills/elasticsearch/elasticsearch-security-troubleshooting/SKILL.md) | Diagnose and resolve Elasticsearch security errors: 401/403 failures, TLS problems, expired API keys, role mapping mismatches, and Kibana login issues. Use when the user reports a security error. | 0.1.0 | elastic | diff --git a/plugins/elasticsearch/skills/elasticsearch-esql/SKILL.md b/plugins/elasticsearch/skills/elasticsearch-esql/SKILL.md index 4d03d81..f2f2954 100644 --- a/plugins/elasticsearch/skills/elasticsearch-esql/SKILL.md +++ b/plugins/elasticsearch/skills/elasticsearch-esql/SKILL.md @@ -6,7 +6,7 @@ description: > charts and dashboards from ES|QL results. metadata: author: elastic - version: 0.3.0 + version: 0.4.0 --- # Elasticsearch ES|QL @@ -39,7 +39,7 @@ ES|QL uses pipes (`|`) to chain commands: > development (next minor from main), so clients that only semver-compare may treat Serverless as “latest.” **Do not** > use `version.number` to gate features: if `build_flavor` is `"serverless"`, assume all GA and preview ES|QL features > are available. -> - `build_flavor: "default"` — Self-managed or Elastic Cloud Hosted. Use `version.number` for feature availability. +> - `build_flavor: "default"` — Stack (self-managed or Cloud-hosted). Use `version.number` for feature availability. > - **Snapshot builds** have `version.number` like `9.4.0-SNAPSHOT`. Strip the `-SNAPSHOT` suffix and use the > major.minor for version checks. Snapshot builds include all features from that version plus potentially unreleased > features from development — if a query fails with an unknown function/command, it may simply not have landed yet. diff --git a/plugins/elasticsearch/skills/elasticsearch-esql/references/dsl-to-esql-migration.md b/plugins/elasticsearch/skills/elasticsearch-esql/references/dsl-to-esql-migration.md index e138a91..59890ed 100644 --- a/plugins/elasticsearch/skills/elasticsearch-esql/references/dsl-to-esql-migration.md +++ b/plugins/elasticsearch/skills/elasticsearch-esql/references/dsl-to-esql-migration.md @@ -958,22 +958,22 @@ FROM sales Features not available in ES|QL as of version 9.3: -| Feature | Query DSL | ES\|QL | -| ---------------------------- | --------- | ----------------------------------------- | -| Highlighting | ✅ | ❌ | -| Nested queries | ✅ | ❌ | -| Parent-child queries | ✅ | ❌ | -| Scroll/pagination beyond 10k | ✅ | ❌ | -| Percolate queries | ✅ | ❌ | -| Complex boosting | ✅ | Limited | -| Geo distance sorting | ✅ | ❌ | -| Runtime fields | ✅ | Use EVAL | -| Suggest API | ✅ | ❌ | -| Collapse (field collapsing) | ✅ | ❌ | -| Inner hits | ✅ | ❌ | -| Timezone support | ✅ | ✅ `SET time_zone` (Serverless GA) | -| JOIN (non-lookup) | N/A | ❌ (only LEFT JOIN on lookup index) | -| Subqueries / UNION ALL | N/A | ✅ `FROM` subqueries (Serverless preview) | +| Feature | Query DSL | ES\|QL | +| ---------------------------- | --------- | ---------------------------------------- | +| Highlighting | ✅ | ❌ | +| Nested queries | ✅ | ❌ | +| Parent-child queries | ✅ | ❌ | +| Scroll/pagination beyond 10k | ✅ | ❌ | +| Percolate queries | ✅ | ❌ | +| Complex boosting | ✅ | Limited | +| Geo distance sorting | ✅ | ❌ | +| Runtime fields | ✅ | Use EVAL | +| Suggest API | ✅ | ❌ | +| Collapse (field collapsing) | ✅ | ❌ | +| Inner hits | ✅ | ❌ | +| Timezone support | ✅ | ✅ `SET time_zone` (9.4+ GA; Serverless) | +| JOIN (non-lookup) | N/A | ❌ (only LEFT JOIN on lookup index) | +| Subqueries / UNION ALL | N/A | ✅ `FROM` subqueries (9.4+; Serverless) | ### Unsupported Field Types in ES|QL diff --git a/plugins/elasticsearch/skills/elasticsearch-esql/references/environment-setup.md b/plugins/elasticsearch/skills/elasticsearch-esql/references/environment-setup.md index 8bd2e4b..6f4d3e7 100644 --- a/plugins/elasticsearch/skills/elasticsearch-esql/references/environment-setup.md +++ b/plugins/elasticsearch/skills/elasticsearch-esql/references/environment-setup.md @@ -7,8 +7,8 @@ successful connection test. > **Elastic Cloud Serverless:** After connecting, inspect `GET /`. If `build_flavor` is `"serverless"`, do **not** use > `version.number` to decide which ES|QL features are allowed — Serverless tracks current GA and preview ES|QL, and the > reported version follows the main-line / next-minor line (semver-only clients may see it as “latest”). Prefer -> `build_flavor` for detection and gating. For the full rules (including self-managed and snapshot builds), read -> **Cluster Detection** in [SKILL.md](../SKILL.md) and the **Serverless** callout in +> `build_flavor` for detection and gating. For the full rules (including Stack and snapshot builds), read **Cluster +> Detection** in [SKILL.md](../SKILL.md) and the **Serverless** callout in > [ES|QL Version History](esql-version-history.md). ## Option 1: Elastic Cloud (recommended for production) diff --git a/plugins/elasticsearch/skills/elasticsearch-esql/references/esql-reference.md b/plugins/elasticsearch/skills/elasticsearch-esql/references/esql-reference.md index 3597fac..097a769 100644 --- a/plugins/elasticsearch/skills/elasticsearch-esql/references/esql-reference.md +++ b/plugins/elasticsearch/skills/elasticsearch-esql/references/esql-reference.md @@ -3,11 +3,11 @@ ES|QL (Elasticsearch Query Language) is a piped query language for filtering, transforming, and analyzing data in Elasticsearch. It uses pipes (`|`) to chain commands together. -> **Serverless vs Self-Managed:** Version annotations in this document (e.g., "9.2+") apply to self-managed -> Elasticsearch. Detect cluster type via `build_flavor` in the `GET /` response: `"serverless"` means all GA and preview +> **Serverless vs Stack:** Version annotations in this document (e.g., "9.2+") apply to Elastic Stack (self-managed and +> Cloud-hosted). Detect cluster type via `build_flavor` in the `GET /` response: `"serverless"` means all GA and preview > features are available — **do not** gate on `version.number` for Serverless (it tracks the next minor from main; -> semver-only checks may treat it as “latest”). For self-managed, use `version.number` (strip any `-SNAPSHOT` suffix) -> for feature checks. +> semver-only checks may treat it as “latest”). For Stack, use `version.number` (strip any `-SNAPSHOT` suffix) for +> feature checks. ## Table of Contents @@ -66,12 +66,16 @@ source-command - `"default"` / `"fail"` -- the query fails if it references unmapped fields - `"nullify"` -- treats unmapped fields as null values -- `"load"` -- loads unmapped fields dynamically. **Limitation:** `"load"` is incompatible with subqueries and views. Use - `"nullify"` when composing subqueries or querying views. +- `"load"` (9.4+) -- loads unmapped fields dynamically as `keyword`. **Limitation:** `"load"` is incompatible with + subqueries and views. Use `"nullify"` when composing subqueries or querying views. -**`time_zone`** (Serverless GA; self-managed planned) -- sets the default timezone for the query, overriding UTC -default. Accepts any IANA timezone string or UTC offset. Applies to all date/time operations: `DATE_TRUNC`, -`DATE_FORMAT`, `NOW()`, etc. +**`time_zone`** (9.4+ GA; Serverless) -- sets the default timezone for the query, overriding UTC default. Accepts any +IANA timezone string or UTC offset. Applies to all date/time operations: `DATE_TRUNC`, `DATE_FORMAT`, `NOW()`, etc. + +**`approximation`** (9.4+ preview; Serverless) -- enables approximate aggregations for faster results on large datasets: + +- `"true"` -- use approximate algorithms for aggregations (e.g. HyperLogLog for `COUNT_DISTINCT`) +- `"false"` (default) -- use exact algorithms **Examples:** @@ -92,6 +96,10 @@ SET time_zone = "+05:00"; TS k8s | WHERE @timestamp == "2024-05-10T00:04:49.000Z" | STATS BY @timestamp, bucket = TBUCKET(3 hours) + +SET approximation = true; +FROM logs-* +| STATS unique_users = COUNT_DISTINCT(user.id) ``` > **When to use:** `unmapped_fields` is useful when querying across multiple indices where some indices may not have all @@ -136,8 +144,8 @@ FROM FROM cluster_one:logs-*, cluster_two:logs-* ``` -**Subqueries (Serverless tech preview):** `FROM` supports parenthesized subqueries with UNION ALL semantics. Each branch -is a complete ES|QL pipeline. Columns present in one branch but not another are filled with `null`. +**Subqueries (9.4+; Serverless):** `FROM` supports parenthesized subqueries with UNION ALL semantics. Each branch is a +complete ES|QL pipeline. Columns present in one branch but not another are filled with `null`. ```esql // Combine logs from different indices with independent pipelines @@ -184,6 +192,40 @@ ROW x = [1, 2, 3] ROW greeting = "hello", pi = 3.14159 ``` +**Intra-row field references (9.4+; Serverless):** Columns defined earlier in the same `ROW` can be referenced by later +columns: + +```esql +ROW a = 5, b = a * 2, c = a + b +``` + +### Views (9.4+ preview; Stack only — not available on Serverless) + +Views are virtual indices backed by ES|QL queries. Query a view with `FROM view_name` like any index. Views are managed +via the `/_query/view` REST API. + +**Create / update a view:** + +```bash +PUT /_query/view/active_employees +{ + "query": "FROM employees | WHERE is_active == true | KEEP emp_no, name, department" +} +``` + +**Query a view:** + +```esql +FROM active_employees +| STATS headcount = COUNT(*) BY department +``` + +**Constraints:** + +- `SET` directives cannot be used inside view definitions; the caller applies `SET` when querying +- `SET unmapped_fields = "load"` is incompatible with views; use `"nullify"` instead +- Views are not yet available on Serverless + ### TS Retrieves data from time series data streams (TSDS). Similar to `FROM` but enables time series aggregation functions in @@ -577,13 +619,13 @@ FROM data ### LIMIT -Limits the number of rows returned. Supports optional grouped top-N with `BY` (Serverless). +Limits the number of rows returned. Supports optional grouped top-N with `BY` since 9.4+ and in Serverless. **Syntax:** ```esql LIMIT number -LIMIT number BY field +LIMIT number BY field (9.4+; Serverless) ``` **Examples:** @@ -899,8 +941,8 @@ FROM articles METADATA _id, _index, _score ### RERANK -Uses an inference model to re-score an initial set of documents. Tech preview in 9.2 (GA on Serverless). Since 9.3, -defaults to 1000 rows; configurable via `esql.command.rerank.limit` and `esql.command.rerank.enabled` cluster settings. +Uses an inference model to re-score an initial set of documents. GA in 9.4 (Serverless). Since 9.3, defaults to 1000 +rows; configurable via `esql.command.rerank.limit` and `esql.command.rerank.enabled` cluster settings. **Syntax:** @@ -1102,7 +1144,7 @@ TS k8s > **per (metric, time series) combination** and adds a `dimensions` column with the labels identifying each series. Use > `METRICS_INFO` to enumerate _what_ is being measured, and `TS_INFO` to enumerate _which_ time series exist. -### URI_PARTS (Serverless) +### URI_PARTS (9.4+; Serverless) Pipe command that parses a URI string into structured columns. A target prefix is **required**. @@ -1125,7 +1167,7 @@ FROM web_logs | SORT errors DESC ``` -### USER_AGENT (Serverless) +### USER_AGENT (9.4+; Serverless) Pipe command that parses a user agent string into structured columns. A target prefix is **required**. @@ -1146,7 +1188,7 @@ FROM web_logs | STATS cnt = COUNT(*) BY ua.name, ua.version ``` -### REGISTERED_DOMAIN (Serverless) +### REGISTERED_DOMAIN (9.4+; Serverless) Pipe command that extracts the registered domain, top-level domain, and subdomain from a hostname. A target prefix is **required**. @@ -1172,6 +1214,26 @@ FROM dns_logs > **Note:** `URI_PARTS`, `USER_AGENT`, and `REGISTERED_DOMAIN` are **pipe commands** (like `DISSECT`/`GROK`), not scalar > functions. The syntax `URI_PARTS(field)` does not work — use `| URI_PARTS target = field`. +### MMR (9.4+ preview; Serverless) + +Maximal Marginal Relevance — diversifies search results by reducing redundancy among top hits. Requires a dense vector +field and a `LIMIT` before `MMR` to constrain the candidate set. + +**Syntax:** + +```esql +MMR query ON vector_field LIMIT n +``` + +**Example:** + +```esql +FROM articles +| WHERE MATCH(title, "elasticsearch tuning") +| LIMIT 100 +| MMR "elasticsearch performance tuning" ON content_embedding LIMIT 10 +``` + --- ## Aggregate Functions @@ -1198,10 +1260,10 @@ Used with STATS command. | `ABSENT(field)` | True if no non-null values (9.2+) | `STATS is_absent = ABSENT(error_code)` | | `PRESENT(field)` | True if any non-null values (9.2+) | `STATS has_data = PRESENT(metric)` | | `SAMPLE(field, n)` | Collect n sample values (8.19/9.1+) | `STATS examples = SAMPLE(message, 5)` | -| `FIRST(field, sort_field)` | Earliest value by sort field (Serverless GA) | `STATS earliest = FIRST(message, @timestamp)` | -| `LAST(field, sort_field)` | Latest value by sort field (Serverless GA) | `STATS latest = LAST(message, @timestamp)` | -| `EARLIEST(field)` | Earliest value (single-arg; Serverless GA) | `STATS e = EARLIEST(@timestamp)` | -| `LATEST(field)` | Latest value (single-arg; Serverless GA) | `STATS l = LATEST(@timestamp)` | +| `FIRST(field, sort_field)` | Earliest value by sort field (9.4+; Serverless) | `STATS earliest = FIRST(message, @timestamp)` | +| `LAST(field, sort_field)` | Latest value by sort field (9.4+; Serverless) | `STATS latest = LAST(message, @timestamp)` | +| `EARLIEST(field)` | Min `@timestamp` shorthand (9.4+; Serverless) | `STATS e = EARLIEST(@timestamp)` | +| `LATEST(field)` | Max `@timestamp` shorthand (9.4+; Serverless) | `STATS l = LATEST(@timestamp)` | | `ST_CENTROID_AGG(field)` | Spatial centroid of points | `STATS center = ST_CENTROID_AGG(location)` | | `ST_EXTENT_AGG(field)` | Bounding box of geometries (8.18/9.0+, preview) | `STATS bbox = ST_EXTENT_AGG(location)` | @@ -1308,40 +1370,40 @@ TS metrics ## String Functions -| Function | Description | Example | -| --------------------------- | ---------------------------------------------- | -------------------------------------------------------------------- | -| `LENGTH(s)` | String length | `EVAL len = LENGTH(name)` | -| `CONCAT(s1, s2, ...)` | Concatenate strings | `EVAL full = CONCAT(first, " ", last)` | -| `SUBSTRING(s, start, len)` | Extract substring | `EVAL sub = SUBSTRING(text, 1, 10)` | -| `LEFT(s, n)` | Left n characters | `EVAL l = LEFT(text, 5)` | -| `RIGHT(s, n)` | Right n characters | `EVAL r = RIGHT(text, 5)` | -| `TRIM(s)` | Remove whitespace | `EVAL clean = TRIM(input)` | -| `LTRIM(s)` | Trim left | `EVAL clean = LTRIM(input)` | -| `RTRIM(s)` | Trim right | `EVAL clean = RTRIM(input)` | -| `TO_UPPER(s)` | Uppercase | `EVAL upper = TO_UPPER(name)` | -| `TO_LOWER(s)` | Lowercase | `EVAL lower = TO_LOWER(name)` | -| `REPLACE(s, old, new)` | Replace text | `EVAL fixed = REPLACE(msg, "err", "error")` | -| `SPLIT(s, delim)` | Split into array | `EVAL parts = SPLIT(path, "/")` | -| `STARTS_WITH(s, prefix)` | Check prefix | `WHERE STARTS_WITH(url, "https")` | -| `ENDS_WITH(s, suffix)` | Check suffix | `WHERE ENDS_WITH(file, ".log")` | -| `CONTAINS(s, substr)` | Check contains | `WHERE CONTAINS(message, "error")` | -| `LOCATE(substr, s)` | Find position | `EVAL pos = LOCATE("@", email)` | -| `REVERSE(s)` | Reverse string | `EVAL rev = REVERSE(text)` | -| `REPEAT(s, n)` | Repeat string | `EVAL sep = REPEAT("-", 10)` | -| `SPACE(n)` | N spaces | `EVAL spaces = SPACE(5)` | -| `BIT_LENGTH(s)` | Bit length (8.17+) | `EVAL bits = BIT_LENGTH(name)` | -| `BYTE_LENGTH(s)` | Byte length (8.17+) | `EVAL bytes = BYTE_LENGTH(name)` | -| `CHUNK(field, settings)` | Split text into chunks (9.3+, preview) | `EVAL chunks = CHUNK(body, {"strategy":"word","max_chunk_size":50})` | -| `HASH(alg, s)` | Hash string (8.18/9.0+) | `EVAL h = HASH("SHA-256", msg)` | -| `MD5(s)` | MD5 hash (8.18/9.0+) | `EVAL h = MD5(content)` | -| `SHA1(s)` | SHA-1 hash (8.18/9.0+) | `EVAL h = SHA1(content)` | -| `SHA256(s)` | SHA-256 hash (8.18/9.0+) | `EVAL h = SHA256(content)` | -| `FROM_BASE64(s)` | Decode base64 | `EVAL decoded = FROM_BASE64(encoded)` | -| `TO_BASE64(s)` | Encode to base64 | `EVAL encoded = TO_BASE64(data)` | -| `URL_DECODE(s)` | URL-decode (9.2+) | `EVAL decoded = URL_DECODE(url)` | -| `URL_ENCODE(s)` | URL-encode (9.2+) | `EVAL encoded = URL_ENCODE(text)` | -| `URL_ENCODE_COMPONENT(s)` | URL-encode for URI components (9.2+) | `EVAL encoded = URL_ENCODE_COMPONENT(text)` | -| `JSON_EXTRACT(field, path)` | Extract value from JSON string (Serverless GA) | `EVAL name = JSON_EXTRACT(raw, "$.user.name")` | +| Function | Description | Example | +| --------------------------- | --------------------------------------------------------- | -------------------------------------------------------------------- | +| `LENGTH(s)` | String length | `EVAL len = LENGTH(name)` | +| `CONCAT(s1, s2, ...)` | Concatenate strings | `EVAL full = CONCAT(first, " ", last)` | +| `SUBSTRING(s, start, len)` | Extract substring | `EVAL sub = SUBSTRING(text, 1, 10)` | +| `LEFT(s, n)` | Left n characters | `EVAL l = LEFT(text, 5)` | +| `RIGHT(s, n)` | Right n characters | `EVAL r = RIGHT(text, 5)` | +| `TRIM(s)` | Remove whitespace | `EVAL clean = TRIM(input)` | +| `LTRIM(s)` | Trim left | `EVAL clean = LTRIM(input)` | +| `RTRIM(s)` | Trim right | `EVAL clean = RTRIM(input)` | +| `TO_UPPER(s)` | Uppercase | `EVAL upper = TO_UPPER(name)` | +| `TO_LOWER(s)` | Lowercase | `EVAL lower = TO_LOWER(name)` | +| `REPLACE(s, old, new)` | Replace text | `EVAL fixed = REPLACE(msg, "err", "error")` | +| `SPLIT(s, delim)` | Split into array | `EVAL parts = SPLIT(path, "/")` | +| `STARTS_WITH(s, prefix)` | Check prefix | `WHERE STARTS_WITH(url, "https")` | +| `ENDS_WITH(s, suffix)` | Check suffix | `WHERE ENDS_WITH(file, ".log")` | +| `CONTAINS(s, substr)` | Check contains | `WHERE CONTAINS(message, "error")` | +| `LOCATE(substr, s)` | Find position | `EVAL pos = LOCATE("@", email)` | +| `REVERSE(s)` | Reverse string | `EVAL rev = REVERSE(text)` | +| `REPEAT(s, n)` | Repeat string | `EVAL sep = REPEAT("-", 10)` | +| `SPACE(n)` | N spaces | `EVAL spaces = SPACE(5)` | +| `BIT_LENGTH(s)` | Bit length (8.17+) | `EVAL bits = BIT_LENGTH(name)` | +| `BYTE_LENGTH(s)` | Byte length (8.17+) | `EVAL bytes = BYTE_LENGTH(name)` | +| `CHUNK(field, settings)` | Split text into chunks (9.3+, preview) | `EVAL chunks = CHUNK(body, {"strategy":"word","max_chunk_size":50})` | +| `HASH(alg, s)` | Hash string (8.18/9.0+) | `EVAL h = HASH("SHA-256", msg)` | +| `MD5(s)` | MD5 hash (8.18/9.0+) | `EVAL h = MD5(content)` | +| `SHA1(s)` | SHA-1 hash (8.18/9.0+) | `EVAL h = SHA1(content)` | +| `SHA256(s)` | SHA-256 hash (8.18/9.0+) | `EVAL h = SHA256(content)` | +| `FROM_BASE64(s)` | Decode base64 | `EVAL decoded = FROM_BASE64(encoded)` | +| `TO_BASE64(s)` | Encode to base64 | `EVAL encoded = TO_BASE64(data)` | +| `URL_DECODE(s)` | URL-decode (9.2+) | `EVAL decoded = URL_DECODE(url)` | +| `URL_ENCODE(s)` | URL-encode (9.2+) | `EVAL encoded = URL_ENCODE(text)` | +| `URL_ENCODE_COMPONENT(s)` | URL-encode for URI components (9.2+) | `EVAL encoded = URL_ENCODE_COMPONENT(text)` | +| `JSON_EXTRACT(field, path)` | Extract value from JSON string (9.4+ preview; Serverless) | `EVAL name = JSON_EXTRACT(raw, "$.user.name")` | **JSON_EXTRACT with \_source — flattened field workaround:** @@ -1466,25 +1528,30 @@ FROM network_logs ## Spatial Functions -| Function | Description | Example | -| ------------------------- | ----------------------------- | --------------------------------------------------------- | -| `ST_DISTANCE(p1, p2)` | Distance between points | `EVAL dist = ST_DISTANCE(loc, TO_GEOPOINT("POINT(0 0)"))` | -| `ST_INTERSECTS(g1, g2)` | Geometries intersect | `WHERE ST_INTERSECTS(geo, boundary)` | -| `ST_DISJOINT(g1, g2)` | Geometries don't intersect | `WHERE ST_DISJOINT(geo, zone)` | -| `ST_CONTAINS(g1, g2)` | g1 contains g2 | `WHERE ST_CONTAINS(region, point)` | -| `ST_WITHIN(g1, g2)` | g1 within g2 | `WHERE ST_WITHIN(point, region)` | -| `ST_X(point)` | X coordinate / longitude | `EVAL lon = ST_X(location)` | -| `ST_Y(point)` | Y coordinate / latitude | `EVAL lat = ST_Y(location)` | -| `ST_ENVELOPE(geo)` | Bounding box (8.18/9.0+) | `EVAL bbox = ST_ENVELOPE(shape)` | -| `ST_XMAX(geo)` | Max X / longitude (8.18/9.0+) | `EVAL max_lon = ST_XMAX(shape)` | -| `ST_XMIN(geo)` | Min X / longitude (8.18/9.0+) | `EVAL min_lon = ST_XMIN(shape)` | -| `ST_YMAX(geo)` | Max Y / latitude (8.18/9.0+) | `EVAL max_lat = ST_YMAX(shape)` | -| `ST_YMIN(geo)` | Min Y / latitude (8.18/9.0+) | `EVAL min_lat = ST_YMIN(shape)` | -| `ST_GEOHASH(point, prec)` | Encode as geohash (9.2+) | `EVAL hash = ST_GEOHASH(location, 5)` | -| `ST_GEOHEX(point, prec)` | Encode as geohex (9.2+) | `EVAL hex = ST_GEOHEX(location, 5)` | -| `ST_GEOTILE(point, prec)` | Encode as geotile (9.2+) | `EVAL tile = ST_GEOTILE(location, 10)` | -| `ST_NPOINTS(geo)` | Number of points | `EVAL n = ST_NPOINTS(shape)` | -| `ST_SIMPLIFY(geo, tol)` | Simplify geometry | `EVAL simple = ST_SIMPLIFY(shape, 100)` | +| Function | Description | Example | +| --------------------------------------- | ----------------------------------- | --------------------------------------------------------- | +| `ST_DISTANCE(p1, p2)` | Distance between points | `EVAL dist = ST_DISTANCE(loc, TO_GEOPOINT("POINT(0 0)"))` | +| `ST_INTERSECTS(g1, g2)` | Geometries intersect | `WHERE ST_INTERSECTS(geo, boundary)` | +| `ST_DISJOINT(g1, g2)` | Geometries don't intersect | `WHERE ST_DISJOINT(geo, zone)` | +| `ST_CONTAINS(g1, g2)` | g1 contains g2 | `WHERE ST_CONTAINS(region, point)` | +| `ST_WITHIN(g1, g2)` | g1 within g2 | `WHERE ST_WITHIN(point, region)` | +| `ST_X(point)` | X coordinate / longitude | `EVAL lon = ST_X(location)` | +| `ST_Y(point)` | Y coordinate / latitude | `EVAL lat = ST_Y(location)` | +| `ST_ENVELOPE(geo)` | Bounding box (8.18/9.0+) | `EVAL bbox = ST_ENVELOPE(shape)` | +| `ST_XMAX(geo)` | Max X / longitude (8.18/9.0+) | `EVAL max_lon = ST_XMAX(shape)` | +| `ST_XMIN(geo)` | Min X / longitude (8.18/9.0+) | `EVAL min_lon = ST_XMIN(shape)` | +| `ST_YMAX(geo)` | Max Y / latitude (8.18/9.0+) | `EVAL max_lat = ST_YMAX(shape)` | +| `ST_YMIN(geo)` | Min Y / latitude (8.18/9.0+) | `EVAL min_lat = ST_YMIN(shape)` | +| `ST_GEOHASH(point, prec)` | Encode as geohash (9.2+) | `EVAL hash = ST_GEOHASH(location, 5)` | +| `ST_GEOHEX(point, prec)` | Encode as geohex (9.2+) | `EVAL hex = ST_GEOHEX(location, 5)` | +| `ST_GEOTILE(point, prec)` | Encode as geotile (9.2+) | `EVAL tile = ST_GEOTILE(location, 10)` | +| `ST_NPOINTS(geo)` | Number of points | `EVAL n = ST_NPOINTS(shape)` | +| `ST_SIMPLIFY(geo, tol)` | Simplify geometry | `EVAL simple = ST_SIMPLIFY(shape, 100)` | +| `ST_DIMENSION(geo)` | Dimension (0/1/2) (9.4+) | `EVAL dim = ST_DIMENSION(shape)` | +| `ST_GEOMETRYTYPE(geo)` | Geometry type string (9.4+) | `EVAL gtype = ST_GEOMETRYTYPE(shape)` | +| `ST_ISEMPTY(geo)` | True if empty (9.4+) | `WHERE NOT ST_ISEMPTY(shape)` | +| `ST_BUFFER(geo, dist)` | Buffer around geometry (9.4+) | `EVAL area = ST_BUFFER(point, 1000)` | +| `ST_SIMPLIFYPRESERVETOPOLOGY(geo, tol)` | Simplify preserving topology (9.4+) | `EVAL s = ST_SIMPLIFYPRESERVETOPOLOGY(shape, 100)` | --- @@ -1508,29 +1575,30 @@ For vector search and similarity operations on `dense_vector` and `semantic_text For handling fields with multiple values. -| Function | Description | Example | -| ------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------- | -| `MV_COUNT(field)` | Count values | `EVAL n = MV_COUNT(tags)` | -| `MV_FIRST(field)` | First value | `EVAL first_val = MV_FIRST(values)` | -| `MV_LAST(field)` | Last value | `EVAL last_val = MV_LAST(values)` | -| `MV_MIN(field)` | Minimum | `EVAL min = MV_MIN(scores)` | -| `MV_MAX(field)` | Maximum | `EVAL max = MV_MAX(scores)` | -| `MV_SUM(field)` | Sum | `EVAL total = MV_SUM(amounts)` | -| `MV_AVG(field)` | Average | `EVAL avg = MV_AVG(scores)` | -| `MV_MEDIAN(field)` | Median | `EVAL med = MV_MEDIAN(values)` | -| `MV_CONCAT(field, delim)` | Join to string | `EVAL str = MV_CONCAT(tags, ", ")` | -| `MV_DEDUPE(field)` | Remove duplicates | `EVAL unique = MV_DEDUPE(tags)` | -| `MV_SORT(field)` | Sort values | `EVAL sorted = MV_SORT(values)` | -| `MV_SLICE(field, start, end)` | Slice array | `EVAL slice = MV_SLICE(arr, 0, 3)` | -| `MV_ZIP(f1, f2)` | Zip arrays (both must be keyword/text) | `EVAL zipped = MV_ZIP(keys, values)` | -| `MV_APPEND(f1, f2)` | Concatenate MVs | `EVAL all = MV_APPEND(tags1, tags2)` | -| `MV_CONTAINS(f1, f2)` | All values in f2 present in f1 (9.2+) | `EVAL has = MV_CONTAINS(perms, required)` | -| `MV_INTERSECTION(f1, f2)` | Values present in both (9.3+) | `EVAL common = MV_INTERSECTION(a, b)` | -| `MV_INTERSECTS(f1, f2)` | Any value in f2 present in f1 (Serverless; self-managed 9.4) | `EVAL overlap = MV_INTERSECTS(a, b)` | -| `MV_UNION(f1, f2)` | Deduplicated union (Serverless; self-managed 9.4) | `EVAL merged = MV_UNION(a, b)` | -| `MV_PERCENTILE(field, p)` | Percentile of MV | `EVAL p95 = MV_PERCENTILE(vals, 95)` | -| `MV_PSERIES_WEIGHTED_SUM(field, p)` | P-series weighted sum (both args must be double) | `EVAL ws = MV_PSERIES_WEIGHTED_SUM(vals, 2.0)` | -| `MV_MEDIAN_ABSOLUTE_DEVIATION(field)` | MAD of MV | `EVAL mad = MV_MEDIAN_ABSOLUTE_DEVIATION(vals)` | +| Function | Description | Example | +| ------------------------------------- | ------------------------------------------------ | ----------------------------------------------- | +| `MV_COUNT(field)` | Count values | `EVAL n = MV_COUNT(tags)` | +| `MV_FIRST(field)` | First value | `EVAL first_val = MV_FIRST(values)` | +| `MV_LAST(field)` | Last value | `EVAL last_val = MV_LAST(values)` | +| `MV_MIN(field)` | Minimum | `EVAL min = MV_MIN(scores)` | +| `MV_MAX(field)` | Maximum | `EVAL max = MV_MAX(scores)` | +| `MV_SUM(field)` | Sum | `EVAL total = MV_SUM(amounts)` | +| `MV_AVG(field)` | Average | `EVAL avg = MV_AVG(scores)` | +| `MV_MEDIAN(field)` | Median | `EVAL med = MV_MEDIAN(values)` | +| `MV_CONCAT(field, delim)` | Join to string | `EVAL str = MV_CONCAT(tags, ", ")` | +| `MV_DEDUPE(field)` | Remove duplicates | `EVAL unique = MV_DEDUPE(tags)` | +| `MV_SORT(field)` | Sort values | `EVAL sorted = MV_SORT(values)` | +| `MV_SLICE(field, start, end)` | Slice array | `EVAL slice = MV_SLICE(arr, 0, 3)` | +| `MV_ZIP(f1, f2)` | Zip arrays (both must be keyword/text) | `EVAL zipped = MV_ZIP(keys, values)` | +| `MV_APPEND(f1, f2)` | Concatenate MVs | `EVAL all = MV_APPEND(tags1, tags2)` | +| `MV_CONTAINS(f1, f2)` | All values in f2 present in f1 (9.2+) | `EVAL has = MV_CONTAINS(perms, required)` | +| `MV_INTERSECTION(f1, f2)` | Values present in both (9.3+) | `EVAL common = MV_INTERSECTION(a, b)` | +| `MV_INTERSECTS(f1, f2)` | Any value in f2 present in f1 (9.4+; Serverless) | `EVAL overlap = MV_INTERSECTS(a, b)` | +| `MV_UNION(f1, f2)` | Deduplicated union (9.4+; Serverless) | `EVAL merged = MV_UNION(a, b)` | +| `MV_DIFFERENCE(f1, f2)` | Values in f1 not in f2 (9.4+; Serverless) | `EVAL diff = MV_DIFFERENCE(a, b)` | +| `MV_PERCENTILE(field, p)` | Percentile of MV | `EVAL p95 = MV_PERCENTILE(vals, 95)` | +| `MV_PSERIES_WEIGHTED_SUM(field, p)` | P-series weighted sum (both args must be double) | `EVAL ws = MV_PSERIES_WEIGHTED_SUM(vals, 2.0)` | +| `MV_MEDIAN_ABSOLUTE_DEVIATION(field)` | MAD of MV | `EVAL mad = MV_MEDIAN_ABSOLUTE_DEVIATION(vals)` | --- @@ -1755,6 +1823,7 @@ regular index fields. | `_ignored` | keyword | Fields that were ignored when the document was indexed | | `_index_mode` | keyword | Index mode (`standard`, `lookup`, `logsdb`, `time_series` etc.) | | `_source` | special | Original JSON document body. Use `JSON_EXTRACT` to access flattened or unmapped fields | +| `_size` | integer | Document size in bytes (9.4+; requires `mapper-size` plugin) | ```esql FROM logs METADATA _id, _index, _version diff --git a/plugins/elasticsearch/skills/elasticsearch-esql/references/esql-version-history.md b/plugins/elasticsearch/skills/elasticsearch-esql/references/esql-version-history.md index 13ef9a1..b5c2f69 100644 --- a/plugins/elasticsearch/skills/elasticsearch-esql/references/esql-version-history.md +++ b/plugins/elasticsearch/skills/elasticsearch-esql/references/esql-version-history.md @@ -27,58 +27,61 @@ determine compatibility when writing queries for specific Elasticsearch deployme ## Version Timeline Overview -| Version | Release | Status | Key Additions | -| ------- | -------- | ------------ | ------------------------------------------------------------------------------------------------------- | -| 8.11 | Nov 2023 | Tech Preview | Initial ES\|QL release | -| 8.12 | Jan 2024 | Tech Preview | Spatial types, PROFILE | -| 8.13 | Mar 2024 | Tech Preview | Async queries, cross-cluster ENRICH | -| 8.14 | May 2024 | **GA** | Spatial functions, regex optimization | -| 8.15 | Aug 2024 | GA | Type casting (`::`), Arrow output | -| 8.16 | Oct 2024 | GA | Per-aggregation WHERE, new math/string functions | -| 8.17 | Dec 2024 | GA | MATCH, QSTR full-text functions | -| 8.18 | Feb 2025 | GA | LOOKUP JOIN (preview), scoring, KQL | -| 8.19 | Apr 2025 | GA | MATCH_PHRASE, FORK, CHANGE_POINT (preview) | -| 9.0 | Feb 2025 | GA | Released with 8.18 features | -| 9.1 | Jun 2025 | GA | Full-text functions GA, FORK (preview) | -| 9.2 | Oct 2025 | GA | Multi-field joins, TS, INLINE STATS (preview), CHANGE_POINT GA, FUSE (preview), RERANK (preview) | -| 9.3 | Jan 2026 | GA | INLINE STATS GA, SET directive (preview), Lucene-pushable JOIN predicates | -| 9.4 | May 2026 | GA | TS GA, time series functions GA, WITHOUT/METRICS_INFO/TS_INFO GA, PROMQL (preview), MV_EXPAND/VALUES GA | +| Version | Release | Status | Key Additions | +| ------- | -------- | ------------ | ----------------------------------------------------------------------------------------------------------------------------------------- | +| 8.11 | Nov 2023 | Tech Preview | Initial ES\|QL release | +| 8.12 | Jan 2024 | Tech Preview | Spatial types, PROFILE | +| 8.13 | Mar 2024 | Tech Preview | Async queries, cross-cluster ENRICH | +| 8.14 | May 2024 | **GA** | Spatial functions, regex optimization | +| 8.15 | Aug 2024 | GA | Type casting (`::`), Arrow output | +| 8.16 | Oct 2024 | GA | Per-aggregation WHERE, new math/string functions | +| 8.17 | Dec 2024 | GA | MATCH, QSTR full-text functions | +| 8.18 | Feb 2025 | GA | LOOKUP JOIN (preview), scoring, KQL | +| 8.19 | Apr 2025 | GA | MATCH_PHRASE, FORK, CHANGE_POINT (preview) | +| 9.0 | Feb 2025 | GA | Released with 8.18 features | +| 9.1 | Jun 2025 | GA | Full-text functions GA, FORK (preview) | +| 9.2 | Oct 2025 | GA | Multi-field joins, TS, INLINE STATS (preview), CHANGE_POINT GA, FUSE (preview), RERANK (preview) | +| 9.3 | Jan 2026 | GA | INLINE STATS GA, SET directive (preview), Lucene-pushable JOIN predicates | +| 9.4 | May 2026 | GA | TS/TBUCKET GA, time series funcs GA, WITHOUT/METRICS_INFO/TS_INFO, PROMQL (preview), Views, RERANK GA, MV_EXPAND/VALUES GA, SET time_zone | ## Feature Availability by Version ### Commands -| Command | Introduced | GA | Notes | -| -------------- | ---------- | -------- | ----------------------------------------------- | -| `FROM` | 8.11 | 8.14 | Source command | -| `WHERE` | 8.11 | 8.14 | Filtering | -| `EVAL` | 8.11 | 8.14 | Computed columns | -| `STATS ... BY` | 8.11 | 8.14 | Aggregations with grouping | -| `SORT` | 8.11 | 8.14 | Ordering results | -| `LIMIT` | 8.11 | 8.14 | Result set size | -| `KEEP` | 8.11 | 8.14 | Column selection | -| `DROP` | 8.11 | 8.14 | Column removal | -| `RENAME` | 8.11 | 8.14 | Column renaming | -| `DISSECT` | 8.11 | 8.14 | Pattern extraction | -| `GROK` | 8.11 | 8.14 | Log parsing | -| `ENRICH` | 8.11 | 8.14 | Data enrichment | -| `MV_EXPAND` | 8.11 | 9.4 | Multi-value expansion (GA) | -| `SHOW` | 8.11 | 8.14 | Metadata display | -| `ROW` | 8.11 | 8.14 | Literal row creation | -| `LOOKUP JOIN` | 8.18/9.0 | 8.19/9.1 | SQL-style LEFT JOIN with lookup indices | -| `INLINE STATS` | 9.2 | 9.3 | Inline aggregations (like window functions) | -| `FORK` | 8.19/9.1 | Preview | Multiple execution branches | -| `FUSE` | 9.2 | Preview | Combine results from FORK branches | -| `TS` | 9.2 | 9.4 | Time series source command | -| `PROMQL` | 9.4 | Preview | Source command using PromQL syntax on TSDS | -| `METRICS_INFO` | 9.4 | 9.4 | TSDS metric catalogue (after `TS`) | -| `TS_INFO` | 9.4 | 9.4 | Per-(metric, time series) metadata (after `TS`) | -| `RERANK` | 9.2 | Preview | Re-score results with inference | -| `COMPLETION` | 9.2 | 9.2 | LLM text generation | -| `SAMPLE` | 8.19/9.1 | Preview | Random sampling | -| `URI_PARTS` | Srvless | Srvless | Parse URI into structured columns | -| `USER_AGENT` | Srvless | Srvless | Parse user agent into structured columns | -| `REG_DOMAIN` | Srvless | Srvless | `REGISTERED_DOMAIN`: extract from hostname | +| Command | Introduced | GA | Notes | +| ------------------- | ---------- | -------- | ---------------------------------------------------------------- | +| `FROM` | 8.11 | 8.14 | Source command | +| `WHERE` | 8.11 | 8.14 | Filtering | +| `EVAL` | 8.11 | 8.14 | Computed columns | +| `STATS ... BY` | 8.11 | 8.14 | Aggregations with grouping | +| `SORT` | 8.11 | 8.14 | Ordering results | +| `LIMIT` | 8.11 | 8.14 | Result set size | +| `KEEP` | 8.11 | 8.14 | Column selection | +| `DROP` | 8.11 | 8.14 | Column removal | +| `RENAME` | 8.11 | 8.14 | Column renaming | +| `DISSECT` | 8.11 | 8.14 | Pattern extraction | +| `GROK` | 8.11 | 8.14 | Log parsing | +| `ENRICH` | 8.11 | 8.14 | Data enrichment | +| `MV_EXPAND` | 8.11 | 9.4 | Multi-value expansion (GA) | +| `SHOW` | 8.11 | 8.14 | Metadata display | +| `ROW` | 8.11 | 8.14 | Literal row creation | +| `LOOKUP JOIN` | 8.18/9.0 | 8.19/9.1 | SQL-style LEFT JOIN with lookup indices | +| `INLINE STATS` | 9.2 | 9.3 | Inline aggregations (like window functions) | +| `FORK` | 8.19/9.1 | 9.4 | Multiple execution branches | +| `FUSE` | 9.2 | Preview | Combine results from FORK branches | +| `TS` | 9.2 | 9.4 | Time series source command (GA in 9.4) | +| `PROMQL` | 9.4 | Preview | Source command using PromQL syntax on TSDS | +| `METRICS_INFO` | 9.4 | 9.4 | TSDS metric catalogue (after `TS`) | +| `TS_INFO` | 9.4 | 9.4 | Per-(metric, time series) metadata (after `TS`) | +| `RERANK` | 9.2 | 9.4 | Re-score results with inference (GA in 9.4) | +| `COMPLETION` | 9.2 | 9.2 | LLM text generation | +| `SAMPLE` | 8.19/9.1 | Preview | Random sampling | +| `CHANGE_POINT` | 8.19/9.1 | 9.2 | Spike/dip detection (Platinum license) | +| `MMR` | 9.4 | Preview | Maximal Marginal Relevance diversification | +| `URI_PARTS` | 9.4 | 9.4 | Parse URI into structured columns (pipe command) | +| `USER_AGENT` | 9.4 | 9.4 | Parse user agent into structured columns (pipe command) | +| `REGISTERED_DOMAIN` | 9.4 | 9.4 | `REGISTERED_DOMAIN`: extract from hostname (pipe command) | +| Views | 9.4 | Preview | Virtual indices from ES\|QL queries (Stack only, not Serverless) | ### Full-Text Search Functions @@ -97,18 +100,23 @@ determine compatibility when writing queries for specific Elasticsearch deployme ### Spatial Functions -| Function | Introduced | Notes | -| ---------------------- | ---------- | ---------------------------- | -| `GEO_POINT` type | 8.12 | Basic spatial type support | -| `CARTESIAN_POINT` type | 8.12 | Cartesian coordinate support | -| `ST_INTERSECTS` | 8.14 | Geometry intersection test | -| `ST_CONTAINS` | 8.14 | Containment test | -| `ST_DISJOINT` | 8.14 | Disjoint test | -| `ST_WITHIN` | 8.14 | Within test | -| `ST_X`, `ST_Y` | 8.14 | Coordinate extraction | -| `ST_DISTANCE` | 8.15 | Distance calculation | -| `ST_EXTENT_AGG` | 8.18/9.0 | Bounding box aggregation | -| `ST_ENVELOPE` | 8.18/9.0 | Bounding box for geometry | +| Function | Introduced | Notes | +| ----------------------------- | ---------- | ------------------------------------- | +| `GEO_POINT` type | 8.12 | Basic spatial type support | +| `CARTESIAN_POINT` type | 8.12 | Cartesian coordinate support | +| `ST_INTERSECTS` | 8.14 | Geometry intersection test | +| `ST_CONTAINS` | 8.14 | Containment test | +| `ST_DISJOINT` | 8.14 | Disjoint test | +| `ST_WITHIN` | 8.14 | Within test | +| `ST_X`, `ST_Y` | 8.14 | Coordinate extraction | +| `ST_DISTANCE` | 8.15 | Distance calculation | +| `ST_EXTENT_AGG` | 8.18/9.0 | Bounding box aggregation | +| `ST_ENVELOPE` | 8.18/9.0 | Bounding box for geometry | +| `ST_DIMENSION` | 9.4 | Geometry dimension (0/1/2) | +| `ST_GEOMETRYTYPE` | 9.4 | Geometry type as string | +| `ST_ISEMPTY` | 9.4 | Test if geometry is empty | +| `ST_BUFFER` | 9.4 | Buffer around geometry | +| `ST_SIMPLIFYPRESERVETOPOLOGY` | 9.4 | Simplify geometry preserving topology | ### Date/Time Functions @@ -125,52 +133,61 @@ determine compatibility when writing queries for specific Elasticsearch deployme ### String Functions -| Function | Introduced | Notes | -| --------------------------- | ---------- | --------------------------- | -| `LEFT`, `RIGHT` | 8.11 | Substring extraction | -| `SUBSTRING` | 8.11 | Position-based extraction | -| `CONCAT` | 8.11 | String concatenation | -| `TRIM`, `LTRIM`, `RTRIM` | 8.11 | Whitespace removal | -| `TO_UPPER`, `TO_LOWER` | 8.13 | Case conversion | -| `LOCATE` | 8.14 | Find substring position | -| `SPACE` | 8.16 | Generate spaces | -| `REVERSE` | 8.16 | Reverse string | -| `BIT_LENGTH`, `BYTE_LENGTH` | 8.17 | String length in bits/bytes | -| `STARTS_WITH`, `ENDS_WITH` | 8.11 | Prefix/suffix matching | -| `CONTAINS` | 9.2 | Substring containment check | +| Function | Introduced | Notes | +| --------------------------- | ---------- | ---------------------------------------------------- | +| `LEFT`, `RIGHT` | 8.11 | Substring extraction | +| `SUBSTRING` | 8.11 | Position-based extraction | +| `CONCAT` | 8.11 | String concatenation | +| `TRIM`, `LTRIM`, `RTRIM` | 8.11 | Whitespace removal | +| `TO_UPPER`, `TO_LOWER` | 8.13 | Case conversion | +| `LOCATE` | 8.14 | Find substring position | +| `SPACE` | 8.16 | Generate spaces | +| `REVERSE` | 8.16 | Reverse string | +| `BIT_LENGTH`, `BYTE_LENGTH` | 8.17 | String length in bits/bytes | +| `STARTS_WITH`, `ENDS_WITH` | 8.11 | Prefix/suffix matching | +| `CONTAINS` | 9.2 | Substring containment check | +| `JSON_EXTRACT` | 9.4 | Extract value from JSON string by JSONPath (preview) | ### Multi-Value Functions -| Function | Introduced | Notes | -| ------------------------- | ---------- | ---------------------- | -| `MV_COUNT` | 8.11 | Count values | -| `MV_CONCAT` | 8.11 | Join values | -| `MV_FIRST`, `MV_LAST` | 8.13 | First/last value | -| `MV_MIN`, `MV_MAX` | 8.11 | Min/max value | -| `MV_SUM`, `MV_AVG` | 8.11 | Sum/average | -| `MV_MEDIAN` | 8.11 | Median value | -| `MV_SORT` | 8.14 | Sort multi-values | -| `MV_SLICE` | 8.14 | Slice multi-values | -| `MV_PERCENTILE` | 8.16 | Percentile calculation | -| `MV_PSERIES_WEIGHTED_SUM` | 8.16 | Weighted sum | +| Function | Introduced | Notes | +| ------------------------- | ---------- | --------------------------------------- | +| `MV_COUNT` | 8.11 | Count values | +| `MV_CONCAT` | 8.11 | Join values | +| `MV_FIRST`, `MV_LAST` | 8.13 | First/last value | +| `MV_MIN`, `MV_MAX` | 8.11 | Min/max value | +| `MV_SUM`, `MV_AVG` | 8.11 | Sum/average | +| `MV_MEDIAN` | 8.11 | Median value | +| `MV_SORT` | 8.14 | Sort multi-values | +| `MV_SLICE` | 8.14 | Slice multi-values | +| `MV_PERCENTILE` | 8.16 | Percentile calculation | +| `MV_PSERIES_WEIGHTED_SUM` | 8.16 | Weighted sum | +| `MV_DIFFERENCE` | 9.4 | Set difference of two MV fields | +| `MV_UNION` | 9.4 | Set union of two MV fields | +| `MV_INTERSECTION` | 9.4 | Set intersection of two MV fields | +| `MV_INTERSECTS` | 9.4 | True if MV fields share a value | +| `MV_CONTAINS` | 9.4 | True if first MV contains all of second | ### Aggregation Functions -| Function | Introduced | Notes | -| ------------------------------------- | ---------- | ------------------------------- | -| `COUNT`, `COUNT_DISTINCT` | 8.11 | Counting | -| `SUM`, `AVG` | 8.11 | Basic aggregations | -| `MIN`, `MAX` | 8.11 | Extended to strings/IPs in 8.16 | -| `MEDIAN`, `MEDIAN_ABSOLUTE_DEVIATION` | 8.11 | Statistical | -| `PERCENTILE` | 8.11 | Percentile calculation | -| `TOP` | 8.15 | Top N values | -| `VALUES` | 8.14 | Unique values (GA in 9.4) | -| `ST_EXTENT_AGG` | 8.18/9.0 | Spatial bounding box | -| `WEIGHTED_AVG` | 8.16 | Weighted average | -| `STD_DEV` | 8.18/9.0 | Standard deviation | -| `VARIANCE` | 8.18/9.0 | Variance | -| `FIRST` / `EARLIEST` | Serverless | Earliest value by sort field | -| `LAST` / `LATEST` | Serverless | Latest value by sort field | +| Function | Introduced | Notes | +| ------------------------------------- | ---------- | ---------------------------------------------------- | +| `COUNT`, `COUNT_DISTINCT` | 8.11 | Counting | +| `SUM`, `AVG` | 8.11 | Basic aggregations | +| `MIN`, `MAX` | 8.11 | Extended to strings/IPs in 8.16 | +| `MEDIAN`, `MEDIAN_ABSOLUTE_DEVIATION` | 8.11 | Statistical | +| `PERCENTILE` | 8.11 | Percentile calculation | +| `TOP` | 8.15 | Top N values | +| `VALUES` | 8.14 | Unique values (GA in 9.4) | +| `ST_EXTENT_AGG` | 8.18/9.0 | Spatial bounding box | +| `WEIGHTED_AVG` | 8.16 | Weighted average | +| `STD_DEV` | 8.18/9.0 | Standard deviation | +| `VARIANCE` | 8.18/9.0 | Variance | +| `FIRST(value, sort_field)` | 9.4 | Value from row with earliest sort field | +| `LAST(value, sort_field)` | 9.4 | Value from row with latest sort field | +| `EARLIEST(@timestamp)` | 9.4 | Min `@timestamp` (1-arg shorthand) | +| `LATEST(@timestamp)` | 9.4 | Max `@timestamp` (1-arg shorthand) | +| `SPARKLINE` | Serverless | Histogram sparkline (Serverless only, not 9.4 Stack) | ### Grouping Functions @@ -270,13 +287,10 @@ ES|QL **does not support cursor-based pagination** like the Search API's `search - Use `STATS` to aggregate at query time - For exports, use Search API with `search_after` instead -### Time Zone Support (Limited before Serverless / 9.4) +### Time Zone Support (GA in 9.4+; Serverless) -ES|QL has **limited timezone support** on self-managed clusters prior to 9.4. All dates are processed in UTC internally -and there is no per-function timezone argument. - -On **Serverless**, ES|QL supports query-wide timezone via the `SET time_zone` directive (GA on Serverless). This accepts -IANA timezone strings and UTC offsets, and applies to all date/time operations including `DATE_TRUNC`, `DATE_FORMAT`, +ES|QL supports query-wide timezone via the `SET time_zone` directive (GA in 9.4+; Serverless). This accepts IANA +timezone strings and UTC offsets, and applies to all date/time operations including `DATE_TRUNC`, `DATE_FORMAT`, `NOW()`, bucketing, and display. ```esql @@ -290,16 +304,13 @@ FROM logs-* - No per-function timezone argument — `DATE_TRUNC(1 hour, @timestamp, "America/New_York")` does **not** work - `DATE_FORMAT` and `DATE_PARSE` do not accept timezone parameters directly; use `SET time_zone` instead -- GitHub tracking issue: [#107560](https://github.com/elastic/elasticsearch/issues/107560) - -**Self-managed before 9.4:** -- `SET time_zone` only accepts UTC offsets (`"+05:00"`), not IANA timezone strings -- Workaround: use `EVAL` to add/subtract hours manually: +**Versions before 9.4:** No timezone support. All dates are processed in UTC. Workaround: use `EVAL` to add/subtract +hours manually: - ```esql - | EVAL local_time = timestamp + 1 hour - ``` +```esql +| EVAL local_time = timestamp + 1 hour +``` ### Nested Fields (Not Supported) @@ -347,8 +358,8 @@ These field types are not supported or have limitations: ### Subqueries (Limited) -ES|QL supports **subqueries in `FROM`** (Serverless tech preview) for combining results from multiple pipelines (UNION -ALL semantics). These are non-correlated — each branch is independent. +ES|QL supports **subqueries in `FROM`** (9.4+; Serverless) for combining results from multiple pipelines (UNION ALL +semantics). These are non-correlated — each branch is independent. ```esql FROM @@ -385,12 +396,15 @@ Use `INLINE STATS` (9.2+) for per-row vs. aggregate comparison patterns. ## API Endpoints -| Endpoint | Version | Notes | -| --------------------------- | ------- | ----------------------- | -| `POST /_query` | 8.11 | Synchronous query | -| `POST /_query/async` | 8.13 | Async query submission | -| `GET /_query/async/{id}` | 8.13 | Get async query results | -| `DELETE /_query/async/{id}` | 8.13 | Cancel async query | +| Endpoint | Version | Notes | +| ---------------------------- | ------- | -------------------------------- | +| `POST /_query` | 8.11 | Synchronous query | +| `POST /_query/async` | 8.13 | Async query submission | +| `GET /_query/async/{id}` | 8.13 | Get async query results | +| `DELETE /_query/async/{id}` | 8.13 | Cancel async query | +| `PUT /_query/view/{name}` | 9.4 | Create/update view (Stack only) | +| `GET /_query/view/{name}` | 9.4 | Get view definition (Stack only) | +| `DELETE /_query/view/{name}` | 9.4 | Delete view (Stack only) | ## Performance Tips by Version @@ -430,33 +444,21 @@ Use `INLINE STATS` (9.2+) for per-row vs. aggregate comparison patterns. ### 9.4+ -- `TS` source command and **all** time series aggregation functions are now **GA** — both the 9.2-introduced set - (`RATE`, `IRATE`, `INCREASE`, `DELTA`, `IDELTA`, `*_OVER_TIME`, `PRESENT_OVER_TIME`, `ABSENT_OVER_TIME`) and the - 9.3-introduced set (`DERIV`, `PERCENTILE_OVER_TIME`, `STDDEV_OVER_TIME`, `VARIANCE_OVER_TIME`). -- `TBUCKET` grouping function is **GA**. -- New `WITHOUT(...)` grouping function (GA) for time series queries: `BY WITHOUT(dim1, ...)` groups by every dimension - except the listed ones; `BY WITHOUT()` (no args) is equivalent to the implicit "group by all dimensions" behavior. -- New `METRICS_INFO` and `TS_INFO` processing commands (both **GA**) for discovering the metric catalogue and dimension - labels of TSDS data without inspecting index mappings. Both must come after a `TS` source command and must appear - before pipeline-breaking commands (`STATS`/`SORT`/`LIMIT`). `METRICS_INFO` returns one row per distinct metric - signature; `TS_INFO` returns one row per (metric, time series) combination with the identifying dimension labels. -- Sliding window parameter (`RATE(field, 10m)`) accepts arbitrary durations — no longer limited to multiples of the - `TBUCKET` interval. Note: a single query cannot mix windows smaller than the bucket for one metric with windows larger - than the bucket for another metric. -- New `PROMQL` source command (preview) to run Prometheus Query Language directly against TSDS indices, with implicit - range selectors and a Kibana-aware `step`/`buckets` model. See [promql-command.md](promql-command.md). Prefer `PROMQL` - only when the user explicitly thinks in PromQL or is migrating Prometheus dashboards/alerts; otherwise prefer `TS`. -- `MV_EXPAND` is GA -- `VALUES` aggregation is GA +- `TS`, `TBUCKET`, and all time series aggregation functions are **GA** — safe for production metrics queries +- Use `METRICS_INFO` / `TS_INFO` to discover TSDS schemas instead of inspecting mappings or field capabilities +- Use `WITHOUT(dim, ...)` to group by all dimensions except specific ones — avoids enumerating every dimension manually +- Sliding window accepts arbitrary durations — `RATE(field, 7m)` with `TBUCKET(5 minute)` now works +- Use `SET time_zone` with IANA strings for timezone-aware queries instead of manual `EVAL` offset arithmetic +- Use `SET approximation = true` for faster `COUNT_DISTINCT` on large datasets when exact counts aren't needed +- Use `SET unmapped_fields = "load"` to query fields missing from some indices without errors +- Use `FIRST`/`LAST` (or `EARLIEST`/`LATEST`) instead of `SORT` + `LIMIT 1` for grouped first/last-value queries +- Use `PROMQL` when porting Prometheus dashboards/alerts; otherwise prefer `TS` for native ES|QL ### Serverless (latest) -- `SET time_zone` with IANA timezone strings for query-wide timezone support (GA) -- `LIMIT n BY field` for grouped top-N queries -- `URI_PARTS`, `USER_AGENT`, `REGISTERED_DOMAIN` pipe commands for parsing structured strings -- `FROM` subqueries for combining results from multiple pipelines (tech preview) -- `EARLIEST`/`LATEST` aliases for `FIRST`/`LAST` aggregations -- `JSON_EXTRACT` on `METADATA _source` for accessing flattened field sub-keys +Serverless includes all 9.4 features and may have additional preview features: + +- `SPARKLINE` — histogram sparkline aggregation (Serverless only, not 9.4 Stack) ## Version Detection diff --git a/plugins/elasticsearch/skills/elasticsearch-esql/references/generation-tips.md b/plugins/elasticsearch/skills/elasticsearch-esql/references/generation-tips.md index e4192a0..d6071d6 100644 --- a/plugins/elasticsearch/skills/elasticsearch-esql/references/generation-tips.md +++ b/plugins/elasticsearch/skills/elasticsearch-esql/references/generation-tips.md @@ -4,8 +4,8 @@ Guidelines for generating accurate ES|QL queries from natural language. > **Cluster detection:** Check `build_flavor` in the `GET /` response. For Serverless (`"serverless"`), **do not** > version-gate: `version.number` tracks the next minor from main (semver-only clients may see it as “latest”), but -> feature availability is not determined by that string — use `build_flavor` as the signal. For self-managed -> (`"default"`), use `version.number` for feature checks (strip `-SNAPSHOT` suffix on pre-release builds). +> feature availability is not determined by that string — use `build_flavor` as the signal. For Stack (`"default"`), use +> `version.number` for feature checks (strip `-SNAPSHOT` suffix on pre-release builds). ## Table of Contents @@ -153,6 +153,10 @@ For time series data streams (TSDS), use `TS` instead of `FROM` to enable time s TS metrics-* // Time series source — enables RATE, AVG_OVER_TIME, etc. ``` +**When the question asks about rates, throughput, CPU/memory trends, or metric comparisons**, prefer a `metrics-*` or +TSDS index with `TS` over a general log index with `FROM`. Check the schema — if an index has `Index mode: time_series`, +always use `TS`. + ### 2. Determine Time Range **Question:** What time period should be covered? @@ -469,14 +473,18 @@ The `schema` command displays the data stream name when the index is a TSDS back **2. TBUCKET takes only a duration — not @timestamp:** -`TBUCKET` is not `DATE_TRUNC`. Do not pass `@timestamp`: +`TBUCKET` is not `DATE_TRUNC`. Do not pass `@timestamp`. Always assign a column alias so you can reference it in `SORT`: ```esql // WRONG — DATE_TRUNC-style syntax | STATS avg_cpu = AVG(cpu) BY bucket = TBUCKET(@timestamp, 5 minutes) -// CORRECT — duration only, timestamp is implicit +// WRONG — no alias makes SORT difficult +| STATS avg_cpu = AVG(cpu) BY TBUCKET(5 minutes) + +// CORRECT — duration only with alias for SORT | STATS avg_cpu = AVG(cpu) BY bucket = TBUCKET(5 minutes) +| SORT bucket ``` **3. Counter fields need RATE() wrapped in an outer aggregation:** @@ -494,9 +502,15 @@ TS metrics-tsds | STATS request_rate = SUM(RATE(requests)) BY TBUCKET(1 hour), host ``` -For gauge fields, use `AVG()` or `MAX()` as the outer function: +For gauge fields, use `AVG()` or `MAX()` as the outer function. Prefer the plain form — the inner `LAST_OVER_TIME` is +implicit and sufficient for most gauge queries: ```esql +// Preferred — plain aggregation (implicit LAST_OVER_TIME) +TS metrics-tsds +| STATS avg_cpu = AVG(cpu) BY TBUCKET(5 minutes), service.name + +// Only use explicit *_OVER_TIME when you need specific window behavior TS metrics-tsds | STATS avg_cpu = AVG(AVG_OVER_TIME(cpu)) BY TBUCKET(5 minutes), service.name ``` @@ -532,7 +546,7 @@ ES|QL before 9.2**. There is no fallback. When the cluster is pre-9.2 and the question requires per-row vs. aggregate comparison, explain that `INLINE STATS` is needed and suggest the user either upgrade or perform the comparison client-side. -### Pipe Commands: URI_PARTS, USER_AGENT, REGISTERED_DOMAIN (Serverless) +### Pipe Commands: URI_PARTS, USER_AGENT, REGISTERED_DOMAIN (9.4+; Serverless) These are **pipe commands** (like `DISSECT`/`GROK`), not scalar functions. They must appear on their own pipeline stage with `target = expression` syntax. A target prefix is mandatory. @@ -555,7 +569,7 @@ When the user asks to "parse URLs", "extract domains", or "parse user agents", r | Parse a user agent string | `USER_AGENT` | | Extract registered domain | `REGISTERED_DOMAIN` | -### Grouped Top-N with LIMIT BY (Serverless) +### Grouped Top-N with LIMIT BY (9.4+; Serverless) `LIMIT n BY field` keeps the top N rows per group after sorting. The number comes **before** `BY`. @@ -572,8 +586,8 @@ This replaces the common `INLINE STATS` + rank-and-filter pattern for simple gro ### Subqueries in FROM vs FORK -**Subqueries** (Serverless tech preview) combine results from **different** data sources (UNION ALL semantics). **FORK** -runs **different analyses** on the **same** data source. +**Subqueries** (9.4+; Serverless) combine results from **different** data sources (UNION ALL semantics). **FORK** runs +**different analyses** on the **same** data source. | Scenario | Use | | ------------------------------------- | ---------- | diff --git a/plugins/elasticsearch/skills/elasticsearch-esql/references/query-patterns.md b/plugins/elasticsearch/skills/elasticsearch-esql/references/query-patterns.md index 50ff4bd..9203a1b 100644 --- a/plugins/elasticsearch/skills/elasticsearch-esql/references/query-patterns.md +++ b/plugins/elasticsearch/skills/elasticsearch-esql/references/query-patterns.md @@ -437,7 +437,7 @@ FROM flights | KEEP flight_id, destination, distance, avg_dist ``` -### Grouped Top-N with LIMIT BY (Serverless) +### Grouped Top-N with LIMIT BY (9.4+; Serverless) ```text "top 3 error types per service" @@ -452,7 +452,7 @@ FROM logs-* ```text "most recent event per user" → -// Serverless: use LATEST to get the most recent value per group +// 9.4+/Serverless: use LATEST to get the most recent value per group FROM events-* | STATS last_action = LATEST(event.action), last_ts = LATEST(@timestamp) BY user.name @@ -463,7 +463,7 @@ FROM events-* | KEEP user.name, @timestamp, event.action ``` -### Subquery Composition (Serverless tech preview) +### Subquery Composition (9.4+; Serverless) ```text "combine web server errors and application errors into one view" diff --git a/plugins/elasticsearch/skills/elasticsearch-esql/references/time-series-queries.md b/plugins/elasticsearch/skills/elasticsearch-esql/references/time-series-queries.md index a4d5f08..ed7eece 100644 --- a/plugins/elasticsearch/skills/elasticsearch-esql/references/time-series-queries.md +++ b/plugins/elasticsearch/skills/elasticsearch-esql/references/time-series-queries.md @@ -88,6 +88,12 @@ TS metrics | STATS AVG(memory_usage) TS metrics | STATS AVG(LAST_OVER_TIME(memory_usage)) ``` +**When to use `*_OVER_TIME` vs plain aggregations for gauges:** For simple gauge queries (average CPU, max memory), +prefer `AVG(cpu)` over `AVG(AVG_OVER_TIME(cpu))` — the implicit `LAST_OVER_TIME` is sufficient and produces cleaner +queries. Use explicit `*_OVER_TIME` only when you need a specific window behavior: `AVG_OVER_TIME` to average all +samples (not just the last), `MIN_OVER_TIME`/`MAX_OVER_TIME` to find extremes within each time series before +aggregating, or `DELTA`/`DERIV` to compute changes. When in doubt, omit the inner function. + Since 9.3 (preview), use a time series function directly without an outer aggregation to get one value per time series per bucket. The result is implicitly grouped by all dimensions of each time series and includes a `_timeseries` column with the dimension key/value pairs — see [WITHOUT Grouping Function](#without-grouping-function) for narrowing this @@ -164,11 +170,11 @@ For gauge metrics and general numeric fields (`double`, `integer`, `long`, `aggr | `DERIV` | Derivative over time using linear regression | 9.3 (preview) | GA (9.4) | ```esql -// Average memory per cluster per 5 minutes +// Average memory per cluster per 5 minutes (plain form — implicit LAST_OVER_TIME is sufficient for gauges) // cluster is a dimension of the TSDS index metrics TS metrics | WHERE TRANGE(1 day) -| STATS AVG(AVG_OVER_TIME(memory_usage)) BY cluster, TBUCKET(5 minute) +| STATS AVG(memory_usage) BY cluster, TBUCKET(5 minute) // P95 network cost per cluster per minute // k8s is an example of a TSDS index, to showcase that time series indexes do not have to be called metrics @@ -245,15 +251,18 @@ The interval is a time duration (`1 hour`, `5 minute`, `30s`) or date period (`1 **Availability:** Preview from 9.2 to 9.3, **GA since 9.4**. +Always assign a column alias to `TBUCKET` so it can be referenced in `SORT`: + ```esql -// 1-hour buckets +// 1-hour buckets — alias enables SORT TS metrics -| STATS SUM(RATE(requests)) BY TBUCKET(1 hour), host +| STATS rate = SUM(RATE(requests)) BY bucket = TBUCKET(1 hour), host +| SORT bucket, host -// 5-minute buckets -// service is a dimension of the TSDS index metrics in this example +// 5-minute buckets (plain form for gauge) TS metrics -| STATS AVG(AVG_OVER_TIME(cpu_percent)) BY TBUCKET(5 minute), service +| STATS avg_cpu = AVG(cpu_percent) BY bucket = TBUCKET(5 minute), service +| SORT bucket ``` --- @@ -519,19 +528,20 @@ TS metrics ### Average Gauge per Cluster Over Time ```esql +// Plain form — preferred for gauge metrics TS metrics | WHERE TRANGE(1 day) -| STATS AVG(AVG_OVER_TIME(memory_usage)) BY TBUCKET(5 minute), cluster +| STATS AVG(memory_usage) BY TBUCKET(5 minute), cluster ``` ### Per-Time-Series Averages vs Global Average ```esql -// Average of per-time-series averages (accounts for different series lengths) -TS metrics | STATS AVG(AVG_OVER_TIME(memory_usage)) - -// Average of last values per time series (default behavior) +// Average of last values per time series (default — preferred for most gauge queries) TS metrics | STATS AVG(memory_usage) + +// Average of ALL samples per time series (use AVG_OVER_TIME only when you need this distinction) +TS metrics | STATS AVG(AVG_OVER_TIME(memory_usage)) ``` ### Detect Missing Data diff --git a/plugins/elasticsearch/skills/elasticsearch-onboarding/references/elasticsearch-onboarding-playbook.md b/plugins/elasticsearch/skills/elasticsearch-onboarding/references/elasticsearch-onboarding-playbook.md index c5366f5..b9fb5d3 100644 --- a/plugins/elasticsearch/skills/elasticsearch-onboarding/references/elasticsearch-onboarding-playbook.md +++ b/plugins/elasticsearch/skills/elasticsearch-onboarding/references/elasticsearch-onboarding-playbook.md @@ -4,50 +4,6 @@ You are an Elasticsearch solutions architect embedded in the developer's IDE. Gu working search experience — understanding their intent, recommending the right approach, and generating production-ready code. -## UI Context Hint - -The rule file may contain one or both of these lines at the top, injected by the Kibana onboarding UI at download time. -Read them before the first message — they pre-answer questions you would otherwise ask. - -### `# user-context:` - -Opens with a confirmation instead of a blank question: - -- `# user-context: ai-pipeline` → "Looks like you're building an AI app or pipeline — chatbot, RAG, vector store, or - recommendations. Is that right? Are you building something users interact with directly, or a retrieval layer that - feeds another system like LangChain?" -- `# user-context: document-search` → "Looks like you're building search over documents or content — a knowledge base, - wiki, or docs site. Is that right? Tell me about what you're searching over." -- `# user-context: catalog-ecommerce` → "Looks like you're building browse-and-filter search — products, listings, or a - structured catalog. Is that right? Tell me about your data." -- `# user-context: geo-search` → "Looks like you're building location-based search — 'near me', maps, or geo filters. Is - that right? Tell me about your use case." -- `# user-context: log-search` → "Looks like you're building log or event search — app logs, security events, or IoT - data. Is that right? Tell me about your data pipeline." -- `# user-context: recommendations` → "Looks like you're building a recommendations feature — 'you might also like', - related content, or personalized feeds. Is that right? Tell me about what you're recommending." -- `# user-context: something-else: ` → "Looks like you're building [text] — is that right? Tell me more about what - you're searching over." - -If the developer confirms, proceed directly to Step 2 (skip the use case question in Step 1). If they correct it, -re-route immediately and continue from there. - -If no `# user-context:` hint is present, use the standard First Message flow below. - -### `# deployment:` - -Pre-answers deployment type — do NOT ask about this if the hint is present: - -- `# deployment: serverless` → Treat as Serverless throughout. Version is always latest. `semantic_text` works out of - the box with no inference endpoint setup. -- `# deployment: cloud-hosted` → Treat as Elastic Cloud Hosted (ECH). Detect version via MCP or ask. -- `# deployment: self-managed` → Treat as Self-Managed. Detect version via MCP or ask. - -If both hints are present, incorporate both silently — weave the deployment context into the confirmation message -naturally. For example, if `deployment: serverless` and `user-context: ai-pipeline`: "Looks like you're on Elastic Cloud -Serverless and building an AI pipeline — great combination. `semantic_text` will handle embeddings automatically with no -setup. Is that right?" - ## First Message If the developer's first message is vague or exploratory ("hi," "help," "get started," "search"), jump straight into the @@ -113,15 +69,15 @@ specific project, or maybe you're just exploring what's possible — either way Listen for signals: -| Signal | Approach | Output | -| ----------------------------------------------------------------------------------------------------------- | -------------------- | ------------------------------------ | -| "search bar", "filter by", "facets", "autocomplete" | keyword-search | Ranked results | -| "find similar", "natural language", "meaning-based" | vector-hybrid-search | Ranked results (by meaning) | -| "both keyword and semantic", "hybrid" | vector-hybrid-search | Ranked results (combined) | -| "chatbot", "Q&A", "answer from my docs", "RAG" | rag-chatbot | Generated answers (not just results) | -| "product search", "e-commerce", "catalog" | catalog-ecommerce | Ranked results with facets | -| "vector store", "embeddings", "LangChain", "LlamaIndex", "AI app", "agent", "similarity", "recommendations" | vector-hybrid-search | Vectors for downstream AI | -| "just learning", "exploring", "not sure yet", "new to Elasticsearch" | use-case-library | Guided exploration | +| Signal | Approach | Output | +| ------------------------------------------------------------------------------------------------------------------------------ | -------------------- | ------------------------------------ | +| "search bar", "filter by", "facets", "autocomplete" | keyword-search | Ranked results | +| "find similar", "natural language", "meaning-based" | vector-hybrid-search | Ranked results (by meaning) | +| "both keyword and semantic", "hybrid" | vector-hybrid-search | Ranked results (combined) | +| "chatbot", "Q&A", "answer from my docs", "RAG" | rag-chatbot | Generated answers (not just results) | +| "product search", "e-commerce", "catalog" | catalog-ecommerce | Ranked results with facets | +| "vector store", "vector database", "embeddings", "LangChain", "LlamaIndex", "AI app", "agent", "similarity", "recommendations" | vector-hybrid-search | Vectors for downstream AI | +| "just learning", "exploring", "not sure yet", "new to Elasticsearch" | use-case-library | Guided exploration | **If the developer is exploring or doesn't know what to build**, load the [use-case-library](use-case-library/use-case-library.md) reference and walk through it conversationally. Help them @@ -189,19 +145,31 @@ Python. Use what you learn to determine fields to map, embedding model needs, ingestion path, and client library. -### Step 3: Confirm Version +### Step 3: Confirm Deployment and Version + +Establish where Elasticsearch is running and what version, before recommending an approach or generating code. Don't +re-ask what the conversation has already answered — the developer may have said it, or MCP may have revealed it. + +- **MCP connected** → detect automatically via `GET /` (`version.number`; `version.build_flavor` is `serverless` on + Serverless). Tell the developer what you found. +- **Otherwise** → ask: "Where is your Elasticsearch running — Elastic Cloud Serverless, Elastic Cloud Hosted, + self-managed, or nowhere yet?" -Confirm the Elasticsearch version before recommending an approach or generating code. +Then resolve the version by deployment type: -- **`# deployment: serverless`** or inferred Serverless → version is always latest, skip this question. -- **MCP connected** → detect automatically via `GET /` (`version.number`). Tell the developer what you found. -- **Otherwise** → ask: "What version of Elasticsearch are you running? Find it in Kibana under **Stack Management → - Upgrade assistant**, or paste the output of `GET /` from **Dev Tools**." +- **Serverless** → version is always latest — skip the version question. `semantic_text` works out of the box with no + inference endpoint setup. +- **Cloud Hosted (ECH) or Self-Managed** → detect via MCP, or ask: "What version of Elasticsearch are you running? Find + it in Kibana under **Stack Management → Upgrade assistant**, or paste the output of `GET /` from **Dev Tools**." +- **No deployment yet** → recommend Elastic Cloud Serverless as the fastest path, and match the project type to the use + case from Step 1 — **Elasticsearch** for search use cases; Observability and Security use cases route to their + dedicated project types per Step 1. [Docs](https://www.elastic.co/docs/get-started/introduction). Treat as Serverless + from here on. Use the version to determine available field types (`semantic_text` requires 8.15+), inference endpoints, RRF/ELSER/EIS availability, and which doc version to link. -**Don't generate code until the version is confirmed.** +**Don't generate code until the deployment type and version are confirmed.** ### Step 4: Recommend and Confirm @@ -351,8 +319,8 @@ principles: - **[keyword-search](keyword-search/keyword-search.md)** — Load when the developer needs full-text search, filters, facets, or autocomplete without semantic/vector features. - **[vector-hybrid-search](vector-hybrid-search/vector-hybrid-search.md)** — Load when the developer needs semantic - search, hybrid BM25+vector search, kNN, embeddings, or Elasticsearch as a vector database. This is the primary guide - for any use case involving vectors or meaning-based search. + search, hybrid BM25+vector search, kNN, embeddings, or Elasticsearch as a vector store. This is the primary guide for + any use case involving vectors or meaning-based search. - **[rag-chatbot](rag-chatbot/rag-chatbot.md)** — Load when the developer wants to build a chatbot, Q&A system, or RAG pipeline that generates answers from documents. - **[catalog-ecommerce](catalog-ecommerce/ecommerce.md)** — Load when the developer needs product search with faceted diff --git a/plugins/elasticsearch/skills/elasticsearch-onboarding/references/use-case-library/use-case-library.md b/plugins/elasticsearch/skills/elasticsearch-onboarding/references/use-case-library/use-case-library.md index 5db424e..7e23114 100644 --- a/plugins/elasticsearch/skills/elasticsearch-onboarding/references/use-case-library/use-case-library.md +++ b/plugins/elasticsearch/skills/elasticsearch-onboarding/references/use-case-library/use-case-library.md @@ -196,7 +196,7 @@ project. --- -### 8. Vector Database (for AI/ML Pipelines) +### 8. Vector Store (for AI/ML Pipelines) Store and retrieve embeddings programmatically — code searches, not people. @@ -231,7 +231,7 @@ Store and retrieve embeddings programmatically — code searches, not people. | Customer support search | Hybrid search, synonyms | Aggregations for trend detection | | Location-based search | Geo-point, distance sort, geo filters | Combined with full-text | | Log & event search | Data streams, ILM, ES\|QL, aggregations | Runtime fields | -| Vector database | Dense vectors, kNN/ANN (HNSW) | Quantization, metadata filtering | +| Vector store | Dense vectors, kNN/ANN (HNSW) | Quantization, metadata filtering | ## Non-Search Use Cases diff --git a/plugins/elasticsearch/skills/elasticsearch-onboarding/references/vector-hybrid-search/vector-hybrid-search.md b/plugins/elasticsearch/skills/elasticsearch-onboarding/references/vector-hybrid-search/vector-hybrid-search.md index da27a10..094fb9a 100644 --- a/plugins/elasticsearch/skills/elasticsearch-onboarding/references/vector-hybrid-search/vector-hybrid-search.md +++ b/plugins/elasticsearch/skills/elasticsearch-onboarding/references/vector-hybrid-search/vector-hybrid-search.md @@ -1,36 +1,19 @@ --- name: vector-hybrid-search description: - Complete guide for building vector search, hybrid search, and using Elasticsearch as a vector database for AI - pipelines. Covers the full decision tree from deployment type through production optimization and RAG extension. Use - when a developer wants semantic search, hybrid BM25+vector search, kNN, embeddings, RAG, or Elasticsearch as a vector - store for LangChain/LlamaIndex. + Complete guide for building vector search, hybrid search, and using Elasticsearch as a vector store for AI pipelines. + Covers the full decision tree from deployment type through production optimization and RAG extension. Use when a + developer wants semantic search, hybrid BM25+vector search, kNN, embeddings, RAG, or Elasticsearch as a vector store + for LangChain/LlamaIndex. --- # Vector & Hybrid Search Guide -## UI Context Hint - -If the downloaded skill file contains a `# user-context:` line (set by the getting started UI at download time), read it -before the first message and open with a confirmation rather than a blank question: - -- `# user-context: vector-database` → "Looks like you're building a vector database for an AI pipeline — is that right? - Are you using LangChain, LlamaIndex, or a custom stack?" -- `# user-context: hybrid-search` → "Looks like you're building hybrid search — is that right? Will users be typing - queries directly, or is this powering an AI pipeline?" -- `# user-context: semantic-search` → "Looks like you're building semantic search — is that right? Tell me about what - you're searching over." - -If the developer corrects the use case, re-route immediately. No commitment. - -If no `# user-context:` hint is present, open with: "What are you building — a search experience for users, or a -retrieval backend for an AI pipeline like RAG or LangChain?" - ---- - ## Consumer Fork -Before any other decision, establish who consumes the search results: +Before any other decision, establish who consumes the search results. If the conversation hasn't answered it yet, open +with: "What are you building — a search experience for users, or a retrieval backend for an AI pipeline like RAG or +LangChain?" - **AI pipeline** (code consumes results) → LangChain, LlamaIndex, custom RAG, agent memory, recommendations - **Human-facing search** (people type queries) → search bar, results page, filters, autocomplete @@ -45,14 +28,14 @@ This determines Decision I (App Integration) and whether to offer a frontend at Ask what they're building. Listen for: -| Signal | Use Case | -| --------------------------------------------------------- | ---------------------------------- | -| "semantic search", "meaning-based", "natural language" | Semantic search | -| "BM25 + vector", "hybrid", "keyword and semantic" | Hybrid search | -| "RAG", "chatbot", "Q&A over documents" | RAG — use rag-chatbot skill | -| "LangChain", "LlamaIndex", "vector store", "agent memory" | AI pipeline / vector DB | -| "recommendations", "similar items" | Vector similarity | -| "image search", "multimodal" | Dense vector with image embeddings | +| Signal | Use Case | +| ---------------------------------------------------------------------------- | ---------------------------------- | +| "semantic search", "meaning-based", "natural language" | Semantic search | +| "BM25 + vector", "hybrid", "keyword and semantic" | Hybrid search | +| "RAG", "chatbot", "Q&A over documents" | RAG — use rag-chatbot skill | +| "LangChain", "LlamaIndex", "vector store", "vector database", "agent memory" | AI pipeline / vector store | +| "recommendations", "similar items" | Vector similarity | +| "image search", "multimodal" | Dense vector with image embeddings | **Scale check:** If the developer indicates >1M documents, >10GB, or cost sensitivity, flag quantization early: diff --git a/skills/elasticsearch/elasticsearch-esql/SKILL.md b/skills/elasticsearch/elasticsearch-esql/SKILL.md index 4d03d81..f2f2954 100644 --- a/skills/elasticsearch/elasticsearch-esql/SKILL.md +++ b/skills/elasticsearch/elasticsearch-esql/SKILL.md @@ -6,7 +6,7 @@ description: > charts and dashboards from ES|QL results. metadata: author: elastic - version: 0.3.0 + version: 0.4.0 --- # Elasticsearch ES|QL @@ -39,7 +39,7 @@ ES|QL uses pipes (`|`) to chain commands: > development (next minor from main), so clients that only semver-compare may treat Serverless as “latest.” **Do not** > use `version.number` to gate features: if `build_flavor` is `"serverless"`, assume all GA and preview ES|QL features > are available. -> - `build_flavor: "default"` — Self-managed or Elastic Cloud Hosted. Use `version.number` for feature availability. +> - `build_flavor: "default"` — Stack (self-managed or Cloud-hosted). Use `version.number` for feature availability. > - **Snapshot builds** have `version.number` like `9.4.0-SNAPSHOT`. Strip the `-SNAPSHOT` suffix and use the > major.minor for version checks. Snapshot builds include all features from that version plus potentially unreleased > features from development — if a query fails with an unknown function/command, it may simply not have landed yet. diff --git a/skills/elasticsearch/elasticsearch-esql/references/dsl-to-esql-migration.md b/skills/elasticsearch/elasticsearch-esql/references/dsl-to-esql-migration.md index e138a91..59890ed 100644 --- a/skills/elasticsearch/elasticsearch-esql/references/dsl-to-esql-migration.md +++ b/skills/elasticsearch/elasticsearch-esql/references/dsl-to-esql-migration.md @@ -958,22 +958,22 @@ FROM sales Features not available in ES|QL as of version 9.3: -| Feature | Query DSL | ES\|QL | -| ---------------------------- | --------- | ----------------------------------------- | -| Highlighting | ✅ | ❌ | -| Nested queries | ✅ | ❌ | -| Parent-child queries | ✅ | ❌ | -| Scroll/pagination beyond 10k | ✅ | ❌ | -| Percolate queries | ✅ | ❌ | -| Complex boosting | ✅ | Limited | -| Geo distance sorting | ✅ | ❌ | -| Runtime fields | ✅ | Use EVAL | -| Suggest API | ✅ | ❌ | -| Collapse (field collapsing) | ✅ | ❌ | -| Inner hits | ✅ | ❌ | -| Timezone support | ✅ | ✅ `SET time_zone` (Serverless GA) | -| JOIN (non-lookup) | N/A | ❌ (only LEFT JOIN on lookup index) | -| Subqueries / UNION ALL | N/A | ✅ `FROM` subqueries (Serverless preview) | +| Feature | Query DSL | ES\|QL | +| ---------------------------- | --------- | ---------------------------------------- | +| Highlighting | ✅ | ❌ | +| Nested queries | ✅ | ❌ | +| Parent-child queries | ✅ | ❌ | +| Scroll/pagination beyond 10k | ✅ | ❌ | +| Percolate queries | ✅ | ❌ | +| Complex boosting | ✅ | Limited | +| Geo distance sorting | ✅ | ❌ | +| Runtime fields | ✅ | Use EVAL | +| Suggest API | ✅ | ❌ | +| Collapse (field collapsing) | ✅ | ❌ | +| Inner hits | ✅ | ❌ | +| Timezone support | ✅ | ✅ `SET time_zone` (9.4+ GA; Serverless) | +| JOIN (non-lookup) | N/A | ❌ (only LEFT JOIN on lookup index) | +| Subqueries / UNION ALL | N/A | ✅ `FROM` subqueries (9.4+; Serverless) | ### Unsupported Field Types in ES|QL diff --git a/skills/elasticsearch/elasticsearch-esql/references/environment-setup.md b/skills/elasticsearch/elasticsearch-esql/references/environment-setup.md index 8bd2e4b..6f4d3e7 100644 --- a/skills/elasticsearch/elasticsearch-esql/references/environment-setup.md +++ b/skills/elasticsearch/elasticsearch-esql/references/environment-setup.md @@ -7,8 +7,8 @@ successful connection test. > **Elastic Cloud Serverless:** After connecting, inspect `GET /`. If `build_flavor` is `"serverless"`, do **not** use > `version.number` to decide which ES|QL features are allowed — Serverless tracks current GA and preview ES|QL, and the > reported version follows the main-line / next-minor line (semver-only clients may see it as “latest”). Prefer -> `build_flavor` for detection and gating. For the full rules (including self-managed and snapshot builds), read -> **Cluster Detection** in [SKILL.md](../SKILL.md) and the **Serverless** callout in +> `build_flavor` for detection and gating. For the full rules (including Stack and snapshot builds), read **Cluster +> Detection** in [SKILL.md](../SKILL.md) and the **Serverless** callout in > [ES|QL Version History](esql-version-history.md). ## Option 1: Elastic Cloud (recommended for production) diff --git a/skills/elasticsearch/elasticsearch-esql/references/esql-reference.md b/skills/elasticsearch/elasticsearch-esql/references/esql-reference.md index 3597fac..097a769 100644 --- a/skills/elasticsearch/elasticsearch-esql/references/esql-reference.md +++ b/skills/elasticsearch/elasticsearch-esql/references/esql-reference.md @@ -3,11 +3,11 @@ ES|QL (Elasticsearch Query Language) is a piped query language for filtering, transforming, and analyzing data in Elasticsearch. It uses pipes (`|`) to chain commands together. -> **Serverless vs Self-Managed:** Version annotations in this document (e.g., "9.2+") apply to self-managed -> Elasticsearch. Detect cluster type via `build_flavor` in the `GET /` response: `"serverless"` means all GA and preview +> **Serverless vs Stack:** Version annotations in this document (e.g., "9.2+") apply to Elastic Stack (self-managed and +> Cloud-hosted). Detect cluster type via `build_flavor` in the `GET /` response: `"serverless"` means all GA and preview > features are available — **do not** gate on `version.number` for Serverless (it tracks the next minor from main; -> semver-only checks may treat it as “latest”). For self-managed, use `version.number` (strip any `-SNAPSHOT` suffix) -> for feature checks. +> semver-only checks may treat it as “latest”). For Stack, use `version.number` (strip any `-SNAPSHOT` suffix) for +> feature checks. ## Table of Contents @@ -66,12 +66,16 @@ source-command - `"default"` / `"fail"` -- the query fails if it references unmapped fields - `"nullify"` -- treats unmapped fields as null values -- `"load"` -- loads unmapped fields dynamically. **Limitation:** `"load"` is incompatible with subqueries and views. Use - `"nullify"` when composing subqueries or querying views. +- `"load"` (9.4+) -- loads unmapped fields dynamically as `keyword`. **Limitation:** `"load"` is incompatible with + subqueries and views. Use `"nullify"` when composing subqueries or querying views. -**`time_zone`** (Serverless GA; self-managed planned) -- sets the default timezone for the query, overriding UTC -default. Accepts any IANA timezone string or UTC offset. Applies to all date/time operations: `DATE_TRUNC`, -`DATE_FORMAT`, `NOW()`, etc. +**`time_zone`** (9.4+ GA; Serverless) -- sets the default timezone for the query, overriding UTC default. Accepts any +IANA timezone string or UTC offset. Applies to all date/time operations: `DATE_TRUNC`, `DATE_FORMAT`, `NOW()`, etc. + +**`approximation`** (9.4+ preview; Serverless) -- enables approximate aggregations for faster results on large datasets: + +- `"true"` -- use approximate algorithms for aggregations (e.g. HyperLogLog for `COUNT_DISTINCT`) +- `"false"` (default) -- use exact algorithms **Examples:** @@ -92,6 +96,10 @@ SET time_zone = "+05:00"; TS k8s | WHERE @timestamp == "2024-05-10T00:04:49.000Z" | STATS BY @timestamp, bucket = TBUCKET(3 hours) + +SET approximation = true; +FROM logs-* +| STATS unique_users = COUNT_DISTINCT(user.id) ``` > **When to use:** `unmapped_fields` is useful when querying across multiple indices where some indices may not have all @@ -136,8 +144,8 @@ FROM FROM cluster_one:logs-*, cluster_two:logs-* ``` -**Subqueries (Serverless tech preview):** `FROM` supports parenthesized subqueries with UNION ALL semantics. Each branch -is a complete ES|QL pipeline. Columns present in one branch but not another are filled with `null`. +**Subqueries (9.4+; Serverless):** `FROM` supports parenthesized subqueries with UNION ALL semantics. Each branch is a +complete ES|QL pipeline. Columns present in one branch but not another are filled with `null`. ```esql // Combine logs from different indices with independent pipelines @@ -184,6 +192,40 @@ ROW x = [1, 2, 3] ROW greeting = "hello", pi = 3.14159 ``` +**Intra-row field references (9.4+; Serverless):** Columns defined earlier in the same `ROW` can be referenced by later +columns: + +```esql +ROW a = 5, b = a * 2, c = a + b +``` + +### Views (9.4+ preview; Stack only — not available on Serverless) + +Views are virtual indices backed by ES|QL queries. Query a view with `FROM view_name` like any index. Views are managed +via the `/_query/view` REST API. + +**Create / update a view:** + +```bash +PUT /_query/view/active_employees +{ + "query": "FROM employees | WHERE is_active == true | KEEP emp_no, name, department" +} +``` + +**Query a view:** + +```esql +FROM active_employees +| STATS headcount = COUNT(*) BY department +``` + +**Constraints:** + +- `SET` directives cannot be used inside view definitions; the caller applies `SET` when querying +- `SET unmapped_fields = "load"` is incompatible with views; use `"nullify"` instead +- Views are not yet available on Serverless + ### TS Retrieves data from time series data streams (TSDS). Similar to `FROM` but enables time series aggregation functions in @@ -577,13 +619,13 @@ FROM data ### LIMIT -Limits the number of rows returned. Supports optional grouped top-N with `BY` (Serverless). +Limits the number of rows returned. Supports optional grouped top-N with `BY` since 9.4+ and in Serverless. **Syntax:** ```esql LIMIT number -LIMIT number BY field +LIMIT number BY field (9.4+; Serverless) ``` **Examples:** @@ -899,8 +941,8 @@ FROM articles METADATA _id, _index, _score ### RERANK -Uses an inference model to re-score an initial set of documents. Tech preview in 9.2 (GA on Serverless). Since 9.3, -defaults to 1000 rows; configurable via `esql.command.rerank.limit` and `esql.command.rerank.enabled` cluster settings. +Uses an inference model to re-score an initial set of documents. GA in 9.4 (Serverless). Since 9.3, defaults to 1000 +rows; configurable via `esql.command.rerank.limit` and `esql.command.rerank.enabled` cluster settings. **Syntax:** @@ -1102,7 +1144,7 @@ TS k8s > **per (metric, time series) combination** and adds a `dimensions` column with the labels identifying each series. Use > `METRICS_INFO` to enumerate _what_ is being measured, and `TS_INFO` to enumerate _which_ time series exist. -### URI_PARTS (Serverless) +### URI_PARTS (9.4+; Serverless) Pipe command that parses a URI string into structured columns. A target prefix is **required**. @@ -1125,7 +1167,7 @@ FROM web_logs | SORT errors DESC ``` -### USER_AGENT (Serverless) +### USER_AGENT (9.4+; Serverless) Pipe command that parses a user agent string into structured columns. A target prefix is **required**. @@ -1146,7 +1188,7 @@ FROM web_logs | STATS cnt = COUNT(*) BY ua.name, ua.version ``` -### REGISTERED_DOMAIN (Serverless) +### REGISTERED_DOMAIN (9.4+; Serverless) Pipe command that extracts the registered domain, top-level domain, and subdomain from a hostname. A target prefix is **required**. @@ -1172,6 +1214,26 @@ FROM dns_logs > **Note:** `URI_PARTS`, `USER_AGENT`, and `REGISTERED_DOMAIN` are **pipe commands** (like `DISSECT`/`GROK`), not scalar > functions. The syntax `URI_PARTS(field)` does not work — use `| URI_PARTS target = field`. +### MMR (9.4+ preview; Serverless) + +Maximal Marginal Relevance — diversifies search results by reducing redundancy among top hits. Requires a dense vector +field and a `LIMIT` before `MMR` to constrain the candidate set. + +**Syntax:** + +```esql +MMR query ON vector_field LIMIT n +``` + +**Example:** + +```esql +FROM articles +| WHERE MATCH(title, "elasticsearch tuning") +| LIMIT 100 +| MMR "elasticsearch performance tuning" ON content_embedding LIMIT 10 +``` + --- ## Aggregate Functions @@ -1198,10 +1260,10 @@ Used with STATS command. | `ABSENT(field)` | True if no non-null values (9.2+) | `STATS is_absent = ABSENT(error_code)` | | `PRESENT(field)` | True if any non-null values (9.2+) | `STATS has_data = PRESENT(metric)` | | `SAMPLE(field, n)` | Collect n sample values (8.19/9.1+) | `STATS examples = SAMPLE(message, 5)` | -| `FIRST(field, sort_field)` | Earliest value by sort field (Serverless GA) | `STATS earliest = FIRST(message, @timestamp)` | -| `LAST(field, sort_field)` | Latest value by sort field (Serverless GA) | `STATS latest = LAST(message, @timestamp)` | -| `EARLIEST(field)` | Earliest value (single-arg; Serverless GA) | `STATS e = EARLIEST(@timestamp)` | -| `LATEST(field)` | Latest value (single-arg; Serverless GA) | `STATS l = LATEST(@timestamp)` | +| `FIRST(field, sort_field)` | Earliest value by sort field (9.4+; Serverless) | `STATS earliest = FIRST(message, @timestamp)` | +| `LAST(field, sort_field)` | Latest value by sort field (9.4+; Serverless) | `STATS latest = LAST(message, @timestamp)` | +| `EARLIEST(field)` | Min `@timestamp` shorthand (9.4+; Serverless) | `STATS e = EARLIEST(@timestamp)` | +| `LATEST(field)` | Max `@timestamp` shorthand (9.4+; Serverless) | `STATS l = LATEST(@timestamp)` | | `ST_CENTROID_AGG(field)` | Spatial centroid of points | `STATS center = ST_CENTROID_AGG(location)` | | `ST_EXTENT_AGG(field)` | Bounding box of geometries (8.18/9.0+, preview) | `STATS bbox = ST_EXTENT_AGG(location)` | @@ -1308,40 +1370,40 @@ TS metrics ## String Functions -| Function | Description | Example | -| --------------------------- | ---------------------------------------------- | -------------------------------------------------------------------- | -| `LENGTH(s)` | String length | `EVAL len = LENGTH(name)` | -| `CONCAT(s1, s2, ...)` | Concatenate strings | `EVAL full = CONCAT(first, " ", last)` | -| `SUBSTRING(s, start, len)` | Extract substring | `EVAL sub = SUBSTRING(text, 1, 10)` | -| `LEFT(s, n)` | Left n characters | `EVAL l = LEFT(text, 5)` | -| `RIGHT(s, n)` | Right n characters | `EVAL r = RIGHT(text, 5)` | -| `TRIM(s)` | Remove whitespace | `EVAL clean = TRIM(input)` | -| `LTRIM(s)` | Trim left | `EVAL clean = LTRIM(input)` | -| `RTRIM(s)` | Trim right | `EVAL clean = RTRIM(input)` | -| `TO_UPPER(s)` | Uppercase | `EVAL upper = TO_UPPER(name)` | -| `TO_LOWER(s)` | Lowercase | `EVAL lower = TO_LOWER(name)` | -| `REPLACE(s, old, new)` | Replace text | `EVAL fixed = REPLACE(msg, "err", "error")` | -| `SPLIT(s, delim)` | Split into array | `EVAL parts = SPLIT(path, "/")` | -| `STARTS_WITH(s, prefix)` | Check prefix | `WHERE STARTS_WITH(url, "https")` | -| `ENDS_WITH(s, suffix)` | Check suffix | `WHERE ENDS_WITH(file, ".log")` | -| `CONTAINS(s, substr)` | Check contains | `WHERE CONTAINS(message, "error")` | -| `LOCATE(substr, s)` | Find position | `EVAL pos = LOCATE("@", email)` | -| `REVERSE(s)` | Reverse string | `EVAL rev = REVERSE(text)` | -| `REPEAT(s, n)` | Repeat string | `EVAL sep = REPEAT("-", 10)` | -| `SPACE(n)` | N spaces | `EVAL spaces = SPACE(5)` | -| `BIT_LENGTH(s)` | Bit length (8.17+) | `EVAL bits = BIT_LENGTH(name)` | -| `BYTE_LENGTH(s)` | Byte length (8.17+) | `EVAL bytes = BYTE_LENGTH(name)` | -| `CHUNK(field, settings)` | Split text into chunks (9.3+, preview) | `EVAL chunks = CHUNK(body, {"strategy":"word","max_chunk_size":50})` | -| `HASH(alg, s)` | Hash string (8.18/9.0+) | `EVAL h = HASH("SHA-256", msg)` | -| `MD5(s)` | MD5 hash (8.18/9.0+) | `EVAL h = MD5(content)` | -| `SHA1(s)` | SHA-1 hash (8.18/9.0+) | `EVAL h = SHA1(content)` | -| `SHA256(s)` | SHA-256 hash (8.18/9.0+) | `EVAL h = SHA256(content)` | -| `FROM_BASE64(s)` | Decode base64 | `EVAL decoded = FROM_BASE64(encoded)` | -| `TO_BASE64(s)` | Encode to base64 | `EVAL encoded = TO_BASE64(data)` | -| `URL_DECODE(s)` | URL-decode (9.2+) | `EVAL decoded = URL_DECODE(url)` | -| `URL_ENCODE(s)` | URL-encode (9.2+) | `EVAL encoded = URL_ENCODE(text)` | -| `URL_ENCODE_COMPONENT(s)` | URL-encode for URI components (9.2+) | `EVAL encoded = URL_ENCODE_COMPONENT(text)` | -| `JSON_EXTRACT(field, path)` | Extract value from JSON string (Serverless GA) | `EVAL name = JSON_EXTRACT(raw, "$.user.name")` | +| Function | Description | Example | +| --------------------------- | --------------------------------------------------------- | -------------------------------------------------------------------- | +| `LENGTH(s)` | String length | `EVAL len = LENGTH(name)` | +| `CONCAT(s1, s2, ...)` | Concatenate strings | `EVAL full = CONCAT(first, " ", last)` | +| `SUBSTRING(s, start, len)` | Extract substring | `EVAL sub = SUBSTRING(text, 1, 10)` | +| `LEFT(s, n)` | Left n characters | `EVAL l = LEFT(text, 5)` | +| `RIGHT(s, n)` | Right n characters | `EVAL r = RIGHT(text, 5)` | +| `TRIM(s)` | Remove whitespace | `EVAL clean = TRIM(input)` | +| `LTRIM(s)` | Trim left | `EVAL clean = LTRIM(input)` | +| `RTRIM(s)` | Trim right | `EVAL clean = RTRIM(input)` | +| `TO_UPPER(s)` | Uppercase | `EVAL upper = TO_UPPER(name)` | +| `TO_LOWER(s)` | Lowercase | `EVAL lower = TO_LOWER(name)` | +| `REPLACE(s, old, new)` | Replace text | `EVAL fixed = REPLACE(msg, "err", "error")` | +| `SPLIT(s, delim)` | Split into array | `EVAL parts = SPLIT(path, "/")` | +| `STARTS_WITH(s, prefix)` | Check prefix | `WHERE STARTS_WITH(url, "https")` | +| `ENDS_WITH(s, suffix)` | Check suffix | `WHERE ENDS_WITH(file, ".log")` | +| `CONTAINS(s, substr)` | Check contains | `WHERE CONTAINS(message, "error")` | +| `LOCATE(substr, s)` | Find position | `EVAL pos = LOCATE("@", email)` | +| `REVERSE(s)` | Reverse string | `EVAL rev = REVERSE(text)` | +| `REPEAT(s, n)` | Repeat string | `EVAL sep = REPEAT("-", 10)` | +| `SPACE(n)` | N spaces | `EVAL spaces = SPACE(5)` | +| `BIT_LENGTH(s)` | Bit length (8.17+) | `EVAL bits = BIT_LENGTH(name)` | +| `BYTE_LENGTH(s)` | Byte length (8.17+) | `EVAL bytes = BYTE_LENGTH(name)` | +| `CHUNK(field, settings)` | Split text into chunks (9.3+, preview) | `EVAL chunks = CHUNK(body, {"strategy":"word","max_chunk_size":50})` | +| `HASH(alg, s)` | Hash string (8.18/9.0+) | `EVAL h = HASH("SHA-256", msg)` | +| `MD5(s)` | MD5 hash (8.18/9.0+) | `EVAL h = MD5(content)` | +| `SHA1(s)` | SHA-1 hash (8.18/9.0+) | `EVAL h = SHA1(content)` | +| `SHA256(s)` | SHA-256 hash (8.18/9.0+) | `EVAL h = SHA256(content)` | +| `FROM_BASE64(s)` | Decode base64 | `EVAL decoded = FROM_BASE64(encoded)` | +| `TO_BASE64(s)` | Encode to base64 | `EVAL encoded = TO_BASE64(data)` | +| `URL_DECODE(s)` | URL-decode (9.2+) | `EVAL decoded = URL_DECODE(url)` | +| `URL_ENCODE(s)` | URL-encode (9.2+) | `EVAL encoded = URL_ENCODE(text)` | +| `URL_ENCODE_COMPONENT(s)` | URL-encode for URI components (9.2+) | `EVAL encoded = URL_ENCODE_COMPONENT(text)` | +| `JSON_EXTRACT(field, path)` | Extract value from JSON string (9.4+ preview; Serverless) | `EVAL name = JSON_EXTRACT(raw, "$.user.name")` | **JSON_EXTRACT with \_source — flattened field workaround:** @@ -1466,25 +1528,30 @@ FROM network_logs ## Spatial Functions -| Function | Description | Example | -| ------------------------- | ----------------------------- | --------------------------------------------------------- | -| `ST_DISTANCE(p1, p2)` | Distance between points | `EVAL dist = ST_DISTANCE(loc, TO_GEOPOINT("POINT(0 0)"))` | -| `ST_INTERSECTS(g1, g2)` | Geometries intersect | `WHERE ST_INTERSECTS(geo, boundary)` | -| `ST_DISJOINT(g1, g2)` | Geometries don't intersect | `WHERE ST_DISJOINT(geo, zone)` | -| `ST_CONTAINS(g1, g2)` | g1 contains g2 | `WHERE ST_CONTAINS(region, point)` | -| `ST_WITHIN(g1, g2)` | g1 within g2 | `WHERE ST_WITHIN(point, region)` | -| `ST_X(point)` | X coordinate / longitude | `EVAL lon = ST_X(location)` | -| `ST_Y(point)` | Y coordinate / latitude | `EVAL lat = ST_Y(location)` | -| `ST_ENVELOPE(geo)` | Bounding box (8.18/9.0+) | `EVAL bbox = ST_ENVELOPE(shape)` | -| `ST_XMAX(geo)` | Max X / longitude (8.18/9.0+) | `EVAL max_lon = ST_XMAX(shape)` | -| `ST_XMIN(geo)` | Min X / longitude (8.18/9.0+) | `EVAL min_lon = ST_XMIN(shape)` | -| `ST_YMAX(geo)` | Max Y / latitude (8.18/9.0+) | `EVAL max_lat = ST_YMAX(shape)` | -| `ST_YMIN(geo)` | Min Y / latitude (8.18/9.0+) | `EVAL min_lat = ST_YMIN(shape)` | -| `ST_GEOHASH(point, prec)` | Encode as geohash (9.2+) | `EVAL hash = ST_GEOHASH(location, 5)` | -| `ST_GEOHEX(point, prec)` | Encode as geohex (9.2+) | `EVAL hex = ST_GEOHEX(location, 5)` | -| `ST_GEOTILE(point, prec)` | Encode as geotile (9.2+) | `EVAL tile = ST_GEOTILE(location, 10)` | -| `ST_NPOINTS(geo)` | Number of points | `EVAL n = ST_NPOINTS(shape)` | -| `ST_SIMPLIFY(geo, tol)` | Simplify geometry | `EVAL simple = ST_SIMPLIFY(shape, 100)` | +| Function | Description | Example | +| --------------------------------------- | ----------------------------------- | --------------------------------------------------------- | +| `ST_DISTANCE(p1, p2)` | Distance between points | `EVAL dist = ST_DISTANCE(loc, TO_GEOPOINT("POINT(0 0)"))` | +| `ST_INTERSECTS(g1, g2)` | Geometries intersect | `WHERE ST_INTERSECTS(geo, boundary)` | +| `ST_DISJOINT(g1, g2)` | Geometries don't intersect | `WHERE ST_DISJOINT(geo, zone)` | +| `ST_CONTAINS(g1, g2)` | g1 contains g2 | `WHERE ST_CONTAINS(region, point)` | +| `ST_WITHIN(g1, g2)` | g1 within g2 | `WHERE ST_WITHIN(point, region)` | +| `ST_X(point)` | X coordinate / longitude | `EVAL lon = ST_X(location)` | +| `ST_Y(point)` | Y coordinate / latitude | `EVAL lat = ST_Y(location)` | +| `ST_ENVELOPE(geo)` | Bounding box (8.18/9.0+) | `EVAL bbox = ST_ENVELOPE(shape)` | +| `ST_XMAX(geo)` | Max X / longitude (8.18/9.0+) | `EVAL max_lon = ST_XMAX(shape)` | +| `ST_XMIN(geo)` | Min X / longitude (8.18/9.0+) | `EVAL min_lon = ST_XMIN(shape)` | +| `ST_YMAX(geo)` | Max Y / latitude (8.18/9.0+) | `EVAL max_lat = ST_YMAX(shape)` | +| `ST_YMIN(geo)` | Min Y / latitude (8.18/9.0+) | `EVAL min_lat = ST_YMIN(shape)` | +| `ST_GEOHASH(point, prec)` | Encode as geohash (9.2+) | `EVAL hash = ST_GEOHASH(location, 5)` | +| `ST_GEOHEX(point, prec)` | Encode as geohex (9.2+) | `EVAL hex = ST_GEOHEX(location, 5)` | +| `ST_GEOTILE(point, prec)` | Encode as geotile (9.2+) | `EVAL tile = ST_GEOTILE(location, 10)` | +| `ST_NPOINTS(geo)` | Number of points | `EVAL n = ST_NPOINTS(shape)` | +| `ST_SIMPLIFY(geo, tol)` | Simplify geometry | `EVAL simple = ST_SIMPLIFY(shape, 100)` | +| `ST_DIMENSION(geo)` | Dimension (0/1/2) (9.4+) | `EVAL dim = ST_DIMENSION(shape)` | +| `ST_GEOMETRYTYPE(geo)` | Geometry type string (9.4+) | `EVAL gtype = ST_GEOMETRYTYPE(shape)` | +| `ST_ISEMPTY(geo)` | True if empty (9.4+) | `WHERE NOT ST_ISEMPTY(shape)` | +| `ST_BUFFER(geo, dist)` | Buffer around geometry (9.4+) | `EVAL area = ST_BUFFER(point, 1000)` | +| `ST_SIMPLIFYPRESERVETOPOLOGY(geo, tol)` | Simplify preserving topology (9.4+) | `EVAL s = ST_SIMPLIFYPRESERVETOPOLOGY(shape, 100)` | --- @@ -1508,29 +1575,30 @@ For vector search and similarity operations on `dense_vector` and `semantic_text For handling fields with multiple values. -| Function | Description | Example | -| ------------------------------------- | ------------------------------------------------------------ | ----------------------------------------------- | -| `MV_COUNT(field)` | Count values | `EVAL n = MV_COUNT(tags)` | -| `MV_FIRST(field)` | First value | `EVAL first_val = MV_FIRST(values)` | -| `MV_LAST(field)` | Last value | `EVAL last_val = MV_LAST(values)` | -| `MV_MIN(field)` | Minimum | `EVAL min = MV_MIN(scores)` | -| `MV_MAX(field)` | Maximum | `EVAL max = MV_MAX(scores)` | -| `MV_SUM(field)` | Sum | `EVAL total = MV_SUM(amounts)` | -| `MV_AVG(field)` | Average | `EVAL avg = MV_AVG(scores)` | -| `MV_MEDIAN(field)` | Median | `EVAL med = MV_MEDIAN(values)` | -| `MV_CONCAT(field, delim)` | Join to string | `EVAL str = MV_CONCAT(tags, ", ")` | -| `MV_DEDUPE(field)` | Remove duplicates | `EVAL unique = MV_DEDUPE(tags)` | -| `MV_SORT(field)` | Sort values | `EVAL sorted = MV_SORT(values)` | -| `MV_SLICE(field, start, end)` | Slice array | `EVAL slice = MV_SLICE(arr, 0, 3)` | -| `MV_ZIP(f1, f2)` | Zip arrays (both must be keyword/text) | `EVAL zipped = MV_ZIP(keys, values)` | -| `MV_APPEND(f1, f2)` | Concatenate MVs | `EVAL all = MV_APPEND(tags1, tags2)` | -| `MV_CONTAINS(f1, f2)` | All values in f2 present in f1 (9.2+) | `EVAL has = MV_CONTAINS(perms, required)` | -| `MV_INTERSECTION(f1, f2)` | Values present in both (9.3+) | `EVAL common = MV_INTERSECTION(a, b)` | -| `MV_INTERSECTS(f1, f2)` | Any value in f2 present in f1 (Serverless; self-managed 9.4) | `EVAL overlap = MV_INTERSECTS(a, b)` | -| `MV_UNION(f1, f2)` | Deduplicated union (Serverless; self-managed 9.4) | `EVAL merged = MV_UNION(a, b)` | -| `MV_PERCENTILE(field, p)` | Percentile of MV | `EVAL p95 = MV_PERCENTILE(vals, 95)` | -| `MV_PSERIES_WEIGHTED_SUM(field, p)` | P-series weighted sum (both args must be double) | `EVAL ws = MV_PSERIES_WEIGHTED_SUM(vals, 2.0)` | -| `MV_MEDIAN_ABSOLUTE_DEVIATION(field)` | MAD of MV | `EVAL mad = MV_MEDIAN_ABSOLUTE_DEVIATION(vals)` | +| Function | Description | Example | +| ------------------------------------- | ------------------------------------------------ | ----------------------------------------------- | +| `MV_COUNT(field)` | Count values | `EVAL n = MV_COUNT(tags)` | +| `MV_FIRST(field)` | First value | `EVAL first_val = MV_FIRST(values)` | +| `MV_LAST(field)` | Last value | `EVAL last_val = MV_LAST(values)` | +| `MV_MIN(field)` | Minimum | `EVAL min = MV_MIN(scores)` | +| `MV_MAX(field)` | Maximum | `EVAL max = MV_MAX(scores)` | +| `MV_SUM(field)` | Sum | `EVAL total = MV_SUM(amounts)` | +| `MV_AVG(field)` | Average | `EVAL avg = MV_AVG(scores)` | +| `MV_MEDIAN(field)` | Median | `EVAL med = MV_MEDIAN(values)` | +| `MV_CONCAT(field, delim)` | Join to string | `EVAL str = MV_CONCAT(tags, ", ")` | +| `MV_DEDUPE(field)` | Remove duplicates | `EVAL unique = MV_DEDUPE(tags)` | +| `MV_SORT(field)` | Sort values | `EVAL sorted = MV_SORT(values)` | +| `MV_SLICE(field, start, end)` | Slice array | `EVAL slice = MV_SLICE(arr, 0, 3)` | +| `MV_ZIP(f1, f2)` | Zip arrays (both must be keyword/text) | `EVAL zipped = MV_ZIP(keys, values)` | +| `MV_APPEND(f1, f2)` | Concatenate MVs | `EVAL all = MV_APPEND(tags1, tags2)` | +| `MV_CONTAINS(f1, f2)` | All values in f2 present in f1 (9.2+) | `EVAL has = MV_CONTAINS(perms, required)` | +| `MV_INTERSECTION(f1, f2)` | Values present in both (9.3+) | `EVAL common = MV_INTERSECTION(a, b)` | +| `MV_INTERSECTS(f1, f2)` | Any value in f2 present in f1 (9.4+; Serverless) | `EVAL overlap = MV_INTERSECTS(a, b)` | +| `MV_UNION(f1, f2)` | Deduplicated union (9.4+; Serverless) | `EVAL merged = MV_UNION(a, b)` | +| `MV_DIFFERENCE(f1, f2)` | Values in f1 not in f2 (9.4+; Serverless) | `EVAL diff = MV_DIFFERENCE(a, b)` | +| `MV_PERCENTILE(field, p)` | Percentile of MV | `EVAL p95 = MV_PERCENTILE(vals, 95)` | +| `MV_PSERIES_WEIGHTED_SUM(field, p)` | P-series weighted sum (both args must be double) | `EVAL ws = MV_PSERIES_WEIGHTED_SUM(vals, 2.0)` | +| `MV_MEDIAN_ABSOLUTE_DEVIATION(field)` | MAD of MV | `EVAL mad = MV_MEDIAN_ABSOLUTE_DEVIATION(vals)` | --- @@ -1755,6 +1823,7 @@ regular index fields. | `_ignored` | keyword | Fields that were ignored when the document was indexed | | `_index_mode` | keyword | Index mode (`standard`, `lookup`, `logsdb`, `time_series` etc.) | | `_source` | special | Original JSON document body. Use `JSON_EXTRACT` to access flattened or unmapped fields | +| `_size` | integer | Document size in bytes (9.4+; requires `mapper-size` plugin) | ```esql FROM logs METADATA _id, _index, _version diff --git a/skills/elasticsearch/elasticsearch-esql/references/esql-version-history.md b/skills/elasticsearch/elasticsearch-esql/references/esql-version-history.md index 13ef9a1..b5c2f69 100644 --- a/skills/elasticsearch/elasticsearch-esql/references/esql-version-history.md +++ b/skills/elasticsearch/elasticsearch-esql/references/esql-version-history.md @@ -27,58 +27,61 @@ determine compatibility when writing queries for specific Elasticsearch deployme ## Version Timeline Overview -| Version | Release | Status | Key Additions | -| ------- | -------- | ------------ | ------------------------------------------------------------------------------------------------------- | -| 8.11 | Nov 2023 | Tech Preview | Initial ES\|QL release | -| 8.12 | Jan 2024 | Tech Preview | Spatial types, PROFILE | -| 8.13 | Mar 2024 | Tech Preview | Async queries, cross-cluster ENRICH | -| 8.14 | May 2024 | **GA** | Spatial functions, regex optimization | -| 8.15 | Aug 2024 | GA | Type casting (`::`), Arrow output | -| 8.16 | Oct 2024 | GA | Per-aggregation WHERE, new math/string functions | -| 8.17 | Dec 2024 | GA | MATCH, QSTR full-text functions | -| 8.18 | Feb 2025 | GA | LOOKUP JOIN (preview), scoring, KQL | -| 8.19 | Apr 2025 | GA | MATCH_PHRASE, FORK, CHANGE_POINT (preview) | -| 9.0 | Feb 2025 | GA | Released with 8.18 features | -| 9.1 | Jun 2025 | GA | Full-text functions GA, FORK (preview) | -| 9.2 | Oct 2025 | GA | Multi-field joins, TS, INLINE STATS (preview), CHANGE_POINT GA, FUSE (preview), RERANK (preview) | -| 9.3 | Jan 2026 | GA | INLINE STATS GA, SET directive (preview), Lucene-pushable JOIN predicates | -| 9.4 | May 2026 | GA | TS GA, time series functions GA, WITHOUT/METRICS_INFO/TS_INFO GA, PROMQL (preview), MV_EXPAND/VALUES GA | +| Version | Release | Status | Key Additions | +| ------- | -------- | ------------ | ----------------------------------------------------------------------------------------------------------------------------------------- | +| 8.11 | Nov 2023 | Tech Preview | Initial ES\|QL release | +| 8.12 | Jan 2024 | Tech Preview | Spatial types, PROFILE | +| 8.13 | Mar 2024 | Tech Preview | Async queries, cross-cluster ENRICH | +| 8.14 | May 2024 | **GA** | Spatial functions, regex optimization | +| 8.15 | Aug 2024 | GA | Type casting (`::`), Arrow output | +| 8.16 | Oct 2024 | GA | Per-aggregation WHERE, new math/string functions | +| 8.17 | Dec 2024 | GA | MATCH, QSTR full-text functions | +| 8.18 | Feb 2025 | GA | LOOKUP JOIN (preview), scoring, KQL | +| 8.19 | Apr 2025 | GA | MATCH_PHRASE, FORK, CHANGE_POINT (preview) | +| 9.0 | Feb 2025 | GA | Released with 8.18 features | +| 9.1 | Jun 2025 | GA | Full-text functions GA, FORK (preview) | +| 9.2 | Oct 2025 | GA | Multi-field joins, TS, INLINE STATS (preview), CHANGE_POINT GA, FUSE (preview), RERANK (preview) | +| 9.3 | Jan 2026 | GA | INLINE STATS GA, SET directive (preview), Lucene-pushable JOIN predicates | +| 9.4 | May 2026 | GA | TS/TBUCKET GA, time series funcs GA, WITHOUT/METRICS_INFO/TS_INFO, PROMQL (preview), Views, RERANK GA, MV_EXPAND/VALUES GA, SET time_zone | ## Feature Availability by Version ### Commands -| Command | Introduced | GA | Notes | -| -------------- | ---------- | -------- | ----------------------------------------------- | -| `FROM` | 8.11 | 8.14 | Source command | -| `WHERE` | 8.11 | 8.14 | Filtering | -| `EVAL` | 8.11 | 8.14 | Computed columns | -| `STATS ... BY` | 8.11 | 8.14 | Aggregations with grouping | -| `SORT` | 8.11 | 8.14 | Ordering results | -| `LIMIT` | 8.11 | 8.14 | Result set size | -| `KEEP` | 8.11 | 8.14 | Column selection | -| `DROP` | 8.11 | 8.14 | Column removal | -| `RENAME` | 8.11 | 8.14 | Column renaming | -| `DISSECT` | 8.11 | 8.14 | Pattern extraction | -| `GROK` | 8.11 | 8.14 | Log parsing | -| `ENRICH` | 8.11 | 8.14 | Data enrichment | -| `MV_EXPAND` | 8.11 | 9.4 | Multi-value expansion (GA) | -| `SHOW` | 8.11 | 8.14 | Metadata display | -| `ROW` | 8.11 | 8.14 | Literal row creation | -| `LOOKUP JOIN` | 8.18/9.0 | 8.19/9.1 | SQL-style LEFT JOIN with lookup indices | -| `INLINE STATS` | 9.2 | 9.3 | Inline aggregations (like window functions) | -| `FORK` | 8.19/9.1 | Preview | Multiple execution branches | -| `FUSE` | 9.2 | Preview | Combine results from FORK branches | -| `TS` | 9.2 | 9.4 | Time series source command | -| `PROMQL` | 9.4 | Preview | Source command using PromQL syntax on TSDS | -| `METRICS_INFO` | 9.4 | 9.4 | TSDS metric catalogue (after `TS`) | -| `TS_INFO` | 9.4 | 9.4 | Per-(metric, time series) metadata (after `TS`) | -| `RERANK` | 9.2 | Preview | Re-score results with inference | -| `COMPLETION` | 9.2 | 9.2 | LLM text generation | -| `SAMPLE` | 8.19/9.1 | Preview | Random sampling | -| `URI_PARTS` | Srvless | Srvless | Parse URI into structured columns | -| `USER_AGENT` | Srvless | Srvless | Parse user agent into structured columns | -| `REG_DOMAIN` | Srvless | Srvless | `REGISTERED_DOMAIN`: extract from hostname | +| Command | Introduced | GA | Notes | +| ------------------- | ---------- | -------- | ---------------------------------------------------------------- | +| `FROM` | 8.11 | 8.14 | Source command | +| `WHERE` | 8.11 | 8.14 | Filtering | +| `EVAL` | 8.11 | 8.14 | Computed columns | +| `STATS ... BY` | 8.11 | 8.14 | Aggregations with grouping | +| `SORT` | 8.11 | 8.14 | Ordering results | +| `LIMIT` | 8.11 | 8.14 | Result set size | +| `KEEP` | 8.11 | 8.14 | Column selection | +| `DROP` | 8.11 | 8.14 | Column removal | +| `RENAME` | 8.11 | 8.14 | Column renaming | +| `DISSECT` | 8.11 | 8.14 | Pattern extraction | +| `GROK` | 8.11 | 8.14 | Log parsing | +| `ENRICH` | 8.11 | 8.14 | Data enrichment | +| `MV_EXPAND` | 8.11 | 9.4 | Multi-value expansion (GA) | +| `SHOW` | 8.11 | 8.14 | Metadata display | +| `ROW` | 8.11 | 8.14 | Literal row creation | +| `LOOKUP JOIN` | 8.18/9.0 | 8.19/9.1 | SQL-style LEFT JOIN with lookup indices | +| `INLINE STATS` | 9.2 | 9.3 | Inline aggregations (like window functions) | +| `FORK` | 8.19/9.1 | 9.4 | Multiple execution branches | +| `FUSE` | 9.2 | Preview | Combine results from FORK branches | +| `TS` | 9.2 | 9.4 | Time series source command (GA in 9.4) | +| `PROMQL` | 9.4 | Preview | Source command using PromQL syntax on TSDS | +| `METRICS_INFO` | 9.4 | 9.4 | TSDS metric catalogue (after `TS`) | +| `TS_INFO` | 9.4 | 9.4 | Per-(metric, time series) metadata (after `TS`) | +| `RERANK` | 9.2 | 9.4 | Re-score results with inference (GA in 9.4) | +| `COMPLETION` | 9.2 | 9.2 | LLM text generation | +| `SAMPLE` | 8.19/9.1 | Preview | Random sampling | +| `CHANGE_POINT` | 8.19/9.1 | 9.2 | Spike/dip detection (Platinum license) | +| `MMR` | 9.4 | Preview | Maximal Marginal Relevance diversification | +| `URI_PARTS` | 9.4 | 9.4 | Parse URI into structured columns (pipe command) | +| `USER_AGENT` | 9.4 | 9.4 | Parse user agent into structured columns (pipe command) | +| `REGISTERED_DOMAIN` | 9.4 | 9.4 | `REGISTERED_DOMAIN`: extract from hostname (pipe command) | +| Views | 9.4 | Preview | Virtual indices from ES\|QL queries (Stack only, not Serverless) | ### Full-Text Search Functions @@ -97,18 +100,23 @@ determine compatibility when writing queries for specific Elasticsearch deployme ### Spatial Functions -| Function | Introduced | Notes | -| ---------------------- | ---------- | ---------------------------- | -| `GEO_POINT` type | 8.12 | Basic spatial type support | -| `CARTESIAN_POINT` type | 8.12 | Cartesian coordinate support | -| `ST_INTERSECTS` | 8.14 | Geometry intersection test | -| `ST_CONTAINS` | 8.14 | Containment test | -| `ST_DISJOINT` | 8.14 | Disjoint test | -| `ST_WITHIN` | 8.14 | Within test | -| `ST_X`, `ST_Y` | 8.14 | Coordinate extraction | -| `ST_DISTANCE` | 8.15 | Distance calculation | -| `ST_EXTENT_AGG` | 8.18/9.0 | Bounding box aggregation | -| `ST_ENVELOPE` | 8.18/9.0 | Bounding box for geometry | +| Function | Introduced | Notes | +| ----------------------------- | ---------- | ------------------------------------- | +| `GEO_POINT` type | 8.12 | Basic spatial type support | +| `CARTESIAN_POINT` type | 8.12 | Cartesian coordinate support | +| `ST_INTERSECTS` | 8.14 | Geometry intersection test | +| `ST_CONTAINS` | 8.14 | Containment test | +| `ST_DISJOINT` | 8.14 | Disjoint test | +| `ST_WITHIN` | 8.14 | Within test | +| `ST_X`, `ST_Y` | 8.14 | Coordinate extraction | +| `ST_DISTANCE` | 8.15 | Distance calculation | +| `ST_EXTENT_AGG` | 8.18/9.0 | Bounding box aggregation | +| `ST_ENVELOPE` | 8.18/9.0 | Bounding box for geometry | +| `ST_DIMENSION` | 9.4 | Geometry dimension (0/1/2) | +| `ST_GEOMETRYTYPE` | 9.4 | Geometry type as string | +| `ST_ISEMPTY` | 9.4 | Test if geometry is empty | +| `ST_BUFFER` | 9.4 | Buffer around geometry | +| `ST_SIMPLIFYPRESERVETOPOLOGY` | 9.4 | Simplify geometry preserving topology | ### Date/Time Functions @@ -125,52 +133,61 @@ determine compatibility when writing queries for specific Elasticsearch deployme ### String Functions -| Function | Introduced | Notes | -| --------------------------- | ---------- | --------------------------- | -| `LEFT`, `RIGHT` | 8.11 | Substring extraction | -| `SUBSTRING` | 8.11 | Position-based extraction | -| `CONCAT` | 8.11 | String concatenation | -| `TRIM`, `LTRIM`, `RTRIM` | 8.11 | Whitespace removal | -| `TO_UPPER`, `TO_LOWER` | 8.13 | Case conversion | -| `LOCATE` | 8.14 | Find substring position | -| `SPACE` | 8.16 | Generate spaces | -| `REVERSE` | 8.16 | Reverse string | -| `BIT_LENGTH`, `BYTE_LENGTH` | 8.17 | String length in bits/bytes | -| `STARTS_WITH`, `ENDS_WITH` | 8.11 | Prefix/suffix matching | -| `CONTAINS` | 9.2 | Substring containment check | +| Function | Introduced | Notes | +| --------------------------- | ---------- | ---------------------------------------------------- | +| `LEFT`, `RIGHT` | 8.11 | Substring extraction | +| `SUBSTRING` | 8.11 | Position-based extraction | +| `CONCAT` | 8.11 | String concatenation | +| `TRIM`, `LTRIM`, `RTRIM` | 8.11 | Whitespace removal | +| `TO_UPPER`, `TO_LOWER` | 8.13 | Case conversion | +| `LOCATE` | 8.14 | Find substring position | +| `SPACE` | 8.16 | Generate spaces | +| `REVERSE` | 8.16 | Reverse string | +| `BIT_LENGTH`, `BYTE_LENGTH` | 8.17 | String length in bits/bytes | +| `STARTS_WITH`, `ENDS_WITH` | 8.11 | Prefix/suffix matching | +| `CONTAINS` | 9.2 | Substring containment check | +| `JSON_EXTRACT` | 9.4 | Extract value from JSON string by JSONPath (preview) | ### Multi-Value Functions -| Function | Introduced | Notes | -| ------------------------- | ---------- | ---------------------- | -| `MV_COUNT` | 8.11 | Count values | -| `MV_CONCAT` | 8.11 | Join values | -| `MV_FIRST`, `MV_LAST` | 8.13 | First/last value | -| `MV_MIN`, `MV_MAX` | 8.11 | Min/max value | -| `MV_SUM`, `MV_AVG` | 8.11 | Sum/average | -| `MV_MEDIAN` | 8.11 | Median value | -| `MV_SORT` | 8.14 | Sort multi-values | -| `MV_SLICE` | 8.14 | Slice multi-values | -| `MV_PERCENTILE` | 8.16 | Percentile calculation | -| `MV_PSERIES_WEIGHTED_SUM` | 8.16 | Weighted sum | +| Function | Introduced | Notes | +| ------------------------- | ---------- | --------------------------------------- | +| `MV_COUNT` | 8.11 | Count values | +| `MV_CONCAT` | 8.11 | Join values | +| `MV_FIRST`, `MV_LAST` | 8.13 | First/last value | +| `MV_MIN`, `MV_MAX` | 8.11 | Min/max value | +| `MV_SUM`, `MV_AVG` | 8.11 | Sum/average | +| `MV_MEDIAN` | 8.11 | Median value | +| `MV_SORT` | 8.14 | Sort multi-values | +| `MV_SLICE` | 8.14 | Slice multi-values | +| `MV_PERCENTILE` | 8.16 | Percentile calculation | +| `MV_PSERIES_WEIGHTED_SUM` | 8.16 | Weighted sum | +| `MV_DIFFERENCE` | 9.4 | Set difference of two MV fields | +| `MV_UNION` | 9.4 | Set union of two MV fields | +| `MV_INTERSECTION` | 9.4 | Set intersection of two MV fields | +| `MV_INTERSECTS` | 9.4 | True if MV fields share a value | +| `MV_CONTAINS` | 9.4 | True if first MV contains all of second | ### Aggregation Functions -| Function | Introduced | Notes | -| ------------------------------------- | ---------- | ------------------------------- | -| `COUNT`, `COUNT_DISTINCT` | 8.11 | Counting | -| `SUM`, `AVG` | 8.11 | Basic aggregations | -| `MIN`, `MAX` | 8.11 | Extended to strings/IPs in 8.16 | -| `MEDIAN`, `MEDIAN_ABSOLUTE_DEVIATION` | 8.11 | Statistical | -| `PERCENTILE` | 8.11 | Percentile calculation | -| `TOP` | 8.15 | Top N values | -| `VALUES` | 8.14 | Unique values (GA in 9.4) | -| `ST_EXTENT_AGG` | 8.18/9.0 | Spatial bounding box | -| `WEIGHTED_AVG` | 8.16 | Weighted average | -| `STD_DEV` | 8.18/9.0 | Standard deviation | -| `VARIANCE` | 8.18/9.0 | Variance | -| `FIRST` / `EARLIEST` | Serverless | Earliest value by sort field | -| `LAST` / `LATEST` | Serverless | Latest value by sort field | +| Function | Introduced | Notes | +| ------------------------------------- | ---------- | ---------------------------------------------------- | +| `COUNT`, `COUNT_DISTINCT` | 8.11 | Counting | +| `SUM`, `AVG` | 8.11 | Basic aggregations | +| `MIN`, `MAX` | 8.11 | Extended to strings/IPs in 8.16 | +| `MEDIAN`, `MEDIAN_ABSOLUTE_DEVIATION` | 8.11 | Statistical | +| `PERCENTILE` | 8.11 | Percentile calculation | +| `TOP` | 8.15 | Top N values | +| `VALUES` | 8.14 | Unique values (GA in 9.4) | +| `ST_EXTENT_AGG` | 8.18/9.0 | Spatial bounding box | +| `WEIGHTED_AVG` | 8.16 | Weighted average | +| `STD_DEV` | 8.18/9.0 | Standard deviation | +| `VARIANCE` | 8.18/9.0 | Variance | +| `FIRST(value, sort_field)` | 9.4 | Value from row with earliest sort field | +| `LAST(value, sort_field)` | 9.4 | Value from row with latest sort field | +| `EARLIEST(@timestamp)` | 9.4 | Min `@timestamp` (1-arg shorthand) | +| `LATEST(@timestamp)` | 9.4 | Max `@timestamp` (1-arg shorthand) | +| `SPARKLINE` | Serverless | Histogram sparkline (Serverless only, not 9.4 Stack) | ### Grouping Functions @@ -270,13 +287,10 @@ ES|QL **does not support cursor-based pagination** like the Search API's `search - Use `STATS` to aggregate at query time - For exports, use Search API with `search_after` instead -### Time Zone Support (Limited before Serverless / 9.4) +### Time Zone Support (GA in 9.4+; Serverless) -ES|QL has **limited timezone support** on self-managed clusters prior to 9.4. All dates are processed in UTC internally -and there is no per-function timezone argument. - -On **Serverless**, ES|QL supports query-wide timezone via the `SET time_zone` directive (GA on Serverless). This accepts -IANA timezone strings and UTC offsets, and applies to all date/time operations including `DATE_TRUNC`, `DATE_FORMAT`, +ES|QL supports query-wide timezone via the `SET time_zone` directive (GA in 9.4+; Serverless). This accepts IANA +timezone strings and UTC offsets, and applies to all date/time operations including `DATE_TRUNC`, `DATE_FORMAT`, `NOW()`, bucketing, and display. ```esql @@ -290,16 +304,13 @@ FROM logs-* - No per-function timezone argument — `DATE_TRUNC(1 hour, @timestamp, "America/New_York")` does **not** work - `DATE_FORMAT` and `DATE_PARSE` do not accept timezone parameters directly; use `SET time_zone` instead -- GitHub tracking issue: [#107560](https://github.com/elastic/elasticsearch/issues/107560) - -**Self-managed before 9.4:** -- `SET time_zone` only accepts UTC offsets (`"+05:00"`), not IANA timezone strings -- Workaround: use `EVAL` to add/subtract hours manually: +**Versions before 9.4:** No timezone support. All dates are processed in UTC. Workaround: use `EVAL` to add/subtract +hours manually: - ```esql - | EVAL local_time = timestamp + 1 hour - ``` +```esql +| EVAL local_time = timestamp + 1 hour +``` ### Nested Fields (Not Supported) @@ -347,8 +358,8 @@ These field types are not supported or have limitations: ### Subqueries (Limited) -ES|QL supports **subqueries in `FROM`** (Serverless tech preview) for combining results from multiple pipelines (UNION -ALL semantics). These are non-correlated — each branch is independent. +ES|QL supports **subqueries in `FROM`** (9.4+; Serverless) for combining results from multiple pipelines (UNION ALL +semantics). These are non-correlated — each branch is independent. ```esql FROM @@ -385,12 +396,15 @@ Use `INLINE STATS` (9.2+) for per-row vs. aggregate comparison patterns. ## API Endpoints -| Endpoint | Version | Notes | -| --------------------------- | ------- | ----------------------- | -| `POST /_query` | 8.11 | Synchronous query | -| `POST /_query/async` | 8.13 | Async query submission | -| `GET /_query/async/{id}` | 8.13 | Get async query results | -| `DELETE /_query/async/{id}` | 8.13 | Cancel async query | +| Endpoint | Version | Notes | +| ---------------------------- | ------- | -------------------------------- | +| `POST /_query` | 8.11 | Synchronous query | +| `POST /_query/async` | 8.13 | Async query submission | +| `GET /_query/async/{id}` | 8.13 | Get async query results | +| `DELETE /_query/async/{id}` | 8.13 | Cancel async query | +| `PUT /_query/view/{name}` | 9.4 | Create/update view (Stack only) | +| `GET /_query/view/{name}` | 9.4 | Get view definition (Stack only) | +| `DELETE /_query/view/{name}` | 9.4 | Delete view (Stack only) | ## Performance Tips by Version @@ -430,33 +444,21 @@ Use `INLINE STATS` (9.2+) for per-row vs. aggregate comparison patterns. ### 9.4+ -- `TS` source command and **all** time series aggregation functions are now **GA** — both the 9.2-introduced set - (`RATE`, `IRATE`, `INCREASE`, `DELTA`, `IDELTA`, `*_OVER_TIME`, `PRESENT_OVER_TIME`, `ABSENT_OVER_TIME`) and the - 9.3-introduced set (`DERIV`, `PERCENTILE_OVER_TIME`, `STDDEV_OVER_TIME`, `VARIANCE_OVER_TIME`). -- `TBUCKET` grouping function is **GA**. -- New `WITHOUT(...)` grouping function (GA) for time series queries: `BY WITHOUT(dim1, ...)` groups by every dimension - except the listed ones; `BY WITHOUT()` (no args) is equivalent to the implicit "group by all dimensions" behavior. -- New `METRICS_INFO` and `TS_INFO` processing commands (both **GA**) for discovering the metric catalogue and dimension - labels of TSDS data without inspecting index mappings. Both must come after a `TS` source command and must appear - before pipeline-breaking commands (`STATS`/`SORT`/`LIMIT`). `METRICS_INFO` returns one row per distinct metric - signature; `TS_INFO` returns one row per (metric, time series) combination with the identifying dimension labels. -- Sliding window parameter (`RATE(field, 10m)`) accepts arbitrary durations — no longer limited to multiples of the - `TBUCKET` interval. Note: a single query cannot mix windows smaller than the bucket for one metric with windows larger - than the bucket for another metric. -- New `PROMQL` source command (preview) to run Prometheus Query Language directly against TSDS indices, with implicit - range selectors and a Kibana-aware `step`/`buckets` model. See [promql-command.md](promql-command.md). Prefer `PROMQL` - only when the user explicitly thinks in PromQL or is migrating Prometheus dashboards/alerts; otherwise prefer `TS`. -- `MV_EXPAND` is GA -- `VALUES` aggregation is GA +- `TS`, `TBUCKET`, and all time series aggregation functions are **GA** — safe for production metrics queries +- Use `METRICS_INFO` / `TS_INFO` to discover TSDS schemas instead of inspecting mappings or field capabilities +- Use `WITHOUT(dim, ...)` to group by all dimensions except specific ones — avoids enumerating every dimension manually +- Sliding window accepts arbitrary durations — `RATE(field, 7m)` with `TBUCKET(5 minute)` now works +- Use `SET time_zone` with IANA strings for timezone-aware queries instead of manual `EVAL` offset arithmetic +- Use `SET approximation = true` for faster `COUNT_DISTINCT` on large datasets when exact counts aren't needed +- Use `SET unmapped_fields = "load"` to query fields missing from some indices without errors +- Use `FIRST`/`LAST` (or `EARLIEST`/`LATEST`) instead of `SORT` + `LIMIT 1` for grouped first/last-value queries +- Use `PROMQL` when porting Prometheus dashboards/alerts; otherwise prefer `TS` for native ES|QL ### Serverless (latest) -- `SET time_zone` with IANA timezone strings for query-wide timezone support (GA) -- `LIMIT n BY field` for grouped top-N queries -- `URI_PARTS`, `USER_AGENT`, `REGISTERED_DOMAIN` pipe commands for parsing structured strings -- `FROM` subqueries for combining results from multiple pipelines (tech preview) -- `EARLIEST`/`LATEST` aliases for `FIRST`/`LAST` aggregations -- `JSON_EXTRACT` on `METADATA _source` for accessing flattened field sub-keys +Serverless includes all 9.4 features and may have additional preview features: + +- `SPARKLINE` — histogram sparkline aggregation (Serverless only, not 9.4 Stack) ## Version Detection diff --git a/skills/elasticsearch/elasticsearch-esql/references/generation-tips.md b/skills/elasticsearch/elasticsearch-esql/references/generation-tips.md index e4192a0..d6071d6 100644 --- a/skills/elasticsearch/elasticsearch-esql/references/generation-tips.md +++ b/skills/elasticsearch/elasticsearch-esql/references/generation-tips.md @@ -4,8 +4,8 @@ Guidelines for generating accurate ES|QL queries from natural language. > **Cluster detection:** Check `build_flavor` in the `GET /` response. For Serverless (`"serverless"`), **do not** > version-gate: `version.number` tracks the next minor from main (semver-only clients may see it as “latest”), but -> feature availability is not determined by that string — use `build_flavor` as the signal. For self-managed -> (`"default"`), use `version.number` for feature checks (strip `-SNAPSHOT` suffix on pre-release builds). +> feature availability is not determined by that string — use `build_flavor` as the signal. For Stack (`"default"`), use +> `version.number` for feature checks (strip `-SNAPSHOT` suffix on pre-release builds). ## Table of Contents @@ -153,6 +153,10 @@ For time series data streams (TSDS), use `TS` instead of `FROM` to enable time s TS metrics-* // Time series source — enables RATE, AVG_OVER_TIME, etc. ``` +**When the question asks about rates, throughput, CPU/memory trends, or metric comparisons**, prefer a `metrics-*` or +TSDS index with `TS` over a general log index with `FROM`. Check the schema — if an index has `Index mode: time_series`, +always use `TS`. + ### 2. Determine Time Range **Question:** What time period should be covered? @@ -469,14 +473,18 @@ The `schema` command displays the data stream name when the index is a TSDS back **2. TBUCKET takes only a duration — not @timestamp:** -`TBUCKET` is not `DATE_TRUNC`. Do not pass `@timestamp`: +`TBUCKET` is not `DATE_TRUNC`. Do not pass `@timestamp`. Always assign a column alias so you can reference it in `SORT`: ```esql // WRONG — DATE_TRUNC-style syntax | STATS avg_cpu = AVG(cpu) BY bucket = TBUCKET(@timestamp, 5 minutes) -// CORRECT — duration only, timestamp is implicit +// WRONG — no alias makes SORT difficult +| STATS avg_cpu = AVG(cpu) BY TBUCKET(5 minutes) + +// CORRECT — duration only with alias for SORT | STATS avg_cpu = AVG(cpu) BY bucket = TBUCKET(5 minutes) +| SORT bucket ``` **3. Counter fields need RATE() wrapped in an outer aggregation:** @@ -494,9 +502,15 @@ TS metrics-tsds | STATS request_rate = SUM(RATE(requests)) BY TBUCKET(1 hour), host ``` -For gauge fields, use `AVG()` or `MAX()` as the outer function: +For gauge fields, use `AVG()` or `MAX()` as the outer function. Prefer the plain form — the inner `LAST_OVER_TIME` is +implicit and sufficient for most gauge queries: ```esql +// Preferred — plain aggregation (implicit LAST_OVER_TIME) +TS metrics-tsds +| STATS avg_cpu = AVG(cpu) BY TBUCKET(5 minutes), service.name + +// Only use explicit *_OVER_TIME when you need specific window behavior TS metrics-tsds | STATS avg_cpu = AVG(AVG_OVER_TIME(cpu)) BY TBUCKET(5 minutes), service.name ``` @@ -532,7 +546,7 @@ ES|QL before 9.2**. There is no fallback. When the cluster is pre-9.2 and the question requires per-row vs. aggregate comparison, explain that `INLINE STATS` is needed and suggest the user either upgrade or perform the comparison client-side. -### Pipe Commands: URI_PARTS, USER_AGENT, REGISTERED_DOMAIN (Serverless) +### Pipe Commands: URI_PARTS, USER_AGENT, REGISTERED_DOMAIN (9.4+; Serverless) These are **pipe commands** (like `DISSECT`/`GROK`), not scalar functions. They must appear on their own pipeline stage with `target = expression` syntax. A target prefix is mandatory. @@ -555,7 +569,7 @@ When the user asks to "parse URLs", "extract domains", or "parse user agents", r | Parse a user agent string | `USER_AGENT` | | Extract registered domain | `REGISTERED_DOMAIN` | -### Grouped Top-N with LIMIT BY (Serverless) +### Grouped Top-N with LIMIT BY (9.4+; Serverless) `LIMIT n BY field` keeps the top N rows per group after sorting. The number comes **before** `BY`. @@ -572,8 +586,8 @@ This replaces the common `INLINE STATS` + rank-and-filter pattern for simple gro ### Subqueries in FROM vs FORK -**Subqueries** (Serverless tech preview) combine results from **different** data sources (UNION ALL semantics). **FORK** -runs **different analyses** on the **same** data source. +**Subqueries** (9.4+; Serverless) combine results from **different** data sources (UNION ALL semantics). **FORK** runs +**different analyses** on the **same** data source. | Scenario | Use | | ------------------------------------- | ---------- | diff --git a/skills/elasticsearch/elasticsearch-esql/references/query-patterns.md b/skills/elasticsearch/elasticsearch-esql/references/query-patterns.md index 50ff4bd..9203a1b 100644 --- a/skills/elasticsearch/elasticsearch-esql/references/query-patterns.md +++ b/skills/elasticsearch/elasticsearch-esql/references/query-patterns.md @@ -437,7 +437,7 @@ FROM flights | KEEP flight_id, destination, distance, avg_dist ``` -### Grouped Top-N with LIMIT BY (Serverless) +### Grouped Top-N with LIMIT BY (9.4+; Serverless) ```text "top 3 error types per service" @@ -452,7 +452,7 @@ FROM logs-* ```text "most recent event per user" → -// Serverless: use LATEST to get the most recent value per group +// 9.4+/Serverless: use LATEST to get the most recent value per group FROM events-* | STATS last_action = LATEST(event.action), last_ts = LATEST(@timestamp) BY user.name @@ -463,7 +463,7 @@ FROM events-* | KEEP user.name, @timestamp, event.action ``` -### Subquery Composition (Serverless tech preview) +### Subquery Composition (9.4+; Serverless) ```text "combine web server errors and application errors into one view" diff --git a/skills/elasticsearch/elasticsearch-esql/references/time-series-queries.md b/skills/elasticsearch/elasticsearch-esql/references/time-series-queries.md index a4d5f08..ed7eece 100644 --- a/skills/elasticsearch/elasticsearch-esql/references/time-series-queries.md +++ b/skills/elasticsearch/elasticsearch-esql/references/time-series-queries.md @@ -88,6 +88,12 @@ TS metrics | STATS AVG(memory_usage) TS metrics | STATS AVG(LAST_OVER_TIME(memory_usage)) ``` +**When to use `*_OVER_TIME` vs plain aggregations for gauges:** For simple gauge queries (average CPU, max memory), +prefer `AVG(cpu)` over `AVG(AVG_OVER_TIME(cpu))` — the implicit `LAST_OVER_TIME` is sufficient and produces cleaner +queries. Use explicit `*_OVER_TIME` only when you need a specific window behavior: `AVG_OVER_TIME` to average all +samples (not just the last), `MIN_OVER_TIME`/`MAX_OVER_TIME` to find extremes within each time series before +aggregating, or `DELTA`/`DERIV` to compute changes. When in doubt, omit the inner function. + Since 9.3 (preview), use a time series function directly without an outer aggregation to get one value per time series per bucket. The result is implicitly grouped by all dimensions of each time series and includes a `_timeseries` column with the dimension key/value pairs — see [WITHOUT Grouping Function](#without-grouping-function) for narrowing this @@ -164,11 +170,11 @@ For gauge metrics and general numeric fields (`double`, `integer`, `long`, `aggr | `DERIV` | Derivative over time using linear regression | 9.3 (preview) | GA (9.4) | ```esql -// Average memory per cluster per 5 minutes +// Average memory per cluster per 5 minutes (plain form — implicit LAST_OVER_TIME is sufficient for gauges) // cluster is a dimension of the TSDS index metrics TS metrics | WHERE TRANGE(1 day) -| STATS AVG(AVG_OVER_TIME(memory_usage)) BY cluster, TBUCKET(5 minute) +| STATS AVG(memory_usage) BY cluster, TBUCKET(5 minute) // P95 network cost per cluster per minute // k8s is an example of a TSDS index, to showcase that time series indexes do not have to be called metrics @@ -245,15 +251,18 @@ The interval is a time duration (`1 hour`, `5 minute`, `30s`) or date period (`1 **Availability:** Preview from 9.2 to 9.3, **GA since 9.4**. +Always assign a column alias to `TBUCKET` so it can be referenced in `SORT`: + ```esql -// 1-hour buckets +// 1-hour buckets — alias enables SORT TS metrics -| STATS SUM(RATE(requests)) BY TBUCKET(1 hour), host +| STATS rate = SUM(RATE(requests)) BY bucket = TBUCKET(1 hour), host +| SORT bucket, host -// 5-minute buckets -// service is a dimension of the TSDS index metrics in this example +// 5-minute buckets (plain form for gauge) TS metrics -| STATS AVG(AVG_OVER_TIME(cpu_percent)) BY TBUCKET(5 minute), service +| STATS avg_cpu = AVG(cpu_percent) BY bucket = TBUCKET(5 minute), service +| SORT bucket ``` --- @@ -519,19 +528,20 @@ TS metrics ### Average Gauge per Cluster Over Time ```esql +// Plain form — preferred for gauge metrics TS metrics | WHERE TRANGE(1 day) -| STATS AVG(AVG_OVER_TIME(memory_usage)) BY TBUCKET(5 minute), cluster +| STATS AVG(memory_usage) BY TBUCKET(5 minute), cluster ``` ### Per-Time-Series Averages vs Global Average ```esql -// Average of per-time-series averages (accounts for different series lengths) -TS metrics | STATS AVG(AVG_OVER_TIME(memory_usage)) - -// Average of last values per time series (default behavior) +// Average of last values per time series (default — preferred for most gauge queries) TS metrics | STATS AVG(memory_usage) + +// Average of ALL samples per time series (use AVG_OVER_TIME only when you need this distinction) +TS metrics | STATS AVG(AVG_OVER_TIME(memory_usage)) ``` ### Detect Missing Data diff --git a/skills/elasticsearch/elasticsearch-onboarding/references/elasticsearch-onboarding-playbook.md b/skills/elasticsearch/elasticsearch-onboarding/references/elasticsearch-onboarding-playbook.md index c5366f5..b9fb5d3 100644 --- a/skills/elasticsearch/elasticsearch-onboarding/references/elasticsearch-onboarding-playbook.md +++ b/skills/elasticsearch/elasticsearch-onboarding/references/elasticsearch-onboarding-playbook.md @@ -4,50 +4,6 @@ You are an Elasticsearch solutions architect embedded in the developer's IDE. Gu working search experience — understanding their intent, recommending the right approach, and generating production-ready code. -## UI Context Hint - -The rule file may contain one or both of these lines at the top, injected by the Kibana onboarding UI at download time. -Read them before the first message — they pre-answer questions you would otherwise ask. - -### `# user-context:` - -Opens with a confirmation instead of a blank question: - -- `# user-context: ai-pipeline` → "Looks like you're building an AI app or pipeline — chatbot, RAG, vector store, or - recommendations. Is that right? Are you building something users interact with directly, or a retrieval layer that - feeds another system like LangChain?" -- `# user-context: document-search` → "Looks like you're building search over documents or content — a knowledge base, - wiki, or docs site. Is that right? Tell me about what you're searching over." -- `# user-context: catalog-ecommerce` → "Looks like you're building browse-and-filter search — products, listings, or a - structured catalog. Is that right? Tell me about your data." -- `# user-context: geo-search` → "Looks like you're building location-based search — 'near me', maps, or geo filters. Is - that right? Tell me about your use case." -- `# user-context: log-search` → "Looks like you're building log or event search — app logs, security events, or IoT - data. Is that right? Tell me about your data pipeline." -- `# user-context: recommendations` → "Looks like you're building a recommendations feature — 'you might also like', - related content, or personalized feeds. Is that right? Tell me about what you're recommending." -- `# user-context: something-else: ` → "Looks like you're building [text] — is that right? Tell me more about what - you're searching over." - -If the developer confirms, proceed directly to Step 2 (skip the use case question in Step 1). If they correct it, -re-route immediately and continue from there. - -If no `# user-context:` hint is present, use the standard First Message flow below. - -### `# deployment:` - -Pre-answers deployment type — do NOT ask about this if the hint is present: - -- `# deployment: serverless` → Treat as Serverless throughout. Version is always latest. `semantic_text` works out of - the box with no inference endpoint setup. -- `# deployment: cloud-hosted` → Treat as Elastic Cloud Hosted (ECH). Detect version via MCP or ask. -- `# deployment: self-managed` → Treat as Self-Managed. Detect version via MCP or ask. - -If both hints are present, incorporate both silently — weave the deployment context into the confirmation message -naturally. For example, if `deployment: serverless` and `user-context: ai-pipeline`: "Looks like you're on Elastic Cloud -Serverless and building an AI pipeline — great combination. `semantic_text` will handle embeddings automatically with no -setup. Is that right?" - ## First Message If the developer's first message is vague or exploratory ("hi," "help," "get started," "search"), jump straight into the @@ -113,15 +69,15 @@ specific project, or maybe you're just exploring what's possible — either way Listen for signals: -| Signal | Approach | Output | -| ----------------------------------------------------------------------------------------------------------- | -------------------- | ------------------------------------ | -| "search bar", "filter by", "facets", "autocomplete" | keyword-search | Ranked results | -| "find similar", "natural language", "meaning-based" | vector-hybrid-search | Ranked results (by meaning) | -| "both keyword and semantic", "hybrid" | vector-hybrid-search | Ranked results (combined) | -| "chatbot", "Q&A", "answer from my docs", "RAG" | rag-chatbot | Generated answers (not just results) | -| "product search", "e-commerce", "catalog" | catalog-ecommerce | Ranked results with facets | -| "vector store", "embeddings", "LangChain", "LlamaIndex", "AI app", "agent", "similarity", "recommendations" | vector-hybrid-search | Vectors for downstream AI | -| "just learning", "exploring", "not sure yet", "new to Elasticsearch" | use-case-library | Guided exploration | +| Signal | Approach | Output | +| ------------------------------------------------------------------------------------------------------------------------------ | -------------------- | ------------------------------------ | +| "search bar", "filter by", "facets", "autocomplete" | keyword-search | Ranked results | +| "find similar", "natural language", "meaning-based" | vector-hybrid-search | Ranked results (by meaning) | +| "both keyword and semantic", "hybrid" | vector-hybrid-search | Ranked results (combined) | +| "chatbot", "Q&A", "answer from my docs", "RAG" | rag-chatbot | Generated answers (not just results) | +| "product search", "e-commerce", "catalog" | catalog-ecommerce | Ranked results with facets | +| "vector store", "vector database", "embeddings", "LangChain", "LlamaIndex", "AI app", "agent", "similarity", "recommendations" | vector-hybrid-search | Vectors for downstream AI | +| "just learning", "exploring", "not sure yet", "new to Elasticsearch" | use-case-library | Guided exploration | **If the developer is exploring or doesn't know what to build**, load the [use-case-library](use-case-library/use-case-library.md) reference and walk through it conversationally. Help them @@ -189,19 +145,31 @@ Python. Use what you learn to determine fields to map, embedding model needs, ingestion path, and client library. -### Step 3: Confirm Version +### Step 3: Confirm Deployment and Version + +Establish where Elasticsearch is running and what version, before recommending an approach or generating code. Don't +re-ask what the conversation has already answered — the developer may have said it, or MCP may have revealed it. + +- **MCP connected** → detect automatically via `GET /` (`version.number`; `version.build_flavor` is `serverless` on + Serverless). Tell the developer what you found. +- **Otherwise** → ask: "Where is your Elasticsearch running — Elastic Cloud Serverless, Elastic Cloud Hosted, + self-managed, or nowhere yet?" -Confirm the Elasticsearch version before recommending an approach or generating code. +Then resolve the version by deployment type: -- **`# deployment: serverless`** or inferred Serverless → version is always latest, skip this question. -- **MCP connected** → detect automatically via `GET /` (`version.number`). Tell the developer what you found. -- **Otherwise** → ask: "What version of Elasticsearch are you running? Find it in Kibana under **Stack Management → - Upgrade assistant**, or paste the output of `GET /` from **Dev Tools**." +- **Serverless** → version is always latest — skip the version question. `semantic_text` works out of the box with no + inference endpoint setup. +- **Cloud Hosted (ECH) or Self-Managed** → detect via MCP, or ask: "What version of Elasticsearch are you running? Find + it in Kibana under **Stack Management → Upgrade assistant**, or paste the output of `GET /` from **Dev Tools**." +- **No deployment yet** → recommend Elastic Cloud Serverless as the fastest path, and match the project type to the use + case from Step 1 — **Elasticsearch** for search use cases; Observability and Security use cases route to their + dedicated project types per Step 1. [Docs](https://www.elastic.co/docs/get-started/introduction). Treat as Serverless + from here on. Use the version to determine available field types (`semantic_text` requires 8.15+), inference endpoints, RRF/ELSER/EIS availability, and which doc version to link. -**Don't generate code until the version is confirmed.** +**Don't generate code until the deployment type and version are confirmed.** ### Step 4: Recommend and Confirm @@ -351,8 +319,8 @@ principles: - **[keyword-search](keyword-search/keyword-search.md)** — Load when the developer needs full-text search, filters, facets, or autocomplete without semantic/vector features. - **[vector-hybrid-search](vector-hybrid-search/vector-hybrid-search.md)** — Load when the developer needs semantic - search, hybrid BM25+vector search, kNN, embeddings, or Elasticsearch as a vector database. This is the primary guide - for any use case involving vectors or meaning-based search. + search, hybrid BM25+vector search, kNN, embeddings, or Elasticsearch as a vector store. This is the primary guide for + any use case involving vectors or meaning-based search. - **[rag-chatbot](rag-chatbot/rag-chatbot.md)** — Load when the developer wants to build a chatbot, Q&A system, or RAG pipeline that generates answers from documents. - **[catalog-ecommerce](catalog-ecommerce/ecommerce.md)** — Load when the developer needs product search with faceted diff --git a/skills/elasticsearch/elasticsearch-onboarding/references/use-case-library/use-case-library.md b/skills/elasticsearch/elasticsearch-onboarding/references/use-case-library/use-case-library.md index 5db424e..7e23114 100644 --- a/skills/elasticsearch/elasticsearch-onboarding/references/use-case-library/use-case-library.md +++ b/skills/elasticsearch/elasticsearch-onboarding/references/use-case-library/use-case-library.md @@ -196,7 +196,7 @@ project. --- -### 8. Vector Database (for AI/ML Pipelines) +### 8. Vector Store (for AI/ML Pipelines) Store and retrieve embeddings programmatically — code searches, not people. @@ -231,7 +231,7 @@ Store and retrieve embeddings programmatically — code searches, not people. | Customer support search | Hybrid search, synonyms | Aggregations for trend detection | | Location-based search | Geo-point, distance sort, geo filters | Combined with full-text | | Log & event search | Data streams, ILM, ES\|QL, aggregations | Runtime fields | -| Vector database | Dense vectors, kNN/ANN (HNSW) | Quantization, metadata filtering | +| Vector store | Dense vectors, kNN/ANN (HNSW) | Quantization, metadata filtering | ## Non-Search Use Cases diff --git a/skills/elasticsearch/elasticsearch-onboarding/references/vector-hybrid-search/vector-hybrid-search.md b/skills/elasticsearch/elasticsearch-onboarding/references/vector-hybrid-search/vector-hybrid-search.md index da27a10..094fb9a 100644 --- a/skills/elasticsearch/elasticsearch-onboarding/references/vector-hybrid-search/vector-hybrid-search.md +++ b/skills/elasticsearch/elasticsearch-onboarding/references/vector-hybrid-search/vector-hybrid-search.md @@ -1,36 +1,19 @@ --- name: vector-hybrid-search description: - Complete guide for building vector search, hybrid search, and using Elasticsearch as a vector database for AI - pipelines. Covers the full decision tree from deployment type through production optimization and RAG extension. Use - when a developer wants semantic search, hybrid BM25+vector search, kNN, embeddings, RAG, or Elasticsearch as a vector - store for LangChain/LlamaIndex. + Complete guide for building vector search, hybrid search, and using Elasticsearch as a vector store for AI pipelines. + Covers the full decision tree from deployment type through production optimization and RAG extension. Use when a + developer wants semantic search, hybrid BM25+vector search, kNN, embeddings, RAG, or Elasticsearch as a vector store + for LangChain/LlamaIndex. --- # Vector & Hybrid Search Guide -## UI Context Hint - -If the downloaded skill file contains a `# user-context:` line (set by the getting started UI at download time), read it -before the first message and open with a confirmation rather than a blank question: - -- `# user-context: vector-database` → "Looks like you're building a vector database for an AI pipeline — is that right? - Are you using LangChain, LlamaIndex, or a custom stack?" -- `# user-context: hybrid-search` → "Looks like you're building hybrid search — is that right? Will users be typing - queries directly, or is this powering an AI pipeline?" -- `# user-context: semantic-search` → "Looks like you're building semantic search — is that right? Tell me about what - you're searching over." - -If the developer corrects the use case, re-route immediately. No commitment. - -If no `# user-context:` hint is present, open with: "What are you building — a search experience for users, or a -retrieval backend for an AI pipeline like RAG or LangChain?" - ---- - ## Consumer Fork -Before any other decision, establish who consumes the search results: +Before any other decision, establish who consumes the search results. If the conversation hasn't answered it yet, open +with: "What are you building — a search experience for users, or a retrieval backend for an AI pipeline like RAG or +LangChain?" - **AI pipeline** (code consumes results) → LangChain, LlamaIndex, custom RAG, agent memory, recommendations - **Human-facing search** (people type queries) → search bar, results page, filters, autocomplete @@ -45,14 +28,14 @@ This determines Decision I (App Integration) and whether to offer a frontend at Ask what they're building. Listen for: -| Signal | Use Case | -| --------------------------------------------------------- | ---------------------------------- | -| "semantic search", "meaning-based", "natural language" | Semantic search | -| "BM25 + vector", "hybrid", "keyword and semantic" | Hybrid search | -| "RAG", "chatbot", "Q&A over documents" | RAG — use rag-chatbot skill | -| "LangChain", "LlamaIndex", "vector store", "agent memory" | AI pipeline / vector DB | -| "recommendations", "similar items" | Vector similarity | -| "image search", "multimodal" | Dense vector with image embeddings | +| Signal | Use Case | +| ---------------------------------------------------------------------------- | ---------------------------------- | +| "semantic search", "meaning-based", "natural language" | Semantic search | +| "BM25 + vector", "hybrid", "keyword and semantic" | Hybrid search | +| "RAG", "chatbot", "Q&A over documents" | RAG — use rag-chatbot skill | +| "LangChain", "LlamaIndex", "vector store", "vector database", "agent memory" | AI pipeline / vector store | +| "recommendations", "similar items" | Vector similarity | +| "image search", "multimodal" | Dense vector with image embeddings | **Scale check:** If the developer indicates >1M documents, >10GB, or cost sensitivity, flag quantization early: From bdcd540b6bd2ec3f815719c631c2fd9846ff89c2 Mon Sep 17 00:00:00 2001 From: Elastic Machine Date: Wed, 22 Jul 2026 16:33:26 +0000 Subject: [PATCH 2/2] chore: sync versions from release manifest --- .github/plugin/marketplace.json | 36 +++++++++---------- package.json | 2 +- plugins/cloud/.claude-plugin/plugin.json | 2 +- plugins/cloud/plugin.json | 13 +++++-- .../elasticsearch/.claude-plugin/plugin.json | 2 +- plugins/elasticsearch/plugin.json | 6 ++-- plugins/kibana/.claude-plugin/plugin.json | 2 +- plugins/kibana/plugin.json | 6 ++-- .../observability/.claude-plugin/plugin.json | 2 +- plugins/observability/plugin.json | 6 ++-- plugins/security/.claude-plugin/plugin.json | 2 +- plugins/security/plugin.json | 15 ++++++-- 12 files changed, 58 insertions(+), 36 deletions(-) diff --git a/.github/plugin/marketplace.json b/.github/plugin/marketplace.json index 6bb2b28..69e0499 100644 --- a/.github/plugin/marketplace.json +++ b/.github/plugin/marketplace.json @@ -1,41 +1,41 @@ { - "name": "elastic-agent-skills", - "owner": { - "name": "Elastic" - }, - "metadata": { - "description": "Official Elastic agent skills for Elasticsearch, Kibana, Observability, Security, and Cloud" - }, "plugins": [ { + "version": "0.3.1", "name": "elasticsearch", "source": "./plugins/elasticsearch", - "description": "Elasticsearch skills — ES|QL queries, data ingestion, security (authn, authz, audit), and troubleshooting", - "version": "0.3.0" + "description": "Elasticsearch skills — ES|QL queries, data ingestion, security (authn, authz, audit), and troubleshooting" }, { + "version": "0.3.1", "name": "kibana", "source": "./plugins/kibana", - "description": "Kibana skills — dashboards, alerting rules, connectors, Vega visualizations, Agent Builder, streams, and audit logging", - "version": "0.3.0" + "description": "Kibana skills — dashboards, alerting rules, connectors, Vega visualizations, Agent Builder, streams, and audit logging" }, { + "version": "0.3.1", "name": "observability", "source": "./plugins/observability", - "description": "Elastic Observability skills — OpenTelemetry instrumentation and migration (.NET, Java, Python), LLM observability, log search, SLOs, and service health", - "version": "0.3.0" + "description": "Elastic Observability skills — OpenTelemetry instrumentation and migration (.NET, Java, Python), LLM observability, log search, SLOs, and service health" }, { + "version": "0.3.1", "name": "security", "source": "./plugins/security", - "description": "Elastic Security skills — alert triage, case management, detection rule management, and sample data generation", - "version": "0.3.0" + "description": "Elastic Security skills — alert triage, case management, detection rule management, and sample data generation" }, { + "version": "0.3.1", "name": "cloud", "source": "./plugins/cloud", - "description": "Elastic Cloud skills — project setup, access management, network security, and Serverless project lifecycle", - "version": "0.3.0" + "description": "Elastic Cloud skills — project setup, access management, network security, and Serverless project lifecycle" } - ] + ], + "name": "elastic-agent-skills", + "owner": { + "name": "Elastic" + }, + "metadata": { + "description": "Official Elastic agent skills for Elasticsearch, Kibana, Observability, Security, and Cloud" + } } diff --git a/package.json b/package.json index 9296b3c..a6344a7 100644 --- a/package.json +++ b/package.json @@ -1,5 +1,5 @@ { - "version": "0.3.0", + "version": "0.3.1", "name": "elastic-agent-skills", "description": "Official Elastic agent skills for Elasticsearch, Kibana, Observability, Security, and Cloud", "license": "Apache-2.0", diff --git a/plugins/cloud/.claude-plugin/plugin.json b/plugins/cloud/.claude-plugin/plugin.json index 0d7cb10..1842cbf 100644 --- a/plugins/cloud/.claude-plugin/plugin.json +++ b/plugins/cloud/.claude-plugin/plugin.json @@ -1,5 +1,5 @@ { - "version": "0.3.0", + "version": "0.3.1", "name": "elastic-cloud", "description": "Elastic Cloud skills - project setup, access management, network security, and Serverless project lifecycle", "author": { diff --git a/plugins/cloud/plugin.json b/plugins/cloud/plugin.json index aaed65f..35512a7 100644 --- a/plugins/cloud/plugin.json +++ b/plugins/cloud/plugin.json @@ -1,7 +1,7 @@ { + "version": "0.3.1", "name": "cloud", "description": "Elastic Cloud skills - project setup, access management, network security, and Serverless project lifecycle", - "version": "0.3.0", "author": { "name": "Elastic", "url": "https://www.elastic.co" @@ -9,6 +9,13 @@ "repository": "https://github.com/elastic/agent-skills", "homepage": "https://github.com/elastic/agent-skills/tree/main/plugins/cloud", "license": "Apache-2.0", - "keywords": ["cloud", "serverless", "elastic-cloud", "elastic"], - "skills": ["skills/"] + "keywords": [ + "cloud", + "serverless", + "elastic-cloud", + "elastic" + ], + "skills": [ + "skills/" + ] } diff --git a/plugins/elasticsearch/.claude-plugin/plugin.json b/plugins/elasticsearch/.claude-plugin/plugin.json index 9d65e4c..8b311c7 100644 --- a/plugins/elasticsearch/.claude-plugin/plugin.json +++ b/plugins/elasticsearch/.claude-plugin/plugin.json @@ -1,5 +1,5 @@ { - "version": "0.3.0", + "version": "0.3.1", "name": "elastic-elasticsearch", "description": "Elasticsearch skills - ES|QL queries, data ingestion, security (authn, authz, audit), and troubleshooting", "author": { diff --git a/plugins/elasticsearch/plugin.json b/plugins/elasticsearch/plugin.json index ae037c5..3e04ca5 100644 --- a/plugins/elasticsearch/plugin.json +++ b/plugins/elasticsearch/plugin.json @@ -1,7 +1,7 @@ { + "version": "0.3.1", "name": "elasticsearch", "description": "Elasticsearch skills for GitHub Copilot — translate natural language to ES|QL, ingest data, manage security (authn, authz, audit), and troubleshoot clusters.", - "version": "0.3.0", "author": { "name": "Elastic", "url": "https://www.elastic.co" @@ -17,5 +17,7 @@ "security", "elastic" ], - "skills": ["skills/"] + "skills": [ + "skills/" + ] } diff --git a/plugins/kibana/.claude-plugin/plugin.json b/plugins/kibana/.claude-plugin/plugin.json index af49bcc..dc5fbc5 100644 --- a/plugins/kibana/.claude-plugin/plugin.json +++ b/plugins/kibana/.claude-plugin/plugin.json @@ -1,5 +1,5 @@ { - "version": "0.3.0", + "version": "0.3.1", "name": "elastic-kibana", "description": "Kibana skills - dashboards, alerting rules, connectors, Vega visualizations, Agent Builder, streams, and audit logging", "author": { diff --git a/plugins/kibana/plugin.json b/plugins/kibana/plugin.json index 30933c3..7e651e1 100644 --- a/plugins/kibana/plugin.json +++ b/plugins/kibana/plugin.json @@ -1,7 +1,7 @@ { + "version": "0.3.1", "name": "kibana", "description": "Kibana skills - dashboards, alerting rules, connectors, Vega visualizations, Agent Builder, streams, and audit logging", - "version": "0.3.0", "author": { "name": "Elastic", "url": "https://www.elastic.co" @@ -17,5 +17,7 @@ "vega", "elastic" ], - "skills": ["skills/"] + "skills": [ + "skills/" + ] } diff --git a/plugins/observability/.claude-plugin/plugin.json b/plugins/observability/.claude-plugin/plugin.json index acad7a6..1fe23a0 100644 --- a/plugins/observability/.claude-plugin/plugin.json +++ b/plugins/observability/.claude-plugin/plugin.json @@ -1,5 +1,5 @@ { - "version": "0.3.0", + "version": "0.3.1", "name": "elastic-observability", "description": "Elastic Observability skills - OpenTelemetry instrumentation and migration (.NET, Java, Python), LLM observability, log search, SLOs, and service health", "author": { diff --git a/plugins/observability/plugin.json b/plugins/observability/plugin.json index 6fe0239..a984091 100644 --- a/plugins/observability/plugin.json +++ b/plugins/observability/plugin.json @@ -1,7 +1,7 @@ { + "version": "0.3.1", "name": "observability", "description": "Elastic Observability skills - OpenTelemetry instrumentation and migration (.NET, Java, Python), LLM observability, log search, SLOs, and service health", - "version": "0.3.0", "author": { "name": "Elastic", "url": "https://www.elastic.co" @@ -17,5 +17,7 @@ "slo", "elastic" ], - "skills": ["skills/"] + "skills": [ + "skills/" + ] } diff --git a/plugins/security/.claude-plugin/plugin.json b/plugins/security/.claude-plugin/plugin.json index ad67618..04ca2f7 100644 --- a/plugins/security/.claude-plugin/plugin.json +++ b/plugins/security/.claude-plugin/plugin.json @@ -1,5 +1,5 @@ { - "version": "0.3.0", + "version": "0.3.1", "name": "elastic-security", "description": "Elastic Security skills - alert triage, case management, detection rule management, and sample data generation", "author": { diff --git a/plugins/security/plugin.json b/plugins/security/plugin.json index a4ab879..2164d8e 100644 --- a/plugins/security/plugin.json +++ b/plugins/security/plugin.json @@ -1,7 +1,7 @@ { + "version": "0.3.1", "name": "security", "description": "Elastic Security skills - alert triage, case management, detection rule management, and sample data generation", - "version": "0.3.0", "author": { "name": "Elastic", "url": "https://www.elastic.co" @@ -9,6 +9,15 @@ "repository": "https://github.com/elastic/agent-skills", "homepage": "https://github.com/elastic/agent-skills/tree/main/plugins/security", "license": "Apache-2.0", - "keywords": ["security", "siem", "detection", "alerts", "cases", "elastic"], - "skills": ["skills/"] + "keywords": [ + "security", + "siem", + "detection", + "alerts", + "cases", + "elastic" + ], + "skills": [ + "skills/" + ] }