diff --git a/packages/elasticsearch/_dev/build/docs/README.md b/packages/elasticsearch/_dev/build/docs/README.md index b16b8972c50..d3f9dcf4dd4 100644 --- a/packages/elasticsearch/_dev/build/docs/README.md +++ b/packages/elasticsearch/_dev/build/docs/README.md @@ -87,6 +87,8 @@ ingest-time enrichment. ### Index +The `index` metricset is collected by Metricbeat. From package version 1.22.0, the `elasticsearch.index.mode` and `elasticsearch.index.codec` fields are populated when the collecting Metricbeat version includes [beats#49237](https://github.com/elastic/beats/pull/49237) (Metricbeat **8.19** and later). These fields are written to the source indices consumed by the `index_pivot` transform. + {{event "index"}} {{fields "index"}} @@ -186,6 +188,10 @@ Other visualizations in the dashboard allow you to compare the relative footprin Both approaches can be used in conjunction, allowing you to fine-tune ILM on a data stream basis (if required) to closely match usage patterns. +#### Index mode and codec + +From package version 1.22.0, the Consumption dashboard and `index_pivot` transform also surface index mode and codec. These values come from `elasticsearch.index.mode` and `elasticsearch.index.codec` in the Metricbeat `index` metricset events that the transform reads from (`metrics-elasticsearch.stack_monitoring.index*`, `.monitoring-es-*`, or `metricbeat-*`). Collecting agents must run Metricbeat **8.19** or later, which includes [beats#49237](https://github.com/elastic/beats/pull/49237). When the fields are absent from source documents, ingest pipelines normalize them to `standard` and `default` respectively. + ⚠️ Important notes: * The transform job will process all compatible historical data, which has two implications: 1. if you have pre-8.17.1 data, this will not get picked up by the job and 2. it might take time for "live" data to be available, as the transform job works its way through all documents. You can modify the transform job as you please if need be. diff --git a/packages/elasticsearch/changelog.yml b/packages/elasticsearch/changelog.yml index d845ab4a01c..73e11a5e786 100644 --- a/packages/elasticsearch/changelog.yml +++ b/packages/elasticsearch/changelog.yml @@ -1,7 +1,7 @@ # newer versions go on top - version: "1.22.0" changes: - - description: Wire index mode and codec through the index_pivot transform, normalize missing values to standard/default in the monitoring_indices and index metrics ingest pipelines, and add dashboard visualizations for index mode and codec adoption + - description: Wire index mode and codec through the index_pivot transform, normalize missing values to standard/default in the monitoring_indices and index metrics ingest pipelines, and add dashboard visualizations for index mode and codec adoption. Requires Metricbeat 8.19+ (beats#49237) to collect elasticsearch.index.mode and elasticsearch.index.codec in the index metricset source data. type: enhancement link: https://github.com/elastic/beats/pull/49237 - version: "1.21.1" diff --git a/packages/elasticsearch/data_stream/index/fields/fields.yml b/packages/elasticsearch/data_stream/index/fields/fields.yml index 8c7485fb2fc..349086d06db 100644 --- a/packages/elasticsearch/data_stream/index/fields/fields.yml +++ b/packages/elasticsearch/data_stream/index/fields/fields.yml @@ -22,8 +22,10 @@ type: keyword - name: mode type: keyword + description: Index mode (for example, `standard` or `time_series`). Collected from index settings by Metricbeat 8.19+ ([beats#49237](https://github.com/elastic/beats/pull/49237)). - name: codec type: keyword + description: Index codec (for example, `default` or `best_compression`). Collected from index settings by Metricbeat 8.19+ ([beats#49237](https://github.com/elastic/beats/pull/49237)). - name: name type: keyword dimension: true diff --git a/packages/elasticsearch/docs/README.md b/packages/elasticsearch/docs/README.md index 925d76f7fcf..a4658f7dea6 100644 --- a/packages/elasticsearch/docs/README.md +++ b/packages/elasticsearch/docs/README.md @@ -893,6 +893,8 @@ An example event for `enrich` looks as following: ### Index +The `index` metricset is collected by Metricbeat. From package version 1.22.0, the `elasticsearch.index.mode` and `elasticsearch.index.codec` fields are populated when the collecting Metricbeat version includes [beats#49237](https://github.com/elastic/beats/pull/49237) (Metricbeat **8.19** and later). These fields are written to the source indices consumed by the `index_pivot` transform. + An example event for `index` looks as following: ```json @@ -1061,10 +1063,10 @@ An example event for `index` looks as following: | elasticsearch.cluster.id | Elasticsearch cluster id. | keyword | | | elasticsearch.cluster.name | Elasticsearch cluster name. | keyword | | | elasticsearch.cluster.state.id | Elasticsearch state id. | keyword | | -| elasticsearch.index.codec | | keyword | | +| elasticsearch.index.codec | Index codec (for example, `default` or `best_compression`). Collected from index settings by Metricbeat 8.19+ ([beats#49237](https://github.com/elastic/beats/pull/49237)). | keyword | | | elasticsearch.index.creation_date | | date | | | elasticsearch.index.hidden | | boolean | | -| elasticsearch.index.mode | | keyword | | +| elasticsearch.index.mode | Index mode (for example, `standard` or `time_series`). Collected from index settings by Metricbeat 8.19+ ([beats#49237](https://github.com/elastic/beats/pull/49237)). | keyword | | | elasticsearch.index.name | Index name. | keyword | | | elasticsearch.index.primaries.docs.count | | long | gauge | | elasticsearch.index.primaries.docs.deleted | | long | gauge | @@ -2770,6 +2772,10 @@ Other visualizations in the dashboard allow you to compare the relative footprin Both approaches can be used in conjunction, allowing you to fine-tune ILM on a data stream basis (if required) to closely match usage patterns. +#### Index mode and codec + +From package version 1.22.0, the Consumption dashboard and `index_pivot` transform also surface index mode and codec. These values come from `elasticsearch.index.mode` and `elasticsearch.index.codec` in the Metricbeat `index` metricset events that the transform reads from (`metrics-elasticsearch.stack_monitoring.index*`, `.monitoring-es-*`, or `metricbeat-*`). Collecting agents must run Metricbeat **8.19** or later, which includes [beats#49237](https://github.com/elastic/beats/pull/49237). When the fields are absent from source documents, ingest pipelines normalize them to `standard` and `default` respectively. + ⚠️ Important notes: * The transform job will process all compatible historical data, which has two implications: 1. if you have pre-8.17.1 data, this will not get picked up by the job and 2. it might take time for "live" data to be available, as the transform job works its way through all documents. You can modify the transform job as you please if need be. diff --git a/packages/elasticsearch/kibana/dashboard/elasticsearch-consumption.json b/packages/elasticsearch/kibana/dashboard/elasticsearch-consumption.json index e251c6b53ed..2833870702b 100644 --- a/packages/elasticsearch/kibana/dashboard/elasticsearch-consumption.json +++ b/packages/elasticsearch/kibana/dashboard/elasticsearch-consumption.json @@ -146,7 +146,7 @@ "id": "", "params": { "fontSize": 12, - "markdown": "# Indices and data streams usage dashboard\n\nThis dashboard will require Elasticsearch monitoring data (built-in or Metricbeat module / Elasticsearch integration) to be collected in this cluster, **and** the `logs-elasticsearch.index_pivot-default` [transform](/app/management/data/transform) to be started.\n\nThis dashboard will provide tier-, data stream-, index mode-, and codec-centric consumption information, such as:\n* Ingest volume (written on disk, after indexing by Elasticsearch).\n* Query volume (by count and time).\n* Storage volume.\n\n\nThis will help you review your data usage and ensure alignment between expectations, usage patterns, ILM policy and architecture best practices.", + "markdown": "# Indices and data streams usage dashboard\n\nThis dashboard will require Elasticsearch monitoring data (built-in or Metricbeat module / Elasticsearch integration) to be collected in this cluster, **and** the `logs-elasticsearch.index_pivot-default` [transform](/app/management/data/transform) to be started.\n\nIndex mode and codec panels require Metricbeat **8.19** or later ([beats#49237](https://github.com/elastic/beats/pull/49237)) so that `elasticsearch.index.mode` and `elasticsearch.index.codec` are present in the `index` metricset source data read by the transform.\n\nThis dashboard will provide tier-, data stream-, index mode-, and codec-centric consumption information, such as:\n* Ingest volume (written on disk, after indexing by Elasticsearch).\n* Query volume (by count and time).\n* Storage volume.\n\n\nThis will help you review your data usage and ensure alignment between expectations, usage patterns, ILM policy and architecture best practices.", "openLinksInNewTab": false }, "title": "", @@ -1099,7 +1099,7 @@ "id": "", "params": { "fontSize": 12, - "markdown": "# \ud83d\udd0e Querying", + "markdown": "# 🔎 Querying", "openLinksInNewTab": false }, "title": "", @@ -1757,7 +1757,7 @@ "description": "", "params": { "fontSize": 12, - "markdown": "# \ud83d\udc40 Consumption patterns", + "markdown": "# 👀 Consumption patterns", "openLinksInNewTab": false }, "title": "", @@ -4784,7 +4784,7 @@ "description": "", "params": { "fontSize": 12, - "markdown": "# \ud83d\udcbe Storage", + "markdown": "# 💾 Storage", "openLinksInNewTab": false }, "title": "", @@ -5900,7 +5900,7 @@ "description": "", "params": { "fontSize": 12, - "markdown": "# \ud83d\ude80 Ingest", + "markdown": "# 🚀 Ingest", "openLinksInNewTab": false }, "title": "", @@ -6471,7 +6471,7 @@ "type": "lens", "visualizationType": "lnsXY" }, - "description": "\u26a0\ufe0f Please note that this line can appear \"jittery\" because of the underlying merge activity of the cluster.", + "description": "⚠️ Please note that this line can appear \"jittery\" because of the underlying merge activity of the cluster.", "enhancements": {} }, "gridData": { @@ -6505,7 +6505,7 @@ "id": "", "params": { "fontSize": 12, - "markdown": "# Index mode and codec\n\nReview index mode adoption (for example LogsDB) and compression codec usage alongside storage and ingest patterns.", + "markdown": "# Index mode and codec", "openLinksInNewTab": false }, "title": "", @@ -6981,7 +6981,7 @@ "references": [ { "id": "elasticsearch-monitoringindices", - "name": "indexpattern-datasource-layer-587c1332-dec1-4d27-ad36-beaf4c9a5861", + "name": "indexpattern-datasource-layer-d7e18b52-7e4f-5c3b-a062-3e5f7c9b1d26", "type": "index-pattern" } ], @@ -6991,65 +6991,27 @@ "formBased": { "currentIndexPatternId": "elasticsearch-monitoringindices", "layers": { - "587c1332-dec1-4d27-ad36-beaf4c9a5861": { + "d7e18b52-7e4f-5c3b-a062-3e5f7c9b1d26": { "columnOrder": [ - "54932e2c-488c-4586-8f2d-fb952b1c1005", - "862617fc-1ebe-45ff-a08e-ab4a62558a5b", - "9b494e2a-e553-4634-820e-40f1e4ec6d18", - "7c337400-93e4-4fc8-9a38-832111de654e" + "f1a29384-5b6c-4d7e-8e90-1a2b3c4d5e6f", + "a2b30495-6c7d-5e8f-9f01-2b3c4d5e6f70", + "b3c415a6-7d8e-6f90-a012-3c4d5e6f7081" ], "columns": { - "54932e2c-488c-4586-8f2d-fb952b1c1005": { - "dataType": "string", - "isBucketed": true, - "label": "Top 9 values of elasticsearch.index.datastream", - "operationType": "terms", - "params": { - "exclude": [], - "excludeIsRegex": false, - "include": [], - "includeIsRegex": false, - "missingBucket": true, - "orderDirection": "desc", - "otherBucket": true, - "parentFormat": { - "id": "terms" - }, - "size": 9, - "orderBy": { - "columnId": "7c337400-93e4-4fc8-9a38-832111de654e", - "type": "column" - } - }, - "scale": "ordinal", - "sourceField": "elasticsearch.index.datastream" - }, - "862617fc-1ebe-45ff-a08e-ab4a62558a5b": { - "dataType": "string", + "f1a29384-5b6c-4d7e-8e90-1a2b3c4d5e6f": { + "dataType": "date", "isBucketed": true, - "label": "Top 5 values of elasticsearch.index.mode", - "operationType": "terms", + "label": "@timestamp", + "operationType": "date_histogram", "params": { - "exclude": [], - "excludeIsRegex": false, - "include": [], - "includeIsRegex": false, - "missingBucket": true, - "orderDirection": "desc", - "otherBucket": true, - "parentFormat": { - "id": "terms" - }, - "size": 5, - "orderBy": { - "columnId": "7c337400-93e4-4fc8-9a38-832111de654e", - "type": "column" - } + "dropPartials": false, + "includeEmptyRows": true, + "interval": "auto" }, - "scale": "ordinal", - "sourceField": "elasticsearch.index.mode" + "scale": "interval", + "sourceField": "@timestamp" }, - "9b494e2a-e553-4634-820e-40f1e4ec6d18": { + "a2b30495-6c7d-5e8f-9f01-2b3c4d5e6f70": { "dataType": "string", "isBucketed": true, "label": "Top 5 values of elasticsearch.index.codec", @@ -7067,14 +7029,14 @@ }, "size": 5, "orderBy": { - "columnId": "7c337400-93e4-4fc8-9a38-832111de654e", + "columnId": "b3c415a6-7d8e-6f90-a012-3c4d5e6f7081", "type": "column" } }, "scale": "ordinal", "sourceField": "elasticsearch.index.codec" }, - "7c337400-93e4-4fc8-9a38-832111de654e": { + "b3c415a6-7d8e-6f90-a012-3c4d5e6f7081": { "customLabel": true, "dataType": "number", "isBucketed": false, @@ -7090,7 +7052,8 @@ } }, "scale": "ratio", - "sourceField": "elasticsearch.index.primaries.store.total_data_set_size_in_bytes_delta" + "sourceField": "elasticsearch.index.primaries.store.total_data_set_size_in_bytes_delta", + "timeScale": "s" } }, "ignoreGlobalFilters": false, @@ -7114,47 +7077,83 @@ "query": "" }, "visualization": { - "columns": [ - { - "columnId": "54932e2c-488c-4586-8f2d-fb952b1c1005", - "isTransposed": false - }, - { - "columnId": "862617fc-1ebe-45ff-a08e-ab4a62558a5b", - "isTransposed": false - }, - { - "columnId": "9b494e2a-e553-4634-820e-40f1e4ec6d18", - "isTransposed": false - }, + "axisTitlesVisibilitySettings": { + "x": true, + "yLeft": false, + "yRight": true + }, + "fittingFunction": "None", + "gridlinesVisibilitySettings": { + "x": true, + "yLeft": true, + "yRight": true + }, + "layers": [ { - "columnId": "7c337400-93e4-4fc8-9a38-832111de654e", - "isTransposed": false + "accessors": [ + "b3c415a6-7d8e-6f90-a012-3c4d5e6f7081" + ], + "colorMapping": { + "assignments": [], + "colorMode": { + "type": "categorical" + }, + "paletteId": "eui_amsterdam_color_blind", + "specialAssignments": [ + { + "color": { + "type": "loop" + }, + "rule": { + "type": "other" + }, + "touched": false + } + ] + }, + "layerId": "d7e18b52-7e4f-5c3b-a062-3e5f7c9b1d26", + "layerType": "data", + "position": "top", + "seriesType": "bar_stacked", + "showGridlines": false, + "splitAccessor": "a2b30495-6c7d-5e8f-9f01-2b3c4d5e6f70", + "xAccessor": "f1a29384-5b6c-4d7e-8e90-1a2b3c4d5e6f", + "yConfig": [ + { + "forAccessor": "b3c415a6-7d8e-6f90-a012-3c4d5e6f7081", + "axisMode": "auto" + } + ] } ], - "layerId": "587c1332-dec1-4d27-ad36-beaf4c9a5861", - "layerType": "data", - "sorting": { - "columnId": "7c337400-93e4-4fc8-9a38-832111de654e", - "direction": "desc" - } + "legend": { + "isVisible": true, + "position": "right" + }, + "preferredSeriesType": "bar_stacked", + "title": "Empty XY chart", + "valueLabels": "hide", + "yLeftExtent": { + "mode": "full" + }, + "yLeftScale": "linear" } }, "title": "", "type": "lens", - "visualizationType": "lnsDatatable" + "visualizationType": "lnsXY" }, "enhancements": {} }, "gridData": { "h": 14, - "i": "e428330c-1454-460a-b8ee-9f4ade52b633", + "i": "c8f29a41-6d3e-4b2a-9f71-2d4e6b8a0c15", "w": 48, "x": 0, "y": 174 }, - "panelIndex": "e428330c-1454-460a-b8ee-9f4ade52b633", - "title": "Data stream usage by mode and codec", + "panelIndex": "c8f29a41-6d3e-4b2a-9f71-2d4e6b8a0c15", + "title": "Primary ingest by codec over time", "type": "lens" } ], @@ -7400,7 +7399,7 @@ }, { "id": "elasticsearch-monitoringindices", - "name": "e428330c-1454-460a-b8ee-9f4ade52b633:indexpattern-datasource-layer-587c1332-dec1-4d27-ad36-beaf4c9a5861", + "name": "c8f29a41-6d3e-4b2a-9f71-2d4e6b8a0c15:indexpattern-datasource-layer-d7e18b52-7e4f-5c3b-a062-3e5f7c9b1d26", "type": "index-pattern" } ], diff --git a/packages/elasticsearch/kibana/dashboard/elasticsearch-ea888f80-61e4-11ee-b5a1-0d1803efe5cf.json b/packages/elasticsearch/kibana/dashboard/elasticsearch-ea888f80-61e4-11ee-b5a1-0d1803efe5cf.json index a347e8b95b6..13038ae3951 100644 --- a/packages/elasticsearch/kibana/dashboard/elasticsearch-ea888f80-61e4-11ee-b5a1-0d1803efe5cf.json +++ b/packages/elasticsearch/kibana/dashboard/elasticsearch-ea888f80-61e4-11ee-b5a1-0d1803efe5cf.json @@ -1,18 +1,67 @@ { "attributes": { + "controlGroupInput": { + "chainingSystem": "HIERARCHICAL", + "controlStyle": "oneLine", + "ignoreParentSettingsJSON": { + "ignoreFilters": false, + "ignoreQuery": false, + "ignoreTimerange": false, + "ignoreValidations": false + }, + "panelsJSON": { + "1bd6769a-f2ec-46d7-a23d-d4eab4ec0800": { + "explicitInput": { + "dataViewId": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", + "existsSelected": false, + "fieldName": "elasticsearch.index.name", + "id": "1bd6769a-f2ec-46d7-a23d-d4eab4ec0800", + "searchTechnique": "wildcard", + "selectedOptions": [], + "sort": { + "by": "_count", + "direction": "desc" + }, + "title": "Index" + }, + "grow": true, + "order": 1, + "type": "optionsListControl", + "width": "medium" + }, + "60cae87b-d33a-45e2-9aac-ff1e83d6811e": { + "explicitInput": { + "dataViewId": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", + "existsSelected": false, + "fieldName": "cluster_uuid", + "id": "60cae87b-d33a-45e2-9aac-ff1e83d6811e", + "searchTechnique": "wildcard", + "selectedOptions": [], + "sort": { + "by": "_count", + "direction": "desc" + }, + "title": "Cluster" + }, + "grow": true, + "order": 0, + "type": "optionsListControl", + "width": "medium" + } + }, + "showApplySelections": false + }, "description": "", "kibanaSavedObjectMeta": { "searchSourceJSON": { + "filter": [], "query": { "language": "kuery", "query": "" - }, - "filter": [] + } } }, "optionsJSON": { - "autoApplyFilters": true, - "hidePanelBorders": false, "hidePanelTitles": false, "syncColors": false, "syncCursor": true, @@ -247,20 +296,19 @@ "version": 2, "visualizationType": "lnsXY" }, - "hide_title": false, - "title": "Storage Delta Per Day (Primaries)" + "enhancements": {}, + "hide_title": false }, "gridData": { "h": 15, - "i": "34d6cc78-b35e-4e65-a124-c2a437288204", + "i": "47e0c70a-9115-4cf5-9fa9-020bcfa22074", "w": 42, "x": 6, "y": 0 }, - "panelIndex": "34d6cc78-b35e-4e65-a124-c2a437288204", - "type": "lens", + "panelIndex": "47e0c70a-9115-4cf5-9fa9-020bcfa22074", "title": "Storage Delta Per Day (Primaries)", - "version": "8.10.2" + "type": "lens" }, { "embeddableConfig": { @@ -538,20 +586,19 @@ "version": 2, "visualizationType": "lnsXY" }, - "hide_title": false, - "title": "Cluster Total Size per Index or Data stream (primaries+replicas) " + "enhancements": {}, + "hide_title": false }, "gridData": { "h": 15, - "i": "071e8dc8-55d2-4385-89af-d9c9b4678ee5", + "i": "2fb0c28f-441a-4061-b18f-e25c0fe50ea8", "w": 42, "x": 6, "y": 15 }, - "panelIndex": "071e8dc8-55d2-4385-89af-d9c9b4678ee5", - "type": "lens", + "panelIndex": "2fb0c28f-441a-4061-b18f-e25c0fe50ea8", "title": "Cluster Total Size per Index or Data stream (primaries+replicas) ", - "version": "8.10.2" + "type": "lens" }, { "embeddableConfig": { @@ -793,20 +840,19 @@ "version": 2, "visualizationType": "lnsXY" }, - "hide_title": false, - "title": "Storage Delta Per Index over time (Primaries)" + "enhancements": {}, + "hide_title": false }, "gridData": { "h": 16, - "i": "130521f4-364a-485a-ac0f-8ca29801bae7", + "i": "3b9c7c1d-d838-429a-8151-41fe19ed91ea", "w": 42, "x": 6, "y": 30 }, - "panelIndex": "130521f4-364a-485a-ac0f-8ca29801bae7", - "type": "lens", + "panelIndex": "3b9c7c1d-d838-429a-8151-41fe19ed91ea", "title": "Storage Delta Per Index over time (Primaries)", - "version": "8.10.2" + "type": "lens" }, { "embeddableConfig": { @@ -1044,20 +1090,19 @@ "version": 2, "visualizationType": "lnsDatatable" }, - "hide_title": false, - "title": "Cluster Total Size per Index (primaries+replicas) " + "enhancements": {}, + "hide_title": false }, "gridData": { "h": 29, - "i": "df1fdff3-464a-42cc-8a26-899e05487243", + "i": "a159a9e9-f4bc-4901-a795-fb0485d0850f", "w": 42, "x": 6, "y": 46 }, - "panelIndex": "df1fdff3-464a-42cc-8a26-899e05487243", - "type": "lens", + "panelIndex": "a159a9e9-f4bc-4901-a795-fb0485d0850f", "title": "Cluster Total Size per Index (primaries+replicas) ", - "version": "8.10.2" + "type": "lens" }, { "embeddableConfig": { @@ -1321,20 +1366,19 @@ }, "description": "Index mode and codec are normalized to standard/default at ingest when omitted. For pivot-normalized views, use the Consumption dashboard.", "drilldowns": [], - "hide_title": false, - "title": "Size by index mode and codec" + "enhancements": {}, + "hide_title": false }, "gridData": { "h": 10, - "i": "c15416e6-dbe1-4b3b-8e85-3ed0676d7bf4", + "i": "696750ba-b167-4914-b1db-0a4697b6fb39", "w": 21, "x": 6, "y": 75 }, - "panelIndex": "c15416e6-dbe1-4b3b-8e85-3ed0676d7bf4", - "type": "lens", + "panelIndex": "696750ba-b167-4914-b1db-0a4697b6fb39", "title": "Size by index mode and codec", - "version": "8.10.2" + "type": "lens" }, { "embeddableConfig": { @@ -1556,20 +1600,19 @@ }, "description": "Index mode and codec are normalized to standard/default at ingest when omitted. For pivot-normalized views, use the Consumption dashboard.", "drilldowns": [], - "hide_title": false, - "title": "Distribution of index mode and codec" + "enhancements": {}, + "hide_title": false }, "gridData": { "h": 10, - "i": "3563f2c3-cfec-4108-ab53-6b91f9056d72", + "i": "297fea31-ae92-480a-b181-7edf659f9e01", "w": 21, "x": 27, "y": 75 }, - "panelIndex": "3563f2c3-cfec-4108-ab53-6b91f9056d72", - "type": "lens", + "panelIndex": "297fea31-ae92-480a-b181-7edf659f9e01", "title": "Distribution of index mode and codec", - "version": "8.10.2" + "type": "lens" }, { "embeddableConfig": { @@ -1579,11 +1622,6 @@ "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", "name": "indexpattern-datasource-layer-d41f5e41-9bf4-4d1c-9b83-e584dc9512f2", "type": "index-pattern" - }, - { - "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "a0316cd9-7b90-434c-a176-124b88ef8a78", - "type": "index-pattern" } ], "state": { @@ -1601,9 +1639,10 @@ ], "columns": { "084b5edd-ed2e-4843-a226-1a58cc877ee9": { + "customLabel": true, "dataType": "string", "isBucketed": true, - "label": "Top 5 values of elasticsearch.index.codec", + "label": "Index codec", "operationType": "terms", "params": { "exclude": [], @@ -1620,15 +1659,16 @@ "parentFormat": { "id": "terms" }, - "size": 5 + "size": 10 }, "scale": "ordinal", "sourceField": "elasticsearch.index.codec" }, "0c10e371-9c9a-4e88-b760-d7b0d91b03c3": { + "customLabel": true, "dataType": "string", "isBucketed": true, - "label": "Top 5 values of elasticsearch.index.mode", + "label": "Index mode", "operationType": "terms", "params": { "exclude": [], @@ -1645,7 +1685,7 @@ "parentFormat": { "id": "terms" }, - "size": 5 + "size": 10 }, "scale": "ordinal", "sourceField": "elasticsearch.index.mode" @@ -1669,9 +1709,10 @@ "sourceField": "elasticsearch.index.primaries.store.size_in_bytes" }, "e2397c20-e16d-4ec2-b581-40a6e57a7e60": { + "customLabel": true, "dataType": "string", "isBucketed": true, - "label": "Top 50 values of elasticsearch.index.name", + "label": "Index name", "operationType": "terms", "params": { "exclude": [], @@ -1688,7 +1729,7 @@ "parentFormat": { "id": "terms" }, - "size": 50 + "size": 500 }, "scale": "ordinal", "sourceField": "elasticsearch.index.name" @@ -1772,12 +1813,12 @@ { "columnId": "e2397c20-e16d-4ec2-b581-40a6e57a7e60", "isTransposed": false, - "width": 448 + "width": 694 }, { "columnId": "0c10e371-9c9a-4e88-b760-d7b0d91b03c3", "isTransposed": false, - "width": 374 + "width": 221 }, { "columnId": "084b5edd-ed2e-4843-a226-1a58cc877ee9", @@ -1797,24 +1838,24 @@ } }, "title": "", + "type": "lens", "version": 2, "visualizationType": "lnsDatatable" }, "description": "Index mode and codec are normalized to standard/default at ingest when omitted. For pivot-normalized views, use the Consumption dashboard.", - "hide_title": false, - "title": "Index mode and codec by index" + "enhancements": {}, + "hide_title": false }, "gridData": { "h": 20, - "i": "5a017cba-8244-4277-900b-063210bef9fb", + "i": "8dc4d93f-0385-47dc-b515-28ceeece80e2", "w": 42, "x": 6, "y": 95 }, - "panelIndex": "5a017cba-8244-4277-900b-063210bef9fb", - "type": "lens", + "panelIndex": "8dc4d93f-0385-47dc-b515-28ceeece80e2", "title": "Index mode and codec by index", - "version": "8.10.2" + "type": "lens" }, { "embeddableConfig": { @@ -2047,20 +2088,19 @@ }, "description": "Index mode and codec are normalized to standard/default at ingest when omitted. ", "drilldowns": [], - "hide_title": false, - "title": "Index mode over time" + "enhancements": {}, + "hide_title": false }, "gridData": { "h": 10, - "i": "7f00c828-c4d8-4199-bc71-d399f7e9b2ac", + "i": "465d55d4-c984-4788-bc9f-c7de1e58891a", "w": 21, "x": 6, "y": 85 }, - "panelIndex": "7f00c828-c4d8-4199-bc71-d399f7e9b2ac", - "type": "lens", + "panelIndex": "465d55d4-c984-4788-bc9f-c7de1e58891a", "title": "Index mode over time", - "version": "8.10.2" + "type": "lens" }, { "embeddableConfig": { @@ -2292,24 +2332,27 @@ }, "description": "Index mode and codec are normalized to standard/default at ingest when omitted. ", "drilldowns": [], - "hide_title": false, - "title": "Index codec over time" + "enhancements": {}, + "hide_title": false }, "gridData": { "h": 10, - "i": "67d9b414-bd97-4726-853a-9737ce5eb68b", + "i": "bfe159d6-cd30-4fef-8c92-212e3605974a", "w": 21, "x": 27, "y": 85 }, - "panelIndex": "67d9b414-bd97-4726-853a-9737ce5eb68b", - "type": "lens", + "panelIndex": "bfe159d6-cd30-4fef-8c92-212e3605974a", "title": "Index codec over time", - "version": "8.10.2" + "type": "lens" }, { "embeddableConfig": { - "hide_title": false, + "enhancements": { + "dynamicActions": { + "events": [] + } + }, "savedVis": { "data": { "aggs": [], @@ -2331,124 +2374,112 @@ "title": "", "type": "markdown", "uiState": {} - }, - "title": "Table of Contents" + } }, "gridData": { "h": 115, - "i": "03b8b02a-4f4a-41c3-a2ea-1f8155400209", + "i": "2fe2fa42-46a7-4c87-9c0b-749a40d2fd94", "w": 6, "x": 0, "y": 0 }, - "panelIndex": "03b8b02a-4f4a-41c3-a2ea-1f8155400209", - "type": "visualization", + "panelIndex": "2fe2fa42-46a7-4c87-9c0b-749a40d2fd94", "title": "Table of Contents", - "version": "8.10.2" + "type": "visualization" } ], "refreshInterval": { "pause": true, - "value": 0 + "value": 1000 }, "timeFrom": "now-7d", "timeRestore": true, "timeTo": "now", "title": "[Elasticsearch] Cluster Ingest Per Day - Index & Shard View (Technical Preview/Beta)", - "controlGroupInput": { - "chainingSystem": "HIERARCHICAL", - "controlStyle": "oneLine", - "ignoreParentSettingsJSON": "{\"ignoreFilters\":false,\"ignoreQuery\":false,\"ignoreTimerange\":false,\"ignoreValidations\":false}", - "panelsJSON": "{\"37959a78-8ed3-4ce7-94e1-469e2364810d\":{\"order\":1,\"width\":\"medium\",\"grow\":true,\"type\":\"optionsListControl\",\"explicitInput\":{\"id\":\"37959a78-8ed3-4ce7-94e1-469e2364810d\",\"fieldName\":\"elasticsearch.index.name\",\"title\":\"Index\",\"selectedOptions\":[],\"enhancements\":{},\"existsSelected\":false,\"searchTechnique\":\"wildcard\"}},\"d9fe8897-ed5c-4eb1-9d88-884570d21c64\":{\"order\":0,\"width\":\"medium\",\"grow\":true,\"type\":\"optionsListControl\",\"explicitInput\":{\"id\":\"d9fe8897-ed5c-4eb1-9d88-884570d21c64\",\"fieldName\":\"cluster_uuid\",\"title\":\"Cluster\",\"selectedOptions\":[],\"enhancements\":{},\"existsSelected\":false,\"searchTechnique\":\"wildcard\"}}}" - } + "version": 3 }, "coreMigrationVersion": "8.8.0", - "created_at": "2023-12-11T10:43:01.942Z", + "created_at": "2026-06-15T07:33:37.385Z", "id": "elasticsearch-ea888f80-61e4-11ee-b5a1-0d1803efe5cf", "references": [ { "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "34d6cc78-b35e-4e65-a124-c2a437288204:indexpattern-datasource-layer-585d0337-667f-4573-b541-8e961e8f907d", - "type": "index-pattern" - }, - { - "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "34d6cc78-b35e-4e65-a124-c2a437288204:55b63032-cc90-44e6-a707-b8fe8480d1c2", + "name": "47e0c70a-9115-4cf5-9fa9-020bcfa22074:indexpattern-datasource-layer-585d0337-667f-4573-b541-8e961e8f907d", "type": "index-pattern" }, { "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "071e8dc8-55d2-4385-89af-d9c9b4678ee5:indexpattern-datasource-layer-b60c302b-ed86-4fef-9bd6-bd46bf8bcb7c", + "name": "47e0c70a-9115-4cf5-9fa9-020bcfa22074:55b63032-cc90-44e6-a707-b8fe8480d1c2", "type": "index-pattern" }, { "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "071e8dc8-55d2-4385-89af-d9c9b4678ee5:indexpattern-datasource-layer-3568d57f-7386-4a99-a5dc-c9a0f35f0434", + "name": "2fb0c28f-441a-4061-b18f-e25c0fe50ea8:indexpattern-datasource-layer-b60c302b-ed86-4fef-9bd6-bd46bf8bcb7c", "type": "index-pattern" }, { "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "071e8dc8-55d2-4385-89af-d9c9b4678ee5:ef48d851-fffa-49d1-a25a-a2e97b26790e", + "name": "2fb0c28f-441a-4061-b18f-e25c0fe50ea8:indexpattern-datasource-layer-3568d57f-7386-4a99-a5dc-c9a0f35f0434", "type": "index-pattern" }, { "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "130521f4-364a-485a-ac0f-8ca29801bae7:indexpattern-datasource-layer-585d0337-667f-4573-b541-8e961e8f907d", + "name": "2fb0c28f-441a-4061-b18f-e25c0fe50ea8:ef48d851-fffa-49d1-a25a-a2e97b26790e", "type": "index-pattern" }, { "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "130521f4-364a-485a-ac0f-8ca29801bae7:6043994f-b33e-4a86-94e9-dcacabf0d5d4", + "name": "3b9c7c1d-d838-429a-8151-41fe19ed91ea:indexpattern-datasource-layer-585d0337-667f-4573-b541-8e961e8f907d", "type": "index-pattern" }, { "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "df1fdff3-464a-42cc-8a26-899e05487243:indexpattern-datasource-layer-b60c302b-ed86-4fef-9bd6-bd46bf8bcb7c", + "name": "3b9c7c1d-d838-429a-8151-41fe19ed91ea:6043994f-b33e-4a86-94e9-dcacabf0d5d4", "type": "index-pattern" }, { "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "df1fdff3-464a-42cc-8a26-899e05487243:d45af8cc-e848-4f84-b60a-1ca17691ee01", + "name": "a159a9e9-f4bc-4901-a795-fb0485d0850f:indexpattern-datasource-layer-b60c302b-ed86-4fef-9bd6-bd46bf8bcb7c", "type": "index-pattern" }, { "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "c15416e6-dbe1-4b3b-8e85-3ed0676d7bf4:indexpattern-datasource-layer-d41f5e41-9bf4-4d1c-9b83-e584dc9512f2", + "name": "a159a9e9-f4bc-4901-a795-fb0485d0850f:d45af8cc-e848-4f84-b60a-1ca17691ee01", "type": "index-pattern" }, { "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "3563f2c3-cfec-4108-ab53-6b91f9056d72:indexpattern-datasource-layer-d41f5e41-9bf4-4d1c-9b83-e584dc9512f2", + "name": "696750ba-b167-4914-b1db-0a4697b6fb39:indexpattern-datasource-layer-d41f5e41-9bf4-4d1c-9b83-e584dc9512f2", "type": "index-pattern" }, { "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "5a017cba-8244-4277-900b-063210bef9fb:indexpattern-datasource-layer-d41f5e41-9bf4-4d1c-9b83-e584dc9512f2", + "name": "297fea31-ae92-480a-b181-7edf659f9e01:indexpattern-datasource-layer-d41f5e41-9bf4-4d1c-9b83-e584dc9512f2", "type": "index-pattern" }, { "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "5a017cba-8244-4277-900b-063210bef9fb:a0316cd9-7b90-434c-a176-124b88ef8a78", + "name": "8dc4d93f-0385-47dc-b515-28ceeece80e2:indexpattern-datasource-layer-d41f5e41-9bf4-4d1c-9b83-e584dc9512f2", "type": "index-pattern" }, { "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "7f00c828-c4d8-4199-bc71-d399f7e9b2ac:indexpattern-datasource-layer-d41f5e41-9bf4-4d1c-9b83-e584dc9512f2", + "name": "465d55d4-c984-4788-bc9f-c7de1e58891a:indexpattern-datasource-layer-d41f5e41-9bf4-4d1c-9b83-e584dc9512f2", "type": "index-pattern" }, { "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "67d9b414-bd97-4726-853a-9737ce5eb68b:indexpattern-datasource-layer-d41f5e41-9bf4-4d1c-9b83-e584dc9512f2", + "name": "bfe159d6-cd30-4fef-8c92-212e3605974a:indexpattern-datasource-layer-d41f5e41-9bf4-4d1c-9b83-e584dc9512f2", "type": "index-pattern" }, { "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "controlGroup_d9fe8897-ed5c-4eb1-9d88-884570d21c64:optionsListDataView", + "name": "controlGroup_ce6e7f28-bbd3-43e5-8666-e7c783c26ad9:optionsListDataView", "type": "index-pattern" }, { "id": "befe6dd7-ec0b-4cb7-aa59-e4d5e6f39ae9", - "name": "controlGroup_37959a78-8ed3-4ce7-94e1-469e2364810d:optionsListDataView", + "name": "controlGroup_af8fa402-bd67-4415-90dc-e291163ff3cf:optionsListDataView", "type": "index-pattern" } ],