ess_billing: fix serverless project schema and dashboard breakdowns

packages/ess_billing/changelog.yml

@@ -1,4 +1,12 @@
 # newer versions go on top
+- version: "1.8.0"
+  changes:
+    - description: Fix serverless billing schema — remove null kind field, add deployment_name fallback for serverless projects.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/99999
+    - description: Add include_names query parameter to billing API requests to populate deployment names for serverless projects.
+      type: bugfix
+      link: https://github.com/elastic/integrations/pull/99999
 - version: "1.7.0"
   changes:
     - description: Added deployment tags for ECH

packages/ess_billing/data_stream/billing/_dev/test/pipeline/test-billing.json

@@ -125,6 +125,69 @@
         "forwarded",
         "billing"
       ]
+    },
+    {
+      "@timestamp": "2025-08-20T14:08:02.405Z",
+      "agent": {
+        "ephemeral_id": "082a402e-e376-4e6a-8ae4-9fcd12471589",
+        "id": "e4875ac3-72d6-4ad5-87bc-1e0030128d0a",
+        "name": "elastic-agent-89987",
+        "type": "filebeat",
+        "version": "9.0.4"
+      },
+      "data_stream": {
+        "dataset": "ess_billing.billing",
+        "namespace": "68465",
+        "type": "metrics"
+      },
+      "ecs": {
+        "version": "8.0.0"
+      },
+      "elastic_agent": {
+        "id": "e4875ac3-72d6-4ad5-87bc-1e0030128d0a",
+        "snapshot": false,
+        "version": "9.0.4"
+      },
+      "ess": {
+        "billing": {
+          "deployment_id": "aaa111bbb222ccc333",
+          "deployment_name": "",
+          "deployment_type": "observability",
+          "display_quantity": {
+            "value": 0.025,
+            "formatted_value": "0.025 GBs",
+            "type": "default"
+          },
+          "from": "2025-08-19T00:00:00Z",
+          "kind": null,
+          "name": "Observability Complete - Data Retention (AWS us-east-1)",
+          "organization_id": "1111",
+          "quantities": [],
+          "quantity": {
+            "value": 0.025,
+            "formatted_value": "0.025 GBs"
+          },
+          "rate": {
+            "value": 0.04,
+            "formatted_value": "0.0400 per GB"
+          },
+          "sku": "observability.observability-complete-retained_aws-us-east-1",
+          "to": "2025-08-20T00:00:00Z",
+          "total_ecu": 0.001,
+          "type": "Search AI Lake",
+          "unit": "GB"
+        }
+      },
+      "event": {
+        "dataset": "ess_billing.billing"
+      },
+      "input": {
+        "type": "cel"
+      },
+      "tags": [
+        "forwarded",
+        "billing"
+      ]
     }
   ]
 }

packages/ess_billing/data_stream/billing/_dev/test/pipeline/test-billing.json-expected.json

@@ -166,6 +166,88 @@
             "tags": [
                 "preserve_original_event"
             ]
+        },
+        {
+            "@timestamp": "2025-08-19T00:00:00.000Z",
+            "agent": {
+                "ephemeral_id": "082a402e-e376-4e6a-8ae4-9fcd12471589",
+                "id": "e4875ac3-72d6-4ad5-87bc-1e0030128d0a",
+                "name": "elastic-agent-89987",
+                "type": "filebeat",
+                "version": "9.0.4"
+            },
+            "cloud": {
+                "account": {
+                    "id": "1111"
+                },
+                "geo": {
+                    "location": {
+                        "lat": 39.9612,
+                        "lon": -77.0369
+                    },
+                    "name": "aws-us-east-2"
+                },
+                "instance": {
+                    "id": "aaa111bbb222ccc333",
+                    "name": "aaa111bbb222ccc333"
+                },
+                "provider": "serverless",
+                "region": "aws-us-east-1",
+                "service": {
+                    "type": "observability-complete-retained"
+                }
+            },
+            "data_stream": {
+                "dataset": "ess_billing.billing",
+                "namespace": "68465",
+                "type": "metrics"
+            },
+            "ecs": {
+                "version": "8.0.0"
+            },
+            "elastic_agent": {
+                "id": "e4875ac3-72d6-4ad5-87bc-1e0030128d0a",
+                "snapshot": false,
+                "version": "9.0.4"
+            },
+            "ess": {
+                "billing": {
+                    "deployment_id": "aaa111bbb222ccc333",
+                    "deployment_name": "aaa111bbb222ccc333",
+                    "deployment_type": "observability",
+                    "display_quantity": {
+                        "formatted_value": "0.025 GBs",
+                        "type": "default",
+                        "value": 0.025
+                    },
+                    "from": "2025-08-19T00:00:00Z",
+                    "name": "Observability Complete - Data Retention (AWS us-east-1)",
+                    "organization_id": "1111",
+                    "quantity": {
+                        "formatted_value": "0.025 GBs",
+                        "value": 0.025
+                    },
+                    "rate": {
+                        "formatted_value": "0.0400 per GB",
+                        "value": 0.04
+                    },
+                    "sku": "observability.observability-complete-retained_aws-us-east-1",
+                    "to": "2025-08-20T00:00:00Z",
+                    "total_ecu": 0.001,
+                    "type": "Search AI Lake",
+                    "unit": "GB"
+                }
+            },
+            "event": {
+                "created": "2021-11-11T01:02:03.123456789Z",
+                "dataset": "ess_billing.billing"
+            },
+            "input": {
+                "type": "cel"
+            },
+            "tags": [
+                "preserve_original_event"
+            ]
         }
     ]
 }

packages/ess_billing/data_stream/billing/agent/stream/cel.yml.hbs

@@ -88,6 +88,7 @@ program: |-
               state.url + "?" + {
                 "from": [req.from.format(time_layout.RFC3339)],
                 "to": [req.to.format(time_layout.RFC3339)],
+                "include_names": ["true"],
               }.format_query()
             ).with(
               { "Header": { "Authorization": ["ApiKey " + state.api_key] } }
@@ -167,6 +168,7 @@ program: |-
           state.url + "?" + {
             "from": [req.from.format(time_layout.RFC3339)],
             "to": [req.to.format(time_layout.RFC3339)],
+            "include_names": ["true"],
           }.format_query()
         ).with(
           {

packages/ess_billing/data_stream/billing/elasticsearch/ingest_pipeline/default.yml

@@ -25,6 +25,11 @@ processors:
   - remove:
       field: ess.billing.quantities
       ignore_missing: true
+  - remove:
+      field: ess.billing.kind
+      if: "ctx.ess?.billing?.kind == null"
+      ignore_missing: true
+      tag: remove_null_kind
   - set:
       field: event.created
       copy_from: '@timestamp'
@@ -59,6 +64,11 @@ processors:
       # project names being empty in the billing API. to be removed when Names in billing api fixed
       field: cloud.instance.name
       copy_from: ess.billing.deployment_id
+  - set:
+      field: ess.billing.deployment_name
+      copy_from: ess.billing.deployment_id
+      if: "ctx.ess?.billing?.deployment_name != null && ctx.ess.billing.deployment_name == ''"
+      tag: fallback_deployment_name
   - set:
       field: cloud.instance.name
       copy_from: ess.billing.deployment_name