bugfix: expose prefix cache usage across APIs.

tests/xllm_service/CMakeLists.txt

@@ -1,5 +1,6 @@
 add_subdirectory(chat_template)
 add_subdirectory(common)
 add_subdirectory(http_service)
+add_subdirectory(rpc_service)
 add_subdirectory(scheduler)
 add_subdirectory(tokenizer)

tests/xllm_service/http_service/CMakeLists.txt

@@ -1,5 +1,15 @@
 include(cc_test)
 
+cc_test(
+  NAME
+    usage_proto_adapter_test
+  SRCS
+    usage_proto_adapter_test.cpp
+  DEPS
+    :usage_proto_adapter
+    GTest::gtest_main
+)
+
 cc_test(
   NAME
     chat_json_parser_test

tests/xllm_service/http_service/anthropic_adapter_test.cpp

@@ -807,6 +807,7 @@ TEST(AnthropicAdapterTest, BuildsNonStreamAnthropicJson) {
   usage.num_prompt_tokens = 3;
   usage.num_generated_tokens = 4;
   usage.num_total_tokens = 7;
+  usage.num_cached_tokens = 2;
   output.usage = usage;
 
   xllm::proto::AnthropicMessagesResponse response;
@@ -826,11 +827,35 @@ TEST(AnthropicAdapterTest, BuildsNonStreamAnthropicJson) {
   ASSERT_EQ(json["content"].size(), 1);
   EXPECT_EQ(json["content"][0]["type"], "text");
   EXPECT_EQ(json["content"][0]["text"], "answer");
-  EXPECT_EQ(json["usage"]["input_tokens"], 3);
+  EXPECT_EQ(json["usage"]["input_tokens"], 1);
   EXPECT_EQ(json["usage"]["output_tokens"], 4);
+  EXPECT_EQ(json["usage"]["cache_read_input_tokens"], 2);
+  EXPECT_FALSE(json["usage"].contains("cache_creation_input_tokens"));
   EXPECT_FALSE(json["usage"].contains("total_tokens"));
 }
 
+TEST(AnthropicAdapterTest, BuildsZeroCacheReadUsage) {
+  llm::RequestOutput output;
+  output.request_id = "anthropiccmpl-test";
+  llm::Usage usage;
+  usage.num_prompt_tokens = 3;
+  output.usage = usage;
+
+  xllm::proto::AnthropicMessagesResponse response;
+  auto result = fill_anthropic_resp("test-model", output, &response);
+  ASSERT_TRUE(result.ok) << result.error;
+
+  std::string json_str;
+  std::string error;
+  ASSERT_TRUE(anthropic_json(response, &json_str, &error)) << error;
+  auto json = nlohmann::json::parse(json_str);
+
+  ASSERT_TRUE(json["usage"].contains("cache_read_input_tokens"));
+  EXPECT_EQ(json["usage"]["input_tokens"], 3);
+  EXPECT_EQ(json["usage"]["cache_read_input_tokens"], 0);
+  EXPECT_FALSE(json["usage"].contains("cache_creation_input_tokens"));
+}
+
 TEST(AnthropicAdapterTest, BuildsThinkingNonStreamAnthropicJson) {
   llm::RequestOutput output;
   output.request_id = "anthropiccmpl-test";

tests/xllm_service/http_service/anthropic_stream_encoder_test.cpp

@@ -243,6 +243,7 @@ TEST(AnthropicStreamEncoderTest, FinishClosesBlockAndEmitsMessageDeltaStop) {
   llm::Usage usage;
   usage.num_prompt_tokens = 3;
   usage.num_generated_tokens = 5;
+  usage.num_cached_tokens = 2;
   final.usage = usage;
 
   std::vector<std::string> done;
@@ -254,8 +255,9 @@ TEST(AnthropicStreamEncoderTest, FinishClosesBlockAndEmitsMessageDeltaStop) {
   EXPECT_EQ(events[0]["index"], 0);
   EXPECT_EQ(events[1]["type"], "message_delta");
   EXPECT_EQ(events[1]["delta"]["stop_reason"], "end_turn");
-  EXPECT_EQ(events[1]["usage"]["input_tokens"], 3);
+  EXPECT_EQ(events[1]["usage"]["input_tokens"], 1);
   EXPECT_EQ(events[1]["usage"]["output_tokens"], 5);
+  EXPECT_EQ(events[1]["usage"]["cache_read_input_tokens"], 2);
   EXPECT_EQ(events[2]["type"], "message_stop");
 }
 
@@ -273,6 +275,9 @@ TEST(AnthropicStreamEncoderTest, FinishAfterToolUsesToolStopReason) {
   seq.index = 0;
   seq.finish_reason = "stop";
   final.outputs.push_back(std::move(seq));
+  llm::Usage usage;
+  usage.num_prompt_tokens = 3;
+  final.usage = usage;
 
   std::vector<std::string> done;
   ASSERT_TRUE(encoder.finish(final, &done).ok);
@@ -281,6 +286,10 @@ TEST(AnthropicStreamEncoderTest, FinishAfterToolUsesToolStopReason) {
   EXPECT_EQ(events[0]["type"], "content_block_stop");
   EXPECT_EQ(events[1]["type"], "message_delta");
   EXPECT_EQ(events[1]["delta"]["stop_reason"], "tool_use");
+  ASSERT_TRUE(events[1]["usage"].contains("cache_read_input_tokens"));
+  EXPECT_EQ(events[1]["usage"]["input_tokens"], 3);
+  EXPECT_EQ(events[1]["usage"]["cache_read_input_tokens"], 0);
+  EXPECT_FALSE(events[1]["usage"].contains("cache_creation_input_tokens"));
   EXPECT_EQ(events[2]["type"], "message_stop");
 }
 

tests/xllm_service/http_service/usage_proto_adapter_test.cpp

@@ -0,0 +1,91 @@
+/* Copyright 2025-2026 The xLLM Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://github.com/jd-opensource/xllm-service/blob/main/LICENSE
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "http_service/usage_proto_adapter.h"
+
+#include <gtest/gtest.h>
+
+#include <cstddef>
+#include <vector>
+
+namespace xllm_service {
+namespace {
+
+struct UsageCase {
+  size_t num_prompt_tokens;
+  size_t num_generated_tokens;
+  size_t num_total_tokens;
+  size_t num_cached_tokens;
+};
+
+const std::vector<UsageCase> kUsageCases = {
+    {/*num_prompt_tokens=*/8,
+     /*num_generated_tokens=*/0,
+     /*num_total_tokens=*/8,
+     /*num_cached_tokens=*/0},
+    {/*num_prompt_tokens=*/8,
+     /*num_generated_tokens=*/2,
+     /*num_total_tokens=*/10,
+     /*num_cached_tokens=*/3},
+    {/*num_prompt_tokens=*/8,
+     /*num_generated_tokens=*/2,
+     /*num_total_tokens=*/10,
+     /*num_cached_tokens=*/8},
+};
+
+llm::Usage make_usage(const UsageCase& usage_case) {
+  llm::Usage usage;
+  usage.num_prompt_tokens = usage_case.num_prompt_tokens;
+  usage.num_generated_tokens = usage_case.num_generated_tokens;
+  usage.num_total_tokens = usage_case.num_total_tokens;
+  usage.num_cached_tokens = usage_case.num_cached_tokens;
+  return usage;
+}
+
+TEST(UsageProtoAdapterTest, ConvertsPrefixCacheHitsToOpenAIPromptTokenDetails) {
+  for (const UsageCase& usage_case : kUsageCases) {
+    xllm::proto::Usage proto_usage =
+        to_openai_usage_proto(make_usage(usage_case));
+
+    EXPECT_EQ(proto_usage.prompt_tokens(), usage_case.num_prompt_tokens);
+    EXPECT_EQ(proto_usage.completion_tokens(), usage_case.num_generated_tokens);
+    EXPECT_EQ(proto_usage.total_tokens(), usage_case.num_total_tokens);
+    ASSERT_TRUE(proto_usage.has_prompt_tokens_details());
+    EXPECT_TRUE(proto_usage.prompt_tokens_details().has_cached_tokens());
+    EXPECT_EQ(proto_usage.prompt_tokens_details().cached_tokens(),
+              usage_case.num_cached_tokens);
+  }
+}
+
+TEST(UsageProtoAdapterTest,
+     SplitsAnthropicInputTokensIntoUncachedAndCacheReadTokens) {
+  for (const UsageCase& usage_case : kUsageCases) {
+    xllm::proto::AnthropicUsage proto_usage =
+        to_anthropic_usage_proto(make_usage(usage_case));
+
+    EXPECT_EQ(proto_usage.input_tokens(),
+              usage_case.num_prompt_tokens - usage_case.num_cached_tokens);
+    EXPECT_EQ(proto_usage.cache_read_input_tokens(),
+              usage_case.num_cached_tokens);
+    EXPECT_EQ(proto_usage.output_tokens(), usage_case.num_generated_tokens);
+    EXPECT_EQ(
+        proto_usage.input_tokens() + proto_usage.cache_read_input_tokens(),
+        usage_case.num_prompt_tokens);
+    EXPECT_FALSE(proto_usage.has_cache_creation_input_tokens());
+  }
+}
+
+}  // namespace
+}  // namespace xllm_service

tests/xllm_service/rpc_service/CMakeLists.txt

@@ -0,0 +1,15 @@
+include(cc_test)
+
+cc_test(
+  NAME
+    xllm_rpc_service_test
+  SRCS
+    rpc_service_test.cpp
+  DEPS
+    :disagg_generation_adapter
+    :xllm_rpc_service
+    gflags::gflags
+    glog::glog
+    GTest::gtest_main
+    proto_xllm
+)