From 7c3d7501f2a67cc5760829fe9090ddde9b2e3be6 Mon Sep 17 00:00:00 2001 From: Jiangtian Feng Date: Sat, 30 May 2026 23:50:56 +0800 Subject: [PATCH 1/3] chore(sight): drop dead code + deprecated APIs Hygiene pass over the crate (no behavior change): - Delete compiler-confirmed dead code: the orphaned token-computation cluster in analyzer/unified.rs (compute_prompt_tokens / compute_token_consumption_with_template / compute_output_token_breakdown_from_json, ~265 lines), the unused comm_to_string in sslsniff.rs, and get_pending_calls_for_pid in health/checker. - Remove two `todo!()` public stubs in tokenizer/llm_tok.rs (from_url, messages_to_json) that would panic if ever called and had no callers. - Replace deprecated base64::encode with the Engine API (base64 0.22). - Make AgentSight::set_ffi_sender pub(crate) so it no longer leaks the crate-internal FfiEventSender type. - Drop unused imports (via cargo fix) and silence a few unused-variable bindings. Co-Authored-By: Claude Opus 4.8 --- src/agentsight/src/aggregator/http2.rs | 3 +- src/agentsight/src/aggregator/unified.rs | 2 +- .../src/analyzer/token/extractor/mod.rs | 3 +- src/agentsight/src/analyzer/unified.rs | 271 +----------------- src/agentsight/src/atif/converter.rs | 2 +- src/agentsight/src/bin/cli/token.rs | 1 - src/agentsight/src/genai/builder.rs | 2 +- src/agentsight/src/health/checker.rs | 20 -- src/agentsight/src/parser/http/request.rs | 2 +- src/agentsight/src/parser/http/response.rs | 2 +- src/agentsight/src/parser/proctrace.rs | 2 +- src/agentsight/src/parser/sse/event.rs | 2 +- src/agentsight/src/parser/unified.rs | 2 +- src/agentsight/src/probes/proctrace.rs | 2 +- src/agentsight/src/probes/sslsniff.rs | 13 +- src/agentsight/src/server/handlers.rs | 4 +- src/agentsight/src/storage/sqlite/http.rs | 2 +- .../src/storage/sqlite/interruption.rs | 2 +- src/agentsight/src/tokenizer/llm_tok.rs | 16 -- src/agentsight/src/tokenizer/multi_model.rs | 1 - src/agentsight/src/unified.rs | 6 +- 21 files changed, 23 insertions(+), 337 deletions(-) diff --git a/src/agentsight/src/aggregator/http2.rs b/src/agentsight/src/aggregator/http2.rs index d7ec15f6b..f65bb3f6d 100644 --- a/src/agentsight/src/aggregator/http2.rs +++ b/src/agentsight/src/aggregator/http2.rs @@ -4,12 +4,11 @@ //! by their stream_id and correlating request (client->server) with response (server->client) //! to form complete HTTP/2 request/response pairs. -use std::collections::HashMap; use std::num::NonZeroUsize; use lru::LruCache; use crate::config::DEFAULT_CONNECTION_CAPACITY; use crate::parser::http2::ParsedHttp2Frame; -use crate::parser::sse::{SseParser, SSEParser}; +use crate::parser::sse::SSEParser; use crate::aggregator::http::ConnectionId; use crate::aggregator::result::AggregatedResult; use crate::chrome_trace::{ChromeTraceEvent, ToChromeTraceEvent, ns_to_us}; diff --git a/src/agentsight/src/aggregator/unified.rs b/src/agentsight/src/aggregator/unified.rs index 530827735..a2b628c1c 100644 --- a/src/agentsight/src/aggregator/unified.rs +++ b/src/agentsight/src/aggregator/unified.rs @@ -7,7 +7,7 @@ use super::http::{ConnectionId, ConnectionState, HttpConnectionAggregator}; use super::http2::Http2StreamAggregator; use super::proctrace::ProcessEventAggregator; use super::result::AggregatedResult; -use crate::chrome_trace::{export_trace_events, ToChromeTraceEvent}; +use crate::chrome_trace::export_trace_events; use crate::parser::{ParseResult, ParsedMessage}; /// Unified aggregator for all event types diff --git a/src/agentsight/src/analyzer/token/extractor/mod.rs b/src/agentsight/src/analyzer/token/extractor/mod.rs index d81ac266b..11d63af73 100644 --- a/src/agentsight/src/analyzer/token/extractor/mod.rs +++ b/src/agentsight/src/analyzer/token/extractor/mod.rs @@ -18,7 +18,7 @@ mod anthropic; mod utils; use serde_json::Value; -use super::data::{TokenData, MessageTokenData, ResponseTokenData}; +use super::data::TokenData; /// Extract token data from JSON request/response bodies /// @@ -79,4 +79,3 @@ pub enum Provider { } // Re-export utility functions for internal use -pub use utils::extract_model_from_json; diff --git a/src/agentsight/src/analyzer/unified.rs b/src/agentsight/src/analyzer/unified.rs index d9a078807..2fec8d5a3 100644 --- a/src/agentsight/src/analyzer/unified.rs +++ b/src/agentsight/src/analyzer/unified.rs @@ -26,7 +26,7 @@ use crate::tokenizer::LlmTokenizer; use crate::tokenizer::get_global_tokenizer; use crate::analyzer::token::extract_response_content; -use super::{AuditAnalyzer, TokenParser, MessageParser, AuditRecord, TokenRecord, TokenUsage, ParsedApiMessage, AnalysisResult, PromptTokenCount, HttpRecord}; +use super::{AuditAnalyzer, TokenParser, MessageParser, TokenRecord, TokenUsage, ParsedApiMessage, AnalysisResult, HttpRecord}; use super::result::{TokenConsumptionBreakdown, MessageTokenCount, OutputTokenCount}; /// Token count result for request messages @@ -424,7 +424,7 @@ impl Analyzer { let mut results = Vec::new(); // 1. Audit analysis for process actions (non-HTTP) - if let AggregatedResult::ProcessComplete(process) = result { + if let AggregatedResult::ProcessComplete(_process) = result { if let Some(record) = self.audit.analyze(result) { results.push(AnalysisResult::Audit(record)); } @@ -559,7 +559,7 @@ impl Analyzer { let usage = sse_events.iter().rev() .find_map(|e| self.token.parse_event(e))?; - let mut record = TokenRecord::new( + let record = TokenRecord::new( pid, comm.to_string(), usage.provider.to_string(), @@ -926,78 +926,6 @@ impl Analyzer { .map(AnalysisResult::Token) } - /// Compute prompt tokens for a parsed API message - /// - /// This method uses the tokenizer to compute the actual prompt token count - /// from the request messages. - fn compute_prompt_tokens( - &self, - msg_result: &AnalysisResult, - tokenizer: &LlmTokenizer, - chat_template: &LlmTokenizer, - ) -> Option { - let messages = match msg_result { - AnalysisResult::Message(ParsedApiMessage::OpenAICompletion { request, .. }) => { - request.as_ref()?.messages.clone() - } - _ => return None, - }; - - let provider = match msg_result { - AnalysisResult::Message(msg) => msg.provider().to_string(), - _ => "unknown".to_string(), - }; - - let model = match msg_result { - AnalysisResult::Message(msg) => msg.model().unwrap_or("unknown").to_string(), - _ => "unknown".to_string(), - }; - - let message_count = messages.len(); - - // Convert messages to JSON values - let messages_json: Vec = messages - .iter() - .filter_map(|m| serde_json::to_value(m).ok()) - .collect(); - - // Apply chat template and count tokens - match chat_template.apply_chat_template(&messages_json, true) { - Ok(formatted_prompt) => { - match tokenizer.count(&formatted_prompt) { - Ok(prompt_tokens) => { - // Compute per-message token counts - let per_message_tokens: Vec = messages - .iter() - .filter_map(|m| { - let msg_json = serde_json::to_value(m).ok()?; - let content = msg_json.get("content")?.as_str()?.to_string(); - tokenizer.count(&content).ok() - }) - .collect(); - - Some(PromptTokenCount { - provider, - model, - message_count, - prompt_tokens, - per_message_tokens, - formatted_prompt, - }) - } - Err(e) => { - log::warn!("Failed to count tokens: {}", e); - None - } - } - } - Err(e) => { - log::warn!("Failed to apply chat template: {}", e); - None - } - } - } - /// Get reference to the audit analyzer pub fn audit_analyzer(&self) -> &AuditAnalyzer { &self.audit @@ -1049,199 +977,6 @@ impl Analyzer { self.message.parse_by_path(path, request_body, response_body) } - /// Compute token consumption using apply_chat_template for accurate counting - /// - /// This function directly uses the Jinja2 template to format messages and count tokens, - /// avoiding intermediate conversions and providing more accurate results. - fn compute_token_consumption_with_template( - &self, - messages: &[serde_json::Value], - model: &str, - provider: &str, - tools: Vec, - system_prompt: Option, - response_jsons: &[serde_json::Value], - pid: u32, - comm: String, - ) -> Option { - let (tokenizer, chat_template) = match (&self.tokenizer, &self.chat_template) { - (Some(t), Some(ct)) => (t, ct), - _ => { - log::warn!("Tokenizer or chat template not available, cannot compute accurate token consumption"); - return None; - } - }; - - let mut by_role: std::collections::HashMap = std::collections::HashMap::new(); - let mut per_message: Vec = Vec::new(); - - // Prepare messages for apply_chat_template - let mut template_messages: Vec = messages.to_vec(); - - // Add system prompt as first message if present and not already in messages - if let Some(ref system) = system_prompt { - if !template_messages.is_empty() && template_messages[0].get("role") != Some(&serde_json::Value::String("system".to_string())) { - let system_msg = serde_json::json!({ - "role": "system", - "content": system - }); - template_messages.insert(0, system_msg); - } - } - - // Count tools tokens - let tools_tokens: usize = tools.iter() - .filter_map(|tool| tokenizer.count(tool).ok()) - .sum(); - - // Use apply_chat_template to format all messages and count total tokens - let total_msg_tokens = match chat_template.apply_chat_template(&template_messages, true) { - Ok(formatted) => tokenizer.count(&formatted).unwrap_or(0), - Err(e) => { - log::warn!("Failed to apply chat template: {}", e); - // Fallback: count raw content - messages.iter() - .filter_map(|m| m.get("content").and_then(|c| c.as_str())) - .filter_map(|c| tokenizer.count(c).ok()) - .sum() - } - }; - - // Count per-message tokens using incremental approach - for (i, msg) in messages.iter().enumerate() { - let partial_messages: Vec = template_messages.iter().take(i + 1).cloned().collect(); - let tokens = match chat_template.apply_chat_template(&partial_messages, false) { - Ok(formatted) => tokenizer.count(&formatted).unwrap_or(0), - Err(_) => { - // Fallback: count content only - msg.get("content").and_then(|c| c.as_str()) - .and_then(|c| tokenizer.count(c).ok()) - .unwrap_or(0) - } - }; - - // Calculate this message's tokens by subtracting previous total - let prev_total: usize = per_message.iter().map(|m| m.tokens).sum(); - let msg_tokens = tokens.saturating_sub(prev_total); - - let role = msg.get("role").and_then(|r| r.as_str()).unwrap_or("unknown").to_string(); - *by_role.entry(role.clone()).or_insert(0) += msg_tokens; - - per_message.push(MessageTokenCount { - role, - tokens: msg_tokens, - }); - } - - // Count system prompt tokens separately - let system_prompt_tokens = if let Some(ref system) = system_prompt { - tokenizer.count(system).unwrap_or(system.len() / 4) - } else { - 0 - }; - - // Total input = tools + all messages (system is included in messages) - let total_input = tools_tokens + total_msg_tokens; - - // Compute output token breakdown from response - let (output_by_type, output_per_block) = self.compute_output_token_breakdown_from_json( - response_jsons, - tokenizer, - chat_template, - ); - - let total_output_tokens: usize = output_by_type.values().sum(); - - Some(TokenConsumptionBreakdown { - timestamp_ns: std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_nanos() as u64) - .unwrap_or(0), - pid, - comm, - provider: provider.to_string(), - model: model.to_string(), - total_input_tokens: total_input, - total_output_tokens, - by_role, - per_message, - tools_tokens, - system_prompt_tokens, - output_by_type, - output_per_block, - }) - } - - /// Compute output token breakdown from SSE response JSONs - /// - /// Directly processes SSE chunks using extract_response_content which now supports - /// both "message" and "delta" formats, eliminating the need for aggregate_sse_chunks. - fn compute_output_token_breakdown_from_json( - &self, - response_jsons: &[serde_json::Value], - tokenizer: &LlmTokenizer, - _chat_template: &LlmTokenizer, - ) -> (std::collections::HashMap, Vec) { - let mut output_by_type: std::collections::HashMap = std::collections::HashMap::new(); - let mut output_per_block: Vec = Vec::new(); - - // Accumulate content from all SSE chunks (extract_response_content now supports delta format) - let mut all_content = String::new(); - let mut all_reasoning = String::new(); - let mut all_tool_calls = Vec::new(); - - for chunk in response_jsons { - if let Some((content, reasoning, tool_calls)) = extract_response_content(Some(chunk)) { - if !content.is_empty() { - all_content.push_str(&content); - } - if let Some(r) = reasoning { - if !r.is_empty() { - all_reasoning.push_str(&r); - } - } - for tc in tool_calls { - if !tc.is_empty() { - all_tool_calls.push(tc); - } - } - } - } - - // Handle text content - if !all_content.is_empty() { - let tokens = tokenizer.count(&all_content).unwrap_or(all_content.len() / 4); - *output_by_type.entry("text".to_string()).or_insert(0) += tokens; - output_per_block.push(OutputTokenCount { - content_type: "text".to_string(), - tokens, - }); - } - - // Handle reasoning content - if !all_reasoning.is_empty() { - let tokens = tokenizer.count(&all_reasoning).unwrap_or(all_reasoning.len() / 4); - *output_by_type.entry("reasoning".to_string()).or_insert(0) += tokens; - output_per_block.push(OutputTokenCount { - content_type: "reasoning".to_string(), - tokens, - }); - } - - // Handle tool calls - aggregate all tool calls and count once - if !all_tool_calls.is_empty() { - let aggregated_tool_calls = all_tool_calls.join(""); - let tokens = tokenizer.count(&aggregated_tool_calls).unwrap_or(aggregated_tool_calls.len() / 4); - *output_by_type.entry("tool_calls".to_string()).or_insert(0) += tokens; - output_per_block.push(OutputTokenCount { - content_type: "tool_calls".to_string(), - tokens, - }); - } - - (output_by_type, output_per_block) - } - /// Analyze AggregatedResult and extract token consumption breakdown /// /// This is a convenience method that combines extract_token_data and diff --git a/src/agentsight/src/atif/converter.rs b/src/agentsight/src/atif/converter.rs index 862b64f9c..34bd57548 100644 --- a/src/agentsight/src/atif/converter.rs +++ b/src/agentsight/src/atif/converter.rs @@ -746,7 +746,7 @@ fn ns_to_iso8601(ns: u64) -> String { #[cfg(test)] mod tests { use super::*; - use crate::genai::semantic::{InputMessage, OutputMessage, MessagePart}; + use crate::genai::semantic::{InputMessage, MessagePart}; #[test] fn test_ns_to_iso8601() { diff --git a/src/agentsight/src/bin/cli/token.rs b/src/agentsight/src/bin/cli/token.rs index ae620f621..634f3c1bb 100644 --- a/src/agentsight/src/bin/cli/token.rs +++ b/src/agentsight/src/bin/cli/token.rs @@ -5,7 +5,6 @@ use agentsight::{ SqliteConfig, }; use structopt::StructOpt; -use std::collections::HashMap; /// Token query subcommand #[derive(Debug, StructOpt, Clone)] diff --git a/src/agentsight/src/genai/builder.rs b/src/agentsight/src/genai/builder.rs index 9da392041..73fcc6392 100644 --- a/src/agentsight/src/genai/builder.rs +++ b/src/agentsight/src/genai/builder.rs @@ -1417,7 +1417,7 @@ impl GenAIBuilder { log::debug!("[GenAI] Parsing SSE body with {} chunks", chunks.len()); - for (chunk_idx, chunk) in chunks.iter().enumerate() { + for (_chunk_idx, chunk) in chunks.iter().enumerate() { let choices = chunk.get("choices").and_then(|c| c.as_array()); let choices = match choices { Some(c) => c, diff --git a/src/agentsight/src/health/checker.rs b/src/agentsight/src/health/checker.rs index aa5553eaa..c0d84c46b 100644 --- a/src/agentsight/src/health/checker.rs +++ b/src/agentsight/src/health/checker.rs @@ -341,26 +341,6 @@ impl HealthChecker { } } - /// Query pending LLM calls for a specific PID from genai_events. - /// - /// Returns a list of (call_id, session_id, trace_id, conversation_id) tuples. - fn get_pending_calls_for_pid( - &self, - pid: u32, - ) -> Vec<(String, Option, Option, Option)> { - if let Some(ref genai_store) = self.genai_store { - match genai_store.list_pending_for_pid(pid as i32) { - Ok(calls) => calls, - Err(e) => { - log::warn!("Failed to query pending calls for pid={}: {}", pid, e); - vec![] - } - } - } else { - vec![] - } - } - /// Query pending LLM calls for multiple PIDs at once. /// /// Returns a list of (call_id, session_id, trace_id, conversation_id) tuples. diff --git a/src/agentsight/src/parser/http/request.rs b/src/agentsight/src/parser/http/request.rs index 6d070caba..e184c4f77 100644 --- a/src/agentsight/src/parser/http/request.rs +++ b/src/agentsight/src/parser/http/request.rs @@ -205,7 +205,7 @@ fn format_body(data: &[u8]) -> String { format!("(text, {} bytes)\n{}", data.len(), text) } else { // Binary data - show as base64 - format!("(binary, {} bytes)\n{}", data.len(), base64::encode(data)) + format!("(binary, {} bytes)\n{}", data.len(), base64::Engine::encode(&base64::engine::general_purpose::STANDARD, data)) } } diff --git a/src/agentsight/src/parser/http/response.rs b/src/agentsight/src/parser/http/response.rs index e3516521b..7f0fc337b 100644 --- a/src/agentsight/src/parser/http/response.rs +++ b/src/agentsight/src/parser/http/response.rs @@ -151,6 +151,6 @@ fn format_body(data: &[u8]) -> String { format!("(text, {} bytes)\n{}", data.len(), text) } else { // Binary data - show as base64 - format!("(binary, {} bytes)\n{}", data.len(), base64::encode(data)) + format!("(binary, {} bytes)\n{}", data.len(), base64::Engine::encode(&base64::engine::general_purpose::STANDARD, data)) } } diff --git a/src/agentsight/src/parser/proctrace.rs b/src/agentsight/src/parser/proctrace.rs index ba90ed7c2..be9837253 100644 --- a/src/agentsight/src/parser/proctrace.rs +++ b/src/agentsight/src/parser/proctrace.rs @@ -47,7 +47,7 @@ impl ProcTraceParser { /// Parse a variable-length process event pub fn parse_variable(event: &VariableEvent) -> Option { match event { - VariableEvent::Exec { header, filename, args } => { + VariableEvent::Exec { header, filename: _, args } => { Some(ParsedProcEvent { event_type: ProcEventType::Exec, pid: header.pid, diff --git a/src/agentsight/src/parser/sse/event.rs b/src/agentsight/src/parser/sse/event.rs index d7d512f98..dd4e8865a 100644 --- a/src/agentsight/src/parser/sse/event.rs +++ b/src/agentsight/src/parser/sse/event.rs @@ -182,7 +182,7 @@ fn format_sse_data(data: &[u8]) -> String { format!("(text, {} bytes)\n{}", data.len(), text) } else { // Binary data - show as base64 - format!("(binary, {} bytes)\n{}", data.len(), base64::encode(data)) + format!("(binary, {} bytes)\n{}", data.len(), base64::Engine::encode(&base64::engine::general_purpose::STANDARD, data)) } } diff --git a/src/agentsight/src/parser/unified.rs b/src/agentsight/src/parser/unified.rs index 12409adfc..70e62ff8b 100644 --- a/src/agentsight/src/parser/unified.rs +++ b/src/agentsight/src/parser/unified.rs @@ -50,7 +50,7 @@ impl Parser { pub fn parse_ssl_event(&self, ssl_event: Rc) -> ParseResult { log::debug!("parse_ssl_event: length={}", ssl_event.buf_size()); - let comm = ssl_event.comm.trim_end_matches('\0'); + let _comm = ssl_event.comm.trim_end_matches('\0'); // 1. HTTP/1.x detection (text-based protocols) if ssl_event.is_http() { diff --git a/src/agentsight/src/probes/proctrace.rs b/src/agentsight/src/probes/proctrace.rs index 3f5d8d121..587c81edf 100644 --- a/src/agentsight/src/probes/proctrace.rs +++ b/src/agentsight/src/probes/proctrace.rs @@ -11,7 +11,7 @@ use libbpf_rs::{ }; use std::{ mem::MaybeUninit, - os::fd::{AsFd, AsRawFd}, + os::fd::AsFd, sync::{ Arc, atomic::{AtomicBool, Ordering}, diff --git a/src/agentsight/src/probes/sslsniff.rs b/src/agentsight/src/probes/sslsniff.rs index 0b5a0d599..889c5bc28 100644 --- a/src/agentsight/src/probes/sslsniff.rs +++ b/src/agentsight/src/probes/sslsniff.rs @@ -5,7 +5,7 @@ // Exposes a `SslSniff` struct with a builder-style API. use crate::config; -use anyhow::{Context, Result, bail}; +use anyhow::{Context, Result}; use libbpf_rs::{ Link, MapHandle, RingBufferBuilder, UprobeOpts, skel::{OpenSkel, SkelBuilder}, @@ -15,7 +15,6 @@ use procfs::process::Process; use std::{ collections::{HashMap, HashSet}, fs, - io::Write, mem::{self, MaybeUninit}, path::Path, slice, @@ -684,16 +683,6 @@ fn ssl_libs_from_maps(pid: i32) -> Result> { Ok(results) } -/// Convert a null-terminated byte array (from C `char comm[TASK_COMM_LEN]`) to a `String`. -fn comm_to_string(comm: &[u8]) -> String { - let bytes: Vec = comm - .iter() - .copied() - .take_while(|&b| b != 0) - .collect(); - String::from_utf8_lossy(&bytes).into_owned() -} - // ─── uprobe helpers ─────────────────────────────────────────────────────────── fn make_sym_opts(sym: &str, retprobe: bool) -> UprobeOpts { diff --git a/src/agentsight/src/server/handlers.rs b/src/agentsight/src/server/handlers.rs index 75247d452..002ab2c60 100644 --- a/src/agentsight/src/server/handlers.rs +++ b/src/agentsight/src/server/handlers.rs @@ -1,12 +1,12 @@ //! API request handlers -use actix_web::{delete, get, post, web, HttpResponse, Responder}; +use actix_web::{get, post, web, HttpResponse, Responder}; use serde::{Deserialize, Serialize}; use super::AppState; use crate::health::AgentHealthStatus; use crate::storage::sqlite::{GenAISqliteStore}; -use crate::storage::sqlite::genai::{TimeseriesBucket, ModelTimeseriesBucket, ToolCallTurnInfo}; +use crate::storage::sqlite::genai::{TimeseriesBucket, ModelTimeseriesBucket}; use crate::storage::sqlite::tokenless::{self, TokenlessStatsStore}; // ─── Prometheus helpers ─────────────────────────────────────────────────────── diff --git a/src/agentsight/src/storage/sqlite/http.rs b/src/agentsight/src/storage/sqlite/http.rs index d14a25a8f..da3349513 100644 --- a/src/agentsight/src/storage/sqlite/http.rs +++ b/src/agentsight/src/storage/sqlite/http.rs @@ -2,7 +2,7 @@ //! //! Handles table creation, record insertion, and querying for HTTP request/response records. -use anyhow::{Context, Result}; +use anyhow::Result; use rusqlite::{params, Connection}; use std::path::Path; diff --git a/src/agentsight/src/storage/sqlite/interruption.rs b/src/agentsight/src/storage/sqlite/interruption.rs index 246da97f3..3abf680eb 100644 --- a/src/agentsight/src/storage/sqlite/interruption.rs +++ b/src/agentsight/src/storage/sqlite/interruption.rs @@ -3,7 +3,7 @@ use rusqlite::{params, Connection}; use std::sync::Mutex; -use crate::interruption::{InterruptionEvent, InterruptionType, Severity}; +use crate::interruption::{InterruptionEvent, InterruptionType}; use super::connection::create_connection; // ─── API response types ──────────────────────────────────────────────────────── diff --git a/src/agentsight/src/tokenizer/llm_tok.rs b/src/agentsight/src/tokenizer/llm_tok.rs index 206f59f7b..f20f3fc0e 100644 --- a/src/agentsight/src/tokenizer/llm_tok.rs +++ b/src/agentsight/src/tokenizer/llm_tok.rs @@ -9,7 +9,6 @@ use serde_json::Value; use std::path::Path; use std::sync::Arc; -use crate::analyzer::{MessageRole, OpenAIChatMessage}; use llm_tokenizer::{Decoder as _, Encoder as _, HuggingFaceTokenizer, TokenizerTrait, chat_template::ChatTemplateParams}; /// Unified tokenizer + chat template adapter wrapping `llm-tokenizer` crate. @@ -63,16 +62,6 @@ impl LlmTokenizer { }) } - /// Create a tokenizer from a URL (backward compatibility). - /// - /// This is deprecated in favor of `from_hf` which uses HuggingFace Hub directly. - /// For URLs pointing to HuggingFace (e.g., huggingface.co/...), consider using - /// `from_hf` with the model ID instead. - #[deprecated] - pub fn from_url(url: &str, model_name: &str) -> Result { - todo!() - } - /// Encode text with special tokens. pub fn encode_with_special_tokens(&self, text: &str) -> Result> { let encoding = self.inner.encode(text, true) @@ -111,11 +100,6 @@ impl LlmTokenizer { .map_err(|e| anyhow!("Failed to apply chat template: {}", e)) } - /// Convert OpenAIChatMessage to serde_json::Value for template rendering. - pub fn messages_to_json(messages: &[OpenAIChatMessage]) -> Vec { - todo!() - } - /// Count tokens in text pub fn count(&self, text: &str) -> Result { let encoding = self.inner.encode(text, false) diff --git a/src/agentsight/src/tokenizer/multi_model.rs b/src/agentsight/src/tokenizer/multi_model.rs index 02d2e3769..779da7462 100644 --- a/src/agentsight/src/tokenizer/multi_model.rs +++ b/src/agentsight/src/tokenizer/multi_model.rs @@ -10,7 +10,6 @@ use anyhow::{Result, anyhow}; use hf_hub::api::sync::{Api, ApiBuilder}; use once_cell::sync::OnceCell; use std::collections::HashMap; -use std::path::PathBuf; use std::sync::{Arc, Mutex, MutexGuard}; static GLOBAL_TOKENIZER: OnceCell> = OnceCell::new(); diff --git a/src/agentsight/src/unified.rs b/src/agentsight/src/unified.rs index f1b612ab1..50b3a8ed9 100644 --- a/src/agentsight/src/unified.rs +++ b/src/agentsight/src/unified.rs @@ -738,7 +738,9 @@ impl AgentSight { /// Install an FFI event sender for C API mode. /// When set, completed events are pushed through this channel. - pub fn set_ffi_sender(&mut self, sender: FfiEventSender) { + /// `pub(crate)` because `FfiEventSender` is a crate-internal type and the + /// only caller lives in this crate's FFI layer. + pub(crate) fn set_ffi_sender(&mut self, sender: FfiEventSender) { self.ffi_sender = Some(sender); } @@ -831,7 +833,7 @@ impl AgentSight { for (conn_id, state) in drained { // Destructure to capture both request AND sse_events - let (state_name, request, sse_events) = match state { + let (_state_name, request, sse_events) = match state { ConnectionState::RequestPending { request } => ("RequestPending", request, vec![]), ConnectionState::SseActive { request: Some(req), From 74997da7cfcad1e80e97bacc96f31b534545997b Mon Sep 17 00:00:00 2001 From: Jiangtian Feng Date: Sat, 30 May 2026 23:54:47 +0800 Subject: [PATCH 2/3] chore(sight): silence generated-code warnings The bindgen header + libbpf skeleton blocks (mod bpf { include!(...) }) emit ~100 non_camel_case_types / non_upper_case_globals / dead_code warnings that drowned out real signals. Annotate each of the 7 probe modules' generated include block with a scoped #[allow(...)] instead of touching generated code. Also clear the remaining real warnings: a dead trailing bind_idx increment in token_consumption.rs, the unused format_duration_ns CLI helper, and an unused test binding. The crate now builds warning-free (lib + tests + bins), making a future -D warnings gate viable. Co-Authored-By: Claude Opus 4.8 --- src/agentsight/src/analyzer/message/mod.rs | 2 +- src/agentsight/src/bin/cli/skill_metrics.rs | 16 ---------------- src/agentsight/src/probes/filewatch.rs | 1 + src/agentsight/src/probes/filewrite.rs | 1 + src/agentsight/src/probes/procmon.rs | 1 + src/agentsight/src/probes/proctrace.rs | 1 + src/agentsight/src/probes/sslsniff.rs | 1 + src/agentsight/src/probes/tcpsniff.rs | 1 + src/agentsight/src/probes/udpdns.rs | 1 + .../src/storage/sqlite/token_consumption.rs | 1 - 10 files changed, 8 insertions(+), 18 deletions(-) diff --git a/src/agentsight/src/analyzer/message/mod.rs b/src/agentsight/src/analyzer/message/mod.rs index 367fad4bb..5b5548153 100644 --- a/src/agentsight/src/analyzer/message/mod.rs +++ b/src/agentsight/src/analyzer/message/mod.rs @@ -390,7 +390,7 @@ mod tests { #[test] fn test_full_url_paths() { - let parser = MessageParser::new(); + let _parser = MessageParser::new(); // Should work with full URLs too assert!(MessageParser::is_llm_api_path( diff --git a/src/agentsight/src/bin/cli/skill_metrics.rs b/src/agentsight/src/bin/cli/skill_metrics.rs index d1dded6a6..df3081b30 100644 --- a/src/agentsight/src/bin/cli/skill_metrics.rs +++ b/src/agentsight/src/bin/cli/skill_metrics.rs @@ -367,19 +367,3 @@ fn format_timestamp_ns(ns: i64) -> String { .naive_utc(); dt.format("%m-%d %H:%M").to_string() } - -fn format_duration_ns(ns: i64) -> String { - if ns == 0 { - return "0s".to_string(); - } - let secs = ns as f64 / 1_000_000_000.0; - if secs < 60.0 { - format!("{:.1}s", secs) - } else if secs < 3600.0 { - format!("{:.1}m", secs / 60.0) - } else if secs < 86400.0 { - format!("{:.1}h", secs / 3600.0) - } else { - format!("{:.1}d", secs / 86400.0) - } -} diff --git a/src/agentsight/src/probes/filewatch.rs b/src/agentsight/src/probes/filewatch.rs index e96f44626..acedb7971 100644 --- a/src/agentsight/src/probes/filewatch.rs +++ b/src/agentsight/src/probes/filewatch.rs @@ -15,6 +15,7 @@ use std::{ }; // ─── Generated skeleton ─────────────────────────────────────────────────────── +#[allow(non_camel_case_types, non_upper_case_globals, dead_code, non_snake_case)] mod bpf { include!(concat!(env!("OUT_DIR"), "/filewatch.skel.rs")); include!(concat!(env!("OUT_DIR"), "/filewatch.rs")); diff --git a/src/agentsight/src/probes/filewrite.rs b/src/agentsight/src/probes/filewrite.rs index 741abe524..e1dfefed1 100644 --- a/src/agentsight/src/probes/filewrite.rs +++ b/src/agentsight/src/probes/filewrite.rs @@ -15,6 +15,7 @@ use std::{ }; // ─── Generated skeleton ─────────────────────────────────────────────────────── +#[allow(non_camel_case_types, non_upper_case_globals, dead_code, non_snake_case)] mod bpf { include!(concat!(env!("OUT_DIR"), "/filewrite.skel.rs")); include!(concat!(env!("OUT_DIR"), "/filewrite.rs")); diff --git a/src/agentsight/src/probes/procmon.rs b/src/agentsight/src/probes/procmon.rs index e84c0b5ff..c1ea16046 100644 --- a/src/agentsight/src/probes/procmon.rs +++ b/src/agentsight/src/probes/procmon.rs @@ -15,6 +15,7 @@ use std::{ }; // ─── Generated skeleton ─────────────────────────────────────────────────────── +#[allow(non_camel_case_types, non_upper_case_globals, dead_code, non_snake_case)] mod bpf { include!(concat!(env!("OUT_DIR"), "/procmon.skel.rs")); include!(concat!(env!("OUT_DIR"), "/procmon.rs")); diff --git a/src/agentsight/src/probes/proctrace.rs b/src/agentsight/src/probes/proctrace.rs index 587c81edf..e4aca139f 100644 --- a/src/agentsight/src/probes/proctrace.rs +++ b/src/agentsight/src/probes/proctrace.rs @@ -21,6 +21,7 @@ use std::{ }; // ─── Generated skeleton ─────────────────────────────────────────────────────── +#[allow(non_camel_case_types, non_upper_case_globals, dead_code, non_snake_case)] mod bpf { include!(concat!(env!("OUT_DIR"), "/proctrace.skel.rs")); include!(concat!(env!("OUT_DIR"), "/proctrace.rs")); diff --git a/src/agentsight/src/probes/sslsniff.rs b/src/agentsight/src/probes/sslsniff.rs index 889c5bc28..b8bff9c0a 100644 --- a/src/agentsight/src/probes/sslsniff.rs +++ b/src/agentsight/src/probes/sslsniff.rs @@ -27,6 +27,7 @@ use std::{ }; // ─── Generated skeleton ─────────────────────────────────────────────────────── +#[allow(non_camel_case_types, non_upper_case_globals, dead_code, non_snake_case)] pub mod bpf { include!(concat!(env!("OUT_DIR"), "/sslsniff.skel.rs")); include!(concat!(env!("OUT_DIR"), "/sslsniff.rs")); diff --git a/src/agentsight/src/probes/tcpsniff.rs b/src/agentsight/src/probes/tcpsniff.rs index 3f72422a7..5650f7d89 100644 --- a/src/agentsight/src/probes/tcpsniff.rs +++ b/src/agentsight/src/probes/tcpsniff.rs @@ -26,6 +26,7 @@ use std::{ }; // --- Generated skeleton --- +#[allow(non_camel_case_types, non_upper_case_globals, dead_code, non_snake_case)] mod bpf { include!(concat!(env!("OUT_DIR"), "/tcpsniff.skel.rs")); } diff --git a/src/agentsight/src/probes/udpdns.rs b/src/agentsight/src/probes/udpdns.rs index d522b5b0d..f0bb9e7cb 100644 --- a/src/agentsight/src/probes/udpdns.rs +++ b/src/agentsight/src/probes/udpdns.rs @@ -19,6 +19,7 @@ use std::{ }; // --- Generated skeleton --- +#[allow(non_camel_case_types, non_upper_case_globals, dead_code, non_snake_case)] mod bpf { include!(concat!(env!("OUT_DIR"), "/udpdns.skel.rs")); include!(concat!(env!("OUT_DIR"), "/udpdns.rs")); diff --git a/src/agentsight/src/storage/sqlite/token_consumption.rs b/src/agentsight/src/storage/sqlite/token_consumption.rs index 8e49811cb..a78abffc4 100644 --- a/src/agentsight/src/storage/sqlite/token_consumption.rs +++ b/src/agentsight/src/storage/sqlite/token_consumption.rs @@ -213,7 +213,6 @@ impl TokenConsumptionStore { } if filter.model.is_some() { conditions.push(format!("model = ?{}", bind_idx)); - bind_idx += 1; } let where_clause = if conditions.is_empty() { From 9a3daca5b3bbffb1d85845f0f2d9b6bcb72387ec Mon Sep 17 00:00:00 2001 From: Jiangtian Feng Date: Sun, 31 May 2026 00:02:21 +0800 Subject: [PATCH 3/3] docs(sight): sync probe/Event/test-index docs The docs drifted behind the 7 merged probes and 6 Event variants: - AGENTS.md: Event enum listed 4 of 6 variants and the probe table 4 of 7 probes; add FileWrite/UdpDns + filewrite/udpdns/tcpsniff. - integration-tests/README.md: pointed at nonexistent test_sni.md / test_hermes_sni.md and omitted the real test_dns/test_hermes_dns/test_http/ test_connection_scanner/test_ffi_integration entries. - docs/design-docs/ebpf-probes.md: said "4 probes"; document all 7 (diagram, detail sections, and the ring-buffer source-value table). Co-Authored-By: Claude Opus 4.8 --- src/agentsight/AGENTS.md | 7 ++- .../docs/design-docs/ebpf-probes.md | 45 ++++++++++++++++++- src/agentsight/integration-tests/README.md | 7 ++- 3 files changed, 54 insertions(+), 5 deletions(-) diff --git a/src/agentsight/AGENTS.md b/src/agentsight/AGENTS.md index 6fc894eb6..d460a3675 100644 --- a/src/agentsight/AGENTS.md +++ b/src/agentsight/AGENTS.md @@ -32,8 +32,8 @@ eBPF Probes → Event → Parser → ParsedMessage → Aggregator → Aggregated | 模块 | 位置 | 职责 | 关键类型 | |------|------|------|----------| -| **Probes** | `src/probes/` | eBPF 探针管理 | `Probes`, `ProbesPoller`, `SslSniff`, `ProcMon`, `FileWatch` | -| **Event** | `src/event.rs` | 统一事件枚举 | `Event::{Ssl, Proc, ProcMon, FileWatch}` | +| **Probes** | `src/probes/` | eBPF 探针管理 | `Probes`, `ProbesPoller`, `SslSniff`, `ProcMon`, `FileWatch`, `FileWriteProbe`, `UdpDns`, `TcpSniff` | +| **Event** | `src/event.rs` | 统一事件枚举 | `Event::{Ssl, Proc, ProcMon, FileWatch, FileWrite, UdpDns}` | | **Parser** | `src/parser/` | 协议解析（HTTP/1.x, HTTP/2, SSE, ProcTrace） | `Parser`, `ParsedMessage` | | **Aggregator** | `src/aggregator/` | 请求-响应关联 | `Aggregator`, `AggregatedResult` | | **Analyzer** | `src/analyzer/` | Token/审计/消息分析 | `Analyzer`, `AnalysisResult` | @@ -62,6 +62,9 @@ eBPF Probes → Event → Parser → ParsedMessage → Aggregator → Aggregated | proctrace | `src/bpf/proctrace.bpf.c` | tracepoint on execve 捕获命令行参数 | | procmon | `src/bpf/procmon.bpf.c` | 进程创建/退出事件（Agent 发现） | | filewatch | `src/bpf/filewatch.bpf.c` | 监控 .jsonl 文件打开事件 | +| filewrite | `src/bpf/filewrite.bpf.c` | fentry on vfs_write 捕获 .jsonl 写入内容 | +| udpdns | `src/bpf/udpdns.bpf.c` | fentry on udp_sendmsg 捕获 DNS 查询（域名→IP）| +| tcpsniff | `src/bpf/tcpsniff.bpf.c` | fentry on tcp_recvmsg/sendmsg 捕获明文 HTTP 流量 | 构建时 `build.rs` 通过 `libbpf-cargo` 自动生成 eBPF skeleton。 diff --git a/src/agentsight/docs/design-docs/ebpf-probes.md b/src/agentsight/docs/design-docs/ebpf-probes.md index ad88f0dcb..53f2a8a6a 100644 --- a/src/agentsight/docs/design-docs/ebpf-probes.md +++ b/src/agentsight/docs/design-docs/ebpf-probes.md @@ -2,7 +2,7 @@ ## Overview -AgentSight 使用 4 个 eBPF 探针从内核态捕获数据，所有探针共享同一个 ring buffer 和 `traced_processes` BPF map，由 `Probes` 管理器统一协调。 +AgentSight 使用 7 个 eBPF 探针从内核态捕获数据，所有探针共享同一个 ring buffer 和 `traced_processes` BPF map，由 `Probes` 管理器统一协调。 ## Probe Architecture @@ -13,6 +13,9 @@ graph TB PT[proctrace.bpf.c
tracepoint: sched_process_exec] PM[procmon.bpf.c
tracepoint: sched_process_exec/fork/exit] FW[filewatch.bpf.c
tracepoint: do_sys_open] + FWR[filewrite.bpf.c
fentry: vfs_write] + UD[udpdns.bpf.c
fentry: udp_sendmsg] + TS[tcpsniff.bpf.c
fentry: tcp_recvmsg/sendmsg] end subgraph Shared["Shared BPF Maps"] @@ -24,8 +27,13 @@ graph TB PT -->|write| RB PM -->|write| RB FW -->|write| RB + FWR -->|write| RB + UD -->|write| RB + TS -->|write| RB SSL -->|lookup| TM FW -->|lookup| TM + FWR -->|lookup| TM + UD -->|lookup| TM subgraph Userspace["User Space"] P[Probes Poller Thread] @@ -87,6 +95,39 @@ graph TB **Purpose**: Monitor Agent processes opening .jsonl files for auxiliary Agent session identification. +### 5. filewrite — File Write Capture + +- **BPF Type**: fentry +- **Attach Point**: `vfs_write` +- **Filter**: Only PIDs in `traced_processes` writing to `.jsonl` files +- **Output**: `filewrite_event_t` (pid, filename, written content) +- **Source**: `src/bpf/filewrite.bpf.c`, `src/bpf/filewrite.h` +- **Userspace**: `src/probes/filewrite.rs` + +**Purpose**: Capture written .jsonl content to recover responseId → sessionId mappings. + +### 6. udpdns — DNS Query Capture + +- **BPF Type**: fentry +- **Attach Point**: `udp_sendmsg` +- **Filter**: PIDs in `traced_processes`, UDP destination port 53 +- **Output**: `udpdns_event_t` (queried domain) +- **Source**: `src/bpf/udpdns.bpf.c`, `src/bpf/udpdns.h` +- **Userspace**: `src/probes/udpdns.rs` + +**Purpose**: Resolve configured HTTPS/HTTP domain patterns to IPs at runtime for SSL/TCP attach filtering. + +### 7. tcpsniff — Plaintext HTTP Capture + +- **BPF Type**: fentry/fexit +- **Attach Point**: `tcp_recvmsg` / `tcp_sendmsg` +- **Filter**: Configured destination IP/port targets (`tcp_targets`) +- **Output**: reuses the sslsniff `probe_SSL_data_t` event format +- **Source**: `src/bpf/tcpsniff.bpf.c` +- **Userspace**: `src/probes/tcpsniff.rs` + +**Purpose**: Capture plaintext (non-TLS) HTTP traffic to configured endpoints, e.g. internal MaaS gateways. + ## Shared Resource Design ### Ring Buffer (events_rb) @@ -99,6 +140,8 @@ All probes share one ring buffer, distinguished by `common_event_hdr.source` fie | 2 (EVENT_SOURCE_SSL) | sslsniff event | `SslEvent::from_bpf()` | | 3 (EVENT_SOURCE_PROCMON) | procmon event | `procmon::Event::from_bytes()` | | 4 (EVENT_SOURCE_FILEWATCH) | filewatch event | `FileWatchEvent::from_bytes()` | +| 5 (EVENT_SOURCE_FILEWRITE) | filewrite event | `FileWriteEvent::from_bytes()` | +| 6 (EVENT_SOURCE_UDPDNS) | udpdns event | `UdpDnsEvent::from_bytes()` | **Implementation**: `src/probes/probes.rs:Probes::run()` lines 137-193 — single thread polls ring buffer, dispatches by source field into `Event` enum. diff --git a/src/agentsight/integration-tests/README.md b/src/agentsight/integration-tests/README.md index e683ee801..3a06b5e2e 100644 --- a/src/agentsight/integration-tests/README.md +++ b/src/agentsight/integration-tests/README.md @@ -10,6 +10,9 @@ |------|------| | `RULES.md` | 测试环境、部署流程、通用规则 | | `TEMPLATE.md` | 新建测试用例的模板 | -| `test_sni.md` | TLS SNI 探针加载与域名匹配 | -| `test_hermes_sni.md` | 通过 SNI 捕获 Hermes agent（dashscope.aliyuncs.com） | +| `test_dns.md` | UDP DNS 探针：域名→IP 解析捕获 | +| `test_hermes_dns.md` | 通过 DNS 捕获 Hermes agent（dashscope.aliyuncs.com） | +| `test_http.md` | 明文 HTTP（tcpsniff）流量捕获 | +| `test_connection_scanner.md` | 连接扫描器：活跃连接发现 | +| `test_ffi_integration.md` | C FFI API 集成 | | `test_claude_code.md` | Claude Code BoringSSL 探针、SSE thinking/tool_use 解析、msg_id 会话关联 |