feat: Agent 多模式系统、视觉模型集成、LLM 能力分层与 P3 性能收尾

核心架构变更:

  1. Agent 多模式系统替代 Coordinator
     - 移除 src/agent/coordinator/(Coordinator Agent/Worker/Tools,946 行)
     - 新建 src/agent/modes/:声明式模式抽象(AgentMode/ModeConfig/ToolSet)
     - 三种内置模式:
       - default:通用科研助手,零覆盖保持现有行为
       - deep-research:16 步、启用思考、research 权限、系统性调研
       - literature-reader:白名单工具、只读沙箱、结构化阅读
     - ModeRegistry + ModeConfig 预设 + ToolSet 过滤 + 身份/原则覆盖
     - AgentRuntime::with_mode() 统一入口,模式持久化到 session.mode 字段
     - GET /api/chat/modes 提供模式列表给前端选择器

  2. 视觉模型与图片分析
     - 新增 analyze_image 工具(340 行):本地/URL 图片 → 视觉模型流式分析
     - LlmClient::analyze_image_stream():SSE 增量实时推送
     - 配置:LLM_VISION_MODEL / LLM_VISION_API_KEY / LLM_VISION_API_BASE
     - 前端:粘贴/选择图片附件,重试时复用文件路径
     - Service 层移除 /chat/rag 和 /chat/figure 端点,统一走 Agent SSE
     - Body limit 提升至 100MB 适配大图上传

  3. LLM 三级能力分层
     - Tier 1 (Core) → Tier 2 (Medium) → Tier 3 (Fast),级联回退
     - medium_llm / fast_llm / vision_llm 注入 AppState
     - 资产批量翻译 → Medium LLM + Semaphore(3) 并发控制
     - 记忆提取/上下文压缩子代理 → Fast LLM
     - SubAgentRunner::with_llm_client() 支持注入专用 LLM

  4. 数据库与性能优化
     - SQLite 启用 WAL + busy_timeout(10s) 处理并发写入
     - RAG ingest:DELETE 合并为原子语句 + 批量事务写入
     - Meta sync:save_paper_to_db_tx() 事务化批量插入
     - 翻译词典:first_words HashSet 预过滤 + next_valid_index 跳跃优化
     - read_file 不截断输出 + skip_persist 防止级联磁盘持久化

  5. 工具系统增强
     - ToolContext 增加 tool_call_id + max_output_chars
     - ToolOutput 增加 skip_persist 标记
     - TextDelta SSE 携带可选 tool_call_id 支持工具的流式输出
     - ChatMessage::text() 辅助方法
This commit is contained in:
fmq
2026-06-24 19:52:27 +08:00
parent cec4b8cf7b
commit 85b6429c30
44 changed files with 2307 additions and 1578 deletions
+167 -23
View File
@@ -26,12 +26,59 @@ use crate::agent::runtime::{AgentRuntime, AgentStreamEvent};
pub struct AgentChatRequest {
pub question: String,
pub session_id: Option<String>,
/// 是否启用 LLM 思考模式(默认关闭)
/// Agent 运行模式: "default" / "deep-research" / "literature-reader"
#[serde(default = "default_mode")]
pub mode: String,
/// 是否启用 LLM 思考模式。None = 由 mode 决定,Some(true/false) = 用户显式覆盖。
#[serde(default)]
pub thinking: bool,
/// 是否启用协调者模式(Coordinator delegates to Workers
pub thinking: Option<bool>,
/// 可选的图片附件(base64 编码 + MIME 类型
#[serde(default)]
pub coordinator_mode: bool,
pub image: Option<AttachedImage>,
}
/// 用户附带的图片,用于多模态 Agent 提问。
#[derive(Debug, Deserialize)]
pub struct AttachedImage {
/// base64 编码的图片数据(不含 data:xxx;base64, 前缀)。与 path 互斥。
#[serde(default)]
pub data: String,
/// MIME 类型,如 "image/png"、"image/jpeg"
#[serde(default)]
pub mime_type: String,
/// 已有图片的相对路径(重试时复用已有文件,不再重新 base64 解码存盘)
#[serde(default)]
pub path: Option<String>,
}
fn default_mode() -> String {
"default".to_string()
}
#[derive(Debug, Serialize)]
pub struct AgentModeDto {
pub id: &'static str,
pub name: &'static str,
pub description: &'static str,
pub icon: &'static str,
}
// ── GET /api/chat/modes ──
// 获取可用的智能体运行模式列表
pub async fn get_agent_modes() -> Json<Vec<AgentModeDto>> {
use crate::agent::modes::ModeRegistry;
let registry = ModeRegistry::builtins();
let modes = registry
.list()
.iter()
.map(|m| AgentModeDto {
id: m.id,
name: m.name,
description: m.description,
icon: m.icon,
})
.collect();
Json(modes)
}
pub async fn chat_agent(
@@ -39,21 +86,112 @@ pub async fn chat_agent(
Json(req): Json<AgentChatRequest>,
) -> Result<Sse<impl Stream<Item = Result<Event, Infallible>>>, (StatusCode, String)> {
info!(
"接收到智能体对话请求: question='{}', session_id={:?}",
req.question, req.session_id
"接收到智能体对话请求: question='{}', session_id={:?}, has_image={}",
req.question,
req.session_id,
req.image.is_some()
);
let runtime = AgentRuntime::new(Arc::clone(&state))
.with_thinking(req.thinking)
.with_coordinator_mode(req.coordinator_mode);
// 处理图片附件:保存到磁盘,路径注入 Agent 上下文,前端和 DB 保留原始问题
let question = req.question.clone();
let (image_context, image_path_for_db): (Option<String>, Option<String>) = match req.image {
Some(ref img) => {
// 如果带有 path 字段(重试场景),复用已有文件,不重新保存
let relative_path: String = if let Some(ref existing_path) = img.path {
let full = state.config.library_dir.join(existing_path);
if full.exists() {
info!("重试复用已有图片: {}", existing_path);
existing_path.clone()
} else {
return Err((
StatusCode::BAD_REQUEST,
format!("图片文件不存在: {}", existing_path),
));
}
} else {
if img.data.is_empty() {
return Err((StatusCode::BAD_REQUEST, "图片数据为空".to_string()));
}
if !img.mime_type.starts_with("image/") {
return Err((
StatusCode::BAD_REQUEST,
format!("不支持的图片类型: {}", img.mime_type),
));
}
if state.vision_llm.is_none() {
return Err((
StatusCode::BAD_REQUEST,
"图片分析功能未启用。请配置 LLM_VISION_MODEL 环境变量后重试。".to_string(),
));
}
let ext = img.mime_type.strip_prefix("image/").unwrap_or("png");
let upload_dir = state
.config
.library_dir
.join(".agent_images")
.join("uploads");
std::fs::create_dir_all(&upload_dir).map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
format!("创建上传目录失败: {}", e),
)
})?;
let filename = format!("{}.{}", uuid::Uuid::new_v4(), ext);
let filepath = upload_dir.join(&filename);
use base64::{engine::general_purpose, Engine as _};
let bytes = general_purpose::STANDARD.decode(&img.data).map_err(|e| {
(
StatusCode::BAD_REQUEST,
format!("图片 base64 解码失败: {}", e),
)
})?;
std::fs::write(&filepath, &bytes).map_err(|e| {
(
StatusCode::INTERNAL_SERVER_ERROR,
format!("保存图片失败: {}", e),
)
})?;
let rel = filepath
.strip_prefix(&state.config.library_dir)
.unwrap_or(&filepath)
.display()
.to_string();
info!("用户图片已保存: {}", rel);
rel
};
let ctx = format!(
"用户上传了一张图片,已保存到: {}\n如需分析此图片,请使用 analyze_image 工具,传入 image_path=\"{}\"",
relative_path, relative_path
);
(Some(ctx), Some(relative_path))
}
None => (None, None),
};
let mut runtime = AgentRuntime::new(Arc::clone(&state)).with_mode(&req.mode);
// 只有 mode 未强制固定 thinking 时,用户才可以覆盖
if runtime.mode_fixed_thinking().is_none() {
if let Some(thinking) = req.thinking {
runtime = runtime.with_thinking(thinking);
}
}
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<AgentStreamEvent>();
let question = req.question.clone();
let session_id = req.session_id.clone();
// 在后台 tokio 任务中执行 Agent 循环
tokio::spawn(async move {
match runtime.run_turn(session_id, &question, tx.clone()).await {
match runtime
.run_turn_with_image_context(
session_id,
&question,
image_context,
image_path_for_db,
tx.clone(),
)
.await
{
Ok(sid) => {
info!("智能体对话完成: session_id={}", sid);
}
@@ -96,6 +234,7 @@ pub struct SessionSummary {
pub session_id: String,
pub title: String,
pub model: String,
pub mode: String,
pub turn_count: i32,
pub summary: Option<String>,
pub created_at: String,
@@ -110,7 +249,7 @@ pub async fn list_sessions(
let offset = params.offset.unwrap_or(0);
let rows = sqlx::query(
"SELECT session_id, title, model, turn_count, summary, created_at, updated_at \
"SELECT session_id, title, model, mode, turn_count, summary, created_at, updated_at \
FROM agent_sessions \
WHERE deleted_at IS NULL \
ORDER BY updated_at DESC \
@@ -133,10 +272,11 @@ pub async fn list_sessions(
session_id: r.get(0),
title: r.get(1),
model: r.get(2),
turn_count: r.get(3),
summary: r.get(4),
created_at: r.get(5),
updated_at: r.get(6),
mode: r.get(3),
turn_count: r.get(4),
summary: r.get(5),
created_at: r.get(6),
updated_at: r.get(7),
})
.collect();
@@ -174,7 +314,7 @@ pub async fn get_session(
) -> Result<Json<SessionDetail>, (StatusCode, String)> {
// 查询会话元信息
let session_row = sqlx::query(
"SELECT session_id, title, model, turn_count, summary, created_at, updated_at \
"SELECT session_id, title, model, mode, turn_count, summary, created_at, updated_at \
FROM agent_sessions \
WHERE session_id = ? AND deleted_at IS NULL",
)
@@ -193,10 +333,11 @@ pub async fn get_session(
session_id: session_row.get(0),
title: session_row.get(1),
model: session_row.get(2),
turn_count: session_row.get(3),
summary: session_row.get(4),
created_at: session_row.get(5),
updated_at: session_row.get(6),
mode: session_row.get(3),
turn_count: session_row.get(4),
summary: session_row.get(5),
created_at: session_row.get(6),
updated_at: session_row.get(7),
};
// 查询消息列表(包含 lead 和 subagent 消息,前端按 agent_name/metadata 区分渲染)
@@ -587,13 +728,15 @@ pub struct RetryResponse {
pub new_turn_index: i32,
pub deleted_count: i64,
pub session_id: String,
/// 原消息附带的图片路径(如果有)
pub image_path: Option<String>,
}
pub async fn retry_session(
State(state): State<Arc<AppState>>,
Path(session_id): Path<String>,
) -> Result<Json<RetryResponse>, (StatusCode, String)> {
let (retried_message, new_turn_index) =
let (retried_message, new_turn_index, image_path) =
crate::agent::runtime::session::retry_last_turn(&state.db, &session_id)
.await
.map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?;
@@ -601,8 +744,9 @@ pub async fn retry_session(
Ok(Json(RetryResponse {
retried_message,
new_turn_index,
deleted_count: 0, // 数据库层不便返回,设为 0
deleted_count: 0,
session_id,
image_path,
}))
}