feat: Agent 多模式系统、视觉模型集成、LLM 能力分层与 P3 性能收尾
核心架构变更:
1. Agent 多模式系统替代 Coordinator
- 移除 src/agent/coordinator/(Coordinator Agent/Worker/Tools,946 行)
- 新建 src/agent/modes/:声明式模式抽象(AgentMode/ModeConfig/ToolSet)
- 三种内置模式:
- default:通用科研助手,零覆盖保持现有行为
- deep-research:16 步、启用思考、research 权限、系统性调研
- literature-reader:白名单工具、只读沙箱、结构化阅读
- ModeRegistry + ModeConfig 预设 + ToolSet 过滤 + 身份/原则覆盖
- AgentRuntime::with_mode() 统一入口,模式持久化到 session.mode 字段
- GET /api/chat/modes 提供模式列表给前端选择器
2. 视觉模型与图片分析
- 新增 analyze_image 工具(340 行):本地/URL 图片 → 视觉模型流式分析
- LlmClient::analyze_image_stream():SSE 增量实时推送
- 配置:LLM_VISION_MODEL / LLM_VISION_API_KEY / LLM_VISION_API_BASE
- 前端:粘贴/选择图片附件,重试时复用文件路径
- Service 层移除 /chat/rag 和 /chat/figure 端点,统一走 Agent SSE
- Body limit 提升至 100MB 适配大图上传
3. LLM 三级能力分层
- Tier 1 (Core) → Tier 2 (Medium) → Tier 3 (Fast),级联回退
- medium_llm / fast_llm / vision_llm 注入 AppState
- 资产批量翻译 → Medium LLM + Semaphore(3) 并发控制
- 记忆提取/上下文压缩子代理 → Fast LLM
- SubAgentRunner::with_llm_client() 支持注入专用 LLM
4. 数据库与性能优化
- SQLite 启用 WAL + busy_timeout(10s) 处理并发写入
- RAG ingest:DELETE 合并为原子语句 + 批量事务写入
- Meta sync:save_paper_to_db_tx() 事务化批量插入
- 翻译词典:first_words HashSet 预过滤 + next_valid_index 跳跃优化
- read_file 不截断输出 + skip_persist 防止级联磁盘持久化
5. 工具系统增强
- ToolContext 增加 tool_call_id + max_output_chars
- ToolOutput 增加 skip_persist 标记
- TextDelta SSE 携带可选 tool_call_id 支持工具的流式输出
- ChatMessage::text() 辅助方法
This commit is contained in:
+167
-23
@@ -26,12 +26,59 @@ use crate::agent::runtime::{AgentRuntime, AgentStreamEvent};
|
||||
pub struct AgentChatRequest {
|
||||
pub question: String,
|
||||
pub session_id: Option<String>,
|
||||
/// 是否启用 LLM 思考模式(默认关闭)
|
||||
/// Agent 运行模式: "default" / "deep-research" / "literature-reader"
|
||||
#[serde(default = "default_mode")]
|
||||
pub mode: String,
|
||||
/// 是否启用 LLM 思考模式。None = 由 mode 决定,Some(true/false) = 用户显式覆盖。
|
||||
#[serde(default)]
|
||||
pub thinking: bool,
|
||||
/// 是否启用协调者模式(Coordinator delegates to Workers)
|
||||
pub thinking: Option<bool>,
|
||||
/// 可选的图片附件(base64 编码 + MIME 类型)
|
||||
#[serde(default)]
|
||||
pub coordinator_mode: bool,
|
||||
pub image: Option<AttachedImage>,
|
||||
}
|
||||
|
||||
/// 用户附带的图片,用于多模态 Agent 提问。
|
||||
#[derive(Debug, Deserialize)]
|
||||
pub struct AttachedImage {
|
||||
/// base64 编码的图片数据(不含 data:xxx;base64, 前缀)。与 path 互斥。
|
||||
#[serde(default)]
|
||||
pub data: String,
|
||||
/// MIME 类型,如 "image/png"、"image/jpeg"
|
||||
#[serde(default)]
|
||||
pub mime_type: String,
|
||||
/// 已有图片的相对路径(重试时复用已有文件,不再重新 base64 解码存盘)
|
||||
#[serde(default)]
|
||||
pub path: Option<String>,
|
||||
}
|
||||
|
||||
fn default_mode() -> String {
|
||||
"default".to_string()
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
pub struct AgentModeDto {
|
||||
pub id: &'static str,
|
||||
pub name: &'static str,
|
||||
pub description: &'static str,
|
||||
pub icon: &'static str,
|
||||
}
|
||||
|
||||
// ── GET /api/chat/modes ──
|
||||
// 获取可用的智能体运行模式列表
|
||||
pub async fn get_agent_modes() -> Json<Vec<AgentModeDto>> {
|
||||
use crate::agent::modes::ModeRegistry;
|
||||
let registry = ModeRegistry::builtins();
|
||||
let modes = registry
|
||||
.list()
|
||||
.iter()
|
||||
.map(|m| AgentModeDto {
|
||||
id: m.id,
|
||||
name: m.name,
|
||||
description: m.description,
|
||||
icon: m.icon,
|
||||
})
|
||||
.collect();
|
||||
Json(modes)
|
||||
}
|
||||
|
||||
pub async fn chat_agent(
|
||||
@@ -39,21 +86,112 @@ pub async fn chat_agent(
|
||||
Json(req): Json<AgentChatRequest>,
|
||||
) -> Result<Sse<impl Stream<Item = Result<Event, Infallible>>>, (StatusCode, String)> {
|
||||
info!(
|
||||
"接收到智能体对话请求: question='{}', session_id={:?}",
|
||||
req.question, req.session_id
|
||||
"接收到智能体对话请求: question='{}', session_id={:?}, has_image={}",
|
||||
req.question,
|
||||
req.session_id,
|
||||
req.image.is_some()
|
||||
);
|
||||
|
||||
let runtime = AgentRuntime::new(Arc::clone(&state))
|
||||
.with_thinking(req.thinking)
|
||||
.with_coordinator_mode(req.coordinator_mode);
|
||||
// 处理图片附件:保存到磁盘,路径注入 Agent 上下文,前端和 DB 保留原始问题
|
||||
let question = req.question.clone();
|
||||
let (image_context, image_path_for_db): (Option<String>, Option<String>) = match req.image {
|
||||
Some(ref img) => {
|
||||
// 如果带有 path 字段(重试场景),复用已有文件,不重新保存
|
||||
let relative_path: String = if let Some(ref existing_path) = img.path {
|
||||
let full = state.config.library_dir.join(existing_path);
|
||||
if full.exists() {
|
||||
info!("重试复用已有图片: {}", existing_path);
|
||||
existing_path.clone()
|
||||
} else {
|
||||
return Err((
|
||||
StatusCode::BAD_REQUEST,
|
||||
format!("图片文件不存在: {}", existing_path),
|
||||
));
|
||||
}
|
||||
} else {
|
||||
if img.data.is_empty() {
|
||||
return Err((StatusCode::BAD_REQUEST, "图片数据为空".to_string()));
|
||||
}
|
||||
if !img.mime_type.starts_with("image/") {
|
||||
return Err((
|
||||
StatusCode::BAD_REQUEST,
|
||||
format!("不支持的图片类型: {}", img.mime_type),
|
||||
));
|
||||
}
|
||||
if state.vision_llm.is_none() {
|
||||
return Err((
|
||||
StatusCode::BAD_REQUEST,
|
||||
"图片分析功能未启用。请配置 LLM_VISION_MODEL 环境变量后重试。".to_string(),
|
||||
));
|
||||
}
|
||||
let ext = img.mime_type.strip_prefix("image/").unwrap_or("png");
|
||||
let upload_dir = state
|
||||
.config
|
||||
.library_dir
|
||||
.join(".agent_images")
|
||||
.join("uploads");
|
||||
std::fs::create_dir_all(&upload_dir).map_err(|e| {
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("创建上传目录失败: {}", e),
|
||||
)
|
||||
})?;
|
||||
let filename = format!("{}.{}", uuid::Uuid::new_v4(), ext);
|
||||
let filepath = upload_dir.join(&filename);
|
||||
use base64::{engine::general_purpose, Engine as _};
|
||||
let bytes = general_purpose::STANDARD.decode(&img.data).map_err(|e| {
|
||||
(
|
||||
StatusCode::BAD_REQUEST,
|
||||
format!("图片 base64 解码失败: {}", e),
|
||||
)
|
||||
})?;
|
||||
std::fs::write(&filepath, &bytes).map_err(|e| {
|
||||
(
|
||||
StatusCode::INTERNAL_SERVER_ERROR,
|
||||
format!("保存图片失败: {}", e),
|
||||
)
|
||||
})?;
|
||||
let rel = filepath
|
||||
.strip_prefix(&state.config.library_dir)
|
||||
.unwrap_or(&filepath)
|
||||
.display()
|
||||
.to_string();
|
||||
info!("用户图片已保存: {}", rel);
|
||||
rel
|
||||
};
|
||||
|
||||
let ctx = format!(
|
||||
"用户上传了一张图片,已保存到: {}\n如需分析此图片,请使用 analyze_image 工具,传入 image_path=\"{}\"。",
|
||||
relative_path, relative_path
|
||||
);
|
||||
(Some(ctx), Some(relative_path))
|
||||
}
|
||||
None => (None, None),
|
||||
};
|
||||
|
||||
let mut runtime = AgentRuntime::new(Arc::clone(&state)).with_mode(&req.mode);
|
||||
// 只有 mode 未强制固定 thinking 时,用户才可以覆盖
|
||||
if runtime.mode_fixed_thinking().is_none() {
|
||||
if let Some(thinking) = req.thinking {
|
||||
runtime = runtime.with_thinking(thinking);
|
||||
}
|
||||
}
|
||||
let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<AgentStreamEvent>();
|
||||
|
||||
let question = req.question.clone();
|
||||
let session_id = req.session_id.clone();
|
||||
|
||||
// 在后台 tokio 任务中执行 Agent 循环
|
||||
tokio::spawn(async move {
|
||||
match runtime.run_turn(session_id, &question, tx.clone()).await {
|
||||
match runtime
|
||||
.run_turn_with_image_context(
|
||||
session_id,
|
||||
&question,
|
||||
image_context,
|
||||
image_path_for_db,
|
||||
tx.clone(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(sid) => {
|
||||
info!("智能体对话完成: session_id={}", sid);
|
||||
}
|
||||
@@ -96,6 +234,7 @@ pub struct SessionSummary {
|
||||
pub session_id: String,
|
||||
pub title: String,
|
||||
pub model: String,
|
||||
pub mode: String,
|
||||
pub turn_count: i32,
|
||||
pub summary: Option<String>,
|
||||
pub created_at: String,
|
||||
@@ -110,7 +249,7 @@ pub async fn list_sessions(
|
||||
let offset = params.offset.unwrap_or(0);
|
||||
|
||||
let rows = sqlx::query(
|
||||
"SELECT session_id, title, model, turn_count, summary, created_at, updated_at \
|
||||
"SELECT session_id, title, model, mode, turn_count, summary, created_at, updated_at \
|
||||
FROM agent_sessions \
|
||||
WHERE deleted_at IS NULL \
|
||||
ORDER BY updated_at DESC \
|
||||
@@ -133,10 +272,11 @@ pub async fn list_sessions(
|
||||
session_id: r.get(0),
|
||||
title: r.get(1),
|
||||
model: r.get(2),
|
||||
turn_count: r.get(3),
|
||||
summary: r.get(4),
|
||||
created_at: r.get(5),
|
||||
updated_at: r.get(6),
|
||||
mode: r.get(3),
|
||||
turn_count: r.get(4),
|
||||
summary: r.get(5),
|
||||
created_at: r.get(6),
|
||||
updated_at: r.get(7),
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -174,7 +314,7 @@ pub async fn get_session(
|
||||
) -> Result<Json<SessionDetail>, (StatusCode, String)> {
|
||||
// 查询会话元信息
|
||||
let session_row = sqlx::query(
|
||||
"SELECT session_id, title, model, turn_count, summary, created_at, updated_at \
|
||||
"SELECT session_id, title, model, mode, turn_count, summary, created_at, updated_at \
|
||||
FROM agent_sessions \
|
||||
WHERE session_id = ? AND deleted_at IS NULL",
|
||||
)
|
||||
@@ -193,10 +333,11 @@ pub async fn get_session(
|
||||
session_id: session_row.get(0),
|
||||
title: session_row.get(1),
|
||||
model: session_row.get(2),
|
||||
turn_count: session_row.get(3),
|
||||
summary: session_row.get(4),
|
||||
created_at: session_row.get(5),
|
||||
updated_at: session_row.get(6),
|
||||
mode: session_row.get(3),
|
||||
turn_count: session_row.get(4),
|
||||
summary: session_row.get(5),
|
||||
created_at: session_row.get(6),
|
||||
updated_at: session_row.get(7),
|
||||
};
|
||||
|
||||
// 查询消息列表(包含 lead 和 subagent 消息,前端按 agent_name/metadata 区分渲染)
|
||||
@@ -587,13 +728,15 @@ pub struct RetryResponse {
|
||||
pub new_turn_index: i32,
|
||||
pub deleted_count: i64,
|
||||
pub session_id: String,
|
||||
/// 原消息附带的图片路径(如果有)
|
||||
pub image_path: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn retry_session(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Path(session_id): Path<String>,
|
||||
) -> Result<Json<RetryResponse>, (StatusCode, String)> {
|
||||
let (retried_message, new_turn_index) =
|
||||
let (retried_message, new_turn_index, image_path) =
|
||||
crate::agent::runtime::session::retry_last_turn(&state.db, &session_id)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::BAD_REQUEST, e.to_string()))?;
|
||||
@@ -601,8 +744,9 @@ pub async fn retry_session(
|
||||
Ok(Json(RetryResponse {
|
||||
retried_message,
|
||||
new_turn_index,
|
||||
deleted_count: 0, // 数据库层不便返回,设为 0
|
||||
deleted_count: 0,
|
||||
session_id,
|
||||
image_path,
|
||||
}))
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user