fmq/AstroResearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor: 全栈架构重构与质量硬化

Browse Source 
核心架构重构:
- Config 按职责拆分为 LlmConfig/EmbeddingConfig/VisionConfig/CdsConfig/StorageConfig 五个子结构
- AppState 拆分为 LlmState/DataSourceState/SessionState 三个子结构，消除 50+ 平铺字段
- 新增 ServiceError 结构化错误类型替代 handler 中的 msg.contains() 字符串匹配
- 移除 api::handlers 兼容命名空间，路由直接引用 agent/auth/papers 等模块

认证性能优化:
- login_rate_limiter/upload_rate_limiter 从 Mutex<HashMap> 迁移为 DashMap（无锁）
- 新增 session_last_active: DashMap<String, AtomicU64>，auth 中间件快速路径免写锁
- 会话过期清理改为按间隔触发（300s），避免每次请求全表扫描
- MAX_SESSIONS 1000→10000，SSE 广播通道 256→1024

Agent 工具增强:
- AgentTool trait 新增 is_internal()/display_name()，SSE 事件携带工具元数据
- 新增 GET /chat/tools 端点暴露注册工具列表
- pending_questions/pending_permissions 增加 created_at 时间戳，自动清理过期条目（10min TTL）
- Agent 超时现在正确 abort 后台任务并设置取消令牌

观测数据源修复:
- FITS 解析: APOGEE/DESI 改用 read_image+切片替代 read_rows（修复 fitsio panic）
- ZTF: CIRCLE 参数分隔符 +→空格（修复 IRSA 400），半径自动裁剪至硬上限
- MAST TESS: parse_tic_json 兼容数组/对象两种 API 响应格式
- 统一检索: per_target_limit 默认 50→1，sources 支持 per-source release/version
- Gaia 测光从 VizieR 镜像切换至官方 TAP 服务

RAG 并发优化:
- 向量化降级从串行改为并发 5 条/批（buffer_unordered）
- 混合检索 RRF 合并从借用改为 owned RetrievalResult

安全加固:
- PDF 中间件: URL 解码 %2F/%2E 后判扩展名；文件名过滤非 ASCII + 禁 \ 防头注入
- chat_agent 日志截断问题内容至 50 字符；list_sessions 强制 limit clamp

前端双主题:
- 设计令牌三层架构: primitive→semantic→component，浅色暖纸张学术/暗色 Night Indigo
- useTheme hook + ThemeToggle 侧边栏组件 + main.tsx 防 FOUC 初始化
- 全组件从硬编码 slate 色迁移至语义令牌（bg-surface/text-content/border-subtle 等）
- 新增 ToastContainer 非阻塞通知系统

部署优化:
- deploy.sh 引入 SSH ControlMaster 单次密码复用
This commit is contained in:
fmq
2026-07-11 14:57:40 +08:00
parent 8f1ed6d08c
commit eaf85707b5
142 changed files with 6349 additions and 3470 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/agent/runtime/events.rs
+8
View File
@@ -21,8 +21,12 @@ pub enum AgentStreamEvent {
/// LLM 生成的工具调用 ID，用于全链路关联（前端/审计/持久化）
id: String,
name: String,
#[serde(default)]
display_name: String,
arguments: serde_json::Value,
step: usize,
#[serde(default)]
is_internal: bool,
},
/// 工具执行结果（Observation）
#[serde(rename = "tool_result")]
@@ -30,10 +34,14 @@ pub enum AgentStreamEvent {
/// 对应的工具调用 ID，前端凭此精确匹配 tool_call 条目
tool_call_id: String,
name: String,
#[serde(default)]
display_name: String,
output: String,
is_error: bool,
metadata: serde_json::Value,
step: usize,
#[serde(default)]
is_internal: bool,
},
/// 文本增量流式输出（最终回答或工具流式输出）
#[serde(rename = "text_delta")]
src/agent/runtime/executor/helpers.rs
+9
View File
@@ -111,19 +111,28 @@ pub(super) async fn process_single_result(
additional_contexts: &mut Vec<String>,
db: &SqlitePool,
turn_index: i32,
tool_registry: &crate::agent::tools::ToolRegistry,
) {
use crate::agent::tools::persist::maybe_persist_tool_result;
let elapsed_ms = exec_start.elapsed().as_millis() as u64;
let (is_internal, display_name) = if let Some(tool) = tool_registry.get(tool_name) {
(tool.is_internal(), tool.display_name().to_string())
} else {
(false, tool_name.to_string())
};
// SSE 事件 — 立即推送到前端
let _ = tx.send(AgentStreamEvent::ToolResult {
tool_call_id: tool_call_id.to_string(),
name: tool_name.to_string(),
display_name,
output: output.content.clone(),
is_error: output.is_error,
metadata: output.metadata.clone(),
step,
is_internal,
});
// 输出处理：小结果直接传递，大结果持久化到磁盘并返回 stub
src/agent/runtime/executor/mod.rs
+60 -5
View File
@@ -41,6 +41,7 @@ pub fn validate_and_prepare(
duplicate_detector: &mut DuplicateDetector,
duplicate_threshold: usize,
messages: &mut Vec<ChatMessage>,
tool_registry: &ToolRegistry,
tx: &mpsc::UnboundedSender<AgentStreamEvent>,
db: &SqlitePool,
session_id: &str,
@@ -88,13 +89,20 @@ pub fn validate_and_prepare(
Ok(v) => v,
Err(e) => {
let error_output = format!("工具参数 JSON 解析失败: {}", e);
let (is_internal, display_name) = if let Some(tool) = tool_registry.get(tool_name) {
(tool.is_internal(), tool.display_name().to_string())
} else {
(false, tool_name.clone())
};
let _ = tx.send(AgentStreamEvent::ToolResult {
tool_call_id: call_id.clone(),
name: tool_name.clone(),
display_name,
output: error_output.clone(),
is_error: true,
metadata: serde_json::json!({}),
step,
is_internal,
});
let tool_msg = ChatMessage::tool_result(&call_id, &error_output);
save_tool_message_sync(db, session_id, turn_index, step, &tool_msg);
@@ -158,11 +166,18 @@ pub async fn execute_parallel(
// Phase 1: 发送 ToolCall SSE 事件
for prep in prepared_calls {
let (is_internal, display_name) = if let Some(tool) = tool_registry.get(&prep.tool_name) {
(tool.is_internal(), tool.display_name().to_string())
} else {
(false, prep.tool_name.clone())
};
let _ = tx.send(AgentStreamEvent::ToolCall {
id: prep.tool_call_id.clone(),
name: prep.tool_name.clone(),
display_name,
arguments: prep.args.clone(),
step,
is_internal,
});
}
@@ -260,9 +275,16 @@ pub async fn execute_parallel(
hardline_result.reason
);
let err_output = hardline_result.reason.clone();
let (is_internal, display_name) = if let Some(tool) = tool_registry.get(&prep.tool_name)
{
(tool.is_internal(), tool.display_name().to_string())
} else {
(false, prep.tool_name.clone())
};
let _ = tx.send(AgentStreamEvent::ToolResult {
tool_call_id: prep.tool_call_id.clone(),
name: prep.tool_name.clone(),
display_name,
output: err_output.clone(),
is_error: true,
metadata: serde_json::json!({
@@ -270,6 +292,7 @@ pub async fn execute_parallel(
"hardline_category": hardline_result.category,
}),
step,
is_internal,
});
let err_msg = ChatMessage::tool_result(&prep.tool_call_id, &err_output);
save_tool_message_sync(db, &sid, turn_index, step, &err_msg);
@@ -364,13 +387,21 @@ pub async fn execute_parallel(
);
let err_output =
format!("工具 {} 被权限规则拒绝执行: {}", prep.tool_name, reason);
let (is_internal, display_name) =
if let Some(tool) = tool_registry.get(&prep.tool_name) {
(tool.is_internal(), tool.display_name().to_string())
} else {
(false, prep.tool_name.clone())
};
let _ = tx.send(AgentStreamEvent::ToolResult {
tool_call_id: prep.tool_call_id.clone(),
name: prep.tool_name.clone(),
display_name,
output: err_output.clone(),
is_error: true,
metadata: serde_json::json!({}),
step,
is_internal,
});
let err_msg = ChatMessage::tool_result(&prep.tool_call_id, &err_output);
save_tool_message_sync(db, &sid, turn_index, step, &err_msg);
@@ -413,7 +444,7 @@ pub async fn execute_parallel(
// 存储待处理的权限请求
{
let mut perms = app_state.pending_permissions.lock().await;
let mut perms = app_state.session.pending_permissions.lock().await;
perms.insert(
perm_id.clone(),
PendingPermission {
@@ -422,6 +453,7 @@ pub async fn execute_parallel(
message: message.clone(),
arguments: prep.args.clone(),
response_tx: resp_tx,
created_at: std::time::Instant::now(),
},
);
}
@@ -431,7 +463,12 @@ pub async fn execute_parallel(
let perm_result = tokio::time::timeout(timeout_dur, resp_rx).await;
// 清理待处理的权限请求
app_state.pending_permissions.lock().await.remove(&perm_id);
app_state
.session
.pending_permissions
.lock()
.await
.remove(&perm_id);
match perm_result {
Ok(Ok(response)) if response.allowed => {
@@ -451,13 +488,21 @@ pub async fn execute_parallel(
// 用户拒绝
info!("[Executor] 用户拒绝了工具 {}", prep.tool_name);
let err_output = format!("用户拒绝了工具 {} 的执行", prep.tool_name);
let (is_internal, display_name) =
if let Some(tool) = tool_registry.get(&prep.tool_name) {
(tool.is_internal(), tool.display_name().to_string())
} else {
(false, prep.tool_name.clone())
};
let _ = tx.send(AgentStreamEvent::ToolResult {
tool_call_id: prep.tool_call_id.clone(),
name: prep.tool_name.clone(),
display_name,
output: err_output.clone(),
is_error: true,
metadata: serde_json::json!({}),
step,
is_internal,
});
let err_msg = ChatMessage::tool_result(&prep.tool_call_id, &err_output);
save_tool_message_sync(db, &sid, turn_index, step, &err_msg);
@@ -482,13 +527,21 @@ pub async fn execute_parallel(
warn!("[Executor] 权限请求超时或取消: {}", prep.tool_name);
let err_output =
format!("权限请求超时 (120s): {} 未获得用户确认", prep.tool_name);
let (is_internal, display_name) =
if let Some(tool) = tool_registry.get(&prep.tool_name) {
(tool.is_internal(), tool.display_name().to_string())
} else {
(false, prep.tool_name.clone())
};
let _ = tx.send(AgentStreamEvent::ToolResult {
tool_call_id: prep.tool_call_id.clone(),
name: prep.tool_name.clone(),
display_name,
output: err_output.clone(),
is_error: true,
metadata: serde_json::json!({}),
step,
is_internal,
});
let err_msg = ChatMessage::tool_result(&prep.tool_call_id, &err_output);
save_tool_message_sync(db, &sid, turn_index, step, &err_msg);
@@ -538,7 +591,7 @@ pub async fn execute_parallel(
let cancel_handle = tokio::spawn(async move {
loop {
tokio::time::sleep(std::time::Duration::from_millis(250)).await;
if app_state_ref.cancelled_runs.contains_key(&sid_ref) {
if app_state_ref.session.cancelled_runs.contains_key(&sid_ref) {
cancel_flag.store(true, Ordering::SeqCst);
return;
}
@@ -658,7 +711,7 @@ pub async fn execute_parallel(
exec_start,
tx,
hook_registry,
&app_state.config.library_dir,
&app_state.config.storage.library_dir,
&sid,
agent_name,
step,
@@ -667,6 +720,7 @@ pub async fn execute_parallel(
&mut additional_contexts,
db,
turn_index,
tool_registry,
)
.await;
}
@@ -713,7 +767,7 @@ pub async fn execute_parallel(
exec_start,
tx,
hook_registry,
&app_state.config.library_dir,
&app_state.config.storage.library_dir,
&sid,
agent_name,
step,
@@ -722,6 +776,7 @@ pub async fn execute_parallel(
&mut additional_contexts,
db,
turn_index,
tool_registry,
)
.await;
src/agent/runtime/mod.rs
+30 -22
View File
@@ -134,7 +134,7 @@ impl AgentRuntime {
));
// 视觉模型可用时注册 analyze_image 工具
if app_state.vision_llm.is_some() {
if app_state.llm.vision.is_some() {
tool_registry.add_tool(Box::new(crate::agent::tools::astro::AnalyzeImageTool));
}
@@ -143,7 +143,7 @@ impl AgentRuntime {
// 初始化 checkpoint 管理器（存储在 library_dir/.checkpoints 下）
let checkpoint_enabled = true;
let checkpoint_store = app_state.config.library_dir.join(".checkpoints");
let checkpoint_store = app_state.config.storage.library_dir.join(".checkpoints");
let checkpoint_manager = Arc::new(checkpoint::CheckpointManager::new(
std::fs::canonicalize(&checkpoint_store).unwrap_or(checkpoint_store),
checkpoint_enabled,
@@ -326,7 +326,7 @@ impl AgentRuntime {
tx: mpsc::UnboundedSender<AgentStreamEvent>,
) -> anyhow::Result<String> {
let db = &self.app_state.db;
let llm = &self.app_state.llm;
let llm = &self.app_state.llm.primary;
// Phase 1: 创建或恢复会话
let session_info =
@@ -336,7 +336,7 @@ impl AgentRuntime {
// 构建 hook 注册表（注入依赖，复用 AgentRuntime 的 metrics_data）
let hook_registry = HookRegistry::with_builtins(
db.clone(),
self.app_state.cancelled_runs.clone(),
self.app_state.session.cancelled_runs.clone(),
Some(self.metrics_data.clone()),
);
@@ -404,7 +404,7 @@ impl AgentRuntime {
// Phase 4: 会话收尾（传入实际的终止原因 + trajectory 导出参数）
let system_prompt = self.system_prompt();
let model_name = self.app_state.llm.model().await;
let model_name = self.app_state.llm.primary.model().await;
finalize::finalize_turn(
db,
&session_info.session_id,
@@ -413,7 +413,7 @@ impl AgentRuntime {
&tx,
&hook_registry,
loop_terminal,
Some(&self.app_state.config.library_dir),
Some(&self.app_state.config.storage.library_dir),
Some(&model_name),
Some(&system_prompt),
Some(self.app_state.clone()),
@@ -434,12 +434,13 @@ impl AgentRuntime {
hook_registry: &HookRegistry,
) -> anyhow::Result<(AgentMetrics, Option<TurnTerminal>)> {
let db = &self.app_state.db;
let llm = &self.app_state.llm;
let llm = &self.app_state.llm.primary;
let sid = &session_info.session_id;
let turn_index = session_info.turn_index;
// 注册当前会话的权限检查器（如不存在则从全局配置初始化）
self.app_state
.session
.session_permission_checkers
.entry(sid.clone())
.or_insert_with(|| (*self.permission_checker).clone());
@@ -468,7 +469,7 @@ impl AgentRuntime {
self.checkpoint_manager.new_turn();
// 检查用户取消
let is_cancelled = self.app_state.cancelled_runs.remove(sid).is_some();
let is_cancelled = self.app_state.session.cancelled_runs.remove(sid).is_some();
if is_cancelled {
warn!("[AgentRuntime] 用户手动中止了会话 {} 的智能体执行", sid);
@@ -794,6 +795,7 @@ impl AgentRuntime {
&mut duplicate_detector,
self.config.duplicate_call_threshold,
messages,
&self.tool_registry,
tx,
db,
sid,
@@ -813,6 +815,7 @@ impl AgentRuntime {
// 并行执行工具（带权限检查、checkpoint 和分区器）
let session_checker_snapshot = self
.app_state
.session
.session_permission_checkers
.get(sid)
.map(|r| r.value().clone());
@@ -887,7 +890,7 @@ impl AgentRuntime {
}
if exec_result.was_cancelled {
self.app_state.cancelled_runs.remove(sid);
self.app_state.session.cancelled_runs.remove(sid);
warn!(
"[AgentRuntime] 工具执行期间被用户手动中止，会话 ID: {}",
sid
@@ -947,7 +950,7 @@ impl AgentRuntime {
tx,
step,
session_id,
self.app_state.cancelled_runs.clone(),
self.app_state.session.cancelled_runs.clone(),
self.config.enable_thinking,
)
.await;
@@ -955,7 +958,7 @@ impl AgentRuntime {
match output.status {
StreamStatus::Success => return Some(output),
StreamStatus::Cancelled => {
self.app_state.cancelled_runs.remove(session_id);
self.app_state.session.cancelled_runs.remove(session_id);
warn!(
"[AgentRuntime] 流式调用期间被用户手动中止，会话 ID: {}",
session_id
@@ -1009,7 +1012,7 @@ impl AgentRuntime {
if matches!(error_kind, ErrorKind::Overloaded) {
consecutive_overloads += 1;
if consecutive_overloads >= 3 {
let fallback = &self.app_state.config.llm_fallback_model;
let fallback = &self.app_state.config.llm.fallback_model;
if !fallback.is_empty() {
warn!(
"[AgentRuntime] 连续 {} 次过载，切换到备用模型: {}",
@@ -1017,11 +1020,11 @@ impl AgentRuntime {
);
llm.set_model(fallback.clone()).await;
consecutive_overloads = 0;
} else if !self.app_state.config.llm_fallback_chain.is_empty() {
} else if !self.app_state.config.llm.fallback_chain.is_empty() {
let idx = ((consecutive_overloads as usize - 3)
% self.app_state.config.llm_fallback_chain.len())
.min(self.app_state.config.llm_fallback_chain.len() - 1);
let alt = &self.app_state.config.llm_fallback_chain[idx];
% self.app_state.config.llm.fallback_chain.len())
.min(self.app_state.config.llm.fallback_chain.len() - 1);
let alt = &self.app_state.config.llm.fallback_chain[idx];
warn!(
"[AgentRuntime] 连续 {} 次过载，从链中切换: {}",
consecutive_overloads, alt
@@ -1033,7 +1036,12 @@ impl AgentRuntime {
}
// 检查用户取消
if self.app_state.cancelled_runs.contains_key(session_id) {
if self
.app_state
.session
.cancelled_runs
.contains_key(session_id)
{
warn!("[AgentRuntime] 退避重试期间被用户取消");
let _ = tx.send(AgentStreamEvent::Error {
message: "用户已手动中止执行。".to_string(),
@@ -1049,7 +1057,7 @@ impl AgentRuntime {
tx,
step,
session_id,
self.app_state.cancelled_runs.clone(),
self.app_state.session.cancelled_runs.clone(),
self.config.enable_thinking,
)
.await;
@@ -1060,7 +1068,7 @@ impl AgentRuntime {
return Some(retry_output);
}
StreamStatus::Cancelled => {
self.app_state.cancelled_runs.remove(session_id);
self.app_state.session.cancelled_runs.remove(session_id);
return None;
}
StreamStatus::Error(_) => {
@@ -1155,7 +1163,7 @@ impl AgentRuntime {
tx,
step,
session_id,
self.app_state.cancelled_runs.clone(),
self.app_state.session.cancelled_runs.clone(),
self.config.enable_thinking,
)
.await;
@@ -1168,7 +1176,7 @@ impl AgentRuntime {
return Some(retry_output);
}
StreamStatus::Cancelled => {
self.app_state.cancelled_runs.remove(session_id);
self.app_state.session.cancelled_runs.remove(session_id);
warn!("[AgentRuntime] 恢复期间被用户中止");
let _ = tx.send(AgentStreamEvent::Error {
message: "用户已手动中止执行。".to_string(),
@@ -1335,7 +1343,7 @@ impl AgentRuntime {
};
let today = chrono::Utc::now().format("%Y-%m-%d").to_string();
let model_name = self.app_state.llm.model_name_snapshot();
let model_name = self.app_state.llm.primary.model_name_snapshot();
let mut lines = vec![
"# 环境信息".to_string(),
src/agent/runtime/streaming_executor.rs
+50 -40
View File
@@ -587,58 +587,68 @@ mod tests {
db: pool,
dict: Dictionary::default(),
qiniu,
ads,
arxiv,
vizier,
lamost: crate::clients::lamost::LamostClient::new("https://www.lamost.org", 60)
.unwrap(),
gaia: crate::clients::gaia::GaiaClient::new(
"https://gea.esac.esa.int/tap-server/tap",
"https://gea.esac.esa.int/data-server",
90,
)
.unwrap(),
sdss: crate::clients::sdss::SdssClient::new("https://datalab.noirlab.edu/tap/sync", 90)
.unwrap(),
desi: crate::clients::desi::DesiClient::new(
"https://datalab.noirlab.edu/tap/sync",
120,
)
.unwrap(),
irsa: crate::clients::irsa::IrsaClient::new("https://irsa.ipac.caltech.edu", 60)
.unwrap(),
mast: crate::clients::mast::MastClient::new("https://mast.stsci.edu", 90).unwrap(),
observation_registry: std::sync::Arc::new(
crate::services::observation::ObservationRegistry::default(),
),
llm: llm.clone(),
medium_llm: llm.clone(),
fast_llm: llm.clone(),
vision_llm: None,
embedding,
downloader: Downloader::new().expect("downloader"),
http_client: reqwest::Client::new(),
harvest_status: Arc::new(tokio::sync::Mutex::new(MetaSyncStatus::default())),
batch_status: Arc::new(tokio::sync::Mutex::new(AssetBatchStatus::default())),
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
cancelled_runs: Arc::new(dashmap::DashMap::new()),
skill_registry: Arc::new(tokio::sync::RwLock::new(SkillRegistry::new(PathBuf::from(
"/tmp/sk",
)))),
pending_questions: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
pending_permissions: Arc::new(
tokio::sync::Mutex::new(std::collections::HashMap::new()),
),
session_permission_checkers: Arc::new(dashmap::DashMap::new()),
sse_broadcast: None,
memory_manager: Arc::new(tokio::sync::Mutex::new(MemoryManager::new(PathBuf::from(
"/tmp/test_mem",
)))),
sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
login_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
upload_rate_limiter: Arc::new(
tokio::sync::Mutex::new(std::collections::HashMap::new()),
),
login_rate_limiter: Arc::new(dashmap::DashMap::new()),
upload_rate_limiter: Arc::new(dashmap::DashMap::new()),
llm: crate::api::LlmState {
primary: llm.clone(),
medium: llm.clone(),
fast: llm.clone(),
vision: None,
embedding,
},
sources: crate::api::DataSourceState {
ads,
arxiv,
vizier,
lamost: crate::clients::lamost::LamostClient::new("https://www.lamost.org", 60)
.unwrap(),
gaia: crate::clients::gaia::GaiaClient::new(
"https://gea.esac.esa.int/tap-server/tap",
"https://gea.esac.esa.int/data-server",
90,
)
.unwrap(),
sdss: crate::clients::sdss::SdssClient::new(
"https://datalab.noirlab.edu/tap/sync",
90,
)
.unwrap(),
desi: crate::clients::desi::DesiClient::new(
"https://datalab.noirlab.edu/tap/sync",
120,
)
.unwrap(),
irsa: crate::clients::irsa::IrsaClient::new("https://irsa.ipac.caltech.edu", 60)
.unwrap(),
mast: crate::clients::mast::MastClient::new("https://mast.stsci.edu", 90).unwrap(),
observation_registry: std::sync::Arc::new(
crate::services::observation::ObservationRegistry::default(),
),
},
session: crate::api::SessionState {
sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
session_last_active: Arc::new(dashmap::DashMap::new()),
cancelled_runs: Arc::new(dashmap::DashMap::new()),
session_permission_checkers: Arc::new(dashmap::DashMap::new()),
pending_questions: Arc::new(tokio::sync::Mutex::new(
std::collections::HashMap::new(),
)),
pending_permissions: Arc::new(tokio::sync::Mutex::new(
std::collections::HashMap::new(),
)),
},
});
ToolContext::new(app_state)