feat: RAG 文献问答、天体目标识别、解析器模块化与批量管线扩展

核心新增:
  - RAG 问答系统:Markdown 安全切片器 (LaTeX 保护) + 向量化 + sqlite-vec 检索 + LLM 生成
  - 天体目标识别:IAU 标准正则提取 15+ 星表标识符,CDS SIMBAD/Sesame 查询与本地缓存
  - 多模态 LLM:chat_completion_with_image 支持图表视觉分析
  - CLI Skills Agent (cli.rs):对外暴露 rag/target/ingest 等 5 个子命令
  - 解析器模块化重构:单体 778 行 → 按期刊拆分 (A&A/ar5iv/IOP/Generic/PDF) +
    common.rs 静态正则工具库

  管线与 Schema:
  - AssetSync→AssetBatch 重命名,批量管线新增 embed/target 两个处理阶段
  - 新增 paper_chunks_content (RAG 切片) 和 paper_targets (天体缓存) 两张表
  - StandardPaper 新增 has_vector 字段,所有查询同步更新

  前端:
  - 新增 AI 助手侧边栏 (RAG 问答 + 来源跳转高亮)
  - 最近浏览文献列表 (localStorage 持久化)、跨面板无缝导航
  - SyncPanel 批量阶段扩展为下拉选项,支持向量化/天体识别

  测试与清理:
  - 集成测试合并至 ads.rs 和 llm.rs,新增 chunker + target 单元测试 15 个
  - 删除旧单体 parser.rs、独立测试文件及过期 scratch 脚本
This commit is contained in:
fmq
2026-06-15 13:25:18 +08:00
parent 3f1935678b
commit 22e7e1dcee
45 changed files with 5289 additions and 1411 deletions
+196
View File
@@ -0,0 +1,196 @@
// src/bin/cli.rs
//
// AstroResearch CLI Skills Agent — 向外部 Agent(如 Claude Code)提供
// 标准的命令行工具接口,用于 RAG 问答、天体查询和天体关联操作。
use std::str::FromStr;
use clap::{Parser, Subcommand};
use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
use tracing_subscriber::FmtSubscriber;
use astroresearch::Config;
use astroresearch::clients::llm::{LlmClient, EmbeddingClient};
#[derive(Parser)]
#[command(
name = "astroresearch_cli",
about = "AstroResearch CLI Skills Agent — 天体物理文献智能助手",
version
)]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
/// 跨文献 RAG 智能问答:在所有已导入文献中检索并生成回答
Rag {
/// 用户问题
question: String,
/// 检索 Top-K 数量(默认 5
#[arg(short, long, default_value = "5")]
top_k: usize,
},
/// 查询天体数据:优先读取本地缓存,未命中时自动查询 SIMBAD
TargetQuery {
/// 天体名称(如 "GD 358", "NGC 1234", "HD 209458"
object_name: String,
},
/// 手动关联天体到特定文献
TargetAssociate {
/// 文献 bibcode
bibcode: String,
/// 天体名称
object_name: String,
},
/// 从文本中提取天体标识符(调试/预览用)
ExtractTargets {
/// 输入文本
text: String,
},
/// 向量化导入指定文献的 Markdown 文件
Ingest {
/// 文献 bibcode
bibcode: String,
/// Markdown 文件路径
markdown_path: String,
},
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
// 初始化日志
let subscriber = FmtSubscriber::builder()
.with_max_level(tracing::Level::INFO)
.finish();
tracing_subscriber::util::SubscriberInitExt::init(subscriber);
// SAFETY: 注册静态 sqlite-vec 初始化函数。transmute 是安全的,
// 因为 sqlite3_vec_init 符合 SQLite C API 的自动扩展回调签名
// (sqlite3*, char**, const sqlite3_api_routines*)。
// 该注册必须在开启任何数据库连接前执行。
unsafe {
libsqlite3_sys::sqlite3_auto_extension(Some(std::mem::transmute(
sqlite_vec::sqlite3_vec_init as *const (),
)));
}
let config = Config::from_env();
let options = SqliteConnectOptions::from_str(&config.database_url)?
.foreign_keys(true)
.create_if_missing(true);
let pool = SqlitePoolOptions::new()
.max_connections(2)
.connect_with(options)
.await?;
// 执行迁移
sqlx::migrate!("./migrations")
.run(&pool)
.await?;
let cli = Cli::parse();
match cli.command {
Commands::Rag { question, top_k } => {
let embedding = EmbeddingClient::new(
config.embedding_api_key.clone(),
config.embedding_api_base.clone(),
config.embedding_model.clone(),
);
let llm = LlmClient::new(
config.llm_api_key.clone(),
config.llm_api_base.clone(),
config.llm_model.clone(),
);
let result = astroresearch::services::rag::ask(
&pool, &embedding, &llm, &question, top_k
).await?;
println!("\n📖 回答:\n{}\n", result.answer);
if !result.sources.is_empty() {
println!("📚 参考来源:");
for (i, src) in result.sources.iter().enumerate() {
println!(
" [{}] {} §{} (距离: {:.4})",
i + 1, src.bibcode, src.paragraph_index, src.distance
);
}
}
}
Commands::TargetQuery { object_name } => {
let client = reqwest::Client::new();
let info = astroresearch::services::target::query_target_cached(
&pool, &object_name, None, &client
).await?;
println!("\n🔭 天体信息: {}", info.target_name);
if let Some(ra) = &info.ra {
println!(" RA: {}", ra);
}
if let Some(dec) = &info.dec {
println!(" Dec: {}", dec);
}
if let Some(sp) = &info.spectral_type {
println!(" 光谱类型: {}", sp);
}
if let Some(vmag) = info.v_magnitude {
println!(" 视星等: {:.2}", vmag);
}
if let Some(plx) = info.parallax {
println!(" 视差: {:.2} mas", plx);
}
if !info.aliases.is_empty() {
println!(" 别名: {}", info.aliases.join(", "));
}
}
Commands::TargetAssociate { bibcode, object_name } => {
let client = reqwest::Client::new();
let info = astroresearch::services::target::query_target_cached(
&pool, &object_name, Some(&bibcode), &client
).await?;
println!("✅ 已关联天体 {} -> 文献 {}", info.target_name, bibcode);
}
Commands::ExtractTargets { text } => {
let targets = astroresearch::services::target::extract_targets(&text);
if targets.is_empty() {
println!("未识别到天体标识符。");
} else {
println!("🎯 识别到 {} 个天体:", targets.len());
for t in &targets {
println!(" - {}", t);
}
}
}
Commands::Ingest { bibcode, markdown_path } => {
let content = std::fs::read_to_string(&markdown_path)?;
let embedding = EmbeddingClient::new(
config.embedding_api_key.clone(),
config.embedding_api_base.clone(),
config.embedding_model.clone(),
);
let count = astroresearch::services::rag::ingest_paper(
&pool, &embedding, &bibcode, &content, None
).await?;
println!("✅ 文献 {} 向量化完成,写入 {} 个切片", bibcode, count);
}
}
Ok(())
}
+9
View File
@@ -192,6 +192,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.finish();
tracing::subscriber::set_global_default(subscriber)?;
// SAFETY: sqlite3_vec_init 的函数签名严格符合 SQLite C API 自动扩展
// 回调规范。即使 health_check 不直接使用 vec0,也需要注册以防数据库
// 包含 vec0 虚拟表时连接崩溃。
unsafe {
libsqlite3_sys::sqlite3_auto_extension(Some(std::mem::transmute(
sqlite_vec::sqlite3_vec_init as *const (),
)));
}
let args: Vec<String> = std::env::args().collect();
let fix = args.contains(&"--fix".to_string());
+116
View File
@@ -0,0 +1,116 @@
use std::path::{Path, PathBuf};
use std::time::Instant;
use astroresearch::services::parser::html_to_markdown;
fn main() -> anyhow::Result<()> {
let args: Vec<String> = std::env::args().collect();
let html_dir = Path::new("library/HTML");
let md_dir = Path::new("library/Markdown");
if args.len() > 1 {
// Single file mode
let stem = &args[1];
let html_path = html_dir.join(format!("{}.html", stem));
let md_path = md_dir.join(format!("{}.md", stem));
if !html_path.exists() {
anyhow::bail!("HTML file not found: {:?}", html_path);
}
let front_matter = if md_path.exists() {
extract_front_matter(&std::fs::read_to_string(&md_path)?)
} else { String::new() };
let t0 = Instant::now();
let md_content = html_to_markdown(&html_path)?;
let elapsed = t0.elapsed();
let final_md = if front_matter.is_empty() { md_content }
else { format!("{}\n\n{}", front_matter.trim_end(), md_content) };
std::fs::write(&md_path, &final_md)?;
println!("{}{} [{:.0}ms]", stem, md_path.display(), elapsed.as_secs_f64() * 1000.0);
return Ok(());
}
if !html_dir.exists() {
anyhow::bail!("library/HTML directory not found");
}
let mut html_files: Vec<PathBuf> = std::fs::read_dir(html_dir)?
.filter_map(|e| e.ok())
.map(|e| e.path())
.filter(|p| p.extension().map_or(false, |ext| ext == "html"))
.collect();
html_files.sort();
let total = html_files.len();
println!("Found {} HTML files to reparse\n", total);
let mut success = 0u32;
let mut skipped = 0u32;
let mut failed = 0u32;
for html_path in &html_files {
let stem = html_path.file_stem().and_then(|s| s.to_str()).unwrap_or("");
let md_path = md_dir.join(format!("{}.md", stem));
// Extract front matter from existing MD if present
let front_matter = if md_path.exists() {
match std::fs::read_to_string(&md_path) {
Ok(original) => extract_front_matter(&original),
Err(_) => String::new(),
}
} else {
String::new()
};
// Parse HTML → Markdown
match html_to_markdown(html_path) {
Ok(md_content) => {
let final_md = if front_matter.is_empty() {
md_content
} else {
format!("{}\n\n{}", front_matter.trim_end(), md_content)
};
match std::fs::write(&md_path, &final_md) {
Ok(_) => {
println!("{}{}", stem, md_path.display());
success += 1;
}
Err(e) => {
eprintln!("{} write error: {}", stem, e);
failed += 1;
}
}
}
Err(e) => {
eprintln!("{} parse error: {}", stem, e);
failed += 1;
}
}
}
println!("\n--- Done ---");
println!("{} succeeded", success);
println!("{} failed", failed);
if skipped > 0 {
println!(" ⏭️ {} skipped", skipped);
}
println!(" 📂 {} total", total);
if failed > 0 {
std::process::exit(1);
}
Ok(())
}
fn extract_front_matter(md: &str) -> String {
if !md.starts_with("---") {
return String::new();
}
if let Some(second_dash) = md[3..].find("---") {
let end = 3 + second_dash + 3;
md[..end].to_string()
} else {
String::new()
}
}