feat: RAG 文献问答、天体目标识别、解析器模块化与批量管线扩展
核心新增:
- RAG 问答系统:Markdown 安全切片器 (LaTeX 保护) + 向量化 + sqlite-vec 检索 + LLM 生成
- 天体目标识别:IAU 标准正则提取 15+ 星表标识符,CDS SIMBAD/Sesame 查询与本地缓存
- 多模态 LLM:chat_completion_with_image 支持图表视觉分析
- CLI Skills Agent (cli.rs):对外暴露 rag/target/ingest 等 5 个子命令
- 解析器模块化重构:单体 778 行 → 按期刊拆分 (A&A/ar5iv/IOP/Generic/PDF) +
common.rs 静态正则工具库
管线与 Schema:
- AssetSync→AssetBatch 重命名,批量管线新增 embed/target 两个处理阶段
- 新增 paper_chunks_content (RAG 切片) 和 paper_targets (天体缓存) 两张表
- StandardPaper 新增 has_vector 字段,所有查询同步更新
前端:
- 新增 AI 助手侧边栏 (RAG 问答 + 来源跳转高亮)
- 最近浏览文献列表 (localStorage 持久化)、跨面板无缝导航
- SyncPanel 批量阶段扩展为下拉选项,支持向量化/天体识别
测试与清理:
- 集成测试合并至 ads.rs 和 llm.rs,新增 chunker + target 单元测试 15 个
- 删除旧单体 parser.rs、独立测试文件及过期 scratch 脚本
This commit is contained in:
+196
@@ -0,0 +1,196 @@
|
||||
// src/bin/cli.rs
|
||||
//
|
||||
// AstroResearch CLI Skills Agent — 向外部 Agent(如 Claude Code)提供
|
||||
// 标准的命令行工具接口,用于 RAG 问答、天体查询和天体关联操作。
|
||||
|
||||
use std::str::FromStr;
|
||||
use clap::{Parser, Subcommand};
|
||||
use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
|
||||
use tracing_subscriber::FmtSubscriber;
|
||||
|
||||
use astroresearch::Config;
|
||||
use astroresearch::clients::llm::{LlmClient, EmbeddingClient};
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(
|
||||
name = "astroresearch_cli",
|
||||
about = "AstroResearch CLI Skills Agent — 天体物理文献智能助手",
|
||||
version
|
||||
)]
|
||||
struct Cli {
|
||||
#[command(subcommand)]
|
||||
command: Commands,
|
||||
}
|
||||
|
||||
#[derive(Subcommand)]
|
||||
enum Commands {
|
||||
/// 跨文献 RAG 智能问答:在所有已导入文献中检索并生成回答
|
||||
Rag {
|
||||
/// 用户问题
|
||||
question: String,
|
||||
|
||||
/// 检索 Top-K 数量(默认 5)
|
||||
#[arg(short, long, default_value = "5")]
|
||||
top_k: usize,
|
||||
},
|
||||
|
||||
/// 查询天体数据:优先读取本地缓存,未命中时自动查询 SIMBAD
|
||||
TargetQuery {
|
||||
/// 天体名称(如 "GD 358", "NGC 1234", "HD 209458")
|
||||
object_name: String,
|
||||
},
|
||||
|
||||
/// 手动关联天体到特定文献
|
||||
TargetAssociate {
|
||||
/// 文献 bibcode
|
||||
bibcode: String,
|
||||
/// 天体名称
|
||||
object_name: String,
|
||||
},
|
||||
|
||||
/// 从文本中提取天体标识符(调试/预览用)
|
||||
ExtractTargets {
|
||||
/// 输入文本
|
||||
text: String,
|
||||
},
|
||||
|
||||
/// 向量化导入指定文献的 Markdown 文件
|
||||
Ingest {
|
||||
/// 文献 bibcode
|
||||
bibcode: String,
|
||||
/// Markdown 文件路径
|
||||
markdown_path: String,
|
||||
},
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
// 初始化日志
|
||||
let subscriber = FmtSubscriber::builder()
|
||||
.with_max_level(tracing::Level::INFO)
|
||||
.finish();
|
||||
tracing_subscriber::util::SubscriberInitExt::init(subscriber);
|
||||
|
||||
// SAFETY: 注册静态 sqlite-vec 初始化函数。transmute 是安全的,
|
||||
// 因为 sqlite3_vec_init 符合 SQLite C API 的自动扩展回调签名
|
||||
// (sqlite3*, char**, const sqlite3_api_routines*)。
|
||||
// 该注册必须在开启任何数据库连接前执行。
|
||||
unsafe {
|
||||
libsqlite3_sys::sqlite3_auto_extension(Some(std::mem::transmute(
|
||||
sqlite_vec::sqlite3_vec_init as *const (),
|
||||
)));
|
||||
}
|
||||
|
||||
let config = Config::from_env();
|
||||
|
||||
let options = SqliteConnectOptions::from_str(&config.database_url)?
|
||||
.foreign_keys(true)
|
||||
.create_if_missing(true);
|
||||
|
||||
let pool = SqlitePoolOptions::new()
|
||||
.max_connections(2)
|
||||
.connect_with(options)
|
||||
.await?;
|
||||
|
||||
// 执行迁移
|
||||
sqlx::migrate!("./migrations")
|
||||
.run(&pool)
|
||||
.await?;
|
||||
|
||||
let cli = Cli::parse();
|
||||
|
||||
match cli.command {
|
||||
Commands::Rag { question, top_k } => {
|
||||
let embedding = EmbeddingClient::new(
|
||||
config.embedding_api_key.clone(),
|
||||
config.embedding_api_base.clone(),
|
||||
config.embedding_model.clone(),
|
||||
);
|
||||
let llm = LlmClient::new(
|
||||
config.llm_api_key.clone(),
|
||||
config.llm_api_base.clone(),
|
||||
config.llm_model.clone(),
|
||||
);
|
||||
|
||||
let result = astroresearch::services::rag::ask(
|
||||
&pool, &embedding, &llm, &question, top_k
|
||||
).await?;
|
||||
|
||||
println!("\n📖 回答:\n{}\n", result.answer);
|
||||
if !result.sources.is_empty() {
|
||||
println!("📚 参考来源:");
|
||||
for (i, src) in result.sources.iter().enumerate() {
|
||||
println!(
|
||||
" [{}] {} §{} (距离: {:.4})",
|
||||
i + 1, src.bibcode, src.paragraph_index, src.distance
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Commands::TargetQuery { object_name } => {
|
||||
let client = reqwest::Client::new();
|
||||
let info = astroresearch::services::target::query_target_cached(
|
||||
&pool, &object_name, None, &client
|
||||
).await?;
|
||||
|
||||
println!("\n🔭 天体信息: {}", info.target_name);
|
||||
if let Some(ra) = &info.ra {
|
||||
println!(" RA: {}", ra);
|
||||
}
|
||||
if let Some(dec) = &info.dec {
|
||||
println!(" Dec: {}", dec);
|
||||
}
|
||||
if let Some(sp) = &info.spectral_type {
|
||||
println!(" 光谱类型: {}", sp);
|
||||
}
|
||||
if let Some(vmag) = info.v_magnitude {
|
||||
println!(" 视星等: {:.2}", vmag);
|
||||
}
|
||||
if let Some(plx) = info.parallax {
|
||||
println!(" 视差: {:.2} mas", plx);
|
||||
}
|
||||
if !info.aliases.is_empty() {
|
||||
println!(" 别名: {}", info.aliases.join(", "));
|
||||
}
|
||||
}
|
||||
|
||||
Commands::TargetAssociate { bibcode, object_name } => {
|
||||
let client = reqwest::Client::new();
|
||||
let info = astroresearch::services::target::query_target_cached(
|
||||
&pool, &object_name, Some(&bibcode), &client
|
||||
).await?;
|
||||
|
||||
println!("✅ 已关联天体 {} -> 文献 {}", info.target_name, bibcode);
|
||||
}
|
||||
|
||||
Commands::ExtractTargets { text } => {
|
||||
let targets = astroresearch::services::target::extract_targets(&text);
|
||||
if targets.is_empty() {
|
||||
println!("未识别到天体标识符。");
|
||||
} else {
|
||||
println!("🎯 识别到 {} 个天体:", targets.len());
|
||||
for t in &targets {
|
||||
println!(" - {}", t);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Commands::Ingest { bibcode, markdown_path } => {
|
||||
let content = std::fs::read_to_string(&markdown_path)?;
|
||||
let embedding = EmbeddingClient::new(
|
||||
config.embedding_api_key.clone(),
|
||||
config.embedding_api_base.clone(),
|
||||
config.embedding_model.clone(),
|
||||
);
|
||||
|
||||
let count = astroresearch::services::rag::ingest_paper(
|
||||
&pool, &embedding, &bibcode, &content, None
|
||||
).await?;
|
||||
|
||||
println!("✅ 文献 {} 向量化完成,写入 {} 个切片", bibcode, count);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -192,6 +192,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
.finish();
|
||||
tracing::subscriber::set_global_default(subscriber)?;
|
||||
|
||||
// SAFETY: sqlite3_vec_init 的函数签名严格符合 SQLite C API 自动扩展
|
||||
// 回调规范。即使 health_check 不直接使用 vec0,也需要注册以防数据库
|
||||
// 包含 vec0 虚拟表时连接崩溃。
|
||||
unsafe {
|
||||
libsqlite3_sys::sqlite3_auto_extension(Some(std::mem::transmute(
|
||||
sqlite_vec::sqlite3_vec_init as *const (),
|
||||
)));
|
||||
}
|
||||
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
let fix = args.contains(&"--fix".to_string());
|
||||
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::time::Instant;
|
||||
use astroresearch::services::parser::html_to_markdown;
|
||||
|
||||
fn main() -> anyhow::Result<()> {
|
||||
let args: Vec<String> = std::env::args().collect();
|
||||
let html_dir = Path::new("library/HTML");
|
||||
let md_dir = Path::new("library/Markdown");
|
||||
|
||||
if args.len() > 1 {
|
||||
// Single file mode
|
||||
let stem = &args[1];
|
||||
let html_path = html_dir.join(format!("{}.html", stem));
|
||||
let md_path = md_dir.join(format!("{}.md", stem));
|
||||
if !html_path.exists() {
|
||||
anyhow::bail!("HTML file not found: {:?}", html_path);
|
||||
}
|
||||
let front_matter = if md_path.exists() {
|
||||
extract_front_matter(&std::fs::read_to_string(&md_path)?)
|
||||
} else { String::new() };
|
||||
|
||||
let t0 = Instant::now();
|
||||
let md_content = html_to_markdown(&html_path)?;
|
||||
let elapsed = t0.elapsed();
|
||||
|
||||
let final_md = if front_matter.is_empty() { md_content }
|
||||
else { format!("{}\n\n{}", front_matter.trim_end(), md_content) };
|
||||
std::fs::write(&md_path, &final_md)?;
|
||||
println!("✅ {} → {} [{:.0}ms]", stem, md_path.display(), elapsed.as_secs_f64() * 1000.0);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if !html_dir.exists() {
|
||||
anyhow::bail!("library/HTML directory not found");
|
||||
}
|
||||
|
||||
let mut html_files: Vec<PathBuf> = std::fs::read_dir(html_dir)?
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.path())
|
||||
.filter(|p| p.extension().map_or(false, |ext| ext == "html"))
|
||||
.collect();
|
||||
html_files.sort();
|
||||
|
||||
let total = html_files.len();
|
||||
println!("Found {} HTML files to reparse\n", total);
|
||||
|
||||
let mut success = 0u32;
|
||||
let mut skipped = 0u32;
|
||||
let mut failed = 0u32;
|
||||
|
||||
for html_path in &html_files {
|
||||
let stem = html_path.file_stem().and_then(|s| s.to_str()).unwrap_or("");
|
||||
let md_path = md_dir.join(format!("{}.md", stem));
|
||||
|
||||
// Extract front matter from existing MD if present
|
||||
let front_matter = if md_path.exists() {
|
||||
match std::fs::read_to_string(&md_path) {
|
||||
Ok(original) => extract_front_matter(&original),
|
||||
Err(_) => String::new(),
|
||||
}
|
||||
} else {
|
||||
String::new()
|
||||
};
|
||||
|
||||
// Parse HTML → Markdown
|
||||
match html_to_markdown(html_path) {
|
||||
Ok(md_content) => {
|
||||
let final_md = if front_matter.is_empty() {
|
||||
md_content
|
||||
} else {
|
||||
format!("{}\n\n{}", front_matter.trim_end(), md_content)
|
||||
};
|
||||
|
||||
match std::fs::write(&md_path, &final_md) {
|
||||
Ok(_) => {
|
||||
println!("✅ {} → {}", stem, md_path.display());
|
||||
success += 1;
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("❌ {} write error: {}", stem, e);
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("❌ {} parse error: {}", stem, e);
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
println!("\n--- Done ---");
|
||||
println!(" ✅ {} succeeded", success);
|
||||
println!(" ❌ {} failed", failed);
|
||||
if skipped > 0 {
|
||||
println!(" ⏭️ {} skipped", skipped);
|
||||
}
|
||||
println!(" 📂 {} total", total);
|
||||
|
||||
if failed > 0 {
|
||||
std::process::exit(1);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn extract_front_matter(md: &str) -> String {
|
||||
if !md.starts_with("---") {
|
||||
return String::new();
|
||||
}
|
||||
if let Some(second_dash) = md[3..].find("---") {
|
||||
let end = 3 + second_dash + 3;
|
||||
md[..end].to_string()
|
||||
} else {
|
||||
String::new()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user