Asfmq
22e7e1dcee
核心新增：
- RAG 问答系统：Markdown 安全切片器 (LaTeX 保护) + 向量化 + sqlite-vec 检索 + LLM 生成
- 天体目标识别：IAU 标准正则提取 15+ 星表标识符，CDS SIMBAD/Sesame 查询与本地缓存
- 多模态 LLM：chat_completion_with_image 支持图表视觉分析
- CLI Skills Agent (cli.rs)：对外暴露 rag/target/ingest 等 5 个子命令
- 解析器模块化重构：单体 778 行 → 按期刊拆分 (A&A/ar5iv/IOP/Generic/PDF) +
common.rs 静态正则工具库
管线与 Schema：
- AssetSync→AssetBatch 重命名，批量管线新增 embed/target 两个处理阶段
- 新增 paper_chunks_content (RAG 切片) 和 paper_targets (天体缓存) 两张表
- StandardPaper 新增 has_vector 字段，所有查询同步更新
前端：
- 新增 AI 助手侧边栏 (RAG 问答 + 来源跳转高亮)
- 最近浏览文献列表 (localStorage 持久化)、跨面板无缝导航
- SyncPanel 批量阶段扩展为下拉选项，支持向量化/天体识别
测试与清理：
- 集成测试合并至 ads.rs 和 llm.rs，新增 chunker + target 单元测试 15 个
- 删除旧单体 parser.rs、独立测试文件及过期 scratch 脚本
389 lines
15 KiB
Rust
389 lines
15 KiB
Rust
// src/api/helpers.rs
|
||
use sqlx::{SqlitePool, Row};
|
||
use tracing::info;
|
||
use crate::clients::ads::AdsPaperDoc;
|
||
use crate::clients::arxiv::ArxivPaper;
|
||
use super::StandardPaper;
|
||
|
||
pub fn convert_ads_doc_to_standard(doc: &AdsPaperDoc) -> StandardPaper {
|
||
let title = doc.title.as_ref()
|
||
.and_then(|v: &Vec<String>| v.first())
|
||
.cloned()
|
||
.unwrap_or_else(|| doc.bibcode.clone());
|
||
let authors = doc.author.clone().unwrap_or_default();
|
||
let keywords = doc.keyword.clone().unwrap_or_default();
|
||
let doi = doc.doi.as_ref()
|
||
.and_then(|v: &Vec<String>| v.first())
|
||
.cloned()
|
||
.unwrap_or_default();
|
||
|
||
let mut arxiv_id = String::new();
|
||
if let Some(identifiers) = &doc.identifier {
|
||
for id in identifiers {
|
||
if id.starts_with("arXiv:") {
|
||
arxiv_id = id.replace("arXiv:", "").trim().to_string();
|
||
break;
|
||
}
|
||
}
|
||
}
|
||
if arxiv_id.is_empty() {
|
||
if doc.bibcode.starts_with("arXiv") {
|
||
arxiv_id = doc.bibcode.replace("arXiv", "").trim().to_string();
|
||
}
|
||
}
|
||
|
||
StandardPaper {
|
||
bibcode: doc.bibcode.clone(),
|
||
title,
|
||
authors,
|
||
year: doc.year.clone().unwrap_or_default(),
|
||
pub_journal: doc.pub_journal.clone().unwrap_or_default(),
|
||
keywords,
|
||
abstract_text: doc.abstract_text.clone().unwrap_or_default(),
|
||
doi,
|
||
arxiv_id,
|
||
citation_count: doc.citation_count.unwrap_or(0),
|
||
reference_count: doc.reference_count.unwrap_or(0),
|
||
is_downloaded: false,
|
||
has_pdf: false,
|
||
has_html: false,
|
||
has_markdown: false,
|
||
has_translation: false,
|
||
has_vector: false,
|
||
doctype: doc.doctype.clone().unwrap_or_else(|| "article".to_string()),
|
||
pdf_error: None,
|
||
html_error: None,
|
||
}
|
||
}
|
||
|
||
pub fn convert_arxiv_to_standard(doc: &ArxivPaper) -> StandardPaper {
|
||
StandardPaper {
|
||
bibcode: doc.id.clone(),
|
||
title: doc.title.clone(),
|
||
authors: doc.authors.clone(),
|
||
year: doc.year.clone(),
|
||
pub_journal: "arXiv Preprint".to_string(),
|
||
keywords: Vec::new(),
|
||
abstract_text: doc.abstract_text.clone(),
|
||
doi: doc.doi.clone().unwrap_or_default(),
|
||
arxiv_id: doc.id.clone(),
|
||
citation_count: 0,
|
||
reference_count: 0,
|
||
is_downloaded: false,
|
||
has_pdf: false,
|
||
has_html: false,
|
||
has_markdown: false,
|
||
has_translation: false,
|
||
has_vector: false,
|
||
doctype: "eprint".to_string(),
|
||
pdf_error: None,
|
||
html_error: None,
|
||
}
|
||
}
|
||
|
||
pub async fn save_paper_to_db(db: &SqlitePool, p: &StandardPaper) -> anyhow::Result<()> {
|
||
let authors_json = serde_json::to_string(&p.authors)?;
|
||
let keywords_json = serde_json::to_string(&p.keywords)?;
|
||
|
||
// 1. 如果存在 arxiv_id，检查是否有已存在的相同 arxiv_id 记录以防 duplicate
|
||
if !p.arxiv_id.is_empty() {
|
||
let existing_opt: Option<(String, Option<String>, Option<String>, Option<String>, Option<String>)> = sqlx::query_as(
|
||
"SELECT bibcode, pdf_path, html_path, markdown_path, translation_path FROM papers WHERE arxiv_id = ?"
|
||
)
|
||
.bind(&p.arxiv_id)
|
||
.fetch_optional(db)
|
||
.await?;
|
||
|
||
if let Some((existing_bibcode, _pdf, _html, _md, _tr)) = existing_opt {
|
||
if existing_bibcode != p.bibcode {
|
||
// 发现不同 bibcode 标识的同一篇文献记录，需要进行合并
|
||
// 如果已存在的记录使用的是临时 arXiv ID 作为 bibcode，且新记录使用的是正式 ADS bibcode，我们升级 bibcode 主键
|
||
let is_existing_temp = existing_bibcode == p.arxiv_id;
|
||
let is_new_formal = p.bibcode != p.arxiv_id;
|
||
|
||
if is_existing_temp && is_new_formal {
|
||
info!("发现相同 arXiv ID 的文献，将临时主键 {} 升级为正式 ADS Bibcode: {}", existing_bibcode, p.bibcode);
|
||
sqlx::query(
|
||
"UPDATE papers SET bibcode = ?, title = ?, authors = ?, year = ?, pub = ?, keywords = ?, abstract = ?, doi = ?, citation_count = ?, reference_count = ?, doctype = ? WHERE bibcode = ?"
|
||
)
|
||
.bind(&p.bibcode)
|
||
.bind(&p.title)
|
||
.bind(&authors_json)
|
||
.bind(&p.year)
|
||
.bind(&p.pub_journal)
|
||
.bind(&keywords_json)
|
||
.bind(&p.abstract_text)
|
||
.bind(&p.doi)
|
||
.bind(p.citation_count)
|
||
.bind(p.reference_count)
|
||
.bind(&p.doctype)
|
||
.bind(&existing_bibcode)
|
||
.execute(db)
|
||
.await?;
|
||
|
||
return Ok(());
|
||
} else {
|
||
// 如果已存在的是正式 ADS bibcode，而新插入的是临时 arXiv ID，直接忽略或更新元数据而不更改主键
|
||
info!("发现相同 arXiv ID 的文献 {} 已存在正式记录，忽略临时 arXiv 插入", existing_bibcode);
|
||
return Ok(());
|
||
}
|
||
}
|
||
}
|
||
}
|
||
|
||
// 2. 正常插入/冲突更新
|
||
sqlx::query(
|
||
"INSERT INTO papers (bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, doctype) \
|
||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) \
|
||
ON CONFLICT(bibcode) DO UPDATE SET \
|
||
title=excluded.title, \
|
||
authors=excluded.authors, \
|
||
pub=excluded.pub, \
|
||
keywords=excluded.keywords, \
|
||
abstract=excluded.abstract, \
|
||
doi=excluded.doi, \
|
||
arxiv_id=excluded.arxiv_id, \
|
||
citation_count=excluded.citation_count, \
|
||
reference_count=excluded.reference_count, \
|
||
doctype=excluded.doctype"
|
||
)
|
||
.bind(&p.bibcode)
|
||
.bind(&p.title)
|
||
.bind(authors_json)
|
||
.bind(&p.year)
|
||
.bind(&p.pub_journal)
|
||
.bind(keywords_json)
|
||
.bind(&p.abstract_text)
|
||
.bind(&p.doi)
|
||
.bind(&p.arxiv_id)
|
||
.bind(p.citation_count)
|
||
.bind(p.reference_count)
|
||
.bind(&p.doctype)
|
||
.execute(db)
|
||
.await?;
|
||
|
||
Ok(())
|
||
}
|
||
|
||
pub async fn get_paper_from_db(db: &SqlitePool, library_dir: &std::path::Path, bibcode: &str) -> anyhow::Result<StandardPaper> {
|
||
let r = sqlx::query("SELECT bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, pdf_path, html_path, markdown_path, translation_path, doctype, EXISTS(SELECT 1 FROM paper_chunks_content WHERE bibcode = papers.bibcode) FROM papers WHERE bibcode = ?")
|
||
.bind(bibcode)
|
||
.fetch_one(db)
|
||
.await?;
|
||
|
||
let pdf_path: Option<String> = r.get(11);
|
||
let html_path: Option<String> = r.get(12);
|
||
let markdown_path: Option<String> = r.get(13);
|
||
let translation_path: Option<String> = r.get(14);
|
||
let doctype_val: Option<String> = r.get(15);
|
||
let has_vector: bool = r.get(16);
|
||
|
||
let authors_str: Option<String> = r.get(2);
|
||
let authors: Vec<String> = authors_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default();
|
||
let keywords_str: Option<String> = r.get(5);
|
||
let keywords: Vec<String> = keywords_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default();
|
||
|
||
let is_pdf_exist = pdf_path.as_ref().map(|p| library_dir.join(p).exists()).unwrap_or(false);
|
||
let is_html_exist = html_path.as_ref().map(|p| library_dir.join(p).exists()).unwrap_or(false);
|
||
let is_md_exist = markdown_path.as_ref().map(|p| library_dir.join(p).exists()).unwrap_or(false);
|
||
let is_tr_exist = translation_path.as_ref().map(|p| library_dir.join(p).exists()).unwrap_or(false);
|
||
|
||
let pdf_error = pdf_path.as_ref()
|
||
.filter(|p| p.starts_with("error:"))
|
||
.map(|p| p["error:".len()..].trim().to_string());
|
||
let html_error = html_path.as_ref()
|
||
.filter(|p| p.starts_with("error:"))
|
||
.map(|p| p["error:".len()..].trim().to_string());
|
||
|
||
Ok(StandardPaper {
|
||
bibcode: r.get(0),
|
||
title: r.get(1),
|
||
authors,
|
||
year: r.get(3),
|
||
pub_journal: r.get(4),
|
||
keywords,
|
||
abstract_text: r.get(6),
|
||
doi: r.get(7),
|
||
arxiv_id: r.get(8),
|
||
citation_count: r.get(9),
|
||
reference_count: r.get(10),
|
||
is_downloaded: is_pdf_exist || is_html_exist,
|
||
has_pdf: is_pdf_exist,
|
||
has_html: is_html_exist,
|
||
has_markdown: is_md_exist,
|
||
has_translation: is_tr_exist,
|
||
has_vector,
|
||
doctype: doctype_val.unwrap_or_else(|| "article".to_string()),
|
||
pdf_error,
|
||
html_error,
|
||
})
|
||
}
|
||
|
||
pub async fn check_paper_paths_in_db(
|
||
db: &SqlitePool,
|
||
library_dir: &std::path::Path,
|
||
bibcode: &str
|
||
) -> anyhow::Result<Option<(Option<String>, Option<String>, Option<String>, Option<String>)>> {
|
||
let r_opt = sqlx::query("SELECT pdf_path, html_path, markdown_path, translation_path FROM papers WHERE bibcode = ?")
|
||
.bind(bibcode)
|
||
.fetch_optional(db)
|
||
.await?;
|
||
|
||
if let Some(r) = r_opt {
|
||
let pdf: Option<String> = r.get(0);
|
||
let html: Option<String> = r.get(1);
|
||
let md: Option<String> = r.get(2);
|
||
let tr: Option<String> = r.get(3);
|
||
|
||
let pdf_res = pdf.filter(|p| library_dir.join(p).exists());
|
||
let html_res = html.filter(|p| library_dir.join(p).exists());
|
||
let md_res = md.filter(|p| library_dir.join(p).exists());
|
||
let tr_res = tr.filter(|p| library_dir.join(p).exists());
|
||
|
||
Ok(Some((pdf_res, html_res, md_res, tr_res)))
|
||
} else {
|
||
Ok(None)
|
||
}
|
||
}
|
||
|
||
#[cfg(test)]
|
||
mod tests {
|
||
use super::*;
|
||
use sqlx::sqlite::SqlitePoolOptions;
|
||
|
||
#[test]
|
||
fn test_convert_ads_doc_to_standard() {
|
||
let doc = AdsPaperDoc {
|
||
bibcode: "2026A&A...123..456X".to_string(),
|
||
title: Some(vec!["A Test Title".to_string()]),
|
||
author: Some(vec!["Author A".to_string(), "Author B".to_string()]),
|
||
year: Some("2026".to_string()),
|
||
pub_journal: Some("Astronomy & Astrophysics".to_string()),
|
||
keyword: Some(vec!["Keyword 1".to_string()]),
|
||
abstract_text: Some("This is abstract".to_string()),
|
||
doi: Some(vec!["10.1000/test.doi".to_string()]),
|
||
citation_count: Some(5),
|
||
reference_count: Some(10),
|
||
reference: None,
|
||
citation: None,
|
||
identifier: None,
|
||
doctype: Some("article".to_string()),
|
||
};
|
||
|
||
let paper = convert_ads_doc_to_standard(&doc);
|
||
assert_eq!(paper.bibcode, "2026A&A...123..456X");
|
||
assert_eq!(paper.title, "A Test Title");
|
||
assert_eq!(paper.authors, vec!["Author A", "Author B"]);
|
||
assert_eq!(paper.year, "2026");
|
||
assert_eq!(paper.pub_journal, "Astronomy & Astrophysics");
|
||
assert_eq!(paper.keywords, vec!["Keyword 1"]);
|
||
assert_eq!(paper.abstract_text, "This is abstract");
|
||
assert_eq!(paper.doi, "10.1000/test.doi");
|
||
assert_eq!(paper.arxiv_id, "");
|
||
assert_eq!(paper.citation_count, 5);
|
||
assert_eq!(paper.reference_count, 10);
|
||
}
|
||
|
||
#[test]
|
||
fn test_convert_ads_doc_to_standard_with_arxiv_identifier() {
|
||
let doc = AdsPaperDoc {
|
||
bibcode: "2026MNRAS.530.1234A".to_string(),
|
||
title: Some(vec!["Another Test Title".to_string()]),
|
||
author: Some(vec!["Author A".to_string()]),
|
||
year: Some("2026".to_string()),
|
||
pub_journal: Some("MNRAS".to_string()),
|
||
keyword: None,
|
||
abstract_text: None,
|
||
doi: None,
|
||
citation_count: None,
|
||
reference_count: None,
|
||
reference: None,
|
||
citation: None,
|
||
identifier: Some(vec!["2026MNRAS.530.1234A".to_string(), "arXiv:2606.12345".to_string()]),
|
||
doctype: Some("article".to_string()),
|
||
};
|
||
|
||
let paper = convert_ads_doc_to_standard(&doc);
|
||
assert_eq!(paper.bibcode, "2026MNRAS.530.1234A");
|
||
assert_eq!(paper.arxiv_id, "2606.12345");
|
||
}
|
||
|
||
#[test]
|
||
fn test_convert_arxiv_to_standard() {
|
||
let doc = ArxivPaper {
|
||
id: "2606.12345".to_string(),
|
||
title: "Arxiv Title".to_string(),
|
||
authors: vec!["Author C".to_string()],
|
||
year: "2026".to_string(),
|
||
abstract_text: "Arxiv abstract".to_string(),
|
||
doi: Some("10.1000/arxiv.doi".to_string()),
|
||
pdf_url: "https://arxiv.org/pdf/2606.12345.pdf".to_string(),
|
||
};
|
||
|
||
let paper = convert_arxiv_to_standard(&doc);
|
||
assert_eq!(paper.bibcode, "2606.12345");
|
||
assert_eq!(paper.title, "Arxiv Title");
|
||
assert_eq!(paper.authors, vec!["Author C"]);
|
||
assert_eq!(paper.year, "2026");
|
||
assert_eq!(paper.pub_journal, "arXiv Preprint");
|
||
assert_eq!(paper.doi, "10.1000/arxiv.doi");
|
||
assert_eq!(paper.arxiv_id, "2606.12345");
|
||
}
|
||
|
||
#[tokio::test]
|
||
async fn test_db_operations() -> anyhow::Result<()> {
|
||
let pool = SqlitePoolOptions::new()
|
||
.max_connections(1)
|
||
.connect("sqlite::memory:")
|
||
.await?;
|
||
|
||
// 运行迁移
|
||
sqlx::migrate!("./migrations")
|
||
.run(&pool)
|
||
.await?;
|
||
|
||
let paper = StandardPaper {
|
||
bibcode: "2026A&A...123..456X".to_string(),
|
||
title: "A Test Title".to_string(),
|
||
authors: vec!["Author A".to_string()],
|
||
year: "2026".to_string(),
|
||
pub_journal: "Astronomy & Astrophysics".to_string(),
|
||
keywords: vec!["Keyword 1".to_string()],
|
||
abstract_text: "This is abstract".to_string(),
|
||
doi: "10.1000/test.doi".to_string(),
|
||
arxiv_id: "".to_string(),
|
||
citation_count: 5,
|
||
reference_count: 10,
|
||
is_downloaded: false,
|
||
has_pdf: false,
|
||
has_html: false,
|
||
has_markdown: false,
|
||
has_translation: false,
|
||
has_vector: false,
|
||
doctype: "article".to_string(),
|
||
pdf_error: None,
|
||
html_error: None,
|
||
};
|
||
|
||
// 保存
|
||
save_paper_to_db(&pool, &paper).await?;
|
||
|
||
// 读取
|
||
let retrieved = get_paper_from_db(&pool, std::path::Path::new(""), "2026A&A...123..456X").await?;
|
||
assert_eq!(retrieved.title, paper.title);
|
||
assert_eq!(retrieved.authors, paper.authors);
|
||
assert_eq!(retrieved.keywords, paper.keywords);
|
||
|
||
// 检查路径状态（初始为 None）
|
||
let paths = check_paper_paths_in_db(&pool, std::path::Path::new(""), "2026A&A...123..456X").await?;
|
||
assert!(paths.is_some());
|
||
let (pdf, html, md, tr) = paths.unwrap();
|
||
assert!(pdf.is_none());
|
||
assert!(html.is_none());
|
||
assert!(md.is_none());
|
||
assert!(tr.is_none());
|
||
|
||
Ok(())
|
||
}
|
||
}
|