// src/api/helpers.rs use sqlx::{SqlitePool, Row}; use tracing::info; use crate::clients::ads::AdsPaperDoc; use crate::clients::arxiv::ArxivPaper; use super::StandardPaper; pub fn convert_ads_doc_to_standard(doc: &AdsPaperDoc) -> StandardPaper { let title = doc.title.as_ref() .and_then(|v: &Vec| v.first()) .cloned() .unwrap_or_else(|| doc.bibcode.clone()); let authors = doc.author.clone().unwrap_or_default(); let keywords = doc.keyword.clone().unwrap_or_default(); let doi = doc.doi.as_ref() .and_then(|v: &Vec| v.first()) .cloned() .unwrap_or_default(); let mut arxiv_id = String::new(); if let Some(identifiers) = &doc.identifier { for id in identifiers { if id.starts_with("arXiv:") { arxiv_id = id.replace("arXiv:", "").trim().to_string(); break; } } } if arxiv_id.is_empty() { if doc.bibcode.starts_with("arXiv") { arxiv_id = doc.bibcode.replace("arXiv", "").trim().to_string(); } } StandardPaper { bibcode: doc.bibcode.clone(), title, authors, year: doc.year.clone().unwrap_or_default(), pub_journal: doc.pub_journal.clone().unwrap_or_default(), keywords, abstract_text: doc.abstract_text.clone().unwrap_or_default(), doi, arxiv_id, citation_count: doc.citation_count.unwrap_or(0), reference_count: doc.reference_count.unwrap_or(0), is_downloaded: false, has_pdf: false, has_html: false, has_markdown: false, has_translation: false, has_vector: false, doctype: doc.doctype.clone().unwrap_or_else(|| "article".to_string()), pdf_error: None, html_error: None, } } pub fn convert_arxiv_to_standard(doc: &ArxivPaper) -> StandardPaper { StandardPaper { bibcode: doc.id.clone(), title: doc.title.clone(), authors: doc.authors.clone(), year: doc.year.clone(), pub_journal: "arXiv Preprint".to_string(), keywords: Vec::new(), abstract_text: doc.abstract_text.clone(), doi: doc.doi.clone().unwrap_or_default(), arxiv_id: doc.id.clone(), citation_count: 0, reference_count: 0, is_downloaded: false, has_pdf: false, has_html: false, has_markdown: false, has_translation: false, has_vector: false, doctype: "eprint".to_string(), pdf_error: None, html_error: None, } } pub async fn save_paper_to_db(db: &SqlitePool, p: &StandardPaper) -> anyhow::Result<()> { let authors_json = serde_json::to_string(&p.authors)?; let keywords_json = serde_json::to_string(&p.keywords)?; // 1. 如果存在 arxiv_id,检查是否有已存在的相同 arxiv_id 记录以防 duplicate if !p.arxiv_id.is_empty() { let existing_opt: Option<(String, Option, Option, Option, Option)> = sqlx::query_as( "SELECT bibcode, pdf_path, html_path, markdown_path, translation_path FROM papers WHERE arxiv_id = ?" ) .bind(&p.arxiv_id) .fetch_optional(db) .await?; if let Some((existing_bibcode, _pdf, _html, _md, _tr)) = existing_opt { if existing_bibcode != p.bibcode { // 发现不同 bibcode 标识的同一篇文献记录,需要进行合并 // 如果已存在的记录使用的是临时 arXiv ID 作为 bibcode,且新记录使用的是正式 ADS bibcode,我们升级 bibcode 主键 let is_existing_temp = existing_bibcode == p.arxiv_id; let is_new_formal = p.bibcode != p.arxiv_id; if is_existing_temp && is_new_formal { info!("发现相同 arXiv ID 的文献,将临时主键 {} 升级为正式 ADS Bibcode: {}", existing_bibcode, p.bibcode); sqlx::query( "UPDATE papers SET bibcode = ?, title = ?, authors = ?, year = ?, pub = ?, keywords = ?, abstract = ?, doi = ?, citation_count = ?, reference_count = ?, doctype = ? WHERE bibcode = ?" ) .bind(&p.bibcode) .bind(&p.title) .bind(&authors_json) .bind(&p.year) .bind(&p.pub_journal) .bind(&keywords_json) .bind(&p.abstract_text) .bind(&p.doi) .bind(p.citation_count) .bind(p.reference_count) .bind(&p.doctype) .bind(&existing_bibcode) .execute(db) .await?; return Ok(()); } else { // 如果已存在的是正式 ADS bibcode,而新插入的是临时 arXiv ID,直接忽略或更新元数据而不更改主键 info!("发现相同 arXiv ID 的文献 {} 已存在正式记录,忽略临时 arXiv 插入", existing_bibcode); return Ok(()); } } } } // 2. 正常插入/冲突更新 sqlx::query( "INSERT INTO papers (bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, doctype) \ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) \ ON CONFLICT(bibcode) DO UPDATE SET \ title=excluded.title, \ authors=excluded.authors, \ pub=excluded.pub, \ keywords=excluded.keywords, \ abstract=excluded.abstract, \ doi=excluded.doi, \ arxiv_id=excluded.arxiv_id, \ citation_count=excluded.citation_count, \ reference_count=excluded.reference_count, \ doctype=excluded.doctype" ) .bind(&p.bibcode) .bind(&p.title) .bind(authors_json) .bind(&p.year) .bind(&p.pub_journal) .bind(keywords_json) .bind(&p.abstract_text) .bind(&p.doi) .bind(&p.arxiv_id) .bind(p.citation_count) .bind(p.reference_count) .bind(&p.doctype) .execute(db) .await?; Ok(()) } pub async fn get_paper_from_db(db: &SqlitePool, library_dir: &std::path::Path, bibcode: &str) -> anyhow::Result { let r = sqlx::query("SELECT bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, pdf_path, html_path, markdown_path, translation_path, doctype, EXISTS(SELECT 1 FROM paper_chunks_content WHERE bibcode = papers.bibcode) FROM papers WHERE bibcode = ?") .bind(bibcode) .fetch_one(db) .await?; let pdf_path: Option = r.get(11); let html_path: Option = r.get(12); let markdown_path: Option = r.get(13); let translation_path: Option = r.get(14); let doctype_val: Option = r.get(15); let has_vector: bool = r.get(16); let authors_str: Option = r.get(2); let authors: Vec = authors_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default(); let keywords_str: Option = r.get(5); let keywords: Vec = keywords_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default(); let is_pdf_exist = pdf_path.as_ref().map(|p| library_dir.join(p).exists()).unwrap_or(false); let is_html_exist = html_path.as_ref().map(|p| library_dir.join(p).exists()).unwrap_or(false); let is_md_exist = markdown_path.as_ref().map(|p| library_dir.join(p).exists()).unwrap_or(false); let is_tr_exist = translation_path.as_ref().map(|p| library_dir.join(p).exists()).unwrap_or(false); let pdf_error = pdf_path.as_ref() .filter(|p| p.starts_with("error:")) .map(|p| p["error:".len()..].trim().to_string()); let html_error = html_path.as_ref() .filter(|p| p.starts_with("error:")) .map(|p| p["error:".len()..].trim().to_string()); Ok(StandardPaper { bibcode: r.get(0), title: r.get(1), authors, year: r.get(3), pub_journal: r.get(4), keywords, abstract_text: r.get(6), doi: r.get(7), arxiv_id: r.get(8), citation_count: r.get(9), reference_count: r.get(10), is_downloaded: is_pdf_exist || is_html_exist, has_pdf: is_pdf_exist, has_html: is_html_exist, has_markdown: is_md_exist, has_translation: is_tr_exist, has_vector, doctype: doctype_val.unwrap_or_else(|| "article".to_string()), pdf_error, html_error, }) } pub async fn check_paper_paths_in_db( db: &SqlitePool, library_dir: &std::path::Path, bibcode: &str ) -> anyhow::Result, Option, Option, Option)>> { let r_opt = sqlx::query("SELECT pdf_path, html_path, markdown_path, translation_path FROM papers WHERE bibcode = ?") .bind(bibcode) .fetch_optional(db) .await?; if let Some(r) = r_opt { let pdf: Option = r.get(0); let html: Option = r.get(1); let md: Option = r.get(2); let tr: Option = r.get(3); let pdf_res = pdf.filter(|p| library_dir.join(p).exists()); let html_res = html.filter(|p| library_dir.join(p).exists()); let md_res = md.filter(|p| library_dir.join(p).exists()); let tr_res = tr.filter(|p| library_dir.join(p).exists()); Ok(Some((pdf_res, html_res, md_res, tr_res))) } else { Ok(None) } } #[cfg(test)] mod tests { use super::*; use sqlx::sqlite::SqlitePoolOptions; #[test] fn test_convert_ads_doc_to_standard() { let doc = AdsPaperDoc { bibcode: "2026A&A...123..456X".to_string(), title: Some(vec!["A Test Title".to_string()]), author: Some(vec!["Author A".to_string(), "Author B".to_string()]), year: Some("2026".to_string()), pub_journal: Some("Astronomy & Astrophysics".to_string()), keyword: Some(vec!["Keyword 1".to_string()]), abstract_text: Some("This is abstract".to_string()), doi: Some(vec!["10.1000/test.doi".to_string()]), citation_count: Some(5), reference_count: Some(10), reference: None, citation: None, identifier: None, doctype: Some("article".to_string()), }; let paper = convert_ads_doc_to_standard(&doc); assert_eq!(paper.bibcode, "2026A&A...123..456X"); assert_eq!(paper.title, "A Test Title"); assert_eq!(paper.authors, vec!["Author A", "Author B"]); assert_eq!(paper.year, "2026"); assert_eq!(paper.pub_journal, "Astronomy & Astrophysics"); assert_eq!(paper.keywords, vec!["Keyword 1"]); assert_eq!(paper.abstract_text, "This is abstract"); assert_eq!(paper.doi, "10.1000/test.doi"); assert_eq!(paper.arxiv_id, ""); assert_eq!(paper.citation_count, 5); assert_eq!(paper.reference_count, 10); } #[test] fn test_convert_ads_doc_to_standard_with_arxiv_identifier() { let doc = AdsPaperDoc { bibcode: "2026MNRAS.530.1234A".to_string(), title: Some(vec!["Another Test Title".to_string()]), author: Some(vec!["Author A".to_string()]), year: Some("2026".to_string()), pub_journal: Some("MNRAS".to_string()), keyword: None, abstract_text: None, doi: None, citation_count: None, reference_count: None, reference: None, citation: None, identifier: Some(vec!["2026MNRAS.530.1234A".to_string(), "arXiv:2606.12345".to_string()]), doctype: Some("article".to_string()), }; let paper = convert_ads_doc_to_standard(&doc); assert_eq!(paper.bibcode, "2026MNRAS.530.1234A"); assert_eq!(paper.arxiv_id, "2606.12345"); } #[test] fn test_convert_arxiv_to_standard() { let doc = ArxivPaper { id: "2606.12345".to_string(), title: "Arxiv Title".to_string(), authors: vec!["Author C".to_string()], year: "2026".to_string(), abstract_text: "Arxiv abstract".to_string(), doi: Some("10.1000/arxiv.doi".to_string()), pdf_url: "https://arxiv.org/pdf/2606.12345.pdf".to_string(), }; let paper = convert_arxiv_to_standard(&doc); assert_eq!(paper.bibcode, "2606.12345"); assert_eq!(paper.title, "Arxiv Title"); assert_eq!(paper.authors, vec!["Author C"]); assert_eq!(paper.year, "2026"); assert_eq!(paper.pub_journal, "arXiv Preprint"); assert_eq!(paper.doi, "10.1000/arxiv.doi"); assert_eq!(paper.arxiv_id, "2606.12345"); } #[tokio::test] async fn test_db_operations() -> anyhow::Result<()> { let pool = SqlitePoolOptions::new() .max_connections(1) .connect("sqlite::memory:") .await?; // 运行迁移 sqlx::migrate!("./migrations") .run(&pool) .await?; let paper = StandardPaper { bibcode: "2026A&A...123..456X".to_string(), title: "A Test Title".to_string(), authors: vec!["Author A".to_string()], year: "2026".to_string(), pub_journal: "Astronomy & Astrophysics".to_string(), keywords: vec!["Keyword 1".to_string()], abstract_text: "This is abstract".to_string(), doi: "10.1000/test.doi".to_string(), arxiv_id: "".to_string(), citation_count: 5, reference_count: 10, is_downloaded: false, has_pdf: false, has_html: false, has_markdown: false, has_translation: false, has_vector: false, doctype: "article".to_string(), pdf_error: None, html_error: None, }; // 保存 save_paper_to_db(&pool, &paper).await?; // 读取 let retrieved = get_paper_from_db(&pool, std::path::Path::new(""), "2026A&A...123..456X").await?; assert_eq!(retrieved.title, paper.title); assert_eq!(retrieved.authors, paper.authors); assert_eq!(retrieved.keywords, paper.keywords); // 检查路径状态(初始为 None) let paths = check_paper_paths_in_db(&pool, std::path::Path::new(""), "2026A&A...123..456X").await?; assert!(paths.is_some()); let (pdf, html, md, tr) = paths.unwrap(); assert!(pdf.is_none()); assert!(html.is_none()); assert!(md.is_none()); assert!(tr.is_none()); Ok(()) } }