fmq/AstroResearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: RAG 文献问答、天体目标识别、解析器模块化与批量管线扩展

Browse Source 
核心新增：
  - RAG 问答系统：Markdown 安全切片器 (LaTeX 保护) + 向量化 + sqlite-vec 检索 + LLM 生成
  - 天体目标识别：IAU 标准正则提取 15+ 星表标识符，CDS SIMBAD/Sesame 查询与本地缓存
  - 多模态 LLM：chat_completion_with_image 支持图表视觉分析
  - CLI Skills Agent (cli.rs)：对外暴露 rag/target/ingest 等 5 个子命令
  - 解析器模块化重构：单体 778 行 → 按期刊拆分 (A&A/ar5iv/IOP/Generic/PDF) +
    common.rs 静态正则工具库

  管线与 Schema：
  - AssetSync→AssetBatch 重命名，批量管线新增 embed/target 两个处理阶段
  - 新增 paper_chunks_content (RAG 切片) 和 paper_targets (天体缓存) 两张表
  - StandardPaper 新增 has_vector 字段，所有查询同步更新

  前端：
  - 新增 AI 助手侧边栏 (RAG 问答 + 来源跳转高亮)
  - 最近浏览文献列表 (localStorage 持久化)、跨面板无缝导航
  - SyncPanel 批量阶段扩展为下拉选项，支持向量化/天体识别

  测试与清理：
  - 集成测试合并至 ads.rs 和 llm.rs，新增 chunker + target 单元测试 15 个
  - 删除旧单体 parser.rs、独立测试文件及过期 scratch 脚本
This commit is contained in:
Asfmq 2026-06-15 13:25:18 +08:00
parent 3f1935678b
commit 22e7e1dcee
45 changed files with 5289 additions and 1411 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
.gitignore vendored
View File

@ -1,14 +1,9 @@
# Rust build artifacts # Rust build artifacts
/target/ target/
# Local configuration # Local configuration
.env .env
# SQLite local database
/astro_research.db
/astro_research.db-journal
/astro_research.db-shm
/astro_research.db-wal
# Local literature storage (contains downloaded PDFs, HTML, Markdowns and Translations) # Local literature storage (contains downloaded PDFs, HTML, Markdowns and Translations)
/library/ /library/
@ -20,3 +15,7 @@
.DS_Store .DS_Store
*.suo *.suo
*.swp *.swp
library/MinerU/
libs/
.claude

132
Cargo.lock generated
View File

@ -71,6 +71,56 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "anstream"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
dependencies = [
"anstyle",
"anstyle-parse",
"anstyle-query",
"anstyle-wincon",
"colorchoice",
"is_terminal_polyfill",
"utf8parse",
]
[[package]]
name = "anstyle"
version = "1.0.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000"
[[package]]
name = "anstyle-parse"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
dependencies = [
"utf8parse",
]
[[package]]
name = "anstyle-query"
version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
"windows-sys 0.61.2",
]
[[package]]
name = "anstyle-wincon"
version = "3.0.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys 0.61.2",
]
[[package]] [[package]]
name = "anyhow" name = "anyhow"
version = "1.0.102" version = "1.0.102"
@ -85,11 +135,13 @@ dependencies = [
"axum", "axum",
"base64 0.22.1", "base64 0.22.1",
"chrono", "chrono",
"clap",
"dotenvy", "dotenvy",
"flate2", "flate2",
"futures-util", "futures-util",
"hmac 0.12.1", "hmac 0.12.1",
"html2md", "html2md",
"libsqlite3-sys",
"obscura-browser", "obscura-browser",
"obscura-net", "obscura-net",
"quick-xml", "quick-xml",
@ -99,6 +151,7 @@ dependencies = [
"serde", "serde",
"serde_json", "serde_json",
"sha1 0.10.6", "sha1 0.10.6",
"sqlite-vec",
"sqlx", "sqlx",
"thiserror 1.0.69", "thiserror 1.0.69",
"tokio", "tokio",
@ -527,6 +580,46 @@ dependencies = [
"libloading", "libloading",
] ]
[[package]]
name = "clap"
version = "4.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51"
dependencies = [
"clap_builder",
"clap_derive",
]
[[package]]
name = "clap_builder"
version = "4.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
dependencies = [
"anstream",
"anstyle",
"clap_lex",
"strsim",
]
[[package]]
name = "clap_derive"
version = "4.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9"
dependencies = [
"heck 0.5.0",
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "clap_lex"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
[[package]] [[package]]
name = "cmake" name = "cmake"
version = "0.1.58" version = "0.1.58"
@ -542,6 +635,12 @@ version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a" checksum = "0c9ea0ac24bc397ab3c98583a3c9ba74fa56b09a4449bbe172b9b1ddb016027a"
[[package]]
name = "colorchoice"
version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d07550c9036bf2ae0c684c4297d503f838287c83c53686d05370d0e139ae570"
[[package]] [[package]]
name = "combine" name = "combine"
version = "4.6.7" version = "4.6.7"
@ -1768,6 +1867,12 @@ version = "2.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2"
[[package]]
name = "is_terminal_polyfill"
version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
[[package]] [[package]]
name = "itertools" name = "itertools"
version = "0.13.0" version = "0.13.0"
@ -2289,6 +2394,12 @@ version = "1.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
[[package]]
name = "once_cell_polyfill"
version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
[[package]] [[package]]
name = "openssl-macros" name = "openssl-macros"
version = "0.1.1" version = "0.1.1"
@ -3256,6 +3367,15 @@ dependencies = [
"unicode_categories", "unicode_categories",
] ]
[[package]]
name = "sqlite-vec"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0ba424237a9a5db2f6071f193319e2b6a32f7f3961debb2fbbfe67067abce3f"
dependencies = [
"cc",
]
[[package]] [[package]]
name = "sqlx" name = "sqlx"
version = "0.7.4" version = "0.7.4"
@ -3511,6 +3631,12 @@ dependencies = [
"unicode-properties", "unicode-properties",
] ]
[[package]]
name = "strsim"
version = "0.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
[[package]] [[package]]
name = "strum" name = "strum"
version = "0.27.2" version = "0.27.2"
@ -4122,6 +4248,12 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
[[package]]
name = "utf8parse"
version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
[[package]] [[package]]
name = "uuid" name = "uuid"
version = "1.23.2" version = "1.23.2"

8
Cargo.toml
View File

@ -15,8 +15,9 @@ path = "src/main.rs"
name = "health_check" name = "health_check"
path = "src/bin/health_check.rs" path = "src/bin/health_check.rs"
[[bin]]
name = "astroresearch_cli"
path = "src/bin/cli.rs"
[dependencies] [dependencies]
tokio = { version = "1", features = ["full"] } tokio = { version = "1", features = ["full"] }
axum = { version = "0.7", features = ["macros", "multipart"] } axum = { version = "0.7", features = ["macros", "multipart"] }
@ -47,6 +48,9 @@ uuid = { version = "1.23.2", features = ["v4"] }
tracing-appender = "0.2.5" tracing-appender = "0.2.5"
obscura-browser = { path = "libs/obscura/crates/obscura-browser", optional = true } obscura-browser = { path = "libs/obscura/crates/obscura-browser", optional = true }
obscura-net = { path = "libs/obscura/crates/obscura-net", optional = true } obscura-net = { path = "libs/obscura/crates/obscura-net", optional = true }
libsqlite3-sys = { version = "0.27.0", features = ["bundled"] }
sqlite-vec = "0.1.9"
clap = { version = "4", features = ["derive"] }
[features] [features]
default = [] default = []

94
dashboard/package-lock.json generated
View File

@ -17,6 +17,8 @@
"react-dom": "^19.2.6", "react-dom": "^19.2.6",
"react-markdown": "^10.1.0", "react-markdown": "^10.1.0",
"rehype-katex": "^7.0.1", "rehype-katex": "^7.0.1",
"rehype-raw": "^7.0.0",
"rehype-sanitize": "^6.0.0",
"remark-gfm": "^4.0.1", "remark-gfm": "^4.0.1",
"remark-math": "^6.0.0", "remark-math": "^6.0.0",
"tailwind-merge": "^3.6.0" "tailwind-merge": "^3.6.0"
@ -2449,6 +2451,44 @@
"url": "https://opencollective.com/unified" "url": "https://opencollective.com/unified"
} }
}, },
"node_modules/hast-util-raw": {
"version": "9.1.0",
"resolved": "https://registry.npmjs.org/hast-util-raw/-/hast-util-raw-9.1.0.tgz",
"integrity": "sha512-Y8/SBAHkZGoNkpzqqfCldijcuUKh7/su31kEBp67cFY09Wy0mTRgtsLYsiIxMJxlu0f6AA5SUTbDR8K0rxnbUw==",
"dependencies": {
"@types/hast": "^3.0.0",
"@types/unist": "^3.0.0",
"@ungap/structured-clone": "^1.0.0",
"hast-util-from-parse5": "^8.0.0",
"hast-util-to-parse5": "^8.0.0",
"html-void-elements": "^3.0.0",
"mdast-util-to-hast": "^13.0.0",
"parse5": "^7.0.0",
"unist-util-position": "^5.0.0",
"unist-util-visit": "^5.0.0",
"vfile": "^6.0.0",
"web-namespaces": "^2.0.0",
"zwitch": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-sanitize": {
"version": "5.0.2",
"resolved": "https://registry.npmjs.org/hast-util-sanitize/-/hast-util-sanitize-5.0.2.tgz",
"integrity": "sha512-3yTWghByc50aGS7JlGhk61SPenfE/p1oaFeNwkOOyrscaOkMGrcW9+Cy/QAIOBpZxP1yqDIzFMR0+Np0i0+usg==",
"dependencies": {
"@types/hast": "^3.0.0",
"@ungap/structured-clone": "^1.0.0",
"unist-util-position": "^5.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-to-jsx-runtime": { "node_modules/hast-util-to-jsx-runtime": {
"version": "2.3.6", "version": "2.3.6",
"resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.6.tgz", "resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.6.tgz",
@ -2475,6 +2515,24 @@
"url": "https://opencollective.com/unified" "url": "https://opencollective.com/unified"
} }
}, },
"node_modules/hast-util-to-parse5": {
"version": "8.0.1",
"resolved": "https://registry.npmjs.org/hast-util-to-parse5/-/hast-util-to-parse5-8.0.1.tgz",
"integrity": "sha512-MlWT6Pjt4CG9lFCjiz4BH7l9wmrMkfkJYCxFwKQic8+RTZgWPuWxwAfjJElsXkex7DJjfSJsQIt931ilUgmwdA==",
"dependencies": {
"@types/hast": "^3.0.0",
"comma-separated-tokens": "^2.0.0",
"devlop": "^1.0.0",
"property-information": "^7.0.0",
"space-separated-tokens": "^2.0.0",
"web-namespaces": "^2.0.0",
"zwitch": "^2.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/hast-util-to-text": { "node_modules/hast-util-to-text": {
"version": "4.0.2", "version": "4.0.2",
"resolved": "https://registry.npmjs.org/hast-util-to-text/-/hast-util-to-text-4.0.2.tgz", "resolved": "https://registry.npmjs.org/hast-util-to-text/-/hast-util-to-text-4.0.2.tgz",
@ -2542,6 +2600,15 @@
"url": "https://opencollective.com/unified" "url": "https://opencollective.com/unified"
} }
}, },
"node_modules/html-void-elements": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/html-void-elements/-/html-void-elements-3.0.0.tgz",
"integrity": "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/https-proxy-agent": { "node_modules/https-proxy-agent": {
"version": "5.0.1", "version": "5.0.1",
"resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz", "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
@ -4273,6 +4340,33 @@
"katex": "cli.js" "katex": "cli.js"
} }
}, },
"node_modules/rehype-raw": {
"version": "7.0.0",
"resolved": "https://registry.npmjs.org/rehype-raw/-/rehype-raw-7.0.0.tgz",
"integrity": "sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==",
"dependencies": {
"@types/hast": "^3.0.0",
"hast-util-raw": "^9.0.0",
"vfile": "^6.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/rehype-sanitize": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/rehype-sanitize/-/rehype-sanitize-6.0.0.tgz",
"integrity": "sha512-CsnhKNsyI8Tub6L4sm5ZFsme4puGfc6pYylvXo1AeqaGbjOYyzNv3qZPwvs0oMJ39eryyeOdmxwUIo94IpEhqg==",
"dependencies": {
"@types/hast": "^3.0.0",
"hast-util-sanitize": "^5.0.0"
},
"funding": {
"type": "opencollective",
"url": "https://opencollective.com/unified"
}
},
"node_modules/remark-gfm": { "node_modules/remark-gfm": {
"version": "4.0.1", "version": "4.0.1",
"resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.1.tgz", "resolved": "https://registry.npmjs.org/remark-gfm/-/remark-gfm-4.0.1.tgz",

2
dashboard/package.json
View File

@ -19,6 +19,8 @@
"react-dom": "^19.2.6", "react-dom": "^19.2.6",
"react-markdown": "^10.1.0", "react-markdown": "^10.1.0",
"rehype-katex": "^7.0.1", "rehype-katex": "^7.0.1",
"rehype-raw": "^7.0.0",
"rehype-sanitize": "^6.0.0",
"remark-gfm": "^4.0.1", "remark-gfm": "^4.0.1",
"remark-math": "^6.0.0", "remark-math": "^6.0.0",
"tailwind-merge": "^3.6.0" "tailwind-merge": "^3.6.0"

127
dashboard/src/App.tsx
View File

@ -52,6 +52,34 @@ export default function App() {
const [selectedPaper, setSelectedPaper] = useState<StandardPaper | null>(null); const [selectedPaper, setSelectedPaper] = useState<StandardPaper | null>(null);
const [detailBibcode, setDetailBibcode] = useState<string | null>(null); const [detailBibcode, setDetailBibcode] = useState<string | null>(null);
const [recentlySelected, setRecentlySelected] = useState<StandardPaper[]>(() => {
try {
const saved = localStorage.getItem('astro_recently_selected');
return saved ? JSON.parse(saved) : [];
} catch (e) {
console.error('Failed to parse recently selected papers', e);
return [];
}
});
useEffect(() => {
localStorage.setItem('astro_recently_selected', JSON.stringify(recentlySelected));
}, [recentlySelected]);
const addPaperToRecent = (paper: StandardPaper) => {
setRecentlySelected(prev => {
const filtered = prev.filter(p => p.bibcode !== paper.bibcode);
return [paper, ...filtered].slice(0, 5);
});
};
const openCitation = (paper: StandardPaper) => {
setSelectedPaper(paper);
setActiveTab('citation');
loadCitations(paper.bibcode, true);
addPaperToRecent(paper);
};
// 检索页状态 // 检索页状态
const [searchQuery, setSearchQuery] = useState(''); const [searchQuery, setSearchQuery] = useState('');
const [searchSource, setSearchSource] = useState<'all' | 'ads' | 'arxiv'>('all'); const [searchSource, setSearchSource] = useState<'all' | 'ads' | 'arxiv'>('all');
@ -73,6 +101,7 @@ export default function App() {
const [chineseText, setChineseText] = useState(''); const [chineseText, setChineseText] = useState('');
const [parsing, setParsing] = useState(false); const [parsing, setParsing] = useState(false);
const [translating, setTranslating] = useState(false); const [translating, setTranslating] = useState(false);
const [vectorizing, setVectorizing] = useState(false);
// 引用星系数据状态 // 引用星系数据状态
const [citationNetwork, setCitationNetwork] = useState<CitationNetwork | null>(null); const [citationNetwork, setCitationNetwork] = useState<CitationNetwork | null>(null);
@ -101,6 +130,15 @@ export default function App() {
try { try {
const res = await axios.get<StandardPaper[]>('/api/library'); const res = await axios.get<StandardPaper[]>('/api/library');
setLibrary(res.data); setLibrary(res.data);
const lastReadBibcode = localStorage.getItem('last_read_bibcode');
if (lastReadBibcode) {
const lastReadPaper = res.data.find(p => p.bibcode === lastReadBibcode);
if (lastReadPaper) {
const initialTab = localStorage.getItem('astro_active_tab') || 'search';
openReader(lastReadPaper, initialTab !== 'reader');
}
}
} catch (e) { } catch (e) {
console.error('加载本地文献库失败', e); console.error('加载本地文献库失败', e);
} }
@ -298,6 +336,25 @@ export default function App() {
} }
}; };
// 5.5. 对文献进行向量化分块入库 (独立任务)
const handleVectorize = async (bibcode: string) => {
setVectorizing(true);
try {
const res = await axios.post<{ chunk_count: number }>('/api/embed', { bibcode });
setLibrary(prev => prev.map(p => p.bibcode === bibcode ? { ...p, has_vector: true } : p));
if (selectedPaper?.bibcode === bibcode) {
setSelectedPaper(prev => prev ? { ...prev, has_vector: true } : null);
}
showAlert(`文献向量化分块入库成功，共切片并录入 ${res.data.chunk_count} 个向量块。`, '向量化成功');
} catch (e) {
console.error('文献向量化失败', e);
showAlert('向量化失败，请检查 .env 中的 Embedding API 配置。', '向量化失败');
} finally {
setVectorizing(false);
}
};
// 6. 加载文献引用关系网络 // 6. 加载文献引用关系网络
const loadCitations = useCallback(async (bibcode: string, reset = false) => { const loadCitations = useCallback(async (bibcode: string, reset = false) => {
setLoadingCitations(true); setLoadingCitations(true);
@ -323,13 +380,18 @@ export default function App() {
}, []); }, []);
// 7. 进入阅读器 // 7. 进入阅读器
const openReader = async (paper: StandardPaper) => { const openReader = async (paper: StandardPaper, skipTabSwitch = false) => {
setSelectedPaper(paper); setSelectedPaper(paper);
localStorage.setItem('last_read_bibcode', paper.bibcode);
addPaperToRecent(paper);
setEnglishText(''); setEnglishText('');
setChineseText(''); setChineseText('');
setNotes([]); setNotes([]);
setShowNotesPanel(false); setShowNotesPanel(false);
setActiveTab('reader'); if (!skipTabSwitch) {
setActiveTab('reader');
}
// 获取详情 (包含已有原文及翻译) // 获取详情 (包含已有原文及翻译)
try { try {
@ -355,6 +417,56 @@ export default function App() {
} }
}; };
// 7.5. 从 RAG 问答结果跳转至目标文献段落并高亮
const handleJumpToSource = async (bibcode: string, paragraphIndex: number) => {
setActiveTab('reader');
// 如果当前选中的不是目标文献，则先加载它
if (!selectedPaper || selectedPaper.bibcode !== bibcode) {
setEnglishText('');
setChineseText('');
setNotes([]);
setShowNotesPanel(false);
try {
// 获取 StandardPaper 基础信息与正文
const paperRes = await axios.get<{ paper: StandardPaper, english_content?: string, translation_content?: string }>('/api/paper', {
params: { bibcode }
});
setSelectedPaper(paperRes.data.paper);
if (paperRes.data.english_content) {
setEnglishText(paperRes.data.english_content);
}
if (paperRes.data.translation_content) {
setChineseText(paperRes.data.translation_content);
}
// 加载该文献的所有笔记
const nRes = await axios.get<NoteRecord[]>('/api/notes', { params: { bibcode } });
setNotes(nRes.data);
} catch (e) {
console.error('加载跳转文献失败:', e);
return;
}
}
// 强制开启侧边栏
setShowNotesPanel(true);
// 延迟等 DOM 渲染完成进行滚动高亮
setTimeout(() => {
const element = document.getElementById(`paragraph-block-${paragraphIndex}`);
if (element) {
element.scrollIntoView({ behavior: 'smooth', block: 'center' });
element.classList.add('bg-amber-100', 'transition-all');
setTimeout(() => {
element.classList.remove('bg-amber-100');
}, 2500);
}
}, 400);
};
// 8. 批量导出 BibTeX // 8. 批量导出 BibTeX
const handleExportBibtex = async () => { const handleExportBibtex = async () => {
if (exportingList.length === 0) return; if (exportingList.length === 0) return;
@ -472,6 +584,8 @@ export default function App() {
fetchLibrary={fetchLibrary} fetchLibrary={fetchLibrary}
setActiveTab={setActiveTab} setActiveTab={setActiveTab}
onShowDetail={(paper) => setDetailBibcode(paper.bibcode)} onShowDetail={(paper) => setDetailBibcode(paper.bibcode)}
onOpenReader={openReader}
onOpenCitation={openCitation}
/> />
)} )}
@ -479,10 +593,15 @@ export default function App() {
selectedPaper ? ( selectedPaper ? (
<ReaderPanel <ReaderPanel
selectedPaper={selectedPaper} selectedPaper={selectedPaper}
library={library}
recentlySelected={recentlySelected}
onSwitchPaper={openReader}
parsing={parsing} parsing={parsing}
handleParse={handleParse} handleParse={handleParse}
translating={translating} translating={translating}
handleTranslate={handleTranslate} handleTranslate={handleTranslate}
vectorizing={vectorizing}
handleVectorize={handleVectorize}
showNotesPanel={showNotesPanel} showNotesPanel={showNotesPanel}
setShowNotesPanel={setShowNotesPanel} setShowNotesPanel={setShowNotesPanel}
notes={notes} notes={notes}
@ -500,6 +619,7 @@ export default function App() {
handleCreateNote={handleCreateNote} handleCreateNote={handleCreateNote}
handleDeleteNote={handleDeleteNote} handleDeleteNote={handleDeleteNote}
showConfirm={showConfirm} showConfirm={showConfirm}
onJumpToSource={handleJumpToSource}
/> />
) : ( ) : (
<div className="w-full flex-1 flex flex-col items-center justify-center p-12 bg-white rounded-2xl border border-slate-200 shadow-xs min-h-[450px]"> <div className="w-full flex-1 flex flex-col items-center justify-center p-12 bg-white rounded-2xl border border-slate-200 shadow-xs min-h-[450px]">
@ -524,6 +644,9 @@ export default function App() {
selectedPaper ? ( selectedPaper ? (
<CitationPanel <CitationPanel
selectedPaper={selectedPaper} selectedPaper={selectedPaper}
library={library}
recentlySelected={recentlySelected}
onSwitchPaper={openCitation}
loadingCitations={loadingCitations} loadingCitations={loadingCitations}
citationNetwork={citationNetwork} citationNetwork={citationNetwork}
citationHistory={citationHistory} citationHistory={citationHistory}

89
dashboard/src/features/citation/CitationPanel.tsx
View File

@ -1,11 +1,14 @@
// dashboard/src/features/citation/CitationPanel.tsx // dashboard/src/features/citation/CitationPanel.tsx
import { useState } from 'react'; import { useState } from 'react';
import { Loader, GitFork, RotateCcw } from 'lucide-react'; import { Loader, GitFork, RotateCcw, ChevronDown, History, FileText } from 'lucide-react';
import { CitationGalaxyCanvas } from '../../components/CitationGalaxyCanvas'; import { CitationGalaxyCanvas } from '../../components/CitationGalaxyCanvas';
import type { StandardPaper, CitationNetwork } from '../../types'; import type { StandardPaper, CitationNetwork } from '../../types';
interface CitationPanelProps { interface CitationPanelProps {
selectedPaper: StandardPaper | null; selectedPaper: StandardPaper | null;
library: StandardPaper[];
recentlySelected: StandardPaper[];
onSwitchPaper: (paper: StandardPaper) => void;
loadingCitations: boolean; loadingCitations: boolean;
citationNetwork: CitationNetwork | null; citationNetwork: CitationNetwork | null;
citationHistory: CitationNetwork[]; citationHistory: CitationNetwork[];
@ -15,12 +18,16 @@ interface CitationPanelProps {
export function CitationPanel({ export function CitationPanel({
selectedPaper, selectedPaper,
library,
recentlySelected,
onSwitchPaper,
loadingCitations, loadingCitations,
citationNetwork, citationNetwork,
citationHistory, citationHistory,
loadCitations, loadCitations,
onUncachedClick, onUncachedClick,
}: CitationPanelProps) { }: CitationPanelProps) {
const [showSwitchMenu, setShowSwitchMenu] = useState(false);
const [nodeLimit, setNodeLimit] = useState(50); const [nodeLimit, setNodeLimit] = useState(50);
// 统一节点点击事件：已入库直接拉取，未入库弹窗选择 // 统一节点点击事件：已入库直接拉取，未入库弹窗选择
@ -41,7 +48,85 @@ export function CitationPanel({
<h2 className="text-sm font-bold tracking-wider text-slate-900 uppercase">引用星系拓扑图谱</h2> <h2 className="text-sm font-bold tracking-wider text-slate-900 uppercase">引用星系拓扑图谱</h2>
<p className="text-xs text-slate-500 mt-1">通过图谱层级快速 visual 展现当前文献的“参考文献 - 被引文献”关联脉络</p> <p className="text-xs text-slate-500 mt-1">通过图谱层级快速 visual 展现当前文献的“参考文献 - 被引文献”关联脉络</p>
</div> </div>
<div className="flex gap-4 items-center"> <div className="flex gap-4 items-center relative z-20">
{selectedPaper && (
<div className="relative shrink-0">
<button
type="button"
onClick={() => setShowSwitchMenu(!showSwitchMenu)}
className="btn-console btn-console-secondary px-4 py-2 rounded-lg text-xs font-bold flex items-center gap-2 transition-all cursor-pointer shrink-0"
>
<GitFork className="w-3.5 h-3.5 text-sky-600" />
<span>最近文献</span>
<ChevronDown className={`w-3 h-3 transition-transform ${showSwitchMenu ? 'rotate-180' : ''}`} />
</button>
{showSwitchMenu && (
<>
<div className="fixed inset-0 z-40" onClick={() => setShowSwitchMenu(false)} />
<div className="absolute right-0 mt-1.5 w-72 rounded-xl bg-white border border-slate-200 shadow-xl py-1.5 z-55 text-xs max-h-96 overflow-y-auto scrollbar-thin">
{/* 最近文献 */}
{recentlySelected.length > 0 && (
<div className="border-b border-slate-100 pb-1.5 mb-1.5">
<div className="px-3 py-1 text-[10px] font-bold text-slate-400 uppercase flex items-center gap-1">
<History className="w-3.5 h-3.5 text-slate-400" />
<span>最近选择</span>
</div>
{recentlySelected.map(paper => (
<button
key={`citation-recent-${paper.bibcode}`}
onClick={() => {
onSwitchPaper(paper);
setShowSwitchMenu(false);
}}
className={`w-full px-3 py-2 text-left hover:bg-slate-50 transition-colors flex flex-col gap-0.5 border-l-2 ${
paper.bibcode === selectedPaper.bibcode
? 'border-sky-500 bg-sky-50/30 text-sky-850 font-bold'
: 'border-transparent text-slate-700 font-medium'
}`}
>
<span className="truncate block leading-tight text-left">{paper.title}</span>
<span className="text-[9px] text-slate-400 font-semibold font-mono text-left">{paper.bibcode}</span>
</button>
))}
</div>
)}
{/* 全部已下载文献 */}
<div>
<div className="px-3 py-1 text-[10px] font-bold text-slate-400 uppercase flex items-center gap-1">
<FileText className="w-3.5 h-3.5 text-slate-400" />
<span>全部已下载馆藏</span>
</div>
{library.filter(p => p.is_downloaded).length === 0 ? (
<div className="px-3 py-2 text-slate-400 italic">暂无已下载文献</div>
) : (
library
.filter(p => p.is_downloaded)
.map(paper => (
<button
key={`citation-lib-${paper.bibcode}`}
onClick={() => {
onSwitchPaper(paper);
setShowSwitchMenu(false);
}}
className={`w-full px-3 py-2 text-left hover:bg-slate-50 transition-colors flex flex-col gap-0.5 border-l-2 ${
paper.bibcode === selectedPaper.bibcode
? 'border-sky-500 bg-sky-50/30 text-sky-850 font-bold'
: 'border-transparent text-slate-700 font-medium'
}`}
>
<span className="truncate block leading-tight text-left">{paper.title}</span>
<span className="text-[9px] text-slate-400 font-semibold font-mono text-left">{paper.bibcode}</span>
</button>
))
)}
</div>
</div>
</>
)}
</div>
)}
<div className="flex items-center gap-2 border border-slate-200 bg-white px-3 py-1.5 rounded-lg shadow-sm"> <div className="flex items-center gap-2 border border-slate-200 bg-white px-3 py-1.5 rounded-lg shadow-sm">
<span className="text-[11px] text-slate-500 font-bold">展示节点上限:</span> <span className="text-[11px] text-slate-500 font-bold">展示节点上限:</span>
<input <input

99
dashboard/src/features/library/LibraryPanel.tsx
View File

@ -1,6 +1,6 @@
// dashboard/src/features/library/LibraryPanel.tsx // dashboard/src/features/library/LibraryPanel.tsx
import { useState, useCallback } from 'react'; import { useState, useCallback } from 'react';
import { Library, RotateCw, Search, SlidersHorizontal, X, CheckCircle, AlertTriangle } from 'lucide-react'; import { Library, RotateCw, Search, SlidersHorizontal, X, CheckCircle, AlertTriangle, BookOpen, GitFork } from 'lucide-react';
import type { StandardPaper } from '../../types'; import type { StandardPaper } from '../../types';
import { getDoctypeBadge } from '../search/SearchPanel'; import { getDoctypeBadge } from '../search/SearchPanel';
import { CustomSelect } from '../../components/CustomSelect'; import { CustomSelect } from '../../components/CustomSelect';
@ -10,6 +10,8 @@ interface LibraryPanelProps {
fetchLibrary: () => Promise<void>; fetchLibrary: () => Promise<void>;
setActiveTab: (tab: 'search' | 'library' | 'reader' | 'citation' | 'sync') => void; setActiveTab: (tab: 'search' | 'library' | 'reader' | 'citation' | 'sync') => void;
onShowDetail: (paper: StandardPaper) => void; onShowDetail: (paper: StandardPaper) => void;
onOpenReader: (paper: StandardPaper) => void;
onOpenCitation: (paper: StandardPaper) => void;
} }
export function LibraryPanel({ export function LibraryPanel({
@ -17,9 +19,11 @@ export function LibraryPanel({
fetchLibrary, fetchLibrary,
setActiveTab, setActiveTab,
onShowDetail, onShowDetail,
onOpenReader,
onOpenCitation,
}: LibraryPanelProps) { }: LibraryPanelProps) {
const [searchTerm, setSearchTerm] = useState(''); const [searchTerm, setSearchTerm] = useState('');
const [filterStatus, setFilterStatus] = useState<'all' | 'downloaded' | 'undownloaded' | 'download_failed' | 'no_resource' | 'parsed' | 'translated'>('all'); const [filterStatus, setFilterStatus] = useState<'all' | 'downloaded' | 'undownloaded' | 'download_failed' | 'no_resource' | 'parsed' | 'translated' | 'vectorized'>('all');
const [filterDoctype, setFilterDoctype] = useState<string>('all'); const [filterDoctype, setFilterDoctype] = useState<string>('all');
const [sortBy, setSortBy] = useState<'created' | 'yearDesc' | 'yearAsc' | 'citations' | 'title'>('created'); const [sortBy, setSortBy] = useState<'created' | 'yearDesc' | 'yearAsc' | 'citations' | 'title'>('created');
@ -41,6 +45,7 @@ export function LibraryPanel({
const countNoResource = library.filter(p => !p.is_downloaded && p.pdf_error === 'no_resource' && p.html_error === 'no_resource').length; const countNoResource = library.filter(p => !p.is_downloaded && p.pdf_error === 'no_resource' && p.html_error === 'no_resource').length;
const countParsed = library.filter(p => p.has_markdown).length; const countParsed = library.filter(p => p.has_markdown).length;
const countTranslated = library.filter(p => p.has_translation).length; const countTranslated = library.filter(p => p.has_translation).length;
const countVectorized = library.filter(p => p.has_vector).length;
// 本地检索与筛选过滤 // 本地检索与筛选过滤
const filteredLibrary = library.filter(paper => { const filteredLibrary = library.filter(paper => {
@ -72,6 +77,7 @@ export function LibraryPanel({
} }
if (filterStatus === 'parsed' && !paper.has_markdown) return false; if (filterStatus === 'parsed' && !paper.has_markdown) return false;
if (filterStatus === 'translated' && !paper.has_translation) return false; if (filterStatus === 'translated' && !paper.has_translation) return false;
if (filterStatus === 'vectorized' && !paper.has_vector) return false;
// 3. 文献类型筛选 // 3. 文献类型筛选
if (filterDoctype !== 'all') { if (filterDoctype !== 'all') {
@ -122,6 +128,7 @@ export function LibraryPanel({
}); });
const getStatusScore = (paper: StandardPaper) => { const getStatusScore = (paper: StandardPaper) => {
if (paper.has_vector) return 5;
if (paper.has_translation) return 4; if (paper.has_translation) return 4;
if (paper.has_markdown) return 3; if (paper.has_markdown) return 3;
if (paper.is_downloaded) return 2; if (paper.is_downloaded) return 2;
@ -243,6 +250,7 @@ export function LibraryPanel({
{ value: 'downloaded', label: `已下载 (${countDownloaded})` }, { value: 'downloaded', label: `已下载 (${countDownloaded})` },
{ value: 'parsed', label: `已解析 (${countParsed})` }, { value: 'parsed', label: `已解析 (${countParsed})` },
{ value: 'translated', label: `已翻译 (${countTranslated})` }, { value: 'translated', label: `已翻译 (${countTranslated})` },
{ value: 'vectorized', label: `已向量化 (${countVectorized})` },
{ value: 'undownloaded', label: `未下载 (${countUndownloaded})` }, { value: 'undownloaded', label: `未下载 (${countUndownloaded})` },
{ value: 'download_failed', label: `下载失败 (${countDownloadFailed})` }, { value: 'download_failed', label: `下载失败 (${countDownloadFailed})` },
{ value: 'no_resource', label: `无资源 (${countNoResource})` }, { value: 'no_resource', label: `无资源 (${countNoResource})` },
@ -406,30 +414,34 @@ export function LibraryPanel({
: undefined : undefined
} }
className={`absolute top-0 right-0 px-2 py-0.5 text-[9px] font-bold border-b border-l rounded-bl ${ className={`absolute top-0 right-0 px-2 py-0.5 text-[9px] font-bold border-b border-l rounded-bl ${
paper.has_translation paper.has_vector
? 'bg-emerald-50 text-emerald-700 border-emerald-200' ? 'bg-amber-50 text-amber-700 border-amber-200'
: paper.has_markdown : paper.has_translation
? 'bg-sky-50 text-sky-700 border-sky-200' ? 'bg-emerald-50 text-emerald-700 border-emerald-200'
: paper.is_downloaded : paper.has_markdown
? 'bg-indigo-50 text-indigo-700 border-indigo-200' ? 'bg-sky-50 text-sky-700 border-sky-200'
: (paper.pdf_error === 'no_resource' && paper.html_error === 'no_resource') : paper.is_downloaded
? 'bg-slate-100 text-slate-600 border-slate-200 cursor-help' ? 'bg-indigo-50 text-indigo-700 border-indigo-200'
: (paper.pdf_error || paper.html_error) : (paper.pdf_error === 'no_resource' && paper.html_error === 'no_resource')
? 'bg-rose-50 text-rose-700 border-rose-200 cursor-help' ? 'bg-slate-100 text-slate-600 border-slate-200 cursor-help'
: 'bg-amber-50 text-amber-700 border-amber-200' : (paper.pdf_error || paper.html_error)
? 'bg-rose-50 text-rose-700 border-rose-200 cursor-help'
: 'bg-slate-50 text-slate-500 border-slate-100'
}`} }`}
> >
{paper.has_translation {paper.has_vector
? '已翻译' ? '已向量化'
: paper.has_markdown : paper.has_translation
? '已解析' ? '已翻译'
: paper.is_downloaded : paper.has_markdown
? '已下载' ? '已解析'
: (paper.pdf_error === 'no_resource' && paper.html_error === 'no_resource') : paper.is_downloaded
? '无资源' ? '已下载'
: (paper.pdf_error || paper.html_error) : (paper.pdf_error === 'no_resource' && paper.html_error === 'no_resource')
? '下载失败' ? '无资源'
: '未下载'} : (paper.pdf_error || paper.html_error)
? '下载失败'
: '未下载'}
</div> </div>
<div className="pr-10"> <div className="pr-10">
@ -445,10 +457,35 @@ export function LibraryPanel({
</div> </div>
<div className="flex items-center justify-between border-t border-slate-100 pt-3 mt-4 text-[10px]"> <div className="flex items-center justify-between border-t border-slate-100 pt-3 mt-4 text-[10px]">
<span className="font-mono text-slate-400 select-all" onClick={(e) => e.stopPropagation()}> <div className="flex flex-col gap-1 flex-1 min-w-0">
{paper.bibcode === paper.arxiv_id ? `arXiv:${paper.arxiv_id}` : paper.bibcode} {paper.is_downloaded && (
</span> <div className="flex gap-1.5">
<div className="flex items-center gap-2 shrink-0"> <button
type="button"
onClick={(e) => {
e.stopPropagation();
onOpenReader(paper);
}}
className="px-2 py-0.5 rounded bg-sky-50 hover:bg-sky-100 text-sky-700 border border-sky-200 font-bold transition-all text-[9px] flex items-center gap-1 shadow-2xs cursor-pointer animate-fade-in"
>
<BookOpen className="w-3 h-3" />
<span>阅读</span>
</button>
<button
type="button"
onClick={(e) => {
e.stopPropagation();
onOpenCitation(paper);
}}
className="px-2 py-0.5 rounded bg-slate-50 hover:bg-slate-100 text-slate-700 border border-slate-200 font-bold transition-all text-[9px] flex items-center gap-1 shadow-2xs cursor-pointer animate-fade-in"
>
<GitFork className="w-3 h-3" />
<span>图谱</span>
</button>
</div>
)}
</div>
<div className="flex items-center gap-2 shrink-0 self-end">
<div className="flex items-center gap-1"> <div className="flex items-center gap-1">
<span <span
className={`w-1.5 h-1.5 rounded-full ${paper.has_markdown ? 'bg-sky-500' : 'bg-slate-200'}`} className={`w-1.5 h-1.5 rounded-full ${paper.has_markdown ? 'bg-sky-500' : 'bg-slate-200'}`}
@ -461,6 +498,12 @@ export function LibraryPanel({
/> />
<span className="text-slate-400 text-[9px]">{paper.has_translation ? '翻译' : '未翻译'}</span> <span className="text-slate-400 text-[9px]">{paper.has_translation ? '翻译' : '未翻译'}</span>
</div> </div>
<div className="flex items-center gap-1">
<span
className={`w-1.5 h-1.5 rounded-full ${paper.has_vector ? 'bg-amber-500' : 'bg-slate-200'}`}
/>
<span className="text-slate-400 text-[9px]">{paper.has_vector ? '向量库' : '未向量化'}</span>
</div>
</div> </div>
</div> </div>
</div> </div>

302
dashboard/src/features/reader/AIAssistantPanel.tsx Normal file
View File

@ -0,0 +1,302 @@
// dashboard/src/features/reader/AIAssistantPanel.tsx
import { useState, useRef, useEffect } from 'react';
import ReactMarkdown from 'react-markdown';
import remarkMath from 'remark-math';
import remarkGfm from 'remark-gfm';
import rehypeRaw from 'rehype-raw';
import rehypeKatex from 'rehype-katex';
import rehypeSanitize, { defaultSchema } from 'rehype-sanitize';
import 'katex/dist/katex.min.css';
import { Send, Loader, Sparkles, X, BookOpen, AlertCircle, Compass } from 'lucide-react';
import axios from 'axios';
interface RetrievalSource {
bibcode: string;
paragraph_index: number;
content: string;
distance: number;
}
interface Message {
sender: 'user' | 'ai';
text: string;
sources?: RetrievalSource[];
imageUrl?: string;
}
interface AIAssistantPanelProps {
bibcode: string;
onClose: () => void;
onJumpToSource: (bibcode: string, paragraphIndex: number) => void;
pendingFigure?: { path: string; url: string } | null;
onClearPendingFigure?: () => void;
}
const safeSchema = {
...defaultSchema,
attributes: {
...defaultSchema.attributes,
'*': (defaultSchema.attributes?.['*'] || []).concat(['className', 'style', 'mathvariant', 'display']),
},
tagNames: (defaultSchema.tagNames || []).concat([
'math', 'mrow', 'mi', 'mo', 'mn', 'msup', 'msub', 'msubsup', 'mfrac', 'mover', 'munder', 'munderover', 'mspace', 'mtext', 'annotation'
]),
};
const SUGGESTED_QUESTIONS = [
"对比我馆藏的文献中，针对热亚矮星双星在共同包层抛射过程中恒星风流失速率的各种主流观点差异。",
"有哪些文献提及了脉动白矮星的非径向振动模？",
"简述目前文献中关于 Gaia DR3 视差零点改正的处理方法。",
"文献库中关于双星合并前奏（Precursor）观测特征 of 论述有哪些？"
];
export function AIAssistantPanel({
bibcode,
onClose,
onJumpToSource,
pendingFigure,
onClearPendingFigure
}: AIAssistantPanelProps) {
const [messages, setMessages] = useState<Message[]>([]);
const [input, setInput] = useState('');
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const chatEndRef = useRef<HTMLDivElement>(null);
const scrollToBottom = () => {
chatEndRef.current?.scrollIntoView({ behavior: 'smooth' });
};
useEffect(() => {
scrollToBottom();
}, [messages, loading]);
const handleSend = async (questionText: string) => {
if (!questionText.trim() || loading) return;
setError(null);
const userMsg: Message = {
sender: 'user',
text: questionText,
imageUrl: pendingFigure?.url
};
setMessages(prev => [...prev, userMsg]);
setInput('');
setLoading(true);
const isFigureQuery = !!pendingFigure;
const currentFigurePath = pendingFigure?.path;
if (onClearPendingFigure) {
onClearPendingFigure();
}
try {
if (isFigureQuery && currentFigurePath) {
const res = await axios.post<{ answer: string }>('/api/chat/figure', {
bibcode,
image_path: currentFigurePath,
question: questionText
});
const aiMsg: Message = {
sender: 'ai',
text: res.data.answer
};
setMessages(prev => [...prev, aiMsg]);
} else {
const res = await axios.post<{ answer: string; sources: RetrievalSource[] }>('/api/chat/rag', {
question: questionText,
top_k: 5
});
const aiMsg: Message = {
sender: 'ai',
text: res.data.answer,
sources: res.data.sources
};
setMessages(prev => [...prev, aiMsg]);
}
} catch (err: any) {
console.error('问答请求失败:', err);
setError(err.response?.data?.error || err.message || '网络请求错误，请稍后重试');
} finally {
setLoading(false);
}
};
return (
<div className="console-panel rounded-xl border border-slate-200 bg-slate-50 flex flex-col overflow-hidden relative shadow-sm h-full">
{/* 头部面板 */}
<div className="px-4 py-3.5 border-b border-slate-200 flex items-center justify-between bg-white shrink-0">
<div className="flex items-center gap-1.5">
<Sparkles className="w-4 h-4 text-sky-600 animate-pulse" />
<span className="text-xs font-bold text-slate-800">Astro RAG 学术助手</span>
</div>
<button onClick={onClose} className="text-slate-400 hover:text-slate-600 transition-colors">
<X className="w-4 h-4" />
</button>
</div>
{/* 对话消息区 */}
<div className="flex-1 overflow-y-auto p-4 space-y-4 min-h-0">
{messages.length === 0 ? (
<div className="py-6 space-y-6">
<div className="text-center space-y-2 max-w-sm mx-auto">
<Compass className="w-10 h-10 mx-auto text-sky-500 opacity-60" />
<h3 className="text-xs font-bold text-slate-800">探索馆藏文献知识库</h3>
<p className="text-[11px] text-slate-500 leading-relaxed font-semibold">
基于向量检索（SQLite-Vec）与学术大语言模型，跨越所有已解析的 Markdown 文献为您提供深度知识整合与溯源解答。
</p>
</div>
{/* 推荐提示词 */}
<div className="space-y-2">
<span className="text-[10px] font-bold text-slate-400 tracking-wider uppercase">推荐学术提问:</span>
<div className="space-y-2">
{SUGGESTED_QUESTIONS.map((q, idx) => (
<button
key={idx}
onClick={() => handleSend(q)}
className="w-full text-left bg-white hover:bg-slate-100 border border-slate-200 rounded-lg p-3 text-xs text-slate-700 leading-relaxed font-medium transition-colors shadow-sm cursor-pointer"
>
{q}
</button>
))}
</div>
</div>
</div>
) : (
messages.map((msg, index) => (
<div key={index} className={`flex flex-col ${msg.sender === 'user' ? 'items-end' : 'items-start'} space-y-1`}>
<span className="text-[10px] font-bold text-slate-400 px-1">
{msg.sender === 'user' ? '我' : 'Astro AI'}
</span>
<div
className={`max-w-[90%] rounded-xl px-4 py-3 text-xs leading-relaxed font-medium shadow-sm border ${
msg.sender === 'user'
? 'bg-sky-600 text-white border-sky-600'
: 'bg-white text-slate-800 border-slate-200'
}`}
>
{msg.sender === 'user' ? (
<div className="space-y-2">
{msg.imageUrl && (
<img src={msg.imageUrl} alt="Attached Figure" className="max-w-40 max-h-40 object-contain rounded-md border border-sky-500/30" />
)}
<p className="whitespace-pre-wrap">{msg.text}</p>
</div>
) : (
<div className="prose prose-sm max-w-none text-slate-800 leading-relaxed prose-headings:text-slate-900 prose-headings:font-bold prose-strong:text-slate-900 prose-code:text-sky-700 prose-img:rounded-lg">
<ReactMarkdown
remarkPlugins={[remarkMath, remarkGfm]}
rehypePlugins={[rehypeRaw, [rehypeSanitize, safeSchema], rehypeKatex]}
>
{msg.text}
</ReactMarkdown>
</div>
)}
</div>
{/* 关联来源展示 */}
{msg.sender === 'ai' && msg.sources && msg.sources.length > 0 && (
<div className="w-full mt-2 pl-2 space-y-1.5 border-l-2 border-slate-200">
<span className="text-[9px] font-bold text-slate-400">参考来源文献（点击跳转）:</span>
<div className="grid grid-cols-1 gap-1.5 w-[90%]">
{msg.sources.map((src, sIdx) => (
<button
key={sIdx}
onClick={() => onJumpToSource(src.bibcode, src.paragraph_index)}
className="flex items-center justify-between text-left bg-white hover:bg-slate-100 border border-slate-200 rounded-md px-2.5 py-1.5 text-[10px] text-slate-655 font-semibold transition-all shadow-sm cursor-pointer"
title={src.content}
>
<div className="flex items-center gap-1.5 truncate mr-2">
<BookOpen className="w-3 h-3 text-sky-600 shrink-0" />
<span className="font-bold text-slate-700 truncate">{src.bibcode}</span>
<span className="text-slate-400 text-[9px] font-bold">§{src.paragraph_index + 1}</span>
</div>
<span className="text-slate-400 text-[9px] shrink-0 font-medium">
距: {src.distance.toFixed(3)}
</span>
</button>
))}
</div>
</div>
)}
</div>
))
)}
{loading && (
<div className="flex items-center space-x-2 text-slate-500 pl-2">
<Loader className="w-4 h-4 animate-spin text-sky-600" />
<span className="text-[10px] font-bold">文献检索及综述生成中...</span>
</div>
)}
{error && (
<div className="flex items-start gap-2 bg-red-50 border border-red-200 text-red-700 rounded-lg p-3 text-xs w-[90%] font-semibold">
<AlertCircle className="w-4 h-4 text-red-500 shrink-0 mt-0.5" />
<div>
<div className="font-bold">查询发生错误</div>
<div className="text-[10px] opacity-80 mt-0.5">{error}</div>
</div>
</div>
)}
<div ref={chatEndRef} />
</div>
{/* 待提问图片预览 */}
{pendingFigure && (
<div className="px-3 py-2 bg-slate-100 border-t border-slate-200 flex items-center justify-between shrink-0">
<div className="flex items-center gap-2">
<img
src={pendingFigure.url}
alt="Preview"
className="w-10 h-10 object-cover rounded-md border border-slate-300"
/>
<div className="text-[10px] text-slate-550">
<span className="font-bold text-slate-700 block">已选中图表插图</span>
<span className="font-mono text-slate-400 truncate max-w-48 block">{pendingFigure.path.split('/').pop()}</span>
</div>
</div>
<button
type="button"
onClick={onClearPendingFigure}
className="text-slate-400 hover:text-slate-655 p-1 cursor-pointer"
>
<X className="w-3.5 h-3.5" />
</button>
</div>
)}
{/* 底栏输入区 */}
<form
onSubmit={(e) => {
e.preventDefault();
handleSend(input);
}}
className="p-3 border-t border-slate-200 bg-white shrink-0"
>
<div className="flex gap-2 relative items-center">
<input
type="text"
value={input}
onChange={(e) => setInput(e.target.value)}
disabled={loading}
placeholder={pendingFigure ? "针对选中图表提问..." : "向 AI 馆藏助手提问..."}
className="flex-1 bg-slate-50 border border-slate-200 rounded-xl text-xs text-slate-900 placeholder-slate-400 pl-3 pr-10 py-2.5 focus:outline-none focus:bg-white focus:border-sky-500 focus:ring-1 focus:ring-sky-500/10 leading-relaxed font-semibold transition-all disabled:opacity-60"
/>
<button
type="submit"
disabled={!input.trim() || loading}
className="absolute right-1.5 p-1.5 rounded-lg bg-sky-600 hover:bg-sky-700 text-white disabled:bg-slate-200 disabled:text-slate-400 transition-colors cursor-pointer"
>
<Send className="w-3.5 h-3.5" />
</button>
</div>
</form>
</div>
);
}

878
dashboard/src/features/reader/ReaderPanel.tsx
View File

File diff suppressed because it is too large Load Diff

170
dashboard/src/features/sync/SyncPanel.tsx
View File

@ -5,7 +5,7 @@ import { RefreshCw, Play, Info, AlertTriangle, CheckCircle, Loader, StopCircle,
import type { SavedSyncQuery } from '../../types'; import type { SavedSyncQuery } from '../../types';
import { CustomSelect } from '../../components/CustomSelect'; import { CustomSelect } from '../../components/CustomSelect';
interface ProcessStatus { interface BatchStatus {
active: boolean; active: boolean;
total: number; total: number;
downloaded: number; downloaded: number;
@ -14,7 +14,7 @@ interface ProcessStatus {
parse_failed: number; parse_failed: number;
current_bibcode: string; current_bibcode: string;
logs: string[]; logs: string[];
action?: 'download' | 'parse' | 'translate'; action?: 'download' | 'parse' | 'translate' | 'embed' | 'target';
} }
interface HarvestStatus { interface HarvestStatus {
@ -43,7 +43,7 @@ export function SyncPanel() {
const pollIntervalRef = useRef<any>(null); const pollIntervalRef = useRef<any>(null);
// 批量下载与解析相关状态 // 批量下载与解析相关状态
const [targetPhase, setTargetPhase] = useState<'download' | 'parse' | 'translate'>('download'); const [targetPhase, setTargetPhase] = useState<'download' | 'parse' | 'translate' | 'embed' | 'target'>('download');
const [batchLimitCount, setBatchLimitCount] = useState<number>(100); const [batchLimitCount, setBatchLimitCount] = useState<number>(100);
const [sortOrder, setSortOrder] = useState<'default' | 'pub_year_desc' | 'created_at_desc'>('default'); const [sortOrder, setSortOrder] = useState<'default' | 'pub_year_desc' | 'created_at_desc'>('default');
const [skipCompleted, setSkipCompleted] = useState<boolean>(true); const [skipCompleted, setSkipCompleted] = useState<boolean>(true);
@ -51,7 +51,7 @@ export function SyncPanel() {
const [skipPrecedingFailed, setSkipPrecedingFailed] = useState<boolean>(false); const [skipPrecedingFailed, setSkipPrecedingFailed] = useState<boolean>(false);
const [skipPrecedingUncompleted, setSkipPrecedingUncompleted] = useState<boolean>(false); const [skipPrecedingUncompleted, setSkipPrecedingUncompleted] = useState<boolean>(false);
const [processStatus, setProcessStatus] = useState<ProcessStatus>({ const [batchStatus, setBatchStatus] = useState<BatchStatus>({
active: false, active: false,
total: 0, total: 0,
downloaded: 0, downloaded: 0,
@ -61,8 +61,8 @@ export function SyncPanel() {
current_bibcode: '', current_bibcode: '',
logs: [], logs: [],
}); });
const [processError, setProcessError] = useState<string | null>(null); const [batchError, setBatchError] = useState<string | null>(null);
const processPollIntervalRef = useRef<any>(null); const batchPollIntervalRef = useRef<any>(null);
const logsContainerRef = useRef<HTMLDivElement | null>(null); const logsContainerRef = useRef<HTMLDivElement | null>(null);
const [showBuilder, setShowBuilder] = useState(false); const [showBuilder, setShowBuilder] = useState(false);
@ -203,30 +203,30 @@ export function SyncPanel() {
}, []); }, []);
// 批量下载与解析相关的网络操作 // 批量下载与解析相关的网络操作
const fetchProcessStatus = async () => { const fetchBatchStatus = async () => {
try { try {
const res = await axios.get<ProcessStatus>(`/api/sync/asset/status?t=${Date.now()}`); const res = await axios.get<BatchStatus>(`/api/batch/asset/status?t=${Date.now()}`);
setProcessStatus(res.data); setBatchStatus(res.data);
if (res.data.active) { if (res.data.active) {
startProcessPolling(); startBatchPolling();
} else if (processPollIntervalRef.current) { } else if (batchPollIntervalRef.current) {
clearInterval(processPollIntervalRef.current); clearInterval(batchPollIntervalRef.current);
processPollIntervalRef.current = null; batchPollIntervalRef.current = null;
} }
} catch (e) { } catch (e) {
console.error('获取处理状态失败', e); console.error('获取处理状态失败', e);
} }
}; };
const startProcessPolling = () => { const startBatchPolling = () => {
if (processPollIntervalRef.current) return; if (batchPollIntervalRef.current) return;
processPollIntervalRef.current = setInterval(fetchProcessStatus, 1000); batchPollIntervalRef.current = setInterval(fetchBatchStatus, 1000);
}; };
const handleStartProcess = async () => { const handleStartBatch = async () => {
setProcessError(null); setBatchError(null);
try { try {
await axios.post('/api/sync/asset/run', { await axios.post('/api/batch/asset/run', {
target_phase: targetPhase, target_phase: targetPhase,
limit_count: batchLimitCount, limit_count: batchLimitCount,
sort_order: sortOrder, sort_order: sortOrder,
@ -235,30 +235,30 @@ export function SyncPanel() {
skip_preceding_failed: skipPrecedingFailed, skip_preceding_failed: skipPrecedingFailed,
skip_preceding_uncompleted: skipPrecedingUncompleted, skip_preceding_uncompleted: skipPrecedingUncompleted,
}); });
fetchProcessStatus(); fetchBatchStatus();
startProcessPolling(); startBatchPolling();
} catch (e: any) { } catch (e: any) {
console.error(e); console.error(e);
setProcessError(e.response?.data || '启动批量任务失败。'); setBatchError(e.response?.data || '启动批量任务失败。');
} }
}; };
const handleStopProcess = async () => { const handleStopBatch = async () => {
try { try {
await axios.post('/api/sync/asset/stop'); await axios.post('/api/batch/asset/stop');
fetchProcessStatus(); fetchBatchStatus();
} catch (e: any) { } catch (e: any) {
console.error(e); console.error(e);
setProcessError(e.response?.data || '停止任务失败。'); setBatchError(e.response?.data || '停止任务失败。');
} }
}; };
useEffect(() => { useEffect(() => {
fetchProcessStatus(); fetchBatchStatus();
return () => { return () => {
if (processPollIntervalRef.current) { if (batchPollIntervalRef.current) {
clearInterval(processPollIntervalRef.current); clearInterval(batchPollIntervalRef.current);
} }
}; };
}, []); }, []);
@ -273,7 +273,7 @@ export function SyncPanel() {
container.scrollTop = container.scrollHeight; container.scrollTop = container.scrollHeight;
} }
} }
}, [processStatus.logs]); }, [batchStatus.logs]);
// 估算文献总量 // 估算文献总量
const handleEstimate = async () => { const handleEstimate = async () => {
@ -579,35 +579,29 @@ export function SyncPanel() {
</p> </p>
</div> </div>
{processError && ( {batchError && (
<div className="p-4 rounded-lg bg-red-50 border border-red-200 flex gap-3 text-xs text-red-750 items-start"> <div className="p-4 rounded-lg bg-red-50 border border-red-200 flex gap-3 text-xs text-red-750 items-start">
<AlertTriangle className="w-4 h-4 shrink-0 mt-0.5" /> <AlertTriangle className="w-4 h-4 shrink-0 mt-0.5" />
<div>{processError}</div> <div>{batchError}</div>
</div> </div>
)} )}
<div className="grid grid-cols-1 md:grid-cols-3 gap-6"> <div className="grid grid-cols-1 md:grid-cols-3 gap-6">
<div className="space-y-2"> <div className="space-y-2">
<label className="text-xs font-bold text-slate-700 block">目标阶段</label> <label className="text-xs font-bold text-slate-700 block">目标阶段</label>
<div className="flex flex-col gap-1.5"> <CustomSelect
{[ value={targetPhase}
{ id: 'download', label: '下载' }, disabled={batchStatus.active}
{ id: 'parse', label: '解析' }, onChange={val => setTargetPhase(val as any)}
{ id: 'translate', label: '翻译' }, className="w-full"
].map(phase => ( options={[
<label key={phase.id} className="flex items-center gap-2 text-xs font-medium text-slate-700 cursor-pointer"> { value: 'download', label: '下载' },
<input { value: 'parse', label: '解析' },
type="radio" { value: 'translate', label: '翻译' },
name="targetPhase" { value: 'embed', label: '向量化' },
checked={targetPhase === phase.id} { value: 'target', label: '天体识别' },
disabled={processStatus.active} ]}
onChange={() => setTargetPhase(phase.id as any)} />
className="text-sky-650 focus:ring-sky-500"
/>
<span>{phase.label}</span>
</label>
))}
</div>
</div> </div>
<div className="space-y-2"> <div className="space-y-2">
@ -615,7 +609,7 @@ export function SyncPanel() {
<input <input
type="number" type="number"
value={batchLimitCount} value={batchLimitCount}
disabled={processStatus.active} disabled={batchStatus.active}
onChange={e => setBatchLimitCount(Math.max(1, parseInt(e.target.value) || 0))} onChange={e => setBatchLimitCount(Math.max(1, parseInt(e.target.value) || 0))}
className="w-full px-3 py-2 rounded-lg bg-slate-50 border border-slate-300 text-slate-900 focus:outline-none focus:border-sky-500 transition-all text-xs font-medium" className="w-full px-3 py-2 rounded-lg bg-slate-50 border border-slate-300 text-slate-900 focus:outline-none focus:border-sky-500 transition-all text-xs font-medium"
/> />
@ -625,7 +619,7 @@ export function SyncPanel() {
<label className="text-xs font-bold text-slate-700 block">处理顺序</label> <label className="text-xs font-bold text-slate-700 block">处理顺序</label>
<CustomSelect <CustomSelect
value={sortOrder} value={sortOrder}
disabled={processStatus.active} disabled={batchStatus.active}
onChange={val => setSortOrder(val)} onChange={val => setSortOrder(val)}
className="w-full" className="w-full"
options={[ options={[
@ -644,7 +638,7 @@ export function SyncPanel() {
<input <input
type="checkbox" type="checkbox"
checked={skipCompleted} checked={skipCompleted}
disabled={processStatus.active} disabled={batchStatus.active}
onChange={e => setSkipCompleted(e.target.checked)} onChange={e => setSkipCompleted(e.target.checked)}
className="rounded text-sky-650 focus:ring-sky-500" className="rounded text-sky-650 focus:ring-sky-500"
/> />
@ -655,7 +649,7 @@ export function SyncPanel() {
<input <input
type="checkbox" type="checkbox"
checked={skipFailed} checked={skipFailed}
disabled={processStatus.active} disabled={batchStatus.active}
onChange={e => setSkipFailed(e.target.checked)} onChange={e => setSkipFailed(e.target.checked)}
className="rounded text-sky-650 focus:ring-sky-500" className="rounded text-sky-650 focus:ring-sky-500"
/> />
@ -666,7 +660,7 @@ export function SyncPanel() {
<input <input
type="checkbox" type="checkbox"
checked={skipPrecedingFailed} checked={skipPrecedingFailed}
disabled={processStatus.active} disabled={batchStatus.active}
onChange={e => setSkipPrecedingFailed(e.target.checked)} onChange={e => setSkipPrecedingFailed(e.target.checked)}
className="rounded text-sky-650 focus:ring-sky-500" className="rounded text-sky-650 focus:ring-sky-500"
/> />
@ -677,7 +671,7 @@ export function SyncPanel() {
<input <input
type="checkbox" type="checkbox"
checked={skipPrecedingUncompleted} checked={skipPrecedingUncompleted}
disabled={processStatus.active} disabled={batchStatus.active}
onChange={e => setSkipPrecedingUncompleted(e.target.checked)} onChange={e => setSkipPrecedingUncompleted(e.target.checked)}
className="rounded text-sky-650 focus:ring-sky-500" className="rounded text-sky-650 focus:ring-sky-500"
/> />
@ -688,10 +682,10 @@ export function SyncPanel() {
<div className="flex justify-end pt-2 border-t border-slate-100"> <div className="flex justify-end pt-2 border-t border-slate-100">
<div className="w-full md:w-1/3 flex"> <div className="w-full md:w-1/3 flex">
{processStatus.active ? ( {batchStatus.active ? (
<button <button
type="button" type="button"
onClick={handleStopProcess} onClick={handleStopBatch}
className="w-full py-2 rounded-lg bg-red-600 hover:bg-red-700 text-white text-xs font-bold flex items-center justify-center gap-2 transition-all shadow-sm" className="w-full py-2 rounded-lg bg-red-600 hover:bg-red-700 text-white text-xs font-bold flex items-center justify-center gap-2 transition-all shadow-sm"
> >
<StopCircle className="w-3.5 h-3.5" /> <StopCircle className="w-3.5 h-3.5" />
@ -700,7 +694,7 @@ export function SyncPanel() {
) : ( ) : (
<button <button
type="button" type="button"
onClick={handleStartProcess} onClick={handleStartBatch}
className="btn-console btn-console-primary w-full py-2 rounded-lg text-xs font-bold flex items-center justify-center gap-2" className="btn-console btn-console-primary w-full py-2 rounded-lg text-xs font-bold flex items-center justify-center gap-2"
> >
<Play className="w-3.5 h-3.5" /> <Play className="w-3.5 h-3.5" />
@ -711,22 +705,32 @@ export function SyncPanel() {
</div> </div>
{/* 进度与终端日志展示 */} {/* 进度与终端日志展示 */}
{(processStatus.active || processStatus.total > 0) && ( {(batchStatus.active || batchStatus.total > 0) && (
<div className="space-y-4 pt-4 border-t border-slate-200"> <div className="space-y-4 pt-4 border-t border-slate-200">
<div className="space-y-1.5"> <div className="space-y-1.5">
<div className="flex justify-between text-xs font-bold text-slate-600"> <div className="flex justify-between text-xs font-bold text-slate-600">
<span className="flex items-center gap-1"> <span className="flex items-center gap-1">
{processStatus.action === 'download' && <Download className="w-3.5 h-3.5 text-sky-600" />} {batchStatus.action === 'download' && <Download className="w-3.5 h-3.5 text-sky-600" />}
{processStatus.action === 'parse' && <FileText className="w-3.5 h-3.5 text-emerald-600" />} {batchStatus.action === 'parse' && <FileText className="w-3.5 h-3.5 text-emerald-600" />}
{processStatus.action === 'translate' && <RefreshCw className="w-3.5 h-3.5 text-indigo-600" />} {batchStatus.action === 'translate' && <RefreshCw className="w-3.5 h-3.5 text-indigo-600" />}
{processStatus.action === 'download' ? '正文离线下载进度' : processStatus.action === 'parse' ? '结构化排版解析进度' : '中英双语对照翻译进度'} {batchStatus.action === 'embed' && <SlidersHorizontal className="w-3.5 h-3.5 text-amber-600" />}
{batchStatus.action === 'target' && <RefreshCw className="w-3.5 h-3.5 text-sky-600" />}
{batchStatus.action === 'download'
? '正文离线下载进度'
: batchStatus.action === 'parse'
? '结构化排版解析进度'
: batchStatus.action === 'translate'
? '中英双语对照翻译进度'
: batchStatus.action === 'embed'
? '段落向量分块入库进度'
: '天体识别与缓存进度'}
</span> </span>
<span> <span>
{processStatus.action === 'download' ? processStatus.downloaded : processStatus.parsed} / {processStatus.total} 篇 {batchStatus.action === 'download' ? batchStatus.downloaded : batchStatus.parsed} / {batchStatus.total} 篇
{((processStatus.action === 'download' && processStatus.download_failed > 0) || {((batchStatus.action === 'download' && batchStatus.download_failed > 0) ||
(processStatus.action !== 'download' && processStatus.parse_failed > 0)) && ( (batchStatus.action !== 'download' && batchStatus.parse_failed > 0)) && (
<span className="text-red-500 ml-2 font-bold"> <span className="text-red-500 ml-2 font-bold">
(失败 {processStatus.action === 'download' ? processStatus.download_failed : processStatus.parse_failed} 篇) (失败 {batchStatus.action === 'download' ? batchStatus.download_failed : batchStatus.parse_failed} 篇)
</span> </span>
)} )}
</span> </span>
@ -734,18 +738,26 @@ export function SyncPanel() {
<div className="w-full h-2 rounded-full bg-slate-100 overflow-hidden border border-slate-200"> <div className="w-full h-2 rounded-full bg-slate-100 overflow-hidden border border-slate-200">
<div <div
className={`h-full transition-all duration-300 ${ className={`h-full transition-all duration-300 ${
processStatus.action === 'download' ? 'bg-sky-600' : processStatus.action === 'parse' ? 'bg-emerald-600' : 'bg-indigo-600' batchStatus.action === 'download'
? 'bg-sky-600'
: batchStatus.action === 'parse'
? 'bg-emerald-600'
: batchStatus.action === 'translate'
? 'bg-indigo-600'
: batchStatus.action === 'embed'
? 'bg-amber-600'
: 'bg-sky-600'
}`} }`}
style={{ style={{
width: `${ width: `${
processStatus.total > 0 batchStatus.total > 0
? Math.min( ? Math.min(
100, 100,
Math.round( Math.round(
((processStatus.action === 'download' ((batchStatus.action === 'download'
? processStatus.downloaded + processStatus.download_failed ? batchStatus.downloaded + batchStatus.download_failed
: processStatus.parsed + processStatus.parse_failed) / : batchStatus.parsed + batchStatus.parse_failed) /
processStatus.total) * batchStatus.total) *
100 100
) )
) )
@ -756,10 +768,10 @@ export function SyncPanel() {
</div> </div>
</div> </div>
{processStatus.active && processStatus.current_bibcode && ( {batchStatus.active && batchStatus.current_bibcode && (
<div className="text-xs font-bold text-slate-600 flex items-center gap-2"> <div className="text-xs font-bold text-slate-600 flex items-center gap-2">
<Loader className="w-3.5 h-3.5 text-sky-600 animate-spin" /> <Loader className="w-3.5 h-3.5 text-sky-600 animate-spin" />
<span>当前正在处理: <code className="bg-slate-100 px-2 py-0.5 rounded font-mono font-bold text-slate-800 border border-slate-200">{processStatus.current_bibcode}</code></span> <span>当前正在处理: <code className="bg-slate-100 px-2 py-0.5 rounded font-mono font-bold text-slate-800 border border-slate-200">{batchStatus.current_bibcode}</code></span>
</div> </div>
)} )}
@ -767,10 +779,10 @@ export function SyncPanel() {
<div className="space-y-2"> <div className="space-y-2">
<label className="text-xs font-bold text-slate-700 block">实时处理日志流终端</label> <label className="text-xs font-bold text-slate-700 block">实时处理日志流终端</label>
<div ref={logsContainerRef} className="bg-slate-50 text-slate-800 font-mono text-xs p-4 rounded-lg h-48 overflow-y-auto border border-slate-250 space-y-1 scrollbar-thin scrollbar-thumb-slate-300 relative"> <div ref={logsContainerRef} className="bg-slate-50 text-slate-800 font-mono text-xs p-4 rounded-lg h-48 overflow-y-auto border border-slate-250 space-y-1 scrollbar-thin scrollbar-thumb-slate-300 relative">
{processStatus.logs.length === 0 ? ( {batchStatus.logs.length === 0 ? (
<div className="text-slate-400 italic">等待数据流任务启动，暂无日志输出...</div> <div className="text-slate-400 italic">等待数据流任务启动，暂无日志输出...</div>
) : ( ) : (
processStatus.logs.map((log, idx) => ( batchStatus.logs.map((log, idx) => (
<div key={idx} className="whitespace-pre-wrap leading-relaxed"> <div key={idx} className="whitespace-pre-wrap leading-relaxed">
{log} {log}
</div> </div>

1
dashboard/src/types.ts
View File

@ -17,6 +17,7 @@ export interface StandardPaper {
has_html: boolean; has_html: boolean;
has_markdown: boolean; has_markdown: boolean;
has_translation: boolean; has_translation: boolean;
has_vector: boolean;
doctype: string; doctype: string;
pdf_error?: string; pdf_error?: string;
html_error?: string; html_error?: string;

29
migrations/20260613000000_paper_chunks_metadata.sql Normal file
View File

@ -0,0 +1,29 @@
-- migrations/20260613000000_paper_chunks_metadata.sql
-- RAG 文本切片内容表与天体目标缓存表
-- 存储文献 Markdown 段落切片的元数据与文本内容
-- rowid 与 vec_paper_chunks 虚拟表的 rowid 保持 1:1 对齐
CREATE TABLE IF NOT EXISTS paper_chunks_content (
rowid INTEGER PRIMARY KEY AUTOINCREMENT,
bibcode TEXT,
paragraph_index INTEGER,
content TEXT,
FOREIGN KEY(bibcode) REFERENCES papers(bibcode) ON DELETE CASCADE
);
-- 天体目标信息缓存表，从 SIMBAD/Sesame 解析后本地持久化
CREATE TABLE IF NOT EXISTS paper_targets (
id INTEGER PRIMARY KEY AUTOINCREMENT,
bibcode TEXT,
target_name TEXT, -- 标准化天体名称 (如 GD 358)
ra TEXT, -- 赤道坐标：赤经 (Right Ascension)
dec TEXT, -- 赤道坐标：赤纬 (Declination)
parallax REAL, -- 视差 (mas，毫角秒)
spectral_type TEXT, -- 光谱分类 (如 DA1)
v_magnitude REAL, -- 视星等
aliases TEXT, -- 天体别名 (JSON 数组文本)
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY(bibcode) REFERENCES papers(bibcode) ON DELETE CASCADE,
UNIQUE(bibcode, target_name)
);
CREATE INDEX IF NOT EXISTS idx_paper_targets_name ON paper_targets(target_name);

151
scratch/audit_anomalies.py
View File

@ -1,151 +0,0 @@
# scratch/audit_anomalies.py
import os
import re
LOG_PATH = "/home/fmq/.gemini/antigravity/brain/4e405818-ae6d-46f6-a14a-d59613a4ee1c/.system_generated/tasks/task-914.log"
LIBRARY_DIR = "/home/fmq/program/AstroResearch/library"
def parse_flagged_files(log_path):
html_files = []
pdf_files = []
if not os.path.exists(log_path):
print(f"Log path not found: {log_path}")
return html_files, pdf_files
with open(log_path, 'r', encoding='utf-8') as f:
content = f.read()
# Match lines like: ❌ 发现磁盘上损坏的 HTML 文件: "HTML/2010AIPC.1273..269M.html"
html_matches = re.findall(r'发现磁盘上损坏的 HTML 文件:\s*"([^"]+)"', content)
pdf_matches = re.findall(r'发现磁盘上损坏的 PDF 文件:\s*"([^"]+)"', content)
# Remove duplicates
html_files = sorted(list(set(html_matches)))
pdf_files = sorted(list(set(pdf_matches)))
return html_files, pdf_files
def get_html_title(text):
match = re.search(r'<title[^>]*>(.*?)</title>', text, re.IGNORECASE | re.DOTALL)
if match:
return match.group(1).strip()
return "No Title"
def audit_html(file_path):
if not os.path.exists(file_path):
return "File Missing", 0, ""
size = os.path.getsize(file_path)
if size == 0:
return "Empty File (0 bytes)", size, ""
try:
with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
content = f.read()
except Exception as e:
return f"Read Error: {e}", size, ""
title = get_html_title(content)
lower = content.to_lowercase() if hasattr(content, 'to_lowercase') else content.lower()
# Determine reason
if "just a moment" in lower or "please wait while we verify" in lower:
return "Cloudflare Turnstile WAF Block Page", size, title
if "radware bot manager" in lower:
return "Radware Bot Manager Captcha Page", size, title
if "aws waf" in lower or "awswafintegration" in lower:
return "AWS WAF Block Page", size, title
if "purchase access" in lower or "buy-box" in lower or "subscription required" in lower:
return "Publisher Paywall / Purchase Prompt", size, title
if "redirecting" in lower or "http-equiv=\"refresh\"" in lower:
return "HTML Redirect Page", size, title
if "conversion to html had a fatal error" in lower:
return "ar5iv Conversion Failed Stub Page", size, title
# Check sections/references
has_sections = any(x in lower for x in ["ltx_title_section", "class=\"section\"", "## introduction", "<h2>introduction", "<h3>introduction", "class=\"ltx_section\""])
has_bib = any(x in lower for x in ["ltx_bibliography", "class=\"references\"", "<ol class=\"references\"", "<ul class=\"references\"", "id=\"bib\""])
if size < 50000 and not (has_sections or has_bib):
return f"Snippet / Abstract Page (Missing sections/references, size={size}B)", size, title
return "Valid HTML Content?", size, title
def audit_pdf(file_path):
if not os.path.exists(file_path):
return "File Missing", 0, ""
size = os.path.getsize(file_path)
if size == 0:
return "Empty File (0 bytes)", size, ""
try:
with open(file_path, 'rb') as f:
header = f.read(512)
except Exception as e:
return f"Read Error: {e}", size, ""
if not header.startswith(b"%PDF"):
if header.startswith(b"<!") or header.startswith(b"<html") or header.startswith(b"<HTML"):
# It's an HTML file disguised as a PDF
html_text = header.decode('utf-8', errors='ignore')
title = get_html_title(html_text)
lower = html_text.lower()
if "just a moment" in lower or "cloudflare" in lower:
return "HTML Disguised as PDF (Cloudflare WAF Block)", size, title
if "radware" in lower:
return "HTML Disguised as PDF (Radware Captcha)", size, title
if "open journal systems" in lower or "pkp_page_article" in lower:
return "HTML Disguised as PDF (OJS Viewer Page)", size, title
return "HTML Disguised as PDF (Unknown Webpage)", size, title
return "Corrupted / Missing %PDF Header Magic Number", size, ""
# Check tail EOF
try:
with open(file_path, 'rb') as f:
f.seek(max(0, size - 1024))
tail = f.read(1024)
except Exception as e:
return f"Read Error seeking tail: {e}", size, ""
if b"%%EOF" not in tail:
return "Corrupted PDF (Missing tail %%EOF marker)", size, ""
if size < 5000:
return f"PDF Too Small ({size}B, likely error page)", size, ""
return "Valid PDF Content?", size, ""
def main():
html_files, pdf_files = parse_flagged_files(LOG_PATH)
print(f"Parsed {len(html_files)} HTML files and {len(pdf_files)} PDF files from log.")
html_results = []
for rel_path in html_files:
abs_path = os.path.join(LIBRARY_DIR, rel_path)
status, size, title = audit_html(abs_path)
html_results.append((rel_path, status, size, title))
pdf_results = []
for rel_path in pdf_files:
abs_path = os.path.join(LIBRARY_DIR, rel_path)
status, size, title = audit_pdf(abs_path)
pdf_results.append((rel_path, status, size, title))
print("\n--- HTML AUDIT REPORT ---")
print("| File | Audit Status | Size (Bytes) | HTML Title |")
print("| --- | --- | --- | --- |")
for file, status, size, title in html_results:
clean_title = title.replace("|", "\\|").replace("\n", " ")
print(f"| {file} | {status} | {size} | {clean_title} |")
print("\n--- PDF AUDIT REPORT ---")
print("| File | Audit Status | Size (Bytes) | HTML Title (if HTML) |")
print("| --- | --- | --- | --- |")
for file, status, size, title in pdf_results:
clean_title = title.replace("|", "\\|").replace("\n", " ")
print(f"| {file} | {status} | {size} | {clean_title} |")
if __name__ == "__main__":
main()

38
scratch/scan_emojis.py
View File

@ -1,38 +0,0 @@
import os
import re
import sys
# Define a regex for emojis
# This pattern matches common emoji characters
emoji_pattern = re.compile(
"["
"\U00010000-\U0010ffff" # All supplementary Unicode characters (includes most emojis)
"\u2600-\u27bf" # Miscellaneous symbols, dingbats
"\u2300-\u23ff" # Miscellaneous technical
"\u2b50" # Medium white star
"\u2934-\u2935" # Arrows
"\u3297" # Congratulation sign in circle
"\u3299" # Secret sign in circle
"]",
flags=re.UNICODE
)
src_dir = "/home/fmq/program/AstroResearch/dashboard/src"
found = False
for root, dirs, files in os.walk(src_dir):
for file in files:
if file.endswith(('.tsx', '.ts')):
path = os.path.join(root, file)
try:
with open(path, 'r', encoding='utf-8') as f:
for line_num, line in enumerate(f, 1):
matches = emoji_pattern.findall(line)
if matches:
print(f"{path}:{line_num}: {' '.join(matches)} -> {line.strip()}")
found = True
except Exception as e:
print(f"Error reading {path}: {e}")
if not found:
print("No emojis found.")

7
src/api/helpers.rs
View File

@ -49,6 +49,7 @@ pub fn convert_ads_doc_to_standard(doc: &AdsPaperDoc) -> StandardPaper {
has_html: false, has_html: false,
has_markdown: false, has_markdown: false,
has_translation: false, has_translation: false,
has_vector: false,
doctype: doc.doctype.clone().unwrap_or_else(|| "article".to_string()), doctype: doc.doctype.clone().unwrap_or_else(|| "article".to_string()),
pdf_error: None, pdf_error: None,
html_error: None, html_error: None,
@ -73,6 +74,7 @@ pub fn convert_arxiv_to_standard(doc: &ArxivPaper) -> StandardPaper {
has_html: false, has_html: false,
has_markdown: false, has_markdown: false,
has_translation: false, has_translation: false,
has_vector: false,
doctype: "eprint".to_string(), doctype: "eprint".to_string(),
pdf_error: None, pdf_error: None,
html_error: None, html_error: None,
@ -164,7 +166,7 @@ pub async fn save_paper_to_db(db: &SqlitePool, p: &StandardPaper) -> anyhow::Res
} }
pub async fn get_paper_from_db(db: &SqlitePool, library_dir: &std::path::Path, bibcode: &str) -> anyhow::Result<StandardPaper> { pub async fn get_paper_from_db(db: &SqlitePool, library_dir: &std::path::Path, bibcode: &str) -> anyhow::Result<StandardPaper> {
let r = sqlx::query("SELECT bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, pdf_path, html_path, markdown_path, translation_path, doctype FROM papers WHERE bibcode = ?") let r = sqlx::query("SELECT bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, pdf_path, html_path, markdown_path, translation_path, doctype, EXISTS(SELECT 1 FROM paper_chunks_content WHERE bibcode = papers.bibcode) FROM papers WHERE bibcode = ?")
.bind(bibcode) .bind(bibcode)
.fetch_one(db) .fetch_one(db)
.await?; .await?;
@ -174,6 +176,7 @@ pub async fn get_paper_from_db(db: &SqlitePool, library_dir: &std::path::Path, b
let markdown_path: Option<String> = r.get(13); let markdown_path: Option<String> = r.get(13);
let translation_path: Option<String> = r.get(14); let translation_path: Option<String> = r.get(14);
let doctype_val: Option<String> = r.get(15); let doctype_val: Option<String> = r.get(15);
let has_vector: bool = r.get(16);
let authors_str: Option<String> = r.get(2); let authors_str: Option<String> = r.get(2);
let authors: Vec<String> = authors_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default(); let authors: Vec<String> = authors_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default();
@ -209,6 +212,7 @@ pub async fn get_paper_from_db(db: &SqlitePool, library_dir: &std::path::Path, b
has_html: is_html_exist, has_html: is_html_exist,
has_markdown: is_md_exist, has_markdown: is_md_exist,
has_translation: is_tr_exist, has_translation: is_tr_exist,
has_vector,
doctype: doctype_val.unwrap_or_else(|| "article".to_string()), doctype: doctype_val.unwrap_or_else(|| "article".to_string()),
pdf_error, pdf_error,
html_error, html_error,
@ -355,6 +359,7 @@ mod tests {
has_html: false, has_html: false,
has_markdown: false, has_markdown: false,
has_translation: false, has_translation: false,
has_vector: false,
doctype: "article".to_string(), doctype: "article".to_string(),
pdf_error: None, pdf_error: None,
html_error: None, html_error: None,

21
src/api/mod.rs
View File

@ -19,10 +19,11 @@ pub struct AppState {
pub ads: AdsClient, pub ads: AdsClient,
pub arxiv: ArxivClient, pub arxiv: ArxivClient,
pub llm: LlmClient, pub llm: LlmClient,
pub multimodal_llm: LlmClient,
pub embedding: EmbeddingClient, pub embedding: EmbeddingClient,
pub downloader: Downloader, pub downloader: Downloader,
pub harvest_status: Arc<tokio::sync::Mutex<crate::services::batch_sync::MetaSyncStatus>>, pub harvest_status: Arc<tokio::sync::Mutex<crate::services::batch_sync::MetaSyncStatus>>,
pub process_status: Arc<tokio::sync::Mutex<crate::services::batch_sync::AssetSyncStatus>>, pub batch_status: Arc<tokio::sync::Mutex<crate::services::batch_sync::AssetBatchStatus>>,
pub active_bibcode: Arc<tokio::sync::Mutex<Option<String>>>, pub active_bibcode: Arc<tokio::sync::Mutex<Option<String>>>,
} }
@ -45,6 +46,7 @@ pub struct StandardPaper {
pub has_html: bool, pub has_html: bool,
pub has_markdown: bool, pub has_markdown: bool,
pub has_translation: bool, pub has_translation: bool,
pub has_vector: bool,
pub doctype: String, pub doctype: String,
pub pdf_error: Option<String>, pub pdf_error: Option<String>,
pub html_error: Option<String>, pub html_error: Option<String>,
@ -54,6 +56,7 @@ pub mod helpers;
pub mod papers; pub mod papers;
pub mod notes; pub mod notes;
pub mod sync; pub mod sync;
pub mod targets;
// 提供兼容的 handlers 命名空间，避免修改 main.rs / batch_sync.rs 里的导入 // 提供兼容的 handlers 命名空间，避免修改 main.rs / batch_sync.rs 里的导入
pub mod handlers { pub mod handlers {
@ -62,12 +65,12 @@ pub mod handlers {
get_paper_from_db, check_paper_paths_in_db, get_paper_from_db, check_paper_paths_in_db,
}; };
pub use super::papers::{ pub use super::papers::{
search_papers, download_paper, parse_paper, translate_paper, search_papers, download_paper, parse_paper, translate_paper, embed_paper,
get_citation_network, get_paper_detail, get_library, export_citations, get_citation_network, get_paper_detail, get_library, export_citations,
upload_paper_file, mark_no_resource, get_active_bibcode, set_active_bibcode, upload_paper_file, mark_no_resource, get_active_bibcode, set_active_bibcode,
SearchParams, DownloadRequest, ParseRequest, ParseResponse, SearchParams, DownloadRequest, ParseRequest, ParseResponse,
TranslateRequest, TranslateResponse, CitationsResponse, PaperDetailResponse, TranslateRequest, TranslateResponse, CitationsResponse, PaperDetailResponse,
ExportRequest, ExportResponse, MarkNoResourceRequest, ExportRequest, ExportResponse, MarkNoResourceRequest, EmbedRequest, EmbedResponse,
}; };
pub use super::notes::{ pub use super::notes::{
create_note, get_notes, delete_note, create_note, get_notes, delete_note,
@ -75,9 +78,15 @@ pub mod handlers {
}; };
pub use super::sync::{ pub use super::sync::{
run_meta_sync, get_meta_sync_count, get_meta_sync_status, run_meta_sync, get_meta_sync_count, get_meta_sync_status,
run_asset_sync, stop_asset_sync, get_sync_queries, delete_sync_query, run_asset_batch, stop_asset_batch, get_sync_queries, delete_sync_query,
get_asset_sync_status, MetaSyncRunRequest, MetaSyncCountRequest, get_asset_batch_status, MetaSyncRunRequest, MetaSyncCountRequest,
MetaSyncCountResponse, AssetSyncRunRequest, SavedSyncQuery, MetaSyncCountResponse, AssetBatchRunRequest, SavedSyncQuery,
};
pub use super::targets::{
chat_rag, query_target, associate_target, list_targets, extract_paper_targets,
chat_figure,
RagAskRequest, TargetQueryParams, AssociateTargetRequest, TargetListParams, AssociateResponse,
ExtractTargetsRequest, ExtractTargetsResponse, ChatFigureRequest, ChatResponse,
}; };
pub use super::{AppState, StandardPaper}; pub use super::{AppState, StandardPaper};
} }

46
src/api/papers.rs
View File

@ -269,7 +269,7 @@ pub async fn parse_paper(
paper.keywords.join(",") paper.keywords.join(",")
); );
parsed_markdown = format!("{}{}", front_matter, md); parsed_markdown = format!("{}{}", front_matter, md);
let md_filename = format!("{}_en.md", req.bibcode); let md_filename = format!("{}.md", req.bibcode);
let md_dest = state.config.library_dir.join("Markdown").join(&md_filename); let md_dest = state.config.library_dir.join("Markdown").join(&md_filename);
fs::create_dir_all(md_dest.parent().unwrap()).unwrap_or_default(); fs::create_dir_all(md_dest.parent().unwrap()).unwrap_or_default();
if fs::write(&md_dest, &parsed_markdown).is_ok() { if fs::write(&md_dest, &parsed_markdown).is_ok() {
@ -300,7 +300,7 @@ pub async fn parse_paper(
paper.keywords.join(",") paper.keywords.join(",")
); );
parsed_markdown = format!("{}{}", front_matter, md); parsed_markdown = format!("{}{}", front_matter, md);
let md_filename = format!("{}_en.md", req.bibcode); let md_filename = format!("{}.md", req.bibcode);
let md_dest = state.config.library_dir.join("Markdown").join(&md_filename); let md_dest = state.config.library_dir.join("Markdown").join(&md_filename);
fs::create_dir_all(md_dest.parent().unwrap()).unwrap_or_default(); fs::create_dir_all(md_dest.parent().unwrap()).unwrap_or_default();
if fs::write(&md_dest, &parsed_markdown).is_ok() { if fs::write(&md_dest, &parsed_markdown).is_ok() {
@ -559,7 +559,7 @@ pub async fn get_paper_detail(
pub async fn get_library( pub async fn get_library(
State(state): State<Arc<AppState>>, State(state): State<Arc<AppState>>,
) -> Result<Json<Vec<StandardPaper>>, (StatusCode, String)> { ) -> Result<Json<Vec<StandardPaper>>, (StatusCode, String)> {
let rows = sqlx::query("SELECT bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, pdf_path, html_path, markdown_path, translation_path, doctype FROM papers ORDER BY created_at DESC") let rows = sqlx::query("SELECT bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, pdf_path, html_path, markdown_path, translation_path, doctype, EXISTS(SELECT 1 FROM paper_chunks_content WHERE bibcode = papers.bibcode) FROM papers ORDER BY created_at DESC")
.fetch_all(&state.db) .fetch_all(&state.db)
.await .await
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("访问本地数据库失败: {}", e)))?; .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("访问本地数据库失败: {}", e)))?;
@ -571,6 +571,7 @@ pub async fn get_library(
let markdown_path: Option<String> = r.get(13); let markdown_path: Option<String> = r.get(13);
let translation_path: Option<String> = r.get(14); let translation_path: Option<String> = r.get(14);
let doctype_val: Option<String> = r.get(15); let doctype_val: Option<String> = r.get(15);
let has_vector: bool = r.get(16);
let authors_str: Option<String> = r.get(2); let authors_str: Option<String> = r.get(2);
let authors: Vec<String> = authors_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default(); let authors: Vec<String> = authors_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default();
@ -604,6 +605,7 @@ pub async fn get_library(
has_html: is_html_exist, has_html: is_html_exist,
has_markdown: markdown_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false), has_markdown: markdown_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false),
has_translation: translation_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false), has_translation: translation_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false),
has_vector,
doctype: doctype_val.unwrap_or_else(|| "article".to_string()), doctype: doctype_val.unwrap_or_else(|| "article".to_string()),
pdf_error, pdf_error,
html_error, html_error,
@ -863,3 +865,41 @@ pub async fn set_active_bibcode(
StatusCode::OK StatusCode::OK
} }
// ── POST /api/embed ──
#[derive(Debug, Deserialize)]
pub struct EmbedRequest {
pub bibcode: String,
}
#[derive(Debug, Serialize)]
pub struct EmbedResponse {
pub chunk_count: usize,
}
pub async fn embed_paper(
State(state): State<Arc<AppState>>,
Json(req): Json<EmbedRequest>,
) -> Result<Json<EmbedResponse>, (StatusCode, String)> {
info!("接收到文献向量化分块入库指令: {}", req.bibcode);
let (_, _, md_opt, _) = check_paper_paths_in_db(&state.db, &state.config.library_dir, &req.bibcode)
.await
.map_err(|e| (StatusCode::NOT_FOUND, format!("获取文献路径失败: {}", e)))?
.ok_or((StatusCode::NOT_FOUND, "该文献未注册在数据库中".to_string()))?;
let md_rel = md_opt.ok_or((StatusCode::BAD_REQUEST, "文献尚未解析为 Markdown，请先执行解析".to_string()))?;
let md_abs = state.config.library_dir.join(&md_rel);
if !md_abs.exists() {
return Err((StatusCode::NOT_FOUND, "文献 Markdown 文件未找到，请重新解析".to_string()));
}
let markdown_content = fs::read_to_string(&md_abs)
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("读取 Markdown 文件失败: {}", e)))?;
let chunk_count = crate::services::rag::ingest_paper(&state.db, &state.embedding, &req.bibcode, &markdown_content, None)
.await
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("文献向量化失败: {}", e)))?;
Ok(Json(EmbedResponse { chunk_count }))
}

66
src/api/sync.rs
View File

@ -85,10 +85,10 @@ pub async fn get_meta_sync_status(
Json(status.clone()) Json(status.clone())
} }
// ── POST /api/sync/asset/run ── // ── POST /api/batch/asset/run ──
#[derive(Debug, Deserialize)] #[derive(Debug, Deserialize)]
pub struct AssetSyncRunRequest { pub struct AssetBatchRunRequest {
pub target_phase: String, // "download" | "parse" | "translate" pub target_phase: String, // "download" | "parse" | "translate" | "embed" | "target"
pub limit_count: Option<i32>, // 批量处理上限，默认 100 pub limit_count: Option<i32>, // 批量处理上限，默认 100
pub sort_order: Option<String>, // 处理顺序: "default" | "pub_year_desc" | "created_at_desc" pub sort_order: Option<String>, // 处理顺序: "default" | "pub_year_desc" | "created_at_desc"
pub skip_completed: Option<bool>, pub skip_completed: Option<bool>,
@ -105,15 +105,17 @@ struct PaperRecord {
translation_path: Option<String>, translation_path: Option<String>,
year: String, year: String,
created_at: String, created_at: String,
has_vector: bool,
has_target: bool,
} }
pub async fn run_asset_sync( pub async fn run_asset_batch(
State(state): State<Arc<AppState>>, State(state): State<Arc<AppState>>,
Json(req): Json<AssetSyncRunRequest>, Json(req): Json<AssetBatchRunRequest>,
) -> Result<StatusCode, (StatusCode, String)> { ) -> Result<StatusCode, (StatusCode, String)> {
// 检查是否已经在进行批量处理任务 // 检查是否已经在进行批量处理任务
{ {
let status = state.process_status.lock().await; let status = state.batch_status.lock().await;
if status.active { if status.active {
return Err((StatusCode::CONFLICT, "当前已有文献批量任务在后台运行中，请勿重复启动".to_string())); return Err((StatusCode::CONFLICT, "当前已有文献批量任务在后台运行中，请勿重复启动".to_string()));
} }
@ -121,13 +123,15 @@ pub async fn run_asset_sync(
let target_phase = req.target_phase.clone(); let target_phase = req.target_phase.clone();
let action = match target_phase.as_str() { let action = match target_phase.as_str() {
"download" => crate::services::batch_sync::SyncAction::Download, "download" => crate::services::batch_sync::BatchAction::Download,
"parse" => crate::services::batch_sync::SyncAction::Parse, "parse" => crate::services::batch_sync::BatchAction::Parse,
"translate" => crate::services::batch_sync::SyncAction::Translate, "translate" => crate::services::batch_sync::BatchAction::Translate,
"embed" => crate::services::batch_sync::BatchAction::Embed,
"target" => crate::services::batch_sync::BatchAction::Target,
_ => return Err((StatusCode::BAD_REQUEST, "不支持的 target_phase 参数值".to_string())), _ => return Err((StatusCode::BAD_REQUEST, "不支持的 target_phase 参数值".to_string())),
}; };
let rows = sqlx::query("SELECT bibcode, pdf_path, html_path, markdown_path, translation_path, year, datetime(created_at, 'localtime') FROM papers") let rows = sqlx::query("SELECT bibcode, pdf_path, html_path, markdown_path, translation_path, year, datetime(created_at, 'localtime'), EXISTS(SELECT 1 FROM paper_chunks_content WHERE bibcode = papers.bibcode), EXISTS(SELECT 1 FROM paper_targets WHERE bibcode = papers.bibcode) FROM papers")
.fetch_all(&state.db) .fetch_all(&state.db)
.await .await
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("读取数据库失败: {}", e)))?; .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("读取数据库失败: {}", e)))?;
@ -142,6 +146,8 @@ pub async fn run_asset_sync(
translation_path: r.get(4), translation_path: r.get(4),
year: r.get(5), year: r.get(5),
created_at: r.get(6), created_at: r.get(6),
has_vector: r.get(7),
has_target: r.get(8),
}); });
} }
@ -237,6 +243,28 @@ pub async fn run_asset_sync(
continue; continue;
} }
} }
"embed" => {
if skip_completed && rec.has_vector {
continue;
}
if skip_preceding_failed && parse_failed {
continue;
}
if skip_preceding_uncompleted && parse_uncompleted {
continue;
}
}
"target" => {
if skip_completed && rec.has_target {
continue;
}
if skip_preceding_failed && parse_failed {
continue;
}
if skip_preceding_uncompleted && parse_uncompleted {
continue;
}
}
_ => {} _ => {}
} }
target_bibcodes.push(rec.bibcode); target_bibcodes.push(rec.bibcode);
@ -252,7 +280,7 @@ pub async fn run_asset_sync(
} }
// 启动后台处理 // 启动后台处理
crate::services::batch_sync::AssetSync::start_process( crate::services::batch_sync::AssetBatch::start_process(
state.db.clone(), state.db.clone(),
state.config.clone(), state.config.clone(),
Arc::new(state.downloader.clone()), Arc::new(state.downloader.clone()),
@ -260,17 +288,17 @@ pub async fn run_asset_sync(
Arc::new(state.dict.clone()), Arc::new(state.dict.clone()),
action, action,
target_bibcodes, target_bibcodes,
state.process_status.clone(), state.batch_status.clone(),
); );
Ok(StatusCode::ACCEPTED) Ok(StatusCode::ACCEPTED)
} }
// ── POST /api/sync/asset/stop ── // ── POST /api/batch/asset/stop ──
pub async fn stop_asset_sync( pub async fn stop_asset_batch(
State(state): State<Arc<AppState>>, State(state): State<Arc<AppState>>,
) -> StatusCode { ) -> StatusCode {
let mut status = state.process_status.lock().await; let mut status = state.batch_status.lock().await;
if status.active { if status.active {
status.active = false; status.active = false;
status.add_log("用户手动终止了批量处理任务。".to_string()); status.add_log("用户手动终止了批量处理任务。".to_string());
@ -330,10 +358,10 @@ pub async fn delete_sync_query(
Ok(StatusCode::OK) Ok(StatusCode::OK)
} }
// ── GET /api/sync/asset/status ── // ── GET /api/batch/asset/status ──
pub async fn get_asset_sync_status( pub async fn get_asset_batch_status(
State(state): State<Arc<AppState>>, State(state): State<Arc<AppState>>,
) -> Json<crate::services::batch_sync::AssetSyncStatus> { ) -> Json<crate::services::batch_sync::AssetBatchStatus> {
let status = state.process_status.lock().await; let status = state.batch_status.lock().await;
Json(status.clone()) Json(status.clone())
} }

256
src/api/targets.rs Normal file
View File

@ -0,0 +1,256 @@
// src/api/targets.rs
use axum::{
extract::{Query, State},
http::StatusCode,
Json,
};
use serde::{Deserialize, Serialize};
use std::sync::Arc;
use super::AppState;
use crate::services::rag::{ask, RagAnswer};
use crate::services::target::{query_target_cached, TargetInfo};
#[derive(Deserialize)]
pub struct RagAskRequest {
pub question: String,
pub top_k: Option<usize>,
}
#[derive(Deserialize)]
pub struct TargetQueryParams {
pub object_name: String,
}
#[derive(Deserialize)]
pub struct AssociateTargetRequest {
pub bibcode: String,
pub object_name: String,
}
#[derive(Deserialize)]
pub struct TargetListParams {
pub bibcode: String,
}
#[derive(Serialize)]
pub struct AssociateResponse {
pub status: String,
pub target: TargetInfo,
}
// ── POST /api/chat/rag ──
pub async fn chat_rag(
State(state): State<Arc<AppState>>,
Json(req): Json<RagAskRequest>,
) -> Result<Json<RagAnswer>, (StatusCode, String)> {
let top_k = req.top_k.unwrap_or(5);
let answer = ask(&state.db, &state.embedding, &state.llm, &req.question, top_k)
.await
.map_err(|e| {
tracing::error!("RAG 问答执行失败: {}", e);
(StatusCode::INTERNAL_SERVER_ERROR, format!("RAG 问答执行失败: {}", e))
})?;
Ok(Json(answer))
}
// ── GET /api/target/query ──
pub async fn query_target(
State(state): State<Arc<AppState>>,
Query(params): Query<TargetQueryParams>,
) -> Result<Json<TargetInfo>, (StatusCode, String)> {
let client = reqwest::Client::new();
let info = query_target_cached(&state.db, &params.object_name, None, &client)
.await
.map_err(|e| {
tracing::error!("查询天体信息失败 ({}): {}", params.object_name, e);
(StatusCode::INTERNAL_SERVER_ERROR, format!("查询天体信息失败: {}", e))
})?;
Ok(Json(info))
}
// ── POST /api/target/associate ──
pub async fn associate_target(
State(state): State<Arc<AppState>>,
Json(req): Json<AssociateTargetRequest>,
) -> Result<Json<AssociateResponse>, (StatusCode, String)> {
let client = reqwest::Client::new();
// 查询并将结果缓存/关联到对应文献
let info = query_target_cached(&state.db, &req.object_name, Some(&req.bibcode), &client)
.await
.map_err(|e| {
tracing::error!("手动关联天体失败 ({} -> {}): {}", req.object_name, req.bibcode, e);
(StatusCode::INTERNAL_SERVER_ERROR, format!("手动关联天体失败: {}", e))
})?;
Ok(Json(AssociateResponse {
status: "success".to_string(),
target: info,
}))
}
// ── GET /api/target/list ──
pub async fn list_targets(
State(state): State<Arc<AppState>>,
Query(params): Query<TargetListParams>,
) -> Result<Json<Vec<TargetInfo>>, (StatusCode, String)> {
let rows = sqlx::query_as::<_, (String, Option<String>, Option<String>, Option<f64>, Option<String>, Option<f64>, Option<String>)>(
"SELECT target_name, ra, dec, parallax, spectral_type, v_magnitude, aliases FROM paper_targets WHERE bibcode = ? ORDER BY target_name"
)
.bind(&params.bibcode)
.fetch_all(&state.db)
.await
.map_err(|e| {
tracing::error!("获取文献天体关联列表失败 ({}): {}", params.bibcode, e);
(StatusCode::INTERNAL_SERVER_ERROR, format!("获取文献天体关联列表失败: {}", e))
})?;
let targets: Vec<TargetInfo> = rows
.into_iter()
.map(|(name, ra, dec, parallax, spectral_type, v_magnitude, aliases_json)| {
let aliases: Vec<String> = aliases_json
.and_then(|s| serde_json::from_str(&s).ok())
.unwrap_or_default();
TargetInfo {
target_name: name,
ra,
dec,
parallax,
spectral_type,
v_magnitude,
aliases,
}
})
.collect();
Ok(Json(targets))
}
// ── POST /api/target/extract ──
#[derive(Deserialize)]
pub struct ExtractTargetsRequest {
pub bibcode: String,
}
#[derive(Serialize)]
pub struct ExtractTargetsResponse {
pub targets: Vec<TargetInfo>,
}
pub async fn extract_paper_targets(
State(state): State<Arc<AppState>>,
Json(req): Json<ExtractTargetsRequest>,
) -> Result<Json<ExtractTargetsResponse>, (StatusCode, String)> {
tracing::info!("接收到文献天体提取与识别指令: {}", req.bibcode);
let (_, _, md_opt, _) = crate::api::helpers::check_paper_paths_in_db(&state.db, &state.config.library_dir, &req.bibcode)
.await
.map_err(|e| (StatusCode::NOT_FOUND, format!("获取文献路径失败: {}", e)))?
.ok_or((StatusCode::NOT_FOUND, "该文献未注册在数据库中".to_string()))?;
let md_rel = md_opt.ok_or((StatusCode::BAD_REQUEST, "文献尚未解析为 Markdown，请先执行解析".to_string()))?;
let md_abs = state.config.library_dir.join(&md_rel);
if !md_abs.exists() {
return Err((StatusCode::NOT_FOUND, "文献 Markdown 文件未找到，请重新解析".to_string()));
}
let markdown_content = std::fs::read_to_string(&md_abs)
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("读取 Markdown 文件失败: {}", e)))?;
// 重新识别前先清除该文献已有的天体关联记录，确保陈旧和错误绑定的天体得到重置与刷新
if let Err(e) = sqlx::query("DELETE FROM paper_targets WHERE bibcode = ?")
.bind(&req.bibcode)
.execute(&state.db)
.await
{
tracing::error!("清除文献 {} 的旧天体关联失败: {}", req.bibcode, e);
}
let client = reqwest::Client::new();
let targets = crate::services::target::extract_and_cache_targets(&state.db, &markdown_content, &req.bibcode, &client).await;
Ok(Json(ExtractTargetsResponse { targets }))
}
// ── POST /api/chat/figure ──
#[derive(Deserialize)]
pub struct ChatFigureRequest {
pub bibcode: String,
pub image_path: String,
pub question: String,
}
#[derive(Serialize)]
pub struct ChatResponse {
pub answer: String,
}
async fn get_image_bytes(library_dir: &std::path::Path, path: &str) -> anyhow::Result<(Vec<u8>, String)> {
if path.starts_with("http://") || path.starts_with("https://") {
let client = reqwest::Client::new();
let resp = client.get(path).send().await?;
let headers = resp.headers().clone();
let mime_type = headers
.get(reqwest::header::CONTENT_TYPE)
.and_then(|v| v.to_str().ok())
.unwrap_or("image/png")
.to_string();
let bytes = resp.bytes().await?.to_vec();
Ok((bytes, mime_type))
} else {
let abs_path = library_dir.join(path);
if !abs_path.exists() {
return Err(anyhow::anyhow!("本地图片未找到: {:?}", abs_path));
}
let ext = abs_path
.extension()
.and_then(|e| e.to_str())
.unwrap_or("png")
.to_lowercase();
let mime_type = match ext.as_str() {
"jpg" | "jpeg" => "image/jpeg",
"gif" => "image/gif",
"webp" => "image/webp",
"svg" => "image/svg+xml",
_ => "image/png",
}.to_string();
let bytes = std::fs::read(&abs_path)?;
Ok((bytes, mime_type))
}
}
pub async fn chat_figure(
State(state): State<Arc<AppState>>,
Json(req): Json<ChatFigureRequest>,
) -> Result<Json<ChatResponse>, (StatusCode, String)> {
tracing::info!("接收到图表多模态提问: bibcode={}, image_path={}, question={}", req.bibcode, req.image_path, req.question);
let (bytes, mime_type) = get_image_bytes(&state.config.library_dir, &req.image_path)
.await
.map_err(|e| {
tracing::error!("获取图片失败: {}", e);
(StatusCode::BAD_REQUEST, format!("获取图片失败: {}", e))
})?;
use base64::{Engine as _, engine::general_purpose};
let base64_data = general_purpose::STANDARD.encode(&bytes);
let system_prompt = "You are a professional astronomer and physicist. You are helping a researcher analyze a scientific plot, diagram, or chart extracted from a theoretical or observational astrophysics paper. Please provide an expert, detailed, and clear explanation of the figure in Chinese based on the image provided and the user's question. Format equations in standard LaTeX.";
let answer = state.multimodal_llm.chat_completion_with_image(
system_prompt,
&req.question,
&base64_data,
&mime_type,
).await.map_err(|e| {
tracing::error!("多模态大模型请求失败: {}", e);
(StatusCode::INTERNAL_SERVER_ERROR, format!("大模型图表解析失败: {}", e))
})?;
Ok(Json(ChatResponse { answer }))
}

196
src/bin/cli.rs Normal file
View File

@ -0,0 +1,196 @@
// src/bin/cli.rs
//
// AstroResearch CLI Skills Agent — 向外部 Agent（如 Claude Code）提供
// 标准的命令行工具接口，用于 RAG 问答、天体查询和天体关联操作。
use std::str::FromStr;
use clap::{Parser, Subcommand};
use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
use tracing_subscriber::FmtSubscriber;
use astroresearch::Config;
use astroresearch::clients::llm::{LlmClient, EmbeddingClient};
#[derive(Parser)]
#[command(
name = "astroresearch_cli",
about = "AstroResearch CLI Skills Agent — 天体物理文献智能助手",
version
)]
struct Cli {
#[command(subcommand)]
command: Commands,
}
#[derive(Subcommand)]
enum Commands {
/// 跨文献 RAG 智能问答：在所有已导入文献中检索并生成回答
Rag {
/// 用户问题
question: String,
/// 检索 Top-K 数量（默认 5）
#[arg(short, long, default_value = "5")]
top_k: usize,
},
/// 查询天体数据：优先读取本地缓存，未命中时自动查询 SIMBAD
TargetQuery {
/// 天体名称（如 "GD 358", "NGC 1234", "HD 209458"）
object_name: String,
},
/// 手动关联天体到特定文献
TargetAssociate {
/// 文献 bibcode
bibcode: String,
/// 天体名称
object_name: String,
},
/// 从文本中提取天体标识符（调试/预览用）
ExtractTargets {
/// 输入文本
text: String,
},
/// 向量化导入指定文献的 Markdown 文件
Ingest {
/// 文献 bibcode
bibcode: String,
/// Markdown 文件路径
markdown_path: String,
},
}
#[tokio::main]
async fn main() -> anyhow::Result<()> {
// 初始化日志
let subscriber = FmtSubscriber::builder()
.with_max_level(tracing::Level::INFO)
.finish();
tracing_subscriber::util::SubscriberInitExt::init(subscriber);
// SAFETY: 注册静态 sqlite-vec 初始化函数。transmute 是安全的，
// 因为 sqlite3_vec_init 符合 SQLite C API 的自动扩展回调签名
// (sqlite3*, char**, const sqlite3_api_routines*)。
// 该注册必须在开启任何数据库连接前执行。
unsafe {
libsqlite3_sys::sqlite3_auto_extension(Some(std::mem::transmute(
sqlite_vec::sqlite3_vec_init as *const (),
)));
}
let config = Config::from_env();
let options = SqliteConnectOptions::from_str(&config.database_url)?
.foreign_keys(true)
.create_if_missing(true);
let pool = SqlitePoolOptions::new()
.max_connections(2)
.connect_with(options)
.await?;
// 执行迁移
sqlx::migrate!("./migrations")
.run(&pool)
.await?;
let cli = Cli::parse();
match cli.command {
Commands::Rag { question, top_k } => {
let embedding = EmbeddingClient::new(
config.embedding_api_key.clone(),
config.embedding_api_base.clone(),
config.embedding_model.clone(),
);
let llm = LlmClient::new(
config.llm_api_key.clone(),
config.llm_api_base.clone(),
config.llm_model.clone(),
);
let result = astroresearch::services::rag::ask(
&pool, &embedding, &llm, &question, top_k
).await?;
println!("\n📖 回答:\n{}\n", result.answer);
if !result.sources.is_empty() {
println!("📚 参考来源:");
for (i, src) in result.sources.iter().enumerate() {
println!(
" [{}] {} §{} (距离: {:.4})",
i + 1, src.bibcode, src.paragraph_index, src.distance
);
}
}
}
Commands::TargetQuery { object_name } => {
let client = reqwest::Client::new();
let info = astroresearch::services::target::query_target_cached(
&pool, &object_name, None, &client
).await?;
println!("\n🔭 天体信息: {}", info.target_name);
if let Some(ra) = &info.ra {
println!(" RA: {}", ra);
}
if let Some(dec) = &info.dec {
println!(" Dec: {}", dec);
}
if let Some(sp) = &info.spectral_type {
println!(" 光谱类型: {}", sp);
}
if let Some(vmag) = info.v_magnitude {
println!(" 视星等: {:.2}", vmag);
}
if let Some(plx) = info.parallax {
println!(" 视差: {:.2} mas", plx);
}
if !info.aliases.is_empty() {
println!(" 别名: {}", info.aliases.join(", "));
}
}
Commands::TargetAssociate { bibcode, object_name } => {
let client = reqwest::Client::new();
let info = astroresearch::services::target::query_target_cached(
&pool, &object_name, Some(&bibcode), &client
).await?;
println!("✅ 已关联天体 {} -> 文献 {}", info.target_name, bibcode);
}
Commands::ExtractTargets { text } => {
let targets = astroresearch::services::target::extract_targets(&text);
if targets.is_empty() {
println!("未识别到天体标识符。");
} else {
println!("🎯 识别到 {} 个天体:", targets.len());
for t in &targets {
println!(" - {}", t);
}
}
}
Commands::Ingest { bibcode, markdown_path } => {
let content = std::fs::read_to_string(&markdown_path)?;
let embedding = EmbeddingClient::new(
config.embedding_api_key.clone(),
config.embedding_api_base.clone(),
config.embedding_model.clone(),
);
let count = astroresearch::services::rag::ingest_paper(
&pool, &embedding, &bibcode, &content, None
).await?;
println!("✅ 文献 {} 向量化完成，写入 {} 个切片", bibcode, count);
}
}
Ok(())
}

9
src/bin/health_check.rs
View File

@ -192,6 +192,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
.finish(); .finish();
tracing::subscriber::set_global_default(subscriber)?; tracing::subscriber::set_global_default(subscriber)?;
// SAFETY: sqlite3_vec_init 的函数签名严格符合 SQLite C API 自动扩展
// 回调规范。即使 health_check 不直接使用 vec0，也需要注册以防数据库
// 包含 vec0 虚拟表时连接崩溃。
unsafe {
libsqlite3_sys::sqlite3_auto_extension(Some(std::mem::transmute(
sqlite_vec::sqlite3_vec_init as *const (),
)));
}
let args: Vec<String> = std::env::args().collect(); let args: Vec<String> = std::env::args().collect();
let fix = args.contains(&"--fix".to_string()); let fix = args.contains(&"--fix".to_string());

116
src/bin/reparse.rs Normal file
View File

@ -0,0 +1,116 @@
use std::path::{Path, PathBuf};
use std::time::Instant;
use astroresearch::services::parser::html_to_markdown;
fn main() -> anyhow::Result<()> {
let args: Vec<String> = std::env::args().collect();
let html_dir = Path::new("library/HTML");
let md_dir = Path::new("library/Markdown");
if args.len() > 1 {
// Single file mode
let stem = &args[1];
let html_path = html_dir.join(format!("{}.html", stem));
let md_path = md_dir.join(format!("{}.md", stem));
if !html_path.exists() {
anyhow::bail!("HTML file not found: {:?}", html_path);
}
let front_matter = if md_path.exists() {
extract_front_matter(&std::fs::read_to_string(&md_path)?)
} else { String::new() };
let t0 = Instant::now();
let md_content = html_to_markdown(&html_path)?;
let elapsed = t0.elapsed();
let final_md = if front_matter.is_empty() { md_content }
else { format!("{}\n\n{}", front_matter.trim_end(), md_content) };
std::fs::write(&md_path, &final_md)?;
println!("✅ {} → {} [{:.0}ms]", stem, md_path.display(), elapsed.as_secs_f64() * 1000.0);
return Ok(());
}
if !html_dir.exists() {
anyhow::bail!("library/HTML directory not found");
}
let mut html_files: Vec<PathBuf> = std::fs::read_dir(html_dir)?
.filter_map(|e| e.ok())
.map(|e| e.path())
.filter(|p| p.extension().map_or(false, |ext| ext == "html"))
.collect();
html_files.sort();
let total = html_files.len();
println!("Found {} HTML files to reparse\n", total);
let mut success = 0u32;
let mut skipped = 0u32;
let mut failed = 0u32;
for html_path in &html_files {
let stem = html_path.file_stem().and_then(|s| s.to_str()).unwrap_or("");
let md_path = md_dir.join(format!("{}.md", stem));
// Extract front matter from existing MD if present
let front_matter = if md_path.exists() {
match std::fs::read_to_string(&md_path) {
Ok(original) => extract_front_matter(&original),
Err(_) => String::new(),
}
} else {
String::new()
};
// Parse HTML → Markdown
match html_to_markdown(html_path) {
Ok(md_content) => {
let final_md = if front_matter.is_empty() {
md_content
} else {
format!("{}\n\n{}", front_matter.trim_end(), md_content)
};
match std::fs::write(&md_path, &final_md) {
Ok(_) => {
println!("✅ {} → {}", stem, md_path.display());
success += 1;
}
Err(e) => {
eprintln!("❌ {} write error: {}", stem, e);
failed += 1;
}
}
}
Err(e) => {
eprintln!("❌ {} parse error: {}", stem, e);
failed += 1;
}
}
}
println!("\n--- Done ---");
println!(" ✅ {} succeeded", success);
println!(" ❌ {} failed", failed);
if skipped > 0 {
println!(" ⏭️ {} skipped", skipped);
}
println!(" 📂 {} total", total);
if failed > 0 {
std::process::exit(1);
}
Ok(())
}
fn extract_front_matter(md: &str) -> String {
if !md.starts_with("---") {
return String::new();
}
if let Some(second_dash) = md[3..].find("---") {
let end = 3 + second_dash + 3;
md[..end].to_string()
} else {
String::new()
}
}

64
src/clients/ads.rs
View File

@ -218,3 +218,67 @@ struct RawSearchResponse {
struct RawDocs { struct RawDocs {
docs: Vec<RawDoc>, docs: Vec<RawDoc>,
} }
#[cfg(test)]
mod tests {
use super::*;
use crate::clients::arxiv::ArxivClient;
/// 真实检索集成测试 —— 需要配置 ADS_API_KEY，默认跳过
#[tokio::test]
#[ignore]
async fn test_live_search_comparisons() -> anyhow::Result<()> {
let config = crate::Config::from_env();
if config.ads_api_key.is_empty() {
println!("警告: 未在环境配置中检测到 ADS_API_KEY，跳过 ADS 集成测试。");
}
let ads = AdsClient::new(config.ads_api_key.clone());
let arxiv = ArxivClient::new();
println!("================= 开始真实检索逻辑集成测试 =================");
// 测试 1: 比较 OR 与 AND 逻辑的数据量差异
let query_or = "\"hot subdwarf\" OR Gaia";
let query_and = "\"hot subdwarf\" AND Gaia";
if !config.ads_api_key.is_empty() {
let count_ads_or = ads.get_total_count(query_or).await?;
let count_ads_and = ads.get_total_count(query_and).await?;
println!("NASA ADS 平台:");
println!(" - (OR) \"{}\" 匹配数: {} 篇", query_or, count_ads_or);
println!(" - (AND) \"{}\" 匹配数: {} 篇", query_and, count_ads_and);
assert!(count_ads_or > count_ads_and, "错误: OR 结果应该多于 AND");
}
let count_arxiv_or = arxiv.get_total_count(query_or).await?;
let count_arxiv_and = arxiv.get_total_count(query_and).await?;
println!("arXiv 平台:");
println!(" - (OR) \"{}\" 匹配数: {} 篇", query_or, count_arxiv_or);
println!(" - (AND) \"{}\" 匹配数: {} 篇", query_and, count_arxiv_and);
assert!(count_arxiv_or > count_arxiv_and, "错误: arXiv 的 OR 结果应该多于 AND");
// 测试 2: 比较基础词组与含有 NOT 排除条件的数据量差异
let query_base = "\"hot subdwarf\"";
let query_not = "\"hot subdwarf\" NOT \"white dwarf\"";
if !config.ads_api_key.is_empty() {
let count_ads_base = ads.get_total_count(query_base).await?;
let count_ads_not = ads.get_total_count(query_not).await?;
println!("NASA ADS 平台:");
println!(" - (基础) \"{}\" 匹配数: {} 篇", query_base, count_ads_base);
println!(" - (排除) \"{}\" 匹配数: {} 篇", query_not, count_ads_not);
assert!(count_ads_base >= count_ads_not, "错误: 基础结果应该大于或等于排除后的结果");
}
let count_arxiv_base = arxiv.get_total_count(query_base).await?;
let count_arxiv_not = arxiv.get_total_count(query_not).await?;
println!("arXiv 平台:");
println!(" - (基础) \"{}\" 匹配数: {} 篇", query_base, count_arxiv_base);
println!(" - (排除) \"{}\" 匹配数: {} 篇", query_not, count_arxiv_not);
assert!(count_arxiv_base >= count_arxiv_not, "错误: arXiv 基础结果应该大于或等于排除后的结果");
println!("================= 真实检索逻辑集成测试全部通过 =================");
Ok(())
}
}

131
src/clients/llm.rs
View File

@ -87,6 +87,84 @@ impl LlmClient {
Err(anyhow::anyhow!("大模型返回空翻译选项集")) Err(anyhow::anyhow!("大模型返回空翻译选项集"))
} }
} }
pub async fn chat_completion_with_image(
&self,
system_prompt: &str,
user_prompt: &str,
image_base64: &str,
mime_type: &str,
) -> anyhow::Result<String> {
let url = format!("{}/chat/completions", self.api_base);
let image_url_val = if image_base64.starts_with("data:") {
image_base64.to_string()
} else {
format!("data:{};base64,{}", mime_type, image_base64)
};
let payload = serde_json::json!({
"model": self.model,
"messages": [
{
"role": "system",
"content": system_prompt
},
{
"role": "user",
"content": [
{
"type": "text",
"text": user_prompt
},
{
"type": "image_url",
"image_url": {
"url": image_url_val
}
}
]
}
],
"temperature": 0.3
});
let response = self.client.post(&url)
.header("Authorization", format!("Bearer {}", self.api_key))
.header("Content-Type", "application/json")
.json(&payload)
.send()
.await?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
error!("LLM 多模态接口调用失败: 状态码={}, 报错={}", status, body);
return Err(anyhow::anyhow!("大模型多模态接口返回错误状态: {}", status));
}
#[derive(Deserialize)]
struct Message {
content: String,
}
#[derive(Deserialize)]
struct Choice {
message: Message,
}
#[derive(Deserialize)]
struct LLMResponse {
choices: Vec<Choice>,
}
let res_data: LLMResponse = response.json().await?;
if let Some(choice) = res_data.choices.first() {
Ok(choice.message.content.clone())
} else {
Err(anyhow::anyhow!("大模型返回空多模态回答选项集"))
}
}
} }
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
@ -187,4 +265,57 @@ mod tests {
assert_eq!(client.api_base(), "base"); assert_eq!(client.api_base(), "base");
assert_eq!(client.model(), "model"); assert_eq!(client.model(), "model");
} }
/// 真实网络集成测试 —— 需要配置 LLM_API_KEY 和 EMBEDDING_API_KEY
#[tokio::test]
#[ignore]
async fn test_live_llm_and_embedding() -> anyhow::Result<()> {
let config = crate::Config::from_env();
println!("================= 开始大模型与向量模型真实网络集成测试 =================");
// 1. 测试 LlmClient
if config.llm_api_key.is_empty() {
println!("警告: 未在环境配置中检测到 LLM_API_KEY，跳过 LlmClient 集成测试。");
} else {
println!("测试大模型: {} (API Base: {})", config.llm_model, config.llm_api_base);
let llm = LlmClient::new(
config.llm_api_key.clone(),
config.llm_api_base.clone(),
config.llm_model.clone(),
);
match llm.chat_completion("You are a helpful assistant.", "Say Hello!").await {
Ok(reply) => {
println!("LlmClient 响应成功: {}", reply.trim());
assert!(!reply.trim().is_empty(), "错误: 大模型返回了空响应");
}
Err(e) => panic!("LlmClient 接口调用失败: {}", e),
}
}
// 2. 测试 EmbeddingClient
if config.embedding_api_key.is_empty() {
println!("警告: 未在环境配置中检测到 EMBEDDING_API_KEY，跳过 EmbeddingClient 集成测试。");
} else {
println!("测试向量模型: {} (API Base: {})", config.embedding_model, config.embedding_api_base);
let embedding_client = EmbeddingClient::new(
config.embedding_api_key.clone(),
config.embedding_api_base.clone(),
config.embedding_model.clone(),
);
let test_text = "active galactic nucleus";
match embedding_client.create_embedding(test_text).await {
Ok(vector) => {
println!("EmbeddingClient 响应成功！向量维度: {}", vector.len());
assert!(!vector.is_empty(), "错误: 向量数据为空");
let preview_len = std::cmp::min(5, vector.len());
println!("前 {} 个向量数值样例: {:?}", preview_len, &vector[..preview_len]);
}
Err(e) => panic!("EmbeddingClient 接口调用失败: {}", e),
}
}
println!("================= 大模型与向量模型真实网络集成测试完成 =================");
Ok(())
}
} }

5
src/clients/qiniu.rs
View File

@ -62,9 +62,8 @@ impl QiniuClient {
return Err(anyhow::anyhow!("本地 .env 文件中未正确配置七牛云参数")); return Err(anyhow::anyhow!("本地 .env 文件中未正确配置七牛云参数"));
} }
// 使用毫秒级时间戳防重名覆盖，并放置在 astroresearch 虚拟文件夹下 // 将图片放在 astroresearch 目录下，文件名由调用方保证唯一性
let timestamp = chrono::Utc::now().timestamp_millis(); let key = format!("astroresearch/{}", filename);
let key = format!("astroresearch/{}_{}", timestamp, filename);
let token = self.generate_upload_token(&key); let token = self.generate_upload_token(&key);
info!("正在上传文献提取图片到七牛云: key='{}'", key); info!("正在上传文献提取图片到七牛云: key='{}'", key);

13
src/lib.rs
View File

@ -10,6 +10,9 @@ pub struct Config {
pub llm_api_key: String, // 大语言模型 API Key pub llm_api_key: String, // 大语言模型 API Key
pub llm_api_base: String, // 大语言模型 API 基础地址 pub llm_api_base: String, // 大语言模型 API 基础地址
pub llm_model: String, // 调用的翻译大模型名称 pub llm_model: String, // 调用的翻译大模型名称
pub multimodal_api_key: String, // 多模态模型 API Key (如不设置回退到 llm_api_key)
pub multimodal_api_base: String,// 多模态模型 API 基础地址 (如不设置回退到 llm_api_base)
pub multimodal_model: String, // 多模态模型名称 (如不设置回退到 llm_model)
pub embedding_api_key: String, // 向量模型 API Key pub embedding_api_key: String, // 向量模型 API Key
pub embedding_api_base: String,// 向量模型 API 基础地址 pub embedding_api_base: String,// 向量模型 API 基础地址
pub embedding_model: String, // 向量模型名称 pub embedding_model: String, // 向量模型名称
@ -37,6 +40,13 @@ impl Config {
let llm_model = env::var("LLM_MODEL") let llm_model = env::var("LLM_MODEL")
.unwrap_or_else(|_| "gpt-4o-mini".to_string()); .unwrap_or_else(|_| "gpt-4o-mini".to_string());
let multimodal_api_key = env::var("MULTIMODAL_API_KEY")
.unwrap_or_else(|_| llm_api_key.clone());
let multimodal_api_base = env::var("MULTIMODAL_API_BASE")
.unwrap_or_else(|_| llm_api_base.clone());
let multimodal_model = env::var("MULTIMODAL_MODEL")
.unwrap_or_else(|_| llm_model.clone());
let embedding_api_key = env::var("EMBEDDING_API_KEY") let embedding_api_key = env::var("EMBEDDING_API_KEY")
.unwrap_or_else(|_| llm_api_key.clone()); .unwrap_or_else(|_| llm_api_key.clone());
let embedding_api_base = env::var("EMBEDDING_API_BASE") let embedding_api_base = env::var("EMBEDDING_API_BASE")
@ -66,6 +76,9 @@ impl Config {
llm_api_key, llm_api_key,
llm_api_base, llm_api_base,
llm_model, llm_model,
multimodal_api_key,
multimodal_api_base,
multimodal_model,
embedding_api_key, embedding_api_key,
embedding_api_base, embedding_api_base,
embedding_model, embedding_model,

90
src/main.rs
View File

@ -1,6 +1,7 @@
// src/main.rs // src/main.rs
use std::net::SocketAddr; use std::net::SocketAddr;
use std::str::FromStr;
use std::sync::Arc; use std::sync::Arc;
use axum::{ use axum::{
routing::{get, post}, routing::{get, post},
@ -8,7 +9,7 @@ use axum::{
}; };
use tower_http::cors::{Any, CorsLayer}; use tower_http::cors::{Any, CorsLayer};
use tower_http::services::ServeDir; use tower_http::services::ServeDir;
use sqlx::sqlite::SqlitePoolOptions; use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
use tracing::{info, error}; use tracing::{info, error};
use astroresearch::Config; use astroresearch::Config;
@ -27,6 +28,19 @@ async fn main() -> anyhow::Result<()> {
info!("正在启动 AstroResearch 天文学文献辅助系统后端服务..."); info!("正在启动 AstroResearch 天文学文献辅助系统后端服务...");
// 1.5 静态注册 sqlite-vec 自动扩展
// SAFETY: sqlite3_vec_init 的函数签名严格符合 SQLite C API 自动扩展
// 回调规范 (sqlite3*, char**, const sqlite3_api_routines*)。transmute
// 将该函数指针转换为 sqlite3_auto_extension 所需的 Option<fn()> 类型。
// 该注册必须在任何数据库连接开启之前执行，保证所有 Connection
// 自动拥有 vec0 虚拟表能力。
unsafe {
libsqlite3_sys::sqlite3_auto_extension(Some(std::mem::transmute(
sqlite_vec::sqlite3_vec_init as *const (),
)));
}
info!("sqlite-vec 自动扩展注册完成。");
// 2. 加载环境变量配置 // 2. 加载环境变量配置
let config = Config::from_env(); let config = Config::from_env();
info!("系统配置成功载入。本地 SQLite 连接串: {}", config.database_url); info!("系统配置成功载入。本地 SQLite 连接串: {}", config.database_url);
@ -38,27 +52,17 @@ async fn main() -> anyhow::Result<()> {
std::fs::create_dir_all(config.library_dir.join("Markdown")).unwrap_or_default(); std::fs::create_dir_all(config.library_dir.join("Markdown")).unwrap_or_default();
std::fs::create_dir_all(config.library_dir.join("Translation")).unwrap_or_default(); std::fs::create_dir_all(config.library_dir.join("Translation")).unwrap_or_default();
// 3. 初始化本地 SQLite 数据库文件连接池 // 3. 初始化本地 SQLite 数据库连接池（开启外键约束）
if config.database_url.starts_with("sqlite://") { let options = SqliteConnectOptions::from_str(&config.database_url)?
let db_path = config.database_url.replace("sqlite://", ""); .foreign_keys(true)
if !db_path.contains(":memory:") { .create_if_missing(true);
let path = std::path::Path::new(&db_path);
if !path.exists() {
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent).unwrap_or_default();
}
std::fs::File::create(path)?;
info!("初始化创建本地 SQLite 数据库文件: {:?}", path);
}
}
}
let pool = SqlitePoolOptions::new() let pool = SqlitePoolOptions::new()
.max_connections(5) .max_connections(5)
.connect(&config.database_url) .connect_with(options)
.await?; .await?;
info!("SQLite 数据库连接已建立。"); info!("SQLite 数据库连接已建立（外键约束已启用）。");
// 4. 自动执行数据库迁移脚本 // 4. 自动执行数据库迁移脚本
info!("开始执行 SQL 表结构迁移..."); info!("开始执行 SQL 表结构迁移...");
@ -67,6 +71,35 @@ async fn main() -> anyhow::Result<()> {
.await?; .await?;
info!("数据库迁移执行完成，主表准备就绪。"); info!("数据库迁移执行完成，主表准备就绪。");
// 4.5 动态创建 vec0 向量虚拟表（维度可由环境变量 EMBEDDING_DIM 控制）
let embedding_dim: usize = std::env::var("EMBEDDING_DIM")
.unwrap_or_else(|_| "1536".to_string())
.parse()
.unwrap_or(1536);
// 检测并自愈：如果已存在的 vec_paper_chunks 维度与当前配置不一致，则重建该虚拟表并清空切片内容
let existing_sql: Option<(String,)> = sqlx::query_as(
"SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'vec_paper_chunks'"
)
.fetch_optional(&pool)
.await?;
if let Some((sql,)) = existing_sql {
let expected_pattern = format!("float[{}]", embedding_dim);
if !sql.contains(&expected_pattern) {
info!("检测到已存在的向量表维度不匹配，正在重建以适配当前维度: {}...", embedding_dim);
sqlx::query("DROP TABLE IF EXISTS vec_paper_chunks").execute(&pool).await?;
sqlx::query("DELETE FROM paper_chunks_content").execute(&pool).await?;
}
}
let create_vec_table = format!(
"CREATE VIRTUAL TABLE IF NOT EXISTS vec_paper_chunks USING vec0(embedding float[{}])",
embedding_dim
);
sqlx::query(&create_vec_table).execute(&pool).await?;
info!("vec_paper_chunks 虚拟表就绪（维度={}）。", embedding_dim);
// 5. 异步加载天文学专业名词对照词表 // 5. 异步加载天文学专业名词对照词表
let mut dict = Dictionary::new(); let mut dict = Dictionary::new();
if let Err(e) = dict.load_from_file("dictionary.txt") { if let Err(e) = dict.load_from_file("dictionary.txt") {
@ -89,6 +122,11 @@ async fn main() -> anyhow::Result<()> {
config.llm_api_base.clone(), config.llm_api_base.clone(),
config.llm_model.clone(), config.llm_model.clone(),
); );
let multimodal_llm = LlmClient::new(
config.multimodal_api_key.clone(),
config.multimodal_api_base.clone(),
config.multimodal_model.clone(),
);
let embedding = EmbeddingClient::new( let embedding = EmbeddingClient::new(
config.embedding_api_key.clone(), config.embedding_api_key.clone(),
config.embedding_api_base.clone(), config.embedding_api_base.clone(),
@ -103,10 +141,11 @@ async fn main() -> anyhow::Result<()> {
ads, ads,
arxiv, arxiv,
llm, llm,
multimodal_llm,
embedding, embedding,
downloader, downloader,
harvest_status: Arc::new(tokio::sync::Mutex::new(astroresearch::services::batch_sync::MetaSyncStatus::new())), harvest_status: Arc::new(tokio::sync::Mutex::new(astroresearch::services::batch_sync::MetaSyncStatus::new())),
process_status: Arc::new(tokio::sync::Mutex::new(astroresearch::services::batch_sync::AssetSyncStatus::new())), batch_status: Arc::new(tokio::sync::Mutex::new(astroresearch::services::batch_sync::AssetBatchStatus::new())),
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)), active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
}); });
@ -123,6 +162,7 @@ async fn main() -> anyhow::Result<()> {
.route("/no_resource", post(handlers::mark_no_resource)) .route("/no_resource", post(handlers::mark_no_resource))
.route("/parse", post(handlers::parse_paper)) .route("/parse", post(handlers::parse_paper))
.route("/translate", post(handlers::translate_paper)) .route("/translate", post(handlers::translate_paper))
.route("/embed", post(handlers::embed_paper))
.route("/citations", get(handlers::get_citation_network)) .route("/citations", get(handlers::get_citation_network))
.route("/paper", get(handlers::get_paper_detail)) .route("/paper", get(handlers::get_paper_detail))
.route("/library", get(handlers::get_library)) .route("/library", get(handlers::get_library))
@ -133,12 +173,18 @@ async fn main() -> anyhow::Result<()> {
.route("/sync/meta/count", get(handlers::get_meta_sync_count)) .route("/sync/meta/count", get(handlers::get_meta_sync_count))
.route("/sync/meta/run", post(handlers::run_meta_sync)) .route("/sync/meta/run", post(handlers::run_meta_sync))
.route("/sync/meta/status", get(handlers::get_meta_sync_status)) .route("/sync/meta/status", get(handlers::get_meta_sync_status))
.route("/sync/asset/run", post(handlers::run_asset_sync)) .route("/batch/asset/run", post(handlers::run_asset_batch))
.route("/sync/asset/stop", post(handlers::stop_asset_sync)) .route("/batch/asset/stop", post(handlers::stop_asset_batch))
.route("/sync/asset/status", get(handlers::get_asset_sync_status)) .route("/batch/asset/status", get(handlers::get_asset_batch_status))
.route("/sync/queries", get(handlers::get_sync_queries)) .route("/sync/queries", get(handlers::get_sync_queries))
.route("/sync/queries/:id", axum::routing::delete(handlers::delete_sync_query)) .route("/sync/queries/:id", axum::routing::delete(handlers::delete_sync_query))
.route("/active_bibcode", get(handlers::get_active_bibcode).post(handlers::set_active_bibcode)); .route("/active_bibcode", get(handlers::get_active_bibcode).post(handlers::set_active_bibcode))
.route("/chat/rag", post(handlers::chat_rag))
.route("/chat/figure", post(handlers::chat_figure))
.route("/target/query", get(handlers::query_target))
.route("/target/associate", post(handlers::associate_target))
.route("/target/extract", post(handlers::extract_paper_targets))
.route("/target/list", get(handlers::list_targets));
// 静态文件资源代理托管（当前端打包至 dashboard/dist 后，直接挂载到主域名根路由） // 静态文件资源代理托管（当前端打包至 dashboard/dist 后，直接挂载到主域名根路由）
let serve_dir = ServeDir::new("dashboard/dist") let serve_dir = ServeDir::new("dashboard/dist")

147
src/services/batch/asset.rs
View File

@ -12,15 +12,17 @@ use crate::services::download::Downloader;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[serde(rename_all = "lowercase")] #[serde(rename_all = "lowercase")]
pub enum SyncAction { pub enum BatchAction {
Download, Download,
Parse, Parse,
Translate, Translate,
Embed,
Target,
All, All,
} }
#[derive(Debug, Clone, Serialize)] #[derive(Debug, Clone, Serialize)]
pub struct AssetSyncStatus { pub struct AssetBatchStatus {
pub active: bool, pub active: bool,
pub total: i32, pub total: i32,
pub downloaded: i32, pub downloaded: i32,
@ -29,12 +31,12 @@ pub struct AssetSyncStatus {
pub parse_failed: i32, pub parse_failed: i32,
pub current_bibcode: String, pub current_bibcode: String,
pub logs: Vec<String>, pub logs: Vec<String>,
pub action: Option<SyncAction>, pub action: Option<BatchAction>,
} }
impl AssetSyncStatus { impl AssetBatchStatus {
pub fn new() -> Self { pub fn new() -> Self {
AssetSyncStatus { AssetBatchStatus {
active: false, active: false,
total: 0, total: 0,
downloaded: 0, downloaded: 0,
@ -57,9 +59,9 @@ impl AssetSyncStatus {
} }
} }
pub struct AssetSync; pub struct AssetBatch;
impl AssetSync { impl AssetBatch {
/// 启动后台批量下载与结构化解析任务 /// 启动后台批量下载与结构化解析任务
pub fn start_process( pub fn start_process(
db: SqlitePool, db: SqlitePool,
@ -67,9 +69,9 @@ impl AssetSync {
downloader: Arc<Downloader>, downloader: Arc<Downloader>,
qiniu: Arc<QiniuClient>, qiniu: Arc<QiniuClient>,
dict: Arc<crate::services::translation::Dictionary>, dict: Arc<crate::services::translation::Dictionary>,
action: SyncAction, action: BatchAction,
bibcodes: Vec<String>, bibcodes: Vec<String>,
status: Arc<Mutex<AssetSyncStatus>>, status: Arc<Mutex<AssetBatchStatus>>,
) { ) {
tokio::spawn(async move { tokio::spawn(async move {
let llm_client = crate::clients::llm::LlmClient::new( let llm_client = crate::clients::llm::LlmClient::new(
@ -77,6 +79,11 @@ impl AssetSync {
config.llm_api_base.clone(), config.llm_api_base.clone(),
config.llm_model.clone(), config.llm_model.clone(),
); );
let embedding_client = crate::clients::llm::EmbeddingClient::new(
config.embedding_api_key.clone(),
config.embedding_api_base.clone(),
config.embedding_model.clone(),
);
let total = bibcodes.len() as i32; let total = bibcodes.len() as i32;
{ {
let mut s = status.lock().await; let mut s = status.lock().await;
@ -91,10 +98,12 @@ impl AssetSync {
s.action = Some(action); s.action = Some(action);
let action_desc = match action { let action_desc = match action {
SyncAction::Download => "下载", BatchAction::Download => "下载",
SyncAction::Parse => "解析", BatchAction::Parse => "解析",
SyncAction::Translate => "翻译", BatchAction::Translate => "翻译",
SyncAction::All => "下载与解析", BatchAction::Embed => "向量化",
BatchAction::Target => "天体识别",
BatchAction::All => "下载与解析",
}; };
s.add_log(format!("批量{}任务启动，共 {} 篇文献需处理。", action_desc, total)); s.add_log(format!("批量{}任务启动，共 {} 篇文献需处理。", action_desc, total));
} }
@ -159,18 +168,18 @@ impl AssetSync {
let mut s = status.lock().await; let mut s = status.lock().await;
s.add_log(format!("文献 {} 的类型为 {} (无数字版全文)，跳过下载与解析。", bibcode, doctype_str)); s.add_log(format!("文献 {} 的类型为 {} (无数字版全文)，跳过下载与解析。", bibcode, doctype_str));
// 同样更新处理进度，防止任务进度条卡住 // 同样更新处理进度，防止任务进度条卡住
if action == SyncAction::Download || action == SyncAction::All { if action == BatchAction::Download || action == BatchAction::All {
dl_count += 1; dl_count += 1;
s.downloaded = dl_count; s.downloaded = dl_count;
} }
if action == SyncAction::Parse || action == SyncAction::All { if action == BatchAction::Parse || action == BatchAction::All || action == BatchAction::Translate || action == BatchAction::Embed || action == BatchAction::Target {
s.parsed += 1; s.parsed += 1;
} }
continue; continue;
} }
// 2. 检查并执行下载 // 2. 检查并执行下载
if action == SyncAction::Download || action == SyncAction::All { if action == BatchAction::Download || action == BatchAction::All {
let is_pdf_exist = pdf_path.as_ref().map(|p| config.library_dir.join(p).exists()).unwrap_or(false); let is_pdf_exist = pdf_path.as_ref().map(|p| config.library_dir.join(p).exists()).unwrap_or(false);
let is_html_exist = html_path.as_ref().map(|p| config.library_dir.join(p).exists()).unwrap_or(false); let is_html_exist = html_path.as_ref().map(|p| config.library_dir.join(p).exists()).unwrap_or(false);
@ -266,7 +275,7 @@ impl AssetSync {
} }
// 3. 检查并执行结构化解析（Markdown 转换） // 3. 检查并执行结构化解析（Markdown 转换）
if action == SyncAction::Parse || action == SyncAction::All { if action == BatchAction::Parse || action == BatchAction::All {
let is_md_exist = markdown_path.as_ref().map(|p| config.library_dir.join(p).exists()).unwrap_or(false); let is_md_exist = markdown_path.as_ref().map(|p| config.library_dir.join(p).exists()).unwrap_or(false);
if !is_md_exist { if !is_md_exist {
if pdf_path.is_some() || html_path.is_some() { if pdf_path.is_some() || html_path.is_some() {
@ -505,7 +514,7 @@ impl AssetSync {
} }
// 4. 检查并执行翻译 // 4. 检查并执行翻译
if action == SyncAction::Translate { if action == BatchAction::Translate {
let is_tr_exist = translation_path.as_ref().map(|p| config.library_dir.join(p).exists() && !p.starts_with("error:")).unwrap_or(false); let is_tr_exist = translation_path.as_ref().map(|p| config.library_dir.join(p).exists() && !p.starts_with("error:")).unwrap_or(false);
if !is_tr_exist { if !is_tr_exist {
if let Some(md_rel) = &markdown_path { if let Some(md_rel) = &markdown_path {
@ -614,6 +623,84 @@ impl AssetSync {
s.parsed += 1; s.parsed += 1;
} }
} }
// 5. 检查并执行向量化 (Embedding)
if action == BatchAction::Embed {
let is_md_exist = markdown_path.as_ref().map(|p| config.library_dir.join(p).exists() && !p.starts_with("error:")).unwrap_or(false);
if is_md_exist {
let md_rel = markdown_path.as_ref().unwrap();
let md_abs = config.library_dir.join(md_rel);
{
let mut s = status.lock().await;
s.add_log(format!("文献 {} 开始进行向量化切片入库...", bibcode));
}
match fs::read_to_string(&md_abs) {
Ok(markdown_content) => {
match crate::services::rag::ingest_paper(&db, &embedding_client, &bibcode, &markdown_content, None).await {
Ok(chunk_count) => {
let mut s = status.lock().await;
s.parsed += 1;
s.add_log(format!("文献 {} 向量化成功，共切片入库 {} 个向量块。", bibcode, chunk_count));
}
Err(e) => {
let mut s = status.lock().await;
s.parse_failed += 1;
s.add_log(format!("文献 {} 向量化失败: {}", bibcode, e));
}
}
}
Err(e) => {
let mut s = status.lock().await;
s.parse_failed += 1;
s.add_log(format!("文献 {} 读取英文 Markdown 失败: {}", bibcode, e));
}
}
} else {
let mut s = status.lock().await;
s.parse_failed += 1;
s.add_log(format!("文献 {} 英文 Markdown 文件不存在，跳过向量化。", bibcode));
}
}
// 6. 检查并执行天体识别与缓存
if action == BatchAction::Target {
let is_md_exist = markdown_path.as_ref().map(|p| config.library_dir.join(p).exists() && !p.starts_with("error:")).unwrap_or(false);
if is_md_exist {
let md_rel = markdown_path.as_ref().unwrap();
let md_abs = config.library_dir.join(md_rel);
{
let mut s = status.lock().await;
s.add_log(format!("文献 {} 开始提取并识别天体目标...", bibcode));
}
match fs::read_to_string(&md_abs) {
Ok(markdown_content) => {
// 提取前先清空旧的关联
if let Err(e) = sqlx::query("DELETE FROM paper_targets WHERE bibcode = ?")
.bind(&bibcode)
.execute(&db)
.await
{
warn!("清除文献 {} 的旧天体关联失败: {}", bibcode, e);
}
let client = reqwest::Client::new();
let targets = crate::services::target::extract_and_cache_targets(&db, &markdown_content, &bibcode, &client).await;
let mut s = status.lock().await;
s.parsed += 1;
s.add_log(format!("文献 {} 天体识别完成，共识别并缓存 {} 个天体目标。", bibcode, targets.len()));
}
Err(e) => {
let mut s = status.lock().await;
s.parse_failed += 1;
s.add_log(format!("文献 {} 读取英文 Markdown 失败: {}", bibcode, e));
}
}
} else {
let mut s = status.lock().await;
s.parse_failed += 1;
s.add_log(format!("文献 {} 英文 Markdown 文件不存在，跳过天体识别。", bibcode));
}
}
} }
if !join_handles.is_empty() { if !join_handles.is_empty() {
@ -630,10 +717,12 @@ impl AssetSync {
let mut s = status.lock().await; let mut s = status.lock().await;
s.active = false; s.active = false;
let action_desc = match action { let action_desc = match action {
SyncAction::Download => "下载", BatchAction::Download => "下载",
SyncAction::Parse => "解析", BatchAction::Parse => "解析",
SyncAction::Translate => "翻译", BatchAction::Translate => "翻译",
SyncAction::All => "下载与解析", BatchAction::Embed => "向量化",
BatchAction::Target => "天体识别",
BatchAction::All => "下载与解析",
}; };
s.add_log(format!("批量{}任务顺利完成！", action_desc)); s.add_log(format!("批量{}任务顺利完成！", action_desc));
} }
@ -649,7 +738,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_process_status_log_rotation() { async fn test_process_status_log_rotation() {
let mut status = AssetSyncStatus::new(); let mut status = AssetBatchStatus::new();
assert!(!status.active); assert!(!status.active);
for i in 0..150 { for i in 0..150 {
@ -719,16 +808,16 @@ mod tests {
let downloader = Arc::new(Downloader::new()); let downloader = Arc::new(Downloader::new());
let qiniu = Arc::new(QiniuClient::new("test_access".to_string(), "test_secret".to_string(), "test_bucket".to_string(), "test_domain".to_string())); let qiniu = Arc::new(QiniuClient::new("test_access".to_string(), "test_secret".to_string(), "test_bucket".to_string(), "test_domain".to_string()));
let status = Arc::new(Mutex::new(AssetSyncStatus::new())); let status = Arc::new(Mutex::new(AssetBatchStatus::new()));
let dict = Arc::new(crate::services::translation::Dictionary::new()); let dict = Arc::new(crate::services::translation::Dictionary::new());
AssetSync::start_process( AssetBatch::start_process(
pool.clone(), pool.clone(),
config, config,
downloader, downloader,
qiniu, qiniu,
dict, dict,
SyncAction::All, BatchAction::All,
vec![bibcode.clone()], vec![bibcode.clone()],
status.clone(), status.clone(),
); );
@ -839,16 +928,16 @@ mod tests {
let downloader = Arc::new(Downloader::new()); let downloader = Arc::new(Downloader::new());
let qiniu = Arc::new(QiniuClient::new("test_access".to_string(), "test_secret".to_string(), "test_bucket".to_string(), "test_domain".to_string())); let qiniu = Arc::new(QiniuClient::new("test_access".to_string(), "test_secret".to_string(), "test_bucket".to_string(), "test_domain".to_string()));
let status = Arc::new(Mutex::new(AssetSyncStatus::new())); let status = Arc::new(Mutex::new(AssetBatchStatus::new()));
let dict = Arc::new(crate::services::translation::Dictionary::new()); let dict = Arc::new(crate::services::translation::Dictionary::new());
AssetSync::start_process( AssetBatch::start_process(
pool.clone(), pool.clone(),
config, config,
downloader, downloader,
qiniu, qiniu,
dict, dict,
SyncAction::All, BatchAction::All,
vec![bib1.clone(), bib2.clone()], vec![bib1.clone(), bib2.clone()],
status.clone(), status.clone(),
); );

2
src/services/batch/mod.rs
View File

@ -3,4 +3,4 @@ pub mod meta;
pub mod asset; pub mod asset;
pub use meta::{MetaSyncStatus, MetaSync}; pub use meta::{MetaSyncStatus, MetaSync};
pub use asset::{SyncAction, AssetSyncStatus, AssetSync}; pub use asset::{BatchAction, AssetBatchStatus, AssetBatch};

265
src/services/chunker.rs Normal file
View File

@ -0,0 +1,265 @@
// src/services/chunker.rs
//
// Markdown 文本安全切片器：将长文献 Markdown 分割为适合向量化的文本块，
// 同时保证 LaTeX 公式 ($...$, $$...$$) 不会被截断。
/// 默认目标切片大小（字符数），可通过环境变量 CHUNK_SIZE 覆盖
const DEFAULT_CHUNK_SIZE: usize = 1000;
/// 单个切片的最大允许长度（避免过长公式块导致 Token 超限）
const MAX_CHUNK_SIZE: usize = 3000;
/// 切片重叠字符数，用于保持上下文连贯性
const OVERLAP_SIZE: usize = 100;
/// 一个文本切片，包含其在原文中的段落索引与文本内容
#[derive(Debug, Clone)]
pub struct TextChunk {
/// 段落索引（从 0 开始递增）
pub paragraph_index: usize,
/// 切片文本内容
pub content: String,
}
/// 将 Markdown 文本安全地分割为文本切片。
///
/// 切片策略：
/// 1. 按空行分割 Markdown 为段落
/// 2. 跳过纯标记行（标题、分隔线、空段落）
/// 3. 合并相邻段落，直到达到目标大小
/// 4. 如果单个段落超长，按句子边界二次分割
/// 5. 全程维护 LaTeX 公式上下文，确保 $...$ 和 $$...$$ 不被截断
pub fn chunk_markdown(text: &str, chunk_size: Option<usize>) -> Vec<TextChunk> {
let target_size = chunk_size.unwrap_or(DEFAULT_CHUNK_SIZE);
let paragraphs = split_paragraphs(text);
let mut chunks: Vec<TextChunk> = Vec::new();
let mut current_buf = String::new();
let mut chunk_index: usize = 0;
for para in &paragraphs {
let trimmed = para.trim();
// 跳过纯标记行
if trimmed.is_empty()
|| trimmed.starts_with('#')
|| trimmed.chars().all(|c| c == '-' || c == '=' || c == '*')
{
continue;
}
// 如果追加本段落后超出目标大小，先刷出已有的缓冲区
if !current_buf.is_empty()
&& current_buf.len() + trimmed.len() + 1 > target_size
{
chunks.push(TextChunk {
paragraph_index: chunk_index,
content: current_buf.clone(),
});
chunk_index += 1;
// 保留重叠上下文
current_buf = overlap_tail(&current_buf, OVERLAP_SIZE);
}
// 如果单个段落本身就超出最大限制，按句子二次切割
if trimmed.len() > MAX_CHUNK_SIZE {
if !current_buf.is_empty() {
chunks.push(TextChunk {
paragraph_index: chunk_index,
content: current_buf.clone(),
});
chunk_index += 1;
current_buf.clear();
}
let sub_chunks = split_long_paragraph(trimmed, target_size);
for sc in sub_chunks {
chunks.push(TextChunk {
paragraph_index: chunk_index,
content: sc,
});
chunk_index += 1;
}
continue;
}
if !current_buf.is_empty() {
current_buf.push('\n');
}
current_buf.push_str(trimmed);
}
// 刷出剩余内容
if !current_buf.trim().is_empty() {
chunks.push(TextChunk {
paragraph_index: chunk_index,
content: current_buf,
});
}
chunks
}
/// 按空行分割 Markdown 文本为段落
fn split_paragraphs(text: &str) -> Vec<String> {
let mut paragraphs = Vec::new();
let mut current = String::new();
for line in text.lines() {
if line.trim().is_empty() {
if !current.is_empty() {
paragraphs.push(std::mem::take(&mut current));
}
} else {
if !current.is_empty() {
current.push('\n');
}
current.push_str(line);
}
}
if !current.is_empty() {
paragraphs.push(current);
}
paragraphs
}
/// 将超长段落按句子边界分割，同时保护 LaTeX 公式完整性
fn split_long_paragraph(text: &str, target_size: usize) -> Vec<String> {
let mut result = Vec::new();
let mut current = String::new();
let mut in_inline_math = false;
let mut in_display_math = false;
let chars: Vec<char> = text.chars().collect();
let len = chars.len();
let mut i = 0;
while i < len {
// 检测 display math $$
if i + 1 < len && chars[i] == '$' && chars[i + 1] == '$' {
if in_display_math {
// 关闭 display math
current.push('$');
current.push('$');
i += 2;
in_display_math = false;
continue;
} else {
in_display_math = true;
current.push('$');
current.push('$');
i += 2;
continue;
}
}
// 检测 inline math $（不在 display math 内）
if chars[i] == '$' && !in_display_math {
in_inline_math = !in_inline_math;
current.push('$');
i += 1;
continue;
}
current.push(chars[i]);
// 只在非公式上下文中、到达句子边界时才考虑分割
let is_sentence_end = !in_inline_math
&& !in_display_math
&& is_sentence_boundary(&chars, i);
if is_sentence_end && current.len() >= target_size {
result.push(current.clone());
current = overlap_tail(&current, OVERLAP_SIZE);
}
i += 1;
}
if !current.trim().is_empty() {
result.push(current);
}
result
}
/// 判断当前位置是否为句子边界
fn is_sentence_boundary(chars: &[char], pos: usize) -> bool {
if pos >= chars.len() {
return false;
}
let ch = chars[pos];
// 中英文句号、问号、感叹号后跟空格或 EOF
if ch == '.' || ch == '。' || ch == '?' || ch == '？' || ch == '!' || ch == '！' {
let next = if pos + 1 < chars.len() {
Some(chars[pos + 1])
} else {
None
};
return match next {
None => true,
Some(c) => c.is_whitespace() || c == '\n',
};
}
false
}
/// 从文本尾部取指定长度的重叠片段
fn overlap_tail(text: &str, size: usize) -> String {
if text.len() <= size {
return text.to_string();
}
// 从字符边界安全截取
let start = text.len().saturating_sub(size);
let safe_start = text.ceil_char_boundary(start);
text[safe_start..].to_string()
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_basic_chunking() {
let text = "This is paragraph one.\n\nThis is paragraph two.\n\nThis is paragraph three.";
let chunks = chunk_markdown(text, Some(50));
assert!(!chunks.is_empty());
for chunk in &chunks {
assert!(!chunk.content.is_empty());
}
}
#[test]
fn test_latex_preservation() {
// 确保 inline LaTeX 不会被分割
let text = "The equation $E = mc^2$ is fundamental.\n\nAnother paragraph with $$\\sum_{i=1}^{n} x_i$$ formula.";
let chunks = chunk_markdown(text, Some(200));
for chunk in &chunks {
let dollar_count = chunk.content.matches('$').count();
// $ 符号应该成对出现
assert_eq!(dollar_count % 2, 0, "LaTeX delimiters not balanced in chunk: {}", chunk.content);
}
}
#[test]
fn test_skip_headings() {
let text = "# Title\n\nSome content here.\n\n## Subtitle\n\nMore content here.";
let chunks = chunk_markdown(text, Some(500));
for chunk in &chunks {
assert!(!chunk.content.starts_with('#'), "Heading should be skipped: {}", chunk.content);
}
}
#[test]
fn test_empty_input() {
let chunks = chunk_markdown("", None);
assert!(chunks.is_empty());
}
#[test]
fn test_single_paragraph() {
let text = "A single paragraph of normal length text for testing purposes.";
let chunks = chunk_markdown(text, Some(500));
assert_eq!(chunks.len(), 1);
assert_eq!(chunks[0].content, text);
}
}

3
src/services/mod.rs
View File

@ -4,6 +4,9 @@ pub mod translation;
pub mod query_parser; pub mod query_parser;
pub mod batch; pub mod batch;
pub mod logging; pub mod logging;
pub mod chunker;
pub mod rag;
pub mod target;
pub mod batch_sync { pub mod batch_sync {
pub use super::batch::*; pub use super::batch::*;

778
src/services/parser.rs
View File

@ -1,778 +0,0 @@
// src/parser.rs
use std::fs;
use std::path::Path;
use serde::{Deserialize, Serialize};
use tracing::{info, warn};
use regex::Regex;
use crate::Config;
use crate::clients::qiniu::QiniuClient;
// 清理 HTML 结构，仅提取正文部分并转换为标准 Markdown
pub fn html_to_markdown(html_path: &Path) -> anyhow::Result<String> {
info!("正在解析本地 HTML 并提取 Markdown: {:?}", html_path);
let html_bytes = fs::read(html_path)?;
// 检查是否为 Gzip 压缩文件 (Gzip 幻数: 0x1f 0x8b)
let decompressed_bytes = if html_bytes.starts_with(&[0x1f, 0x8b]) {
use std::io::Read;
let mut decoder = flate2::read::GzDecoder::new(&html_bytes[..]);
let mut buf = Vec::new();
decoder.read_to_end(&mut buf)?;
buf
} else {
html_bytes
};
let html_content = String::from_utf8_lossy(&decompressed_bytes).into_owned();
// 截断页脚及之后的不相关内容以防干扰解析
let mut truncated_html = html_content.as_str();
if let Some(end) = html_content.find("<div class=\"ar5iv-footer\">") {
truncated_html = &html_content[..end];
} else if let Some(end) = html_content.find("<footer") {
truncated_html = &html_content[..end];
}
let mut main_html = truncated_html;
// 定位正文标记块，滤除页眉、页脚等侧栏广告
if let Some(start) = truncated_html.find("<div class=\"ltx_page_main\">") {
main_html = &truncated_html[start..];
} else if let Some(start) = truncated_html.find("<main") {
main_html = &truncated_html[start..];
} else if let Some(start) = truncated_html.find("<article") {
main_html = &truncated_html[start..];
} else if let Some(start) = truncated_html.find("<body") {
main_html = &truncated_html[start..];
}
// 预处理：删除导航条、页眉、侧边栏等不属于正文的结构
let nav_re = Regex::new(r#"(?s)<nav[^>]*>.*?</nav>"#).unwrap();
let preprocessed_html = nav_re.replace_all(main_html, "").to_string();
let ltx_nav_re = Regex::new(r#"(?s)<div[^>]*class="[^"]*ltx_(?:page_navbar|header|navigation)[^"]*"[^>]*>.*?</div>"#).unwrap();
let preprocessed_html = ltx_nav_re.replace_all(&preprocessed_html, "").to_string();
// 预处理：提前用占位符替换 <math ...>...</math> 公式，防止其内部 Latex 语法被标题解析、图注解析或 html2md 破坏
let mut formulas = Vec::new();
let math_re = Regex::new(r#"(?s)<math\s+([^>]*?)>(.*?)</math>"#).unwrap();
let mut placeholder_counter = 0;
let preprocessed_html = math_re.replace_all(&preprocessed_html, |caps: &regex::Captures| {
let attrs = &caps[1];
let alttext_re = Regex::new(r#"alttext="([^"]*)""#).unwrap();
let mut alttext = alttext_re.captures(attrs)
.map(|c| c[1].to_string())
.unwrap_or_default();
// 如果 alttext 为空，尝试从 <annotation encoding="application/x-tex"> 中提取 LaTeX 公式作为备选方案
if alttext.is_empty() {
let annotation_re = Regex::new(r#"(?s)<annotation\s+[^>]*encoding="application/x-tex"[^>]*>(.*?)</annotation>"#).unwrap();
if let Some(ann_caps) = annotation_re.captures(&caps[2]) {
alttext = ann_caps[1].trim().to_string();
}
}
let is_block = attrs.contains("display=\"block\"") || attrs.contains("display='block'");
formulas.push((alttext, is_block));
let placeholder = format!(" MATHPLACEHOLDER{} ", placeholder_counter);
placeholder_counter += 1;
placeholder
}).to_string();
// 预处理：将 ltx_section 标题标记转换为对应层级的 Markdown heading
// h2: section, h3: subsection, h4: subsubsection
let sec_re = Regex::new(r#"(?s)<(?:h[1-6])[^>]*class="[^"]*ltx_title_section[^"]*"[^>]*>(.*?)</(?:h[1-6])>"#).unwrap();
let preprocessed_html = sec_re.replace_all(&preprocessed_html, |caps: &regex::Captures| {
let inner = strip_html_tags(&caps[1]);
format!("\n\n## {}\n\n", inner.trim())
}).to_string();
let subsec_re = Regex::new(r#"(?s)<(?:h[1-6])[^>]*class="[^"]*ltx_title_subsection[^"]*"[^>]*>(.*?)</(?:h[1-6])>"#).unwrap();
let preprocessed_html = subsec_re.replace_all(&preprocessed_html, |caps: &regex::Captures| {
let inner = strip_html_tags(&caps[1]);
format!("\n\n### {}\n\n", inner.trim())
}).to_string();
let subsubsec_re = Regex::new(r#"(?s)<(?:h[1-6])[^>]*class="[^"]*ltx_title_subsubsection[^"]*"[^>]*>(.*?)</(?:h[1-6])>"#).unwrap();
let preprocessed_html = subsubsec_re.replace_all(&preprocessed_html, |caps: &regex::Captures| {
let inner = strip_html_tags(&caps[1]);
format!("\n\n#### {}\n\n", inner.trim())
}).to_string();
// 预处理：将 figcaption 转换为 Markdown 图注格式
let figcaption_re = Regex::new(r#"(?s)<figcaption[^>]*>(.*?)</figcaption>"#).unwrap();
let preprocessed_html = figcaption_re.replace_all(&preprocessed_html, |caps: &regex::Captures| {
let inner = strip_html_tags(&caps[1]);
format!("\n> **Figure:** {}\n", inner.trim())
}).to_string();
// 预处理：将 ltx_caption (LaTeXML figure/table caption) 转换为图注
let ltx_caption_re = Regex::new(r#"(?s)<(?:span|div|p)[^>]*class="[^"]*ltx_caption[^"]*"[^>]*>(.*?)</(?:span|div|p)>"#).unwrap();
let preprocessed_html = ltx_caption_re.replace_all(&preprocessed_html, |caps: &regex::Captures| {
let inner = strip_html_tags(&caps[1]);
format!("\n> **Caption:** {}\n", inner.trim())
}).to_string();
// 预处理：将 ltx_title 文章标题转为 h1
let title_re = Regex::new(r#"(?s)<(?:h[1-6])[^>]*class="[^"]*ltx_title_document[^"]*"[^>]*>(.*?)</(?:h[1-6])>"#).unwrap();
let preprocessed_html = title_re.replace_all(&preprocessed_html, |caps: &regex::Captures| {
let inner = strip_html_tags(&caps[1]);
format!("\n# {}\n\n", inner.trim())
}).to_string();
// 预处理 HTML 中的 sup, sub, inf 标签为更干净的 markdown 格式，解决 html2md 不转换带属性的上下标的问题
let sup_re = Regex::new(r#"(?s)<sup[^>]*>(.*?)</sup>"#).unwrap();
let preprocessed_html = sup_re.replace_all(&preprocessed_html, "^{$1}").to_string();
let sub_re = Regex::new(r#"(?s)<sub[^>]*>(.*?)</sub>"#).unwrap();
let preprocessed_html = sub_re.replace_all(&preprocessed_html, "_{$1}").to_string();
let inf_re = Regex::new(r#"(?s)<inf[^>]*>(.*?)</inf>"#).unwrap();
let preprocessed_html = inf_re.replace_all(&preprocessed_html, "_{$1}").to_string();
// 预处理：去除 <cite> 标签以防止 html2md 将其转为 blockquote (>) 导致行内引用异常断行
let cite_start_re = Regex::new(r#"(?s)<cite[^>]*>"#).unwrap();
let preprocessed_html = cite_start_re.replace_all(&preprocessed_html, "").to_string();
let cite_end_re = Regex::new(r#"(?s)</cite>"#).unwrap();
let preprocessed_html = cite_end_re.replace_all(&preprocessed_html, "").to_string();
// 预处理 HTML 中的 <img> 标签，将相对路径的图片链接补全为 ar5iv 绝对路径，并统一转换为标准 Markdown 图片格式 ![alt](url)
let img_re = Regex::new(r#"(?s)<img\s+([^>]*?)>"#).unwrap();
let preprocessed_html = img_re.replace_all(&preprocessed_html, |caps: &regex::Captures| {
let attrs = &caps[1];
let src_re = Regex::new(r#"src="([^"]*)""#).unwrap();
let src = src_re.captures(attrs)
.map(|c| c[1].to_string())
.unwrap_or_default();
let alt_re = Regex::new(r#"alt="([^"]*)""#).unwrap();
let alt = alt_re.captures(attrs)
.map(|c| c[1].to_string())
.unwrap_or_else(|| "image".to_string());
let absolute_src = if src.starts_with('/') {
format!("https://ar5iv.labs.arxiv.org{}", src)
} else {
src
};
format!("\n\n![{}]({})\n\n", alt, absolute_src)
}).to_string();
// 预处理 HTML 中的 LaTeXML 模拟表格标记，转换模拟的 tabular/tr/td/th 为真正的 table/tr/td 结构，支持复杂嵌套
let preprocessed_html = replace_latexml_tables(&preprocessed_html);
let mut markdown = html2md::parse_html(&preprocessed_html);
// 将公式占位符以逆序还原为原始干净的 LaTeX 格式 ($...$ 或 $$...$$)，避免前缀匹配冲突（例如 MATHPLACEHOLDER1 误匹配 MATHPLACEHOLDER10 的前缀）
for i in (0..formulas.len()).rev() {
let (ref alttext, is_block) = formulas[i];
let placeholder = format!("MATHPLACEHOLDER{}", i);
let replacement = if is_block {
format!(" $${}$$ ", alttext)
} else {
format!(" ${}$ ", alttext)
};
markdown = markdown.replace(&placeholder, &replacement);
}
let cleaned = postprocess_markdown(&markdown);
Ok(cleaned)
}
// 移除 Markdown 垃圾属性标识并清洗每行格式
fn postprocess_markdown(text: &str) -> String {
// 按行清理多余前导/尾随空格，同时保留 fenced 代码块内的缩进
let mut clean_lines = Vec::new();
let mut in_code_block = false;
for line in text.lines() {
let trimmed = line.trim();
if trimmed.starts_with("```") {
in_code_block = !in_code_block;
}
if in_code_block {
clean_lines.push(line.to_string());
} else {
clean_lines.push(trimmed.to_string());
}
}
let mut md = clean_lines.join("\n");
let div_re = Regex::new(r"</?div[^>]*>").unwrap();
let span_re = Regex::new(r"</?span[^>]*>").unwrap();
md = div_re.replace_all(&md, "").to_string();
md = span_re.replace_all(&md, "").to_string();
let empty_brackets = Regex::new(r"\[\]").unwrap();
md = empty_brackets.replace_all(&md, "").to_string();
let excessive_newlines = Regex::new(r"\n{4,}").unwrap();
md = excessive_newlines.replace_all(&md, "\n\n\n").to_string();
// 还原被 html2md 自动转义的标题与引用符号
let unescape_h1 = Regex::new(r"\\#\s+").unwrap();
let unescape_h2 = Regex::new(r"\\##\s+").unwrap();
let unescape_h3 = Regex::new(r"\\###\s+").unwrap();
let unescape_h4 = Regex::new(r"\\####\s+").unwrap();
let unescape_quote = Regex::new(r"\\>\s+").unwrap();
let unescape_bold = Regex::new(r"\\\*\\\*").unwrap();
md = unescape_h1.replace_all(&md, "# ").to_string();
md = unescape_h2.replace_all(&md, "## ").to_string();
md = unescape_h3.replace_all(&md, "### ").to_string();
md = unescape_h4.replace_all(&md, "#### ").to_string();
md = unescape_quote.replace_all(&md, "> ").to_string();
md = unescape_bold.replace_all(&md, "**").to_string();
// 还原 HTML 实体转义符以保证 Markdown/LaTeX 中数学符号（如 < 和 >）正常渲染
md = md
.replace("&lt;", "<")
.replace("&gt;", ">")
.replace("&amp;", "&")
.replace("&quot;", "\"")
.replace("&#39;", "'");
// 还原被 html2md 过度转义的链接与图片 URL 中的下划线/百分号等特殊字符，避免图链损坏
let link_re = Regex::new(r#"(!?\[[^\]]*?\])\(([^)]*?)\)"#).unwrap();
md = link_re.replace_all(&md, |caps: &regex::Captures| {
let label = &caps[1];
let url = &caps[2];
let clean_url = url.replace(r"\_", "_").replace(r"\%", "%");
format!("{}({})", label, clean_url)
}).to_string();
// 清理未定义 LaTeXML 宏带来的 \orgname, \orgdiv, \orgaddress, \articletag, \term 等无意义文本，用空格代替以防单词粘连
let latexml_errs = Regex::new(r"\\{1,2}(?:orgname|orgdiv|orgaddress|articletag|term)").unwrap();
md = latexml_errs.replace_all(&md, " ").to_string();
// 清理标题末尾冗余的井号标记，例如 ###### Keywords: ###### -> ###### Keywords:
let heading_trail_re = Regex::new(r"(?m)^(#{1,6})\s+(.*?)\s+#+$").unwrap();
md = heading_trail_re.replace_all(&md, "$1 $2").to_string();
// 提升低层级标题（特别是 Abstract, Keywords, Glossary, Nomenclature, Acknowledgments, References 等常见顶级区块）为 H2 (##)
let section_promote_re = Regex::new(r"(?mi)^(#{3,6})[ \t]*(Abstract|Keywords|Glossary|Nomenclature|Acknowledgments|References)(:?)[ \t]*$").unwrap();
md = section_promote_re.replace_all(&md, "## $2$3").to_string();
// 消除紧跟在 "## Abstract" 后的冗余 "[Abstract]" 行
let abstract_clean_re = Regex::new(r"(?mi)^##\s+Abstract\s*\n\s*\n\s*\[Abstract\]\s*\n").unwrap();
md = abstract_clean_re.replace_all(&md, "## Abstract\n\n").to_string();
// 将行首的行内 [Glossary] xxx 等转换为标题段落形式
let bracket_inline_re = Regex::new(r"(?mi)^\[(Abstract|Keywords|Glossary|Nomenclature|Acknowledgments|References)\][ \t]+(.+)$").unwrap();
md = bracket_inline_re.replace_all(&md, "## $1\n\n$2").to_string();
// 将独立的 [Nomenclature]、[Glossary] 等行转换为 H2 标题
let bracket_header_re = Regex::new(r"(?mi)^\[(Abstract|Keywords|Glossary|Nomenclature|Acknowledgments|References)\][ \t]*$").unwrap();
md = bracket_header_re.replace_all(&md, "## $1").to_string();
// 清理列表项中冗余的双重项目符号，例如 * • -> *
let bullet_re = Regex::new(r"(?m)^(\s*[\*\-+])\s*•\s*").unwrap();
md = bullet_re.replace_all(&md, "$1 ").to_string();
// 修复因换行而分裂的方括号对，例如 [\n\nNomenclature] -> [Nomenclature]
let bracket_newline_re = Regex::new(r"\[\s*\n+\s*([^\]\n]+?)\]").unwrap();
md = bracket_newline_re.replace_all(&md, "[$1]").to_string();
md.trim().to_string()
}
// 简单移除 HTML 标签，返回纯文本内容（用于标题/图注提取）
fn strip_html_tags(html: &str) -> String {
let tag_re = Regex::new(r"<[^>]+>").unwrap();
let text = tag_re.replace_all(html, "").to_string();
// 解码常见 HTML 实体
text.replace("&amp;", "&")
.replace("&lt;", "<")
.replace("&gt;", ">")
.replace("&quot;", "\"")
.replace("&nbsp;", " ")
.replace("&#39;", "'")
}
#[derive(Serialize)]
struct BatchUploadRequest {
files: Vec<PendingFile>,
language: String,
is_ocr: bool,
model_version: String,
}
#[derive(Serialize)]
struct PendingFile {
name: String,
data_id: String,
}
#[allow(dead_code)]
#[derive(Deserialize)]
struct BatchUploadResponse {
code: i32,
msg: String,
data: BatchUploadData,
}
#[derive(Deserialize)]
struct BatchUploadData {
batch_id: String,
file_urls: Vec<String>,
}
#[allow(dead_code)]
#[derive(Deserialize)]
struct BatchResultResponse {
code: i32,
msg: String,
data: Option<BatchResultData>,
}
#[allow(dead_code)]
#[derive(Deserialize)]
struct BatchResultData {
batch_id: String,
extract_result: Vec<ExtractResult>,
}
#[allow(dead_code)]
#[derive(Deserialize)]
struct ExtractResult {
file_name: String,
state: String,
full_zip_url: Option<String>,
err_msg: Option<String>,
}
// 调用 MinerU 远程接口解析 PDF，并在提取出图片后自动上传至七牛云进行外链替换
pub async fn submit_pdf_to_mineru(
pdf_path: &Path,
config: &Config
) -> anyhow::Result<String> {
info!("正在请求 MinerU 解析本地 PDF 文献: {:?}", pdf_path);
if config.mineru_api_url.is_empty() {
return Err(anyhow::anyhow!("未在环境变量 .env 中配置 MINERU_API_URL"));
}
let pdf_bytes = fs::read(pdf_path)?;
let filename = pdf_path.file_name()
.and_then(|f| f.to_str())
.unwrap_or("paper.pdf")
.to_string();
let bibcode = pdf_path.file_stem()
.and_then(|f| f.to_str())
.unwrap_or("paper")
.to_string();
// 提取 base_url
let base_url = config.mineru_api_url
.replace("/extract/task", "")
.replace("/extract", "")
.trim_end_matches('/')
.to_string();
let client = reqwest::Client::new();
let data_id = uuid::Uuid::new_v4().to_string();
// 1. 获取预签名上传 URL
info!("MinerU: 正在请求批量直传 URL (Bibcode: {})", bibcode);
let upload_req = BatchUploadRequest {
files: vec![PendingFile {
name: filename.clone(),
data_id: data_id.clone(),
}],
language: "en".to_string(),
is_ocr: true,
model_version: "vlm".to_string(),
};
let mut request = client.post(format!("{}/file-urls/batch/", base_url))
.json(&upload_req);
if !config.mineru_api_key.is_empty() {
request = request.header("Authorization", format!("Bearer {}", config.mineru_api_key));
}
let response = request.send().await?;
let status = response.status();
let res_text = response.text().await?;
if !status.is_success() {
return Err(anyhow::anyhow!("请求 MinerU 批量上传 URL 失败 (状态码: {}): {}", status, res_text));
}
let upload_res: BatchUploadResponse = serde_json::from_str(&res_text)?;
if upload_res.code != 0 {
return Err(anyhow::anyhow!("MinerU API 错误: {}", upload_res.msg));
}
let upload_url = upload_res.data.file_urls.first()
.ok_or_else(|| anyhow::anyhow!("MinerU 未返回上传 URL"))?;
// 2. 上传文件 (PUT)
info!("MinerU: 正在直接上传 PDF 字节流至对象存储...");
let put_res = client.put(upload_url)
.body(pdf_bytes)
.send()
.await?;
if !put_res.status().is_success() {
return Err(anyhow::anyhow!("上传 PDF 至 MinerU 对象存储直传 URL 失败: {}", put_res.status()));
}
let batch_id = upload_res.data.batch_id;
Ok(batch_id)
}
pub async fn poll_and_extract_mineru(
batch_id: &str,
bibcode: &str,
qiniu_client: &QiniuClient,
config: &Config
) -> anyhow::Result<String> {
let client = reqwest::Client::new();
let base_url = config.mineru_api_url
.replace("/extract/task", "")
.replace("/extract", "")
.trim_end_matches('/')
.to_string();
let mut poll_count = 0;
let max_polls = 45; // 45 * 10s = 7.5 min
info!("MinerU: 开始轮询任务结果 (Batch ID: {})...", batch_id);
let mut full_zip_url = String::new();
loop {
poll_count += 1;
if poll_count > max_polls {
return Err(anyhow::anyhow!("MinerU 结构化解析超时 (Bibcode: {})", bibcode));
}
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
let mut status_req = client.get(format!("{}/extract-results/batch/{}", base_url, batch_id));
if !config.mineru_api_key.is_empty() {
status_req = status_req.header("Authorization", format!("Bearer {}", config.mineru_api_key));
}
let status_res = status_req.send().await?;
let status_text = status_res.text().await?;
let result_data: BatchResultResponse = serde_json::from_str(&status_text)?;
if let Some(data) = result_data.data {
if let Some(file_result) = data.extract_result.first() {
match file_result.state.as_str() {
"done" => {
info!("MinerU: 解析成功！");
full_zip_url = file_result.full_zip_url.clone().unwrap_or_default();
break;
}
"error" | "failed" => {
let err_msg = file_result.err_msg.clone().unwrap_or_default();
return Err(anyhow::anyhow!("MinerU 批量解析任务失败: {}", err_msg));
}
other => {
info!("MinerU 任务处理中... 当前状态: {}", other);
}
}
} else {
return Err(anyhow::anyhow!("MinerU 轮询响应中未发现文件解析任务结果"));
}
} else {
return Err(anyhow::anyhow!("MinerU 轮询响应数据为空"));
}
}
if full_zip_url.is_empty() {
return Err(anyhow::anyhow!("MinerU 转换成功但未返回结果 ZIP 下载 URL"));
}
// 4. 下载并解压 ZIP
info!("MinerU: 正在下载最终提取压缩包: {}", full_zip_url);
let zip_bytes = client.get(&full_zip_url).send().await?.bytes().await?;
let reader = std::io::Cursor::new(zip_bytes);
let mut archive = zip::ZipArchive::new(reader)?;
let mut markdown = String::new();
let mut image_buffers = std::collections::HashMap::new();
for i in 0..archive.len() {
let mut file = archive.by_index(i)?;
let name = file.name().to_string();
if name.ends_with(".md") {
let mut md_content = String::new();
std::io::Read::read_to_string(&mut file, &mut md_content)?;
markdown = md_content;
} else if file.is_file() {
let lower = name.to_lowercase();
if lower.ends_with(".png") || lower.ends_with(".jpg") || lower.ends_with(".jpeg") || lower.ends_with(".gif") || lower.ends_with(".svg") {
let mut buf = Vec::new();
std::io::copy(&mut file, &mut buf)?;
let file_basename = Path::new(&name)
.file_name()
.and_then(|f| f.to_str())
.unwrap_or(&name)
.to_string();
image_buffers.insert(file_basename, buf);
}
}
}
if markdown.is_empty() {
return Err(anyhow::anyhow!("解析后的压缩包中未发现核心 Markdown 文档"));
}
// 5. 上传图片并重写链接
if !image_buffers.is_empty() {
let local_img_dir = config.library_dir.join("images").join(bibcode);
let _ = fs::create_dir_all(&local_img_dir);
if qiniu_client.is_configured() {
info!("MinerU 批量模式解析出 {} 张本地插图。准备上传至七牛云...", image_buffers.len());
for (img_name, img_bytes) in image_buffers {
let local_path = local_img_dir.join(&img_name);
let _ = fs::write(&local_path, &img_bytes);
match qiniu_client.upload_buffer(img_bytes, &img_name).await {
Ok(qiniu_url) => {
let escaped_img_name = regex::escape(&img_name);
let link_re = Regex::new(&format!(r"\(([^)]*?){}\)", escaped_img_name)).unwrap();
markdown = link_re.replace_all(&markdown, |_: &regex::Captures| {
format!("({})", qiniu_url)
}).to_string();
}
Err(e) => warn!("上传图片至七牛云失败 {}: {}", img_name, e),
}
}
} else {
warn!("未检测到七牛云配置，解析出的图片将保存在本地 images 目录下");
for (img_name, img_bytes) in image_buffers {
let local_path = local_img_dir.join(&img_name);
let _ = fs::write(&local_path, &img_bytes);
let escaped_img_name = regex::escape(&img_name);
let link_re = Regex::new(&format!(r"\(([^)]*?){}\)", escaped_img_name)).unwrap();
let replacement_link = format!("(images/{}/{})", bibcode, img_name);
markdown = link_re.replace_all(&markdown, replacement_link.as_str()).to_string();
}
}
}
Ok(markdown)
}
pub async fn parse_pdf_via_mineru(
pdf_path: &Path,
qiniu_client: &QiniuClient,
config: &Config
) -> anyhow::Result<String> {
let bibcode = pdf_path.file_stem()
.and_then(|f| f.to_str())
.unwrap_or("paper")
.to_string();
let batch_id = submit_pdf_to_mineru(pdf_path, config).await?;
poll_and_extract_mineru(&batch_id, &bibcode, qiniu_client, config).await
}
// 采用栈式解析模型，将 LaTeXML 用 span/div 模拟出的表格容器（ltx_tabular/tbody/thead/tfoot/tr/td/th）还原为真正的 HTML <table> 结构
fn replace_latexml_tables(html: &str) -> String {
use regex::Regex;
let tag_re = Regex::new(r#"(?i)<(span|div)\b([^>]*?)>|</(span|div)>"#).unwrap();
let mut result = String::new();
let mut last_pos = 0;
let mut stack = Vec::new();
for cap in tag_re.captures_iter(html) {
let mat = cap.get(0).unwrap();
result.push_str(&html[last_pos..mat.start()]);
if cap.get(1).is_some() {
let tag_name = cap.get(1).unwrap().as_str().to_lowercase();
let attrs = cap.get(2).unwrap().as_str();
let mut matched_type = None;
if let Some(class_cap) = Regex::new(r#"class="([^"]*)""#).unwrap().captures(attrs) {
let class_str = class_cap[1].to_lowercase();
if class_str.contains("ltx_tabular") {
matched_type = Some("table");
} else if class_str.contains("ltx_tbody") {
matched_type = Some("tbody");
} else if class_str.contains("ltx_thead") {
matched_type = Some("thead");
} else if class_str.contains("ltx_tfoot") {
matched_type = Some("tfoot");
} else if class_str.contains("ltx_tr") {
matched_type = Some("tr");
} else if class_str.contains("ltx_th") {
matched_type = Some("th");
} else if class_str.contains("ltx_td") {
matched_type = Some("td");
}
}
if let Some(t) = matched_type {
result.push_str(&format!("<{}>", t));
stack.push((tag_name, Some(t.to_string())));
} else {
result.push_str(mat.as_str());
stack.push((tag_name, None));
}
} else {
let tag_name = cap.get(3).unwrap().as_str().to_lowercase();
let mut replaced = false;
while let Some((open_name, open_type)) = stack.pop() {
if open_name == tag_name {
if let Some(t) = open_type {
result.push_str(&format!("</{}>", t));
} else {
result.push_str(&format!("</{}>", tag_name));
}
replaced = true;
break;
} else {
if let Some(t) = open_type {
result.push_str(&format!("</{}>", t));
} else {
result.push_str(&format!("</{}>", open_name));
}
}
}
if !replaced {
result.push_str(mat.as_str());
}
}
last_pos = mat.end();
}
result.push_str(&html[last_pos..]);
result
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Write;
#[test]
fn test_postprocess_markdown() {
let dirty = "<div>Hello</div> <span class=\"abc\">World</span> [] &lt;math&gt;\n\n\n\n\nNew Paragraph";
let cleaned = postprocess_markdown(dirty);
assert_eq!(cleaned, "Hello World <math>\n\n\nNew Paragraph");
// Test heading promotion and bracket cleanup
let dirty_abstract = "###### Abstract\n\n[Abstract]\n\nHot subdwarfs are core helium burning stars.";
let cleaned_abstract = postprocess_markdown(dirty_abstract);
assert!(cleaned_abstract.contains("## Abstract\n\nHot subdwarfs are core"));
assert!(!cleaned_abstract.contains("[Abstract]"));
let dirty_keywords = "###### Keywords:\n\nsubdwarfs, gravity";
let cleaned_keywords = postprocess_markdown(dirty_keywords);
assert!(cleaned_keywords.contains("## Keywords:\n\nsubdwarfs, gravity"));
let dirty_glossary = "[Glossary] Hertzsprung-Russell diagram (HRD): info";
let cleaned_glossary = postprocess_markdown(dirty_glossary);
assert_eq!(cleaned_glossary, "## Glossary\n\nHertzsprung-Russell diagram (HRD): info");
let dirty_nomenclature = "[Nomenclature]\n\n| sdB | description |";
let cleaned_nomenclature = postprocess_markdown(dirty_nomenclature);
assert!(cleaned_nomenclature.contains("## Nomenclature\n\n| sdB |"));
}
#[test]
fn test_html_to_markdown() -> anyhow::Result<()> {
let html_content = r#"
<!DOCTYPE html>
<html>
<body>
<div class="ltx_page_main">
<h1>Test Document</h1>
<p>This is a <strong>test</strong> paragraph.</p>
</div>
</body>
</html>
"#;
let mut path = std::env::temp_dir();
path.push("test_doc.html");
{
let mut file = std::fs::File::create(&path)?;
file.write_all(html_content.as_bytes())?;
}
let md = html_to_markdown(&path);
let _ = std::fs::remove_file(&path);
let md_content = md?;
assert!(md_content.contains("Test Document"));
assert!(md_content.contains("This is a **test** paragraph."));
Ok(())
}
#[test]
fn test_html_to_markdown_math_and_table() -> anyhow::Result<()> {
let html_content = r#"
<div class="ltx_page_main">
<p>Here is math: <math alttext="\approx" display="inline"><semantics><mo>≈</mo><annotation-xml><approx></approx></annotation-xml><annotation>\approx</annotation></semantics></math> and block <math alttext="\sum_{i=1}^n" display="block">...</math></p>
<span class="ltx_tabular">
<span class="ltx_tr">
<span class="ltx_td">sdB</span>
<span class="ltx_td">subdwarf B</span>
</span>
</span>
</div>
"#;
let mut path = std::env::temp_dir();
path.push("test_math_table.html");
{
let mut file = std::fs::File::create(&path)?;
file.write_all(html_content.as_bytes())?;
}
let md = html_to_markdown(&path);
let _ = std::fs::remove_file(&path);
let md_content = md?;
// 验证数学公式被成功以未转义的 Latex 格式提取还原
assert!(md_content.contains(r#"$\approx$"#));
assert!(md_content.contains(r#"$$\sum_{i=1}^n$$"#));
// 验证表格被转换成了标准 table
assert!(md_content.contains("sdB"));
assert!(md_content.contains("subdwarf B"));
Ok(())
}
#[test]
fn test_html_to_markdown_math_in_headings_and_captions() -> anyhow::Result<()> {
let html_content = r#"
<div class="ltx_page_main">
<h2 class="ltx_title_section">Heading with math <math alttext="\theta_{eff}" display="inline"><semantics><mo>≈</mo><annotation>\theta_{eff}</annotation></semantics></math></h2>
<figcaption>Figure caption with inline math <math alttext="M_\odot" display="inline"><semantics><mo>≈</mo><annotation>M_\odot</annotation></semantics></math> details.</figcaption>
</div>
"#;
let mut path = std::env::temp_dir();
path.push("test_math_heading_caption.html");
{
let mut file = std::fs::File::create(&path)?;
file.write_all(html_content.as_bytes())?;
}
let md = html_to_markdown(&path);
let _ = std::fs::remove_file(&path);
let md_content = md?;
println!("Markdown content:\n{}", md_content);
// 验证公式占位符在标题和图注内没有被 strip_html_tags 破坏，并能恢复成正确的 Latex
assert!(md_content.contains("## Heading with math"));
assert!(md_content.contains(r#"$\theta_{eff}$"#));
assert!(md_content.contains("> **Figure:** Figure caption with inline math"));
assert!(md_content.contains(r#"$M_\odot$"#));
Ok(())
}
}

107
src/services/parser/aanda.rs Normal file
View File

@ -0,0 +1,107 @@
// A&A / EDP Sciences 期刊解析器
use regex::Regex;
use super::JournalParser;
pub struct AandaParser;
impl AandaParser {
pub fn detect(html: &str) -> bool {
html.contains("www.aanda.org")
|| html.contains("citation_journal_title\" content=\"Astronomy &amp; Astrophysics")
|| html.contains("dc.publisher\" content=\"EDP Sciences")
}
}
impl JournalParser for AandaParser {
fn name(&self) -> &str { "A&A" }
fn extract_body<'a>(&self, html: &'a str) -> &'a str {
if let Some(start) = html.find("<main") {
&html[start..]
} else if let Some(start) = html.find("<article") {
&html[start..]
} else {
html
}
}
fn clean_chrome(&self, html: &str) -> String {
let mut h = html.to_string();
// 通用导航条
h = Regex::new(r#"(?s)<nav[^>]*>.*?</nav>"#).unwrap().replace_all(&h, "").to_string();
// JavaScript 邮件混淆代码
h = Regex::new(r#"(?s)<script[^>]*>.*?</script>"#).unwrap().replace_all(&h, "").to_string();
// A&A 特有的页面框架
for (pattern, replacement) in &[
(r#"(?s)<div[^>]*class="[^"]*breadcrumbs[^"]*"[^>]*>.*?</div>"#, ""), // 面包屑
(r#"(?s)<div[^>]*class="[^"]*menu[^"]*"[^>]*id="bloc"[^>]*>.*?</div>"#, ""), // 内部 TOC
(r#"(?s)<span\s+[^>]*class="[^"]*Z3988[^"]*"[^>]*>.*?</span>"#, ""), // COinS 元数据
(r#"(?s)<nav[^>]*class="[^"]*nav-(?:article|buttons)[^"]*"[^>]*>.*?</nav>"#, ""), // 期刊导航
(r#"(?s)<div\s+[^>]*class="[^"]*special_article[^"]*"[^>]*>.*?</div>"#, ""), // Open Access 徽章
// 元数据表格：从 summary.full 删除到 <div id="article">
(r#"(?s)<div\s+[^>]*class="[^"]*summary\s+full[^"]*"[^>]*>.*?<div\s+id="article""#,
r#"<div id="article""#),
] {
h = Regex::new(pattern).unwrap().replace_all(&h, *replacement).to_string();
}
h
}
fn clean_markers(&self, html: &str) -> String {
// A&A 使用 JS 邮件保护，已由 script 删除阶段处理
html.to_string()
}
fn convert_headings(&self, html: &str) -> String {
let mut h = html.to_string();
// Abstract: <p class="bold">Abstract</p>
h = Regex::new(r#"(?s)<p\s+[^>]*class="[^"]*bold[^"]*"[^>]*>\s*(?:<span[^>]*></span>)?\s*Abstract\s*</p>"#)
.unwrap().replace_all(&h, "\n\n## Abstract\n\n").to_string();
// h2.sec → ##
h = Regex::new(r#"(?s)<h2\s+[^>]*class="[^"]*sec\b[^"]*"[^>]*>(.*?)</h2>"#)
.unwrap().replace_all(&h, |caps: &regex::Captures| {
let inner = super::common::strip_html_tags(&caps[1]);
format!("\n\n## {}\n\n", inner.trim())
}).to_string();
// h3.sec2 → ###
h = Regex::new(r#"(?s)<h3\s+[^>]*class="[^"]*sec2\b[^"]*"[^>]*>(.*?)</h3>"#)
.unwrap().replace_all(&h, |caps: &regex::Captures| {
let inner = super::common::strip_html_tags(&caps[1]);
format!("\n\n### {}\n\n", inner.trim())
}).to_string();
h
}
fn convert_figures(&self, html: &str, base_origin: &str) -> String {
// A&A inset 图注结构：
// <div class="inset"><table><tr><td><a><img src="_small.jpg"></a></td>
// <td class="img-txt">caption</td></tr></table></div>
// 使用 <img src>（而非 <a href>），去掉 _small 缩略图后缀，补全绝对路径
let inset_re = Regex::new(
r#"(?s)<div\s+[^>]*class="[^"]*inset[^"]*"[^>]*>\s*<table[^>]*>\s*<tr[^>]*>\s*<td[^>]*>\s*<a\s+[^>]*href="[^"]*"[^>]*>\s*<img\s+[^>]*src="([^"]*)"[^>]*>\s*</a>\s*</td>\s*<td\s+[^>]*class="[^"]*img-txt[^"]*"[^>]*>\s*(.*?)\s*</td>\s*</tr>\s*</table>\s*</div>"#
).unwrap();
let bo = base_origin;
inset_re.replace_all(html, |caps: &regex::Captures| {
let img_src = &caps[1];
let caption_html = &caps[2];
let full_img_src = img_src.replace("_small.", ".");
let absolute_url = if full_img_src.starts_with('/') {
format!("{}{}", bo, full_img_src)
} else {
full_img_src.to_string()
};
let caption = super::common::strip_html_preserve_links(caption_html);
let clean_caption = caption.trim().replace('\n', " ");
format!("\n\n![{}]({})\n\n", clean_caption, absolute_url)
}).to_string()
}
}

112
src/services/parser/ar5iv.rs Normal file
View File

@ -0,0 +1,112 @@
// ar5iv (LaTeXML) 期刊解析器 —— 覆盖 71% 的 HTML 文献
use regex::Regex;
use super::JournalParser;
pub struct Ar5ivParser;
impl Ar5ivParser {
pub fn detect(html: &str) -> bool {
html.contains("class=\"ltx_") || html.contains("ar5iv.labs.arxiv.org")
}
}
impl JournalParser for Ar5ivParser {
fn name(&self) -> &str { "ar5iv" }
fn extract_body<'a>(&self, html: &'a str) -> &'a str {
if let Some(start) = html.find("<div class=\"ltx_page_main\">") {
&html[start..]
} else if let Some(start) = html.find("<article") {
&html[start..]
} else {
html
}
}
fn clean_chrome(&self, html: &str) -> String {
let mut h = html.to_string();
// 删除导航条、页眉、侧边栏
for pattern in &[
r#"(?s)<nav[^>]*>.*?</nav>"#,
r#"(?s)<div[^>]*class="[^"]*ltx_(?:page_navbar|header|navigation)[^"]*"[^>]*>.*?</div>"#,
// 机构脚注（嵌套 span：外层→ltx_note_outer→ltx_note_content，需匹配到最外层）
r#"(?s)<span\s+[^>]*class="[^"]*ltx_note\s+[^"]*ltx_role_affiliation[^"]*"[^>]*>.*?</span>\s*</span>\s*</span>"#,
r#"(?s)<span\s+[^>]*class="[^"]*ltx_note\s+[^"]*ltx_role_institutetext[^"]*"[^>]*>.*?</span>\s*</span>\s*</span>"#,
r#"(?s)<span\s+[^>]*class="[^"]*ltx_note\s+[^"]*ltx_note_frontmatter[^"]*"[^>]*>.*?</span>\s*</span>\s*</span>"#,
// 首页页眉：[N]Author 格式的运行标题
r#"(?s)\s*<p\s+[^>]*id="p1[^"]*"[^>]*>\s*\[?\d+\]?\s*[^<]*</p>"#,
] {
h = Regex::new(pattern).unwrap().replace_all(&h, "").to_string();
}
h
}
fn clean_markers(&self, html: &str) -> String {
let mut h = html.to_string();
// <sup class="ltx_note_mark">N</sup> → ^{N}
h = Regex::new(r#"<sup\s+[^>]*class="[^"]*ltx_note_mark[^"]*"[^>]*>([^<]*)</sup>"#)
.unwrap().replace_all(&h, "^{$1}").to_string();
// <sup class="ltx_sup">...</sup> → ^{...}
h = Regex::new(r#"<sup\s+[^>]*class="[^"]*ltx_sup[^"]*"[^>]*>(.*?)</sup>"#)
.unwrap().replace_all(&h, "^{$1}").to_string();
// <a class="ltx_ref" href="...">text</a> → [text](url)，供后续内部链接剥离
// 匹配 class 在 href 之前或之后两种顺序
for pat in &[
r#"<a\s+[^>]*class="[^"]*ltx_ref[^"]*"[^>]*href="([^"]*)"[^>]*>(.*?)</a>"#,
r#"<a\s+[^>]*href="([^"]*)"[^>]*class="[^"]*ltx_ref[^"]*"[^>]*>(.*?)</a>"#,
] {
h = Regex::new(pat).unwrap().replace_all(&h, "[$2]($1)").to_string();
}
h
}
fn convert_headings(&self, html: &str) -> String {
let mut h = html.to_string();
// ltx_title_document → # 一级标题
h = Regex::new(r#"(?s)<(?:h[1-6])[^>]*class="[^"]*ltx_title_document[^"]*"[^>]*>(.*?)</(?:h[1-6])>"#)
.unwrap().replace_all(&h, |caps: &regex::Captures| {
let inner = super::common::strip_html_tags(&caps[1]);
format!("\n# {}\n\n", inner.trim())
}).to_string();
// ltx_title_section → ## 二级标题
h = Regex::new(r#"(?s)<(?:h[1-6])[^>]*class="[^"]*ltx_title_section[^"]*"[^>]*>(.*?)</(?:h[1-6])>"#)
.unwrap().replace_all(&h, |caps: &regex::Captures| {
let inner = super::common::strip_html_tags(&caps[1]);
format!("\n\n## {}\n\n", inner.trim())
}).to_string();
// ltx_title_subsection → ### 三级标题
h = Regex::new(r#"(?s)<(?:h[1-6])[^>]*class="[^"]*ltx_title_subsection[^"]*"[^>]*>(.*?)</(?:h[1-6])>"#)
.unwrap().replace_all(&h, |caps: &regex::Captures| {
let inner = super::common::strip_html_tags(&caps[1]);
format!("\n\n### {}\n\n", inner.trim())
}).to_string();
// ltx_title_subsubsection → #### 四级标题
h = Regex::new(r#"(?s)<(?:h[1-6])[^>]*class="[^"]*ltx_title_subsubsection[^"]*"[^>]*>(.*?)</(?:h[1-6])>"#)
.unwrap().replace_all(&h, |caps: &regex::Captures| {
let inner = super::common::strip_html_tags(&caps[1]);
format!("\n\n#### {}\n\n", inner.trim())
}).to_string();
h
}
fn convert_figures(&self, html: &str, _base_origin: &str) -> String {
// ar5iv 的 <figure> + <figcaption> 由 html2md 原生处理。
// 这里将 <figcaption> 预处理为块引用标记以获得更好的格式。
let figcaption_re = Regex::new(r#"(?s)<figcaption[^>]*>(.*?)</figcaption>"#).unwrap();
figcaption_re.replace_all(html, |caps: &regex::Captures| {
let inner = super::common::strip_html_tags(&caps[1]);
format!("\n\n> **Figure:** {}\n", inner.trim())
}).to_string()
}
}

555
src/services/parser/common.rs Normal file
View File

@ -0,0 +1,555 @@
// parser/common.rs — Shared utilities for HTML→Markdown. All regexes compiled once via LazyLock.
use std::sync::LazyLock;
use regex::Regex;
// ── Static regexes ──────────────────────────────────────────────────────────
// extract_html_base_origin
static BASE_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?i)<base\s+[^>]*href="([^"]*)""#).unwrap());
static CANONICAL_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?i)<link\s+[^>]*rel="canonical"[^>]*href="([^"]*)"|<link\s+[^>]*href="([^"]*)"[^>]*rel="canonical""#).unwrap());
static OG_URL_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?i)<meta\s+[^>]*property="og:url"[^>]*content="([^"]*)"|<meta\s+[^>]*content="([^"]*)"[^>]*property="og:url""#).unwrap());
static PRISM_URL_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?i)<meta\s+[^>]*name="prism\.url"[^>]*content="([^"]*)"|<meta\s+[^>]*content="([^"]*)"[^>]*name="prism\.url""#).unwrap());
static CITATION_URL_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?i)<meta\s+[^>]*name="citation_pdf_url"[^>]*content="([^"]*)"|<meta\s+[^>]*content="([^"]*)"[^>]*name="citation_pdf_url""#).unwrap());
// strip_html_tags
static TAG_STRIP_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"<[^>]+>").unwrap());
// strip_html_preserve_links
static A_LINK_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"<a\s+[^>]*href="([^"]*)"[^>]*>(.*?)</a>"#).unwrap());
// entity decoding — static HashMap + regex
static ENTITY_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"&#[0-9]+;|&#x[0-9a-fA-F]+;|&[a-zA-Z]+;").unwrap());
static ENTITY_MAP: LazyLock<std::collections::HashMap<&'static str, &'static str>> = LazyLock::new(|| {
let mut m = std::collections::HashMap::new();
m.insert("&#945;","α");m.insert("&#946;","β");m.insert("&#947;","γ");m.insert("&#948;","δ");m.insert("&#949;","ε");m.insert("&#950;","ζ");
m.insert("&#951;","η");m.insert("&#952;","θ");m.insert("&#953;","ι");m.insert("&#954;","κ");m.insert("&#955;","λ");m.insert("&#956;","μ");
m.insert("&#957;","ν");m.insert("&#958;","ξ");m.insert("&#959;","ο");m.insert("&#960;","π");m.insert("&#961;","ρ");m.insert("&#962;","ς");
m.insert("&#963;","σ");m.insert("&#964;","τ");m.insert("&#965;","υ");m.insert("&#966;","φ");m.insert("&#967;","χ");m.insert("&#968;","ψ");m.insert("&#969;","ω");
m.insert("&#913;","Α");m.insert("&#914;","Β");m.insert("&#915;","Γ");m.insert("&#916;","Δ");m.insert("&#917;","Ε");m.insert("&#918;","Ζ");
m.insert("&#919;","Η");m.insert("&#920;","Θ");m.insert("&#921;","Ι");m.insert("&#922;","Κ");m.insert("&#923;","Λ");m.insert("&#924;","Μ");
m.insert("&#925;","Ν");m.insert("&#926;","Ξ");m.insert("&#927;","Ο");m.insert("&#928;","Π");m.insert("&#929;","Ρ");m.insert("&#931;","Σ");
m.insert("&#932;","Τ");m.insert("&#933;","Υ");m.insert("&#934;","Φ");m.insert("&#935;","Χ");m.insert("&#936;","Ψ");m.insert("&#937;","Ω");
m.insert("&#8804;","≤");m.insert("&#8805;","≥");m.insert("&#8722;","−");m.insert("&#8776;","≈");m.insert("&#8857;","⊙");m.insert("&#9737;","⊙");
m.insert("&#8196;"," ");m.insert("&#8198;"," ");m.insert("&#8721;","∑");m.insert("&#215;","×");m.insert("&#8730;","√");m.insert("&#8734;","∞");
m.insert("&#8747;","∫");m.insert("&#8211;","–");m.insert("&#8212;","—");m.insert("&#8217;","’");m.insert("&#8242;","′");m.insert("&#8243;","″");
m.insert("&#8260;","⁄");m.insert("&#8364;","€");m.insert("&#8800;","≠");m.insert("&#8834;","⊂");m.insert("&#8835;","⊃");
m.insert("&#8838;","⊆");m.insert("&#8839;","⊇");m.insert("&lt;","<");m.insert("&gt;",">");m.insert("&amp;","&");
m
});
// html_math_to_latex sub/sup inner regexes
static ITALIC_INNER_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"<i>(.*?)</i>"#).unwrap());
static SUB_INNER_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"<sub>(.*?)</sub>"#).unwrap());
static SUP_INNER_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"<sup>(.*?)</sup>"#).unwrap());
// convert_html_math_to_latex — dynamic regex cached via OnceLock
use std::sync::OnceLock;
static MATH_REGEX: OnceLock<Regex> = OnceLock::new();
static ITALIC_CHECK_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"<i>([^<]*)</i>"#).unwrap());
// LaTeXML table conversion
static LATEXML_TAG_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?i)<(span|div)\b([^>]*?)>|</(span|div)>"#).unwrap());
static LATEXML_CLASS_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"class="([^"]*)""#).unwrap());
// Table conversion
static TABLE_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?s)<table[^>]*>(.*?)</table>"#).unwrap());
static TR_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?s)<tr[^>]*>(.*?)</tr>"#).unwrap());
static TD_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?s)<(?:td|th)[^>]*>(.*?)</(?:td|th)>"#).unwrap());
// postprocess_markdown
static DIV_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"</?div[^>]*>").unwrap());
static SPAN_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"</?span[^>]*>").unwrap());
static IFRAME_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?s)<iframe[^>]*>.*?</iframe>").unwrap());
static ASTROBJ_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?s)<astrobj[^>]*>(.*?)</astrobj>").unwrap());
static FIG_MERGE_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(!\[[^\]]*\]\([^)]*\))\s*\\>\s*\\\*\\\*(Figure|Caption):\\\*\\\*").unwrap());
static PAGERANGE_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?m)^\^\{†\}\^\{†\}(?:pagerange|pubyear|offprints):.*\n?").unwrap());
static EMPTY_BRACKETS_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\[\]").unwrap());
static INTERNAL_LINK_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(!?)\[([^\]]*?)\]\(([^)]*(?:/articles/aa/[^)]*|#[^)]*))\)"#).unwrap());
static EXCESSIVE_NL_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\n{4,}").unwrap());
static LINK_FIX_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(!?\[[^\]]*?\])\(([^)]*?)\)"#).unwrap());
static LATEXML_MACRO_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\\{1,2}(?:orgname|orgdiv|orgaddress|articletag|term|savesymbol|restoresymbol|volnopage|SInits)\b").unwrap());
/// 清理文首的 LaTeXML 模板垃圾行（如 `second\savesymboldegree\restoresymbol...`）
fn clean_latexml_preamble(md: &str) -> String {
let mut lines: Vec<&str> = md.lines().collect();
// 只清理明确是 LaTeXML 命令残渣的行（含 \ 且不含 $ 或 #）
while let Some(first) = lines.first() {
let trimmed = first.trim();
if trimmed.is_empty() { lines.remove(0); continue; }
let has_slash = trimmed.contains('\\');
let has_math = trimmed.contains('$');
let is_heading = trimmed.starts_with('#');
// 含反斜杠的行，如果既不是数学也不是标题 → LaTeXML 命令残渣
if has_slash && !has_math && !is_heading {
lines.remove(0);
} else {
break;
}
}
lines.join("\n")
}
static HEADING_TRAIL_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?m)^(#{1,6})\s+(.*?)\s+#+$").unwrap());
static SECTION_PROMOTE_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?mi)^(#{3,6})[ \t]*(Abstract|Keywords|Glossary|Nomenclature|Acknowledgments|References)(:?)[ \t]*$").unwrap());
static ABSTRACT_CLEAN_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?mi)^##\s+Abstract\s*\n\s*\n\s*\[Abstract\]\s*\n").unwrap());
static BRACKET_INLINE_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?mi)^\[(Abstract|Keywords|Glossary|Nomenclature|Acknowledgments|References)\][ \t]+(.+)$").unwrap());
static BRACKET_HEADER_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?mi)^\[(Abstract|Keywords|Glossary|Nomenclature|Acknowledgments|References)\][ \t]*$").unwrap());
static BULLET_CLEAN_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"(?m)^(\s*[\*\-+])\s*•\s*").unwrap());
static BRACKET_NEWLINE_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"\[\s*\n+\s*([^\]\n]+)\]").unwrap());
// Unescape patterns
static UNESC_RE: LazyLock<Vec<(Regex, &'static str)>> = LazyLock::new(|| {
vec![
(Regex::new(r"\\#\s+").unwrap(), "# "),
(Regex::new(r"\\##\s+").unwrap(), "## "),
(Regex::new(r"\\###\s+").unwrap(), "### "),
(Regex::new(r"\\####\s+").unwrap(), "#### "),
(Regex::new(r"\\>\s+").unwrap(), "> "),
(Regex::new(r"\\\*\\\*").unwrap(), "**"),
]
});
// cut_acknowledgments_and_references
static TRUNCATE_RE: OnceLock<Regex> = OnceLock::new();
// extract_img_equations
static IMG_EQ_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?s)<span\s+[^>]*class="[^"]*img-equation[^"]*"[^>]*data-latex="([^"]*)"[^>]*>.*?</span>\s*</span>\s*(?:<span\s+class="label-eq">([^<]*)</span>\s*)?</span>"#).unwrap());
// extract_math_blocks
static MATH_BLOCK_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?s)<math\s+([^>]*?)>(.*?)</math>"#).unwrap());
static ALTTEXT_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"alttext="([^"]*)""#).unwrap());
static ANNOTATION_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?s)<annotation\s+[^>]*encoding="application/x-tex"[^>]*>(.*?)</annotation>"#).unwrap());
// convert_img_tags
static IMG_TAG_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?s)<img\s+([^>]*?)>"#).unwrap());
static IMG_SRC_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"src="([^"]*)""#).unwrap());
static IMG_ALT_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"alt="([^"]*)""#).unwrap());
// convert_common_markup
static FIGCAPTION_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?s)<figcaption[^>]*>(.*?)</figcaption>"#).unwrap());
static LTX_SEC_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?s)<(?:h[1-6])[^>]*class="[^"]*ltx_title_section[^"]*"[^>]*>(.*?)</(?:h[1-6])>"#).unwrap());
static LTX_SUBSEC_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?s)<(?:h[1-6])[^>]*class="[^"]*ltx_title_subsection[^"]*"[^>]*>(.*?)</(?:h[1-6])>"#).unwrap());
static LTX_SUBSUBSEC_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?s)<(?:h[1-6])[^>]*class="[^"]*ltx_title_subsubsection[^"]*"[^>]*>(.*?)</(?:h[1-6])>"#).unwrap());
static LTX_CAPTION_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?s)<(?:span|div|p)[^>]*class="[^"]*ltx_caption[^"]*"[^>]*>(.*?)</(?:span|div|p)>"#).unwrap());
static LTX_TITLE_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?s)<(?:h[1-6])[^>]*class="[^"]*ltx_title_document[^"]*"[^>]*>(.*?)</(?:h[1-6])>"#).unwrap());
static CITE_START_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?s)<cite[^>]*>"#).unwrap());
static CITE_END_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?s)</cite>"#).unwrap());
static EMPTY_A_RE: LazyLock<Regex> = LazyLock::new(|| Regex::new(r#"(?s)<a\s+[^>]*>\s*</a>"#).unwrap());
// ── Functions ───────────────────────────────────────────────────────────────
pub fn extract_html_base_origin(html: &str) -> String {
let mut url_str = None;
if let Some(caps) = BASE_RE.captures(html) {
url_str = Some(caps[1].trim().to_string());
} else if let Some(caps) = CANONICAL_RE.captures(html) {
url_str = Some(caps.get(1).or_else(|| caps.get(2)).map(|m| m.as_str().trim().to_string()).unwrap_or_default());
} else if let Some(caps) = OG_URL_RE.captures(html) {
url_str = Some(caps.get(1).or_else(|| caps.get(2)).map(|m| m.as_str().trim().to_string()).unwrap_or_default());
} else if let Some(caps) = PRISM_URL_RE.captures(html) {
url_str = Some(caps.get(1).or_else(|| caps.get(2)).map(|m| m.as_str().trim().to_string()).unwrap_or_default());
} else if let Some(caps) = CITATION_URL_RE.captures(html) {
url_str = Some(caps.get(1).or_else(|| caps.get(2)).map(|m| m.as_str().trim().to_string()).unwrap_or_default());
}
if let Some(mut u) = url_str {
if u.starts_with('/') { u = format!("https://ar5iv.labs.arxiv.org{}", u); }
if let Ok(parsed) = url::Url::parse(&u) {
let scheme = parsed.scheme();
let host = parsed.host_str().unwrap_or("");
if !host.is_empty() { return format!("{}://{}", scheme, host); }
}
}
"https://ar5iv.labs.arxiv.org".to_string()
}
pub fn truncate_footer(html: &str) -> &str {
if let Some(end) = html.find("<div class=\"ar5iv-footer\">") { &html[..end] }
else if let Some(end) = html.find("<footer") { &html[..end] }
else { html }
}
pub fn strip_html_tags(html: &str) -> String {
let text = TAG_STRIP_RE.replace_all(html, "").to_string();
text.replace("&amp;","&").replace("&lt;","<").replace("&gt;",">").replace("&quot;","\"").replace("&nbsp;"," ").replace("&#39;","'")
}
pub fn strip_html_preserve_links(html: &str) -> String {
let text = A_LINK_RE.replace_all(html, "$2").to_string();
let text = TAG_STRIP_RE.replace_all(&text, "").to_string();
text.replace("&amp;","&").replace("&lt;","<").replace("&gt;",">").replace("&quot;","\"").replace("&nbsp;"," ").replace("&#39;","'")
}
pub fn decode_html_numeric_entities(html: &str) -> String {
let entities = &*ENTITY_MAP;
ENTITY_RE.replace_all(html, |caps: &regex::Captures| -> String {
let entity = caps[0].to_owned();
entities.get(entity.as_str()).copied().unwrap_or(&entity).to_owned()
}).to_string()
}
pub fn convert_html_math_to_latex(html: String, mut formulas: Vec<(String, bool)>) -> (String, Vec<(String, bool)>) {
let math_letter = r#"(?:[a-zA-Z]\b|dex\b|log\b|ln\b|sin\b|cos\b|tan\b|eff\b|env\b|obs\b|sys\b|sta\b|WD\b|MS\b|He\b|H\b)"#;
let sub_content = r#"(?:<i>[^<]*</i>|[a-zA-Z0-9α-ωΑ-Ω⊙+\-λθχσϖ])+"#;
let sup_content = r#"(?:<i>[^<]*</i>|[a-zA-Z0-9α-ωΑ-Ω⊙+\-λθχσϖ])+"#;
let math_regex = MATH_REGEX.get_or_init(|| {
Regex::new(&format!(
r#"(?i)(?:<i>[^<]*</i>|{0}<sub>{1}</sub>|{0}<sup>{2}</sup>|[0-9]+(?:\.[0-9]+)?|⊙|log(?: |\s+)?<i>[a-zA-Z]</i>|log\()(?:\s| | |<i>[^<]*</i>|<sub>{1}</sub>|<sup>{2}</sup>|≤|≥|−|=|<|>|≈|/|\+|-|\(|\)|[0-9]+(?:\.[0-9]+)?|⊙)*"#,
math_letter, sub_content, sup_content
)).unwrap()
});
let mut placeholder_counter = formulas.len();
let result = math_regex.replace_all(&html, |caps: &regex::Captures| {
let matched = &caps[0];
let has_text_italic = ITALIC_CHECK_RE.captures_iter(matched).any(|c| {
let content = &c[1];
content.len() > 2 && !content.chars().any(|ch|
('α'..='ω').contains(&ch) || ('Α'..='Ω').contains(&ch) ||
ch == '⊙' || ch == '≤' || ch == '≥' || ch == '−' || ch == '≈'
)
});
if !has_text_italic && (
matched.contains("<sub") || matched.contains("<sup") || matched.contains("<i") ||
matched.contains('≤') || matched.contains('≥') || matched.contains('−') ||
matched.contains('⊙') || matched.contains('≈') || matched.contains("log")) {
let mut math_part = matched;
let mut trailing_part = String::new();
loop {
let trimmed_len = math_part.trim_end().len();
if trimmed_len < math_part.len() {
trailing_part = format!("{}{}", &math_part[trimmed_len..], trailing_part);
math_part = &math_part[..trimmed_len]; continue;
}
if math_part.ends_with(')') {
let open_count = math_part.matches('(').count();
let close_count = math_part.matches(')').count();
if close_count > open_count {
trailing_part = format!("){}", trailing_part);
math_part = &math_part[..math_part.len()-1]; continue;
}
}
break;
}
if math_part.trim().is_empty() { return matched.to_string(); }
let latex = html_math_to_latex(math_part);
formulas.push((latex, false));
let p = format!(" MATHPLACEHOLDER{} ", placeholder_counter);
placeholder_counter += 1;
format!("{}{}", p, trailing_part)
} else { matched.to_string() }
}).to_string();
(result, formulas)
}
fn html_math_to_latex(s: &str) -> String {
let s = s.replace(' '," ").replace(' ',"\\,").replace('\u{2006}',"\\,");
let s = s.replace('≤'," \\le ").replace('≥'," \\ge ").replace('−'," - ")
.replace('≈'," \\approx ").replace('×'," \\times ").replace('≡'," \\equiv ")
.replace('≠'," \\neq ").replace('∑'," \\sum ").replace('√'," \\sqrt ")
.replace('∞'," \\infty ").replace('∫'," \\int ").replace('–'," -- ").replace('—'," --- ");
let s = s.replace('⊙',"{\\odot}");
let s = s.replace('α',"\\alpha").replace('β',"\\beta").replace('γ',"\\gamma").replace('δ',"\\delta").replace('ε',"\\epsilon")
.replace('ζ',"\\zeta").replace('η',"\\eta").replace('θ',"\\theta").replace('ι',"\\iota").replace('κ',"\\kappa")
.replace('λ',"\\lambda").replace('μ',"\\mu").replace('ν',"\\nu").replace('ξ',"\\xi").replace('π',"\\pi")
.replace('ρ',"\\rho").replace('σ',"\\sigma").replace('τ',"\\tau").replace('υ',"\\upsilon").replace('φ',"\\phi")
.replace('χ',"\\chi").replace('ψ',"\\psi").replace('ω',"\\omega").replace('ς',"\\varsigma")
.replace('Γ',"\\Gamma").replace('Δ',"\\Delta").replace('Θ',"\\Theta").replace('Λ',"\\Lambda")
.replace('Ξ',"\\Xi").replace('Π',"\\Pi").replace('Σ',"\\Sigma").replace('Υ',"\\Upsilon").replace('Φ',"\\Phi")
.replace('Ψ',"\\Psi").replace('Ω',"\\Omega");
let s = s.replace("log","\\log ").replace("\\log \\,","\\log ").replace("\\log ","\\log ");
let s = ITALIC_INNER_RE.replace_all(&s, "$1").to_string();
let s = SUB_INNER_RE.replace_all(&s, |caps: &regex::Captures| {
let inner = ITALIC_INNER_RE.replace_all(&caps[1], "$1").to_string();
if inner.chars().all(|c| c.is_alphabetic()) { format!("_{{\\text{{{}}}}}", inner) }
else { format!("_{{{}}}", inner) }
}).to_string();
let s = SUP_INNER_RE.replace_all(&s, |caps: &regex::Captures| {
let inner = ITALIC_INNER_RE.replace_all(&caps[1], "$1").to_string();
if inner.chars().all(|c| c.is_alphabetic()) { format!("^{{\\text{{{}}}}}", inner) }
else { format!("^{{{}}}", inner) }
}).to_string();
s.replace(" K"," \\text{K}").replace("dex"," \\text{dex}").split_whitespace().collect::<Vec<_>>().join(" ")
}
pub fn replace_latexml_tables(html: &str) -> String {
let mut result = String::new();
let mut last_pos = 0;
let mut stack: Vec<(String, Option<String>)> = Vec::new();
for cap in LATEXML_TAG_RE.captures_iter(html) {
let mat = cap.get(0).unwrap();
result.push_str(&html[last_pos..mat.start()]);
if cap.get(1).is_some() {
let tag_name = cap.get(1).unwrap().as_str().to_lowercase();
let attrs = cap.get(2).unwrap().as_str();
let mut matched_type = None;
if let Some(class_cap) = LATEXML_CLASS_RE.captures(attrs) {
let class_str = class_cap[1].to_lowercase();
if class_str.contains("ltx_tabular") { matched_type = Some("table"); }
else if class_str.contains("ltx_tbody") { matched_type = Some("tbody"); }
else if class_str.contains("ltx_thead") { matched_type = Some("thead"); }
else if class_str.contains("ltx_tfoot") { matched_type = Some("tfoot"); }
else if class_str.contains("ltx_tr") { matched_type = Some("tr"); }
else if class_str.contains("ltx_th") { matched_type = Some("th"); }
else if class_str.contains("ltx_td") { matched_type = Some("td"); }
}
if let Some(t) = matched_type {
let t_str = t.to_string();
result.push_str(&format!("<{}>", t_str));
stack.push((tag_name, Some(t_str)));
} else { result.push_str(mat.as_str()); stack.push((tag_name, None)); }
} else {
let tag_name = cap.get(3).unwrap().as_str().to_lowercase();
let mut replaced = false;
while let Some((open_name, open_type)) = stack.pop() {
if open_name == tag_name {
if let Some(t) = open_type { result.push_str(&format!("</{}>", t)); }
else { result.push_str(&format!("</{}>", tag_name)); }
replaced = true; break;
} else if let Some(t) = open_type { result.push_str(&format!("</{}>", t)); }
else { result.push_str(&format!("</{}>", open_name)); }
}
if !replaced { result.push_str(mat.as_str()); }
}
last_pos = mat.end();
}
result.push_str(&html[last_pos..]);
result
}
pub fn convert_html_tables_to_markdown(markdown: &str) -> String {
TABLE_RE.replace_all(markdown, |caps: &regex::Captures| {
let full_table = &caps[0];
if full_table.to_lowercase().contains("colspan") || full_table.to_lowercase().contains("rowspan") {
full_table.to_string()
} else { convert_html_table_to_markdown(full_table) }
}).to_string()
}
fn convert_html_table_to_markdown(html_table: &str) -> String {
let mut md_rows = Vec::new();
let mut col_count = 0;
for tr_cap in TR_RE.captures_iter(html_table) {
let mut cells = Vec::new();
for td_cap in TD_RE.captures_iter(&tr_cap[1]) {
cells.push(strip_html_tags(&td_cap[1]).trim().replace('\n', " "));
}
if !cells.is_empty() {
if col_count == 0 { col_count = cells.len(); }
md_rows.push(format!("| {} |", cells.join(" | ")));
}
}
if md_rows.is_empty() { return html_table.to_string(); }
let separator = format!("|{}|", vec!["---"; col_count].join("|"));
md_rows.insert(1, separator);
md_rows.join("\n")
}
pub fn postprocess_markdown(text: &str) -> String {
let text = convert_html_tables_to_markdown(text);
let mut clean_lines = Vec::new();
let mut in_code_block = false;
for line in text.lines() {
let trimmed = line.trim();
if trimmed.starts_with("```") { in_code_block = !in_code_block; }
if in_code_block { clean_lines.push(line.to_string()); }
else { clean_lines.push(trimmed.to_string()); }
}
let mut md = clean_lines.join("\n");
md = DIV_RE.replace_all(&md, "").to_string();
md = FIG_MERGE_RE.replace_all(&md, "$1\n\n> **$2:**").to_string();
md = PAGERANGE_RE.replace_all(&md, "").to_string();
md = SPAN_RE.replace_all(&md, "").to_string();
md = IFRAME_RE.replace_all(&md, "").to_string();
md = ASTROBJ_RE.replace_all(&md, "$1").to_string();
md = EMPTY_BRACKETS_RE.replace_all(&md, "").to_string();
md = INTERNAL_LINK_RE.replace_all(&md, |caps: &regex::Captures| {
if caps[1].starts_with('!') { caps[0].to_string() } else { caps[2].to_string() }
}).to_string();
md = EXCESSIVE_NL_RE.replace_all(&md, "\n\n\n").to_string();
for (re, repl) in UNESC_RE.iter() {
md = re.replace_all(&md, *repl).to_string();
}
md = md.replace("&lt;","<").replace("&gt;",">").replace("&amp;","&").replace("&quot;","\"").replace("&#39;","'");
md = LINK_FIX_RE.replace_all(&md, |caps: &regex::Captures| {
format!("{}({})", &caps[1], caps[2].replace(r"\_","_").replace(r"\%","%"))
}).to_string();
md = LATEXML_MACRO_RE.replace_all(&md, " ").to_string();
md = HEADING_TRAIL_RE.replace_all(&md, "$1 $2").to_string();
md = SECTION_PROMOTE_RE.replace_all(&md, "## $2$3").to_string();
md = ABSTRACT_CLEAN_RE.replace_all(&md, "## Abstract\n\n").to_string();
md = BRACKET_INLINE_RE.replace_all(&md, "## $1\n\n$2").to_string();
md = BRACKET_HEADER_RE.replace_all(&md, "## $1").to_string();
md = BULLET_CLEAN_RE.replace_all(&md, "$1 ").to_string();
md = BRACKET_NEWLINE_RE.replace_all(&md, "[$1]").to_string();
let md = cut_acknowledgments_and_references(&md);
let md = clean_latexml_preamble(&md);
md.trim().to_string()
}
pub fn cut_acknowledgments_and_references(text: &str) -> String {
let re = TRUNCATE_RE.get_or_init(|| {
Regex::new(r"(?mi)^(?:(?:#+\s*)?(?:[\d\.]+\s+)?(?:acknowledgements?|acknowledgments?|acknowledgment|references?(?:\s+&\s+citations)?|literature\s+cited|bibliography|bibliographie)\s*[:\.]?\s*#*\s*$|\[(?:acknowledgements?|acknowledgments?|acknowledgment|references?|literature\s+cited|bibliography|bibliographie)\]\s*$|\{thebibliography\*?\}\s*$|\{ack(?:nowledgements?)?\}\s*$|^(?:acknowledgements?|acknowledgments?|acknowledgment|references?|literature\s+cited|bibliography|bibliographie)\b\s*[:\.]\s+.*$)").unwrap()
});
if let Some(mat) = re.find(text) { text[..mat.start()].trim_end().to_string() }
else { text.to_string() }
}
pub fn extract_img_equations(html: &str) -> (String, Vec<(String, String)>) {
let mut img_eqs = Vec::new();
let mut counter = 0u32;
let result = IMG_EQ_RE.replace_all(html, |caps: &regex::Captures| {
let latex = caps[1].trim().to_string();
let label = caps.get(2).map(|m| m.as_str().trim().to_string()).unwrap_or_default();
img_eqs.push((latex, label));
let p = format!(" IMGEQPLACEHOLDER{} ", counter);
counter += 1; p
}).to_string();
(result, img_eqs)
}
pub fn extract_math_blocks(html: &str) -> (String, Vec<(String, bool)>) {
let mut formulas = Vec::new();
let mut counter = 0u32;
let result = MATH_BLOCK_RE.replace_all(html, |caps: &regex::Captures| {
let attrs = &caps[1];
let mut alttext = ALTTEXT_RE.captures(attrs).map(|c| c[1].to_string()).unwrap_or_default();
if alttext.is_empty() {
if let Some(ann_caps) = ANNOTATION_RE.captures(&caps[2]) { alttext = ann_caps[1].trim().to_string(); }
}
if alttext.is_empty() { alttext = strip_html_tags(&caps[2]).trim().to_string(); }
let is_block = attrs.contains("display=\"block\"") || attrs.contains("display='block'");
formulas.push((alttext, is_block));
let p = format!(" MATHPLACEHOLDER{} ", counter);
counter += 1; p
}).to_string();
(result, formulas)
}
pub fn convert_img_tags(html: &str, base_origin: &str) -> String {
IMG_TAG_RE.replace_all(html, |caps: &regex::Captures| {
let attrs = &caps[1];
let src = IMG_SRC_RE.captures(attrs).map(|c| c[1].to_string()).unwrap_or_default();
let alt = IMG_ALT_RE.captures(attrs).map(|c| c[1].to_string()).unwrap_or_else(|| "image".to_string());
let absolute_src = if src.starts_with('/') { format!("{}{}", base_origin, src) } else { src };
format!("\n\n![{}]({})\n\n", alt, absolute_src)
}).to_string()
}
pub fn restore_math_placeholders(markdown: &str, formulas: &[(String, bool)]) -> String {
let mut md = markdown.to_string();
for i in (0..formulas.len()).rev() {
let (ref alttext, is_block) = formulas[i];
let placeholder = format!("MATHPLACEHOLDER{}", i);
let replacement = if is_block { format!("\n\n$$\n{}\n$$\n\n", alttext) }
else { format!(" ${}$ ", alttext) };
md = md.replace(&placeholder, &replacement);
}
md
}
pub fn restore_img_equations(markdown: &str, img_eqs: &[(String, String)]) -> String {
let mut md = markdown.to_string();
for i in (0..img_eqs.len()).rev() {
let (ref latex, ref label) = img_eqs[i];
let placeholder = format!("IMGEQPLACEHOLDER{}", i);
let decoded = latex.replace("&amp;","&").replace("&lt;","<").replace("&gt;",">").replace("&quot;","\"").replace("&#39;","'");
let trimmed = decoded.trim();
let mut raw = trimmed.to_string();
if raw.starts_with("$$") && raw.ends_with("$$") { raw = raw[2..raw.len()-2].trim().to_string(); }
else if raw.starts_with('$') && raw.ends_with('$') { raw = raw[1..raw.len()-1].trim().to_string(); }
let replacement = if !label.is_empty() {
format!("\n\n$$\n{} \\tag{{{}}}\n$$\n\n", raw, label.trim_matches(|c| c=='('||c==')').trim())
} else { format!("\n\n$$\n{}\n$$\n\n", raw) };
md = md.replace(&placeholder, &replacement);
}
md
}
pub fn convert_common_markup(html: &str) -> String {
let mut h = html.to_string();
h = FIGCAPTION_RE.replace_all(&h, |caps: &regex::Captures| {
format!("\n\n> **Figure:** {}\n", strip_html_tags(&caps[1]).trim())
}).to_string();
h = LTX_SEC_RE.replace_all(&h, |caps: &regex::Captures| format!("\n\n## {}\n\n", strip_html_tags(&caps[1]).trim())).to_string();
h = LTX_SUBSEC_RE.replace_all(&h, |caps: &regex::Captures| format!("\n\n### {}\n\n", strip_html_tags(&caps[1]).trim())).to_string();
h = LTX_SUBSUBSEC_RE.replace_all(&h, |caps: &regex::Captures| format!("\n\n#### {}\n\n", strip_html_tags(&caps[1]).trim())).to_string();
h = LTX_CAPTION_RE.replace_all(&h, |caps: &regex::Captures| format!("\n> **Caption:** {}\n", strip_html_tags(&caps[1]).trim())).to_string();
h = LTX_TITLE_RE.replace_all(&h, |caps: &regex::Captures| format!("\n# {}\n\n", strip_html_tags(&caps[1]).trim())).to_string();
h = CITE_START_RE.replace_all(&h, "").to_string();
h = CITE_END_RE.replace_all(&h, "").to_string();
h = EMPTY_A_RE.replace_all(&h, "").to_string();
h
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_strip_html_tags() {
assert_eq!(strip_html_tags("<p>Hello</p>"), "Hello");
assert_eq!(strip_html_tags("<a href='x'>Link</a>"), "Link");
assert_eq!(strip_html_tags("A &amp; B"), "A & B");
}
#[test]
fn test_decode_html_numeric_entities() {
let input = "&#955; &#952; &#967; &#945;";
let output = decode_html_numeric_entities(input);
assert!(output.contains('λ') && output.contains('θ') && output.contains('χ') && output.contains('α'));
assert!(!output.contains("&#955;"));
}
#[test]
fn test_cut_acknowledgments() {
assert_eq!(cut_acknowledgments_and_references("text\n## Acknowledgments\nthanks"), "text");
assert_eq!(cut_acknowledgments_and_references("text\n## References\nbib"), "text");
assert_eq!(cut_acknowledgments_and_references("text only"), "text only");
}
#[test]
fn test_postprocess_internal_links() {
let input = "([Maxted et al. 2001](/articles/aa/full_html/.../aa54562-25.html#R45))";
let output = postprocess_markdown(input);
assert!(!output.contains("](/articles/aa/"));
assert!(output.contains("Maxted et al. 2001"));
}
#[test]
fn test_postprocess_bib_links() {
let input = "([1976](#bib.bib16)) and [1984](#bib.bib30)";
let output = postprocess_markdown(input);
assert!(!output.contains("](#bib"));
assert!(output.contains("(1976)"));
}
#[test]
fn test_postprocess_fig_caption() {
let input = r"![Refer to caption](https://x.com/x1.png) \> \*\*Figure:\*\* Fig. 1: Text.";
let output = postprocess_markdown(input);
assert!(output.contains("x1.png)\n\n> **Figure:**"));
}
#[test]
fn test_img_equation_extraction() {
let html = r#"<span class="img-equation" data-latex="$$ \frac{a}{b} $$"><span>...</span></span><span class="label-eq">(1)</span></span>"#;
let (_, eqs) = extract_img_equations(html);
assert_eq!(eqs.len(), 1);
assert_eq!(eqs[0].0, "$$ \\frac{a}{b} $$");
}
#[test]
fn test_math_block_extraction() {
let html = r#"<math alttext="\alpha" display="inline"><semantics><mi>α</mi></semantics></math>"#;
let (_, formulas) = extract_math_blocks(html);
assert_eq!(formulas[0].0, "\\alpha");
assert!(!formulas[0].1);
}
}

47
src/services/parser/generic.rs Normal file
View File

@ -0,0 +1,47 @@
// 通用回退解析器 —— 覆盖 Springer、MNRAS、Zenodo 等未专门适配的期刊
use super::JournalParser;
pub struct GenericParser;
impl GenericParser {
pub fn detect(_html: &str) -> bool {
true // 永远匹配，放在检测链的最末尾
}
}
impl JournalParser for GenericParser {
fn name(&self) -> &str { "通用" }
fn extract_body<'a>(&self, html: &'a str) -> &'a str {
if let Some(start) = html.find("<article") {
&html[start..]
} else if let Some(start) = html.find("<main") {
&html[start..]
} else if let Some(start) = html.find("<body") {
&html[start..]
} else {
html
}
}
fn clean_chrome(&self, html: &str) -> String {
let mut h = html.to_string();
// 删除所有页面通用的垃圾元素
for pat in &[
r#"(?s)<nav[^>]*>.*?</nav>"#,
r#"(?s)<script[^>]*>.*?</script>"#,
r#"(?s)<footer[^>]*>.*?</footer>"#,
r#"(?s)<header[^>]*>.*?</header>"#,
r#"(?s)<noscript[^>]*>.*?</noscript>"#,
] {
h = regex::Regex::new(pat).unwrap().replace_all(&h, "").to_string();
}
h
}
// 以下方法使用默认空操作 —— html2md 原生处理标准 HTML 标记
fn clean_markers(&self, html: &str) -> String { html.to_string() }
fn convert_headings(&self, html: &str) -> String { html.to_string() }
fn convert_figures(&self, html: &str, _: &str) -> String { html.to_string() }
}

115
src/services/parser/iop.rs Normal file
View File

@ -0,0 +1,115 @@
// IOP Publishing / AAS 期刊解析器（7% 的 HTML 文献）
use regex::Regex;
use super::JournalParser;
pub struct IopParser;
impl IopParser {
pub fn detect(html: &str) -> bool {
html.contains("iopscience.iop.org")
|| html.contains("dc.publisher\" content=\"IOP Publishing")
}
}
impl JournalParser for IopParser {
fn name(&self) -> &str { "IOP/AAS" }
fn extract_body<'a>(&self, html: &'a str) -> &'a str {
// IOP 将正文放在 itemprop="articleBody" 的 div 中
if let Some(start) = html.find("<div itemprop=\"articleBody\"") {
&html[start..]
} else if let Some(start) = html.find("<!-- Start article full text -->") {
&html[start..]
} else if let Some(start) = html.find("<main") {
&html[start..]
} else {
html
}
}
fn clean_chrome(&self, html: &str) -> String {
let mut h = html.to_string();
// IOP 页面框架垃圾
for pattern in &[
r#"(?s)<nav[^>]*class="[^"]*wd-main-nav[^"]*"[^>]*>.*?</nav>"#, // 主导航
r#"(?s)<nav[^>]*class="[^"]*content-nav[^"]*"[^>]*>.*?</nav>"#, // 侧边栏
r#"(?s)<header[^>]*class="[^"]*content-grid__full-width[^"]*"[^>]*>.*?</header>"#, // 站点标题
r#"(?s)<div[^>]*class="[^"]*secondary-header[^"]*"[^>]*>.*?</div>"#, // 期刊品牌
r#"(?s)<div[^>]*class="[^"]*article-head[^"]*"[^>]*>.*?</div>\s*</div>"#, // 文章元数据
r#"(?s)<footer[^>]*class="[^"]*footer[^"]*"[^>]*>.*?</footer>"#, // 页脚
r#"(?s)<noscript[^>]*>.*?</noscript>"#, // GTM noscript
r#"(?s)<iframe[^>]*googletagmanager[^>]*>.*?</iframe>"#, // GTM iframe
r#"(?s)<div[^>]*class="[^"]*partner-logo-overlay[^"]*"[^>]*>.*?</div>"#, // AAS logo 弹窗
] {
h = Regex::new(pattern).unwrap().replace_all(&h, "").to_string();
}
h
}
fn convert_headings(&self, html: &str) -> String {
let mut h = html.to_string();
// IOP 节标题：<h2 class="header-anchor">（内含 SVG 图标需跳过）
h = Regex::new(r#"(?s)<h2\s+[^>]*class="[^"]*header-anchor[^"]*"[^>]*>\s*(?:<span[^>]*><svg[^>]*></svg></span>)?\s*(.*?)\s*</h2>"#)
.unwrap().replace_all(&h, |caps: &regex::Captures| {
let inner = super::common::strip_html_tags(&caps[1]);
format!("\n\n## {}\n\n", inner.trim())
}).to_string();
// IOP 子节标题：<h3>
h = Regex::new(r#"(?s)<h3[^>]*>\s*(?:<span[^>]*><svg[^>]*></svg></span>)?\s*(.*?)\s*</h3>"#)
.unwrap().replace_all(&h, |caps: &regex::Captures| {
let inner = super::common::strip_html_tags(&caps[1]);
format!("\n\n### {}\n\n", inner.trim())
}).to_string();
h
}
fn convert_figures(&self, html: &str, _base_origin: &str) -> String {
// IOP 使用双层嵌套 <figure>，图片懒加载 (data-src)，图注含下载链接
let figure_re = Regex::new(
r#"(?s)<figure[^>]*class="[^"]*boxout[^"]*"[^>]*>\s*<figure[^>]*>\s*(.*?)\s*<figcaption[^>]*>\s*(.*?)\s*</figcaption>\s*</figure>\s*</figure>"#
).unwrap();
figure_re.replace_all(html, |caps: &regex::Captures| {
let inner_fig = &caps[1];
let caption_raw = &caps[2];
// 从 data-src 或 src 提取图片 URL
let src = Regex::new(r#"data-src="([^"]*)""#).unwrap()
.captures(inner_fig)
.map(|c| c[1].to_string())
.or_else(|| {
Regex::new(r#"src="([^"]*)""#).unwrap()
.captures(inner_fig)
.map(|c| c[1].to_string())
})
.unwrap_or_default();
// 跳过 base64 占位图
if src.starts_with("data:") {
return caps[0].to_string();
}
// 清理图注：去掉 <strong>Figure N.</strong> 标签和下载链接
let caption_text = Regex::new(r#"(?s)<strong[^>]*>.*?</strong>"#).unwrap()
.replace_all(caption_raw, "").to_string();
let caption_text = Regex::new(r#"(?s)<span[^>]*class="[^"]*btn-multi-block[^"]*"[^>]*>.*?</span>"#).unwrap()
.replace_all(&caption_text, "").to_string();
let caption_text = Regex::new(r#"(?s)<p[^>]*class="[^"]*print-hide[^"]*"[^>]*>.*?</p>"#).unwrap()
.replace_all(&caption_text, "").to_string();
let caption = super::common::strip_html_tags(&caption_text);
format!("\n\n![{}]({})\n\n", caption.trim(), src)
}).to_string()
}
fn clean_markers(&self, html: &str) -> String {
// 删除残留的 JavaScript
Regex::new(r#"(?s)<script[^>]*>.*?</script>"#).unwrap()
.replace_all(html, "").to_string()
}
}

256
src/services/parser/mod.rs Normal file
View File

@ -0,0 +1,256 @@
// JournalParser trait + 期刊自动检测 + HTML→Markdown 解析管线
mod ar5iv;
mod aanda;
mod iop;
mod generic;
pub mod common;
pub mod pdf;
use std::path::Path;
use tracing::info;
use ar5iv::Ar5ivParser;
use aanda::AandaParser;
use iop::IopParser;
use generic::GenericParser;
/// 期刊解析器 trait —— 每种期刊实现自己特有的预处理规则
trait JournalParser {
/// 解析器名称（用于日志）
fn name(&self) -> &str { "未知" }
/// 从完整 HTML 页面中提取正文区域
fn extract_body<'a>(&self, html: &'a str) -> &'a str;
/// 删除页面框架垃圾（导航、面包屑、元数据表格、JS、广告）
fn clean_chrome(&self, html: &str) -> String;
/// 清理期刊特有的标记（机构上标、页眉、引用链接格式）
fn clean_markers(&self, html: &str) -> String;
/// 将期刊特有的标题标记转换为 Markdown 标题
fn convert_headings(&self, html: &str) -> String;
/// 将期刊特有的图注/图片结构转换为 !\[caption\](url)
fn convert_figures(&self, html: &str, base_origin: &str) -> String;
}
/// 自动检测期刊类型并返回对应的解析器
/// 优先级：ar5iv > IOP > A&A > 通用回退
fn detect_parser(html: &str) -> Box<dyn JournalParser> {
if Ar5ivParser::detect(html) { return Box::new(Ar5ivParser); }
if IopParser::detect(html) { return Box::new(IopParser); }
if AandaParser::detect(html) { return Box::new(AandaParser); }
Box::new(GenericParser)
}
// 将 PDF/MinerU 函数重新导出到 parser 模块层级
pub use pdf::{submit_pdf_to_mineru, poll_and_extract_mineru, parse_pdf_via_mineru};
/// HTML → Markdown 核心解析管线
pub fn html_to_markdown(html_path: &Path) -> anyhow::Result<String> {
info!("正在解析 HTML → Markdown: {:?}", html_path);
let html_bytes = std::fs::read(html_path)?;
// Gzip 解压检测
let decompressed = if html_bytes.starts_with(&[0x1f, 0x8b]) {
use std::io::Read;
let mut decoder = flate2::read::GzDecoder::new(&html_bytes[..]);
let mut buf = Vec::new();
decoder.read_to_end(&mut buf)?;
buf
} else {
html_bytes
};
let html_content = String::from_utf8_lossy(&decompressed).into_owned();
let base_origin = common::extract_html_base_origin(&html_content);
// 截断页脚
let html = common::truncate_footer(&html_content);
// 自动检测期刊类型
let parser = detect_parser(html);
info!("检测到期刊类型: {}", parser.name());
// ── 阶段一：期刊特有预处理 ──
let body = parser.extract_body(html);
let html = parser.clean_chrome(body);
let html = parser.clean_markers(&html);
// ── 阶段二：公共预处理（实体解码 + 公式提取必须在标题转换之前） ──
let html = common::decode_html_numeric_entities(&html);
let (html, img_eqs) = common::extract_img_equations(&html);
let (html, math_formulas) = common::extract_math_blocks(&html);
// ── 阶段三：标题与图注转换（公式提取之后执行） ──
let html = parser.convert_headings(&html);
let html = parser.convert_figures(&html, &base_origin);
// 其余通用标记转换
let html = common::convert_common_markup(&html);
let html = common::convert_img_tags(&html, &base_origin);
let html = common::replace_latexml_tables(&html);
// 行内数学公式 → LaTeX 占位符
let (html, all_formulas) = common::convert_html_math_to_latex(html, math_formulas);
// ── 阶段四：核心 html2md 转换 ──
let mut markdown = html2md::parse_html(&html);
// ── 阶段五：公式占位符逆序还原 ──
markdown = common::restore_math_placeholders(&markdown, &all_formulas);
markdown = common::restore_img_equations(&markdown, &img_eqs);
// ── 阶段六：Markdown 后处理 ──
let cleaned = common::postprocess_markdown(&markdown);
Ok(cleaned)
}
// ── 测试 ──────────────────────────────────────────────────────────
#[cfg(test)]
mod tests {
use super::html_to_markdown;
use super::common::{postprocess_markdown, convert_html_tables_to_markdown};
use std::io::Write;
#[test]
fn test_convert_html_tables_to_markdown() {
let raw_md = "Here is a table:\n<table><tr><td>HJD</td><td>Phase</td></tr><tr><td>143.41</td><td>0.42</td></tr></table>\nEnd of table.";
let cleaned = postprocess_markdown(raw_md);
assert!(cleaned.contains("| HJD | Phase |"));
assert!(cleaned.contains("|---|---|"));
assert!(cleaned.contains("| 143.41 | 0.42 |"));
}
#[test]
fn test_convert_html_tables_with_merged_cells() {
let raw_md = "Here is a complex table:\n<table border=\"1\"><tr><td colspan=\"2\">Header</td></tr><tr><td>A</td><td>B</td></tr></table>\nEnd.";
let cleaned = postprocess_markdown(raw_md);
assert!(cleaned.contains("<table border=\"1\">"));
assert!(!cleaned.contains("| Header |"));
}
#[test]
fn test_postprocess_markdown() {
let dirty = "<div>Hello</div> <span class=\"abc\">World</span> <iframe src=\"abc\"></iframe> <astrobj>V391 Peg</astrobj> [] &lt;math&gt;\n\n\n\n\nNew Paragraph";
let cleaned = postprocess_markdown(dirty);
assert_eq!(cleaned, "Hello World V391 Peg <math>\n\n\nNew Paragraph");
let dirty_abstract = "###### Abstract\n\n[Abstract]\n\nHot subdwarfs are core helium burning stars.";
let cleaned_abstract = postprocess_markdown(dirty_abstract);
assert!(cleaned_abstract.contains("## Abstract\n\nHot subdwarfs are core"));
assert!(!cleaned_abstract.contains("[Abstract]"));
let with_ack = "Some content here.\n\n## Acknowledgments\n\nMany thanks to everyone.";
assert_eq!(postprocess_markdown(with_ack), "Some content here.");
let with_ref = "Main text content.\n\n## References\n\n1. Author A, 2024";
assert_eq!(postprocess_markdown(with_ref), "Main text content.");
let with_bib = "Main text.\n\n{thebibliography*}\n\n84\nAuthor C";
assert_eq!(postprocess_markdown(with_bib), "Main text.");
let with_inline_ack = "Main body.\n\nAcknowledgements. This work has been supported by...";
assert_eq!(postprocess_markdown(with_inline_ack), "Main body.");
// 图注与图片连行修复
let merged_fig = r"![alt](https://example.com/img.png) \> \*\*Figure:\*\* This is a caption.";
let fixed_fig = postprocess_markdown(merged_fig);
assert!(fixed_fig.contains("img.png)\n\n> **Figure:**"), "got: '{fixed_fig}'");
// #bib 链接剥离
let bib_txt = "([1976](#bib.bib16)) and [1984](#bib.bib30).";
let bib_fixed = postprocess_markdown(bib_txt);
assert!(!bib_fixed.contains("](#bib"));
assert!(bib_fixed.contains("(1976)"));
// 误报检测：正文中 "Reference objects" 标题不应被截断
let fp = "We compare with reference objects in Sect. 2.\n\n## 2.4 Reference objects\n\nText.";
let fixed = postprocess_markdown(fp);
assert!(fixed.contains("## 2.4 Reference objects"));
}
#[test]
fn test_html_to_markdown() -> anyhow::Result<()> {
let html = r#"<!DOCTYPE html><html><body><div class="ltx_page_main"><h1>Test</h1><p><strong>bold</strong></p></div></body></html>"#;
let mut path = std::env::temp_dir();
path.push("t1.html");
std::fs::write(&path, html)?;
let md = html_to_markdown(&path)?;
let _ = std::fs::remove_file(&path);
assert!(md.contains("Test"));
assert!(md.contains("**bold**"));
Ok(())
}
#[test]
fn test_math_and_table() -> anyhow::Result<()> {
let html = r#"<div class="ltx_page_main"><p><math alttext="\approx" display="inline"><mo>≈</mo></math></p><span class="ltx_tabular"><span class="ltx_tr"><span class="ltx_td">A</span></span></span></div>"#;
let mut path = std::env::temp_dir();
path.push("t2.html");
std::fs::write(&path, html)?;
let md = html_to_markdown(&path)?;
let _ = std::fs::remove_file(&path);
assert!(md.contains(r"$\approx$"));
assert!(md.contains("| A |"));
Ok(())
}
#[test]
fn test_math_in_headings() -> anyhow::Result<()> {
let html = r#"<div class="ltx_page_main"><h2 class="ltx_title_section">H <math alttext="\theta_{eff}" display="inline"><annotation>\theta_{eff}</annotation></math></h2><figcaption>F <math alttext="M_\odot" display="inline"><annotation>M_\odot</annotation></math></figcaption></div>"#;
let mut path = std::env::temp_dir();
path.push("t3.html");
std::fs::write(&path, html)?;
let md = html_to_markdown(&path)?;
let _ = std::fs::remove_file(&path);
assert!(md.contains("## H"));
assert!(md.contains(r"$\theta_{eff}$"));
assert!(md.contains(r"$M_\odot$"));
Ok(())
}
#[test]
fn test_aa_full_html_conversion() -> anyhow::Result<()> {
let html_path = std::path::Path::new("library/HTML/2026A&A...709A..52F.html");
if !html_path.exists() { eprintln!("Skip: file not found"); return Ok(()); }
let md = html_to_markdown(html_path)?;
assert!(!md.contains("var prefix"));
assert!(!md.contains("[Home](/"));
assert!(!md.contains("| Issue |"));
assert!(!md.contains("&#955;"));
assert!(md.trim_start().starts_with("A&A, 709, A52"));
assert!(md.contains("## Abstract"));
assert!(md.contains("## 1. Introduction"));
assert!(md.contains("### 2.1."));
assert!(md.contains("*Aims.*"));
assert!(md.contains("aanda.org/articles/"));
assert!(!md.contains("## Acknowledgments"));
Ok(())
}
#[test]
fn test_ar5iv_full_html_conversion() -> anyhow::Result<()> {
let html_path = std::path::Path::new("library/HTML/1103.1435.html");
if !html_path.exists() { eprintln!("Skip: file not found"); return Ok(()); }
let md = html_to_markdown(html_path)?;
assert!(md.contains("x1.png)\n\n> **Figure:**"));
assert!(!md.contains("[1]"));
assert!(!md.contains("class=\"ltx_ref\""));
assert!(!md.contains("pagerange:"));
assert!(!md.contains("<sup class="));
Ok(())
}
#[test]
fn bench_parse_speed() {
for (label, file) in [
("ar5iv", "library/HTML/1103.1435.html"),
("A&A", "library/HTML/2026A&A...709A..52F.html"),
("small", "library/HTML/0804.1287.html"),
] {
let path = std::path::Path::new(file);
if !path.exists() { continue; }
let start = std::time::Instant::now();
let _ = html_to_markdown(path).unwrap();
let ms = start.elapsed().as_secs_f64() * 1000.0;
println!(" {label:>6}: {ms:.0}ms");
}
}
}

309
src/services/parser/pdf.rs Normal file
View File

@ -0,0 +1,309 @@
// PDF 解析 —— 通过 MinerU 远程 API 提取 Markdown
use std::fs;
use std::path::Path;
use serde::{Deserialize, Serialize};
use tracing::{info, warn};
use regex::Regex;
use crate::Config;
use crate::clients::qiniu::QiniuClient;
// 复用公共后处理函数
use super::common::{convert_html_tables_to_markdown, cut_acknowledgments_and_references};
#[derive(Serialize)]
struct BatchUploadRequest {
files: Vec<PendingFile>,
language: String,
is_ocr: bool,
model_version: String,
}
#[derive(Serialize)]
struct PendingFile {
name: String,
data_id: String,
}
#[allow(dead_code)]
#[derive(Deserialize)]
struct BatchUploadResponse {
code: i32,
msg: String,
data: BatchUploadData,
}
#[derive(Deserialize)]
struct BatchUploadData {
batch_id: String,
file_urls: Vec<String>,
}
#[allow(dead_code)]
#[derive(Deserialize)]
struct BatchResultResponse {
code: i32,
msg: String,
data: Option<BatchResultData>,
}
#[allow(dead_code)]
#[derive(Deserialize)]
struct BatchResultData {
batch_id: String,
extract_result: Vec<ExtractResult>,
}
#[allow(dead_code)]
#[derive(Deserialize)]
struct ExtractResult {
file_name: String,
state: String,
full_zip_url: Option<String>,
err_msg: Option<String>,
}
// 调用 MinerU 远程接口解析 PDF，并在提取出图片后自动上传至七牛云进行外链替换
pub async fn submit_pdf_to_mineru(
pdf_path: &Path,
config: &Config
) -> anyhow::Result<String> {
info!("正在请求 MinerU 解析本地 PDF 文献: {:?}", pdf_path);
if config.mineru_api_url.is_empty() {
return Err(anyhow::anyhow!("未在环境变量 .env 中配置 MINERU_API_URL"));
}
let pdf_bytes = fs::read(pdf_path)?;
let filename = pdf_path.file_name()
.and_then(|f| f.to_str())
.unwrap_or("paper.pdf")
.to_string();
let bibcode = pdf_path.file_stem()
.and_then(|f| f.to_str())
.unwrap_or("paper")
.to_string();
// 提取 base_url
let base_url = config.mineru_api_url
.replace("/extract/task", "")
.replace("/extract", "")
.trim_end_matches('/')
.to_string();
let client = reqwest::Client::new();
let data_id = uuid::Uuid::new_v4().to_string();
// 1. 获取预签名上传 URL
info!("MinerU: 正在请求批量直传 URL (Bibcode: {})", bibcode);
let upload_req = BatchUploadRequest {
files: vec![PendingFile {
name: filename.clone(),
data_id: data_id.clone(),
}],
language: "en".to_string(),
is_ocr: true,
model_version: "vlm".to_string(),
};
let mut request = client.post(format!("{}/file-urls/batch/", base_url))
.json(&upload_req);
if !config.mineru_api_key.is_empty() {
request = request.header("Authorization", format!("Bearer {}", config.mineru_api_key));
}
let response = request.send().await?;
let status = response.status();
let res_text = response.text().await?;
if !status.is_success() {
return Err(anyhow::anyhow!("请求 MinerU 批量上传 URL 失败 (状态码: {}): {}", status, res_text));
}
let upload_res: BatchUploadResponse = serde_json::from_str(&res_text)?;
if upload_res.code != 0 {
return Err(anyhow::anyhow!("MinerU API 错误: {}", upload_res.msg));
}
let upload_url = upload_res.data.file_urls.first()
.ok_or_else(|| anyhow::anyhow!("MinerU 未返回上传 URL"))?;
// 2. 上传文件 (PUT)
info!("MinerU: 正在直接上传 PDF 字节流至对象存储...");
let put_res = client.put(upload_url)
.body(pdf_bytes)
.send()
.await?;
if !put_res.status().is_success() {
return Err(anyhow::anyhow!("上传 PDF 至 MinerU 对象存储直传 URL 失败: {}", put_res.status()));
}
let batch_id = upload_res.data.batch_id;
Ok(batch_id)
}
pub async fn poll_and_extract_mineru(
batch_id: &str,
bibcode: &str,
qiniu_client: &QiniuClient,
config: &Config
) -> anyhow::Result<String> {
let client = reqwest::Client::new();
let base_url = config.mineru_api_url
.replace("/extract/task", "")
.replace("/extract", "")
.trim_end_matches('/')
.to_string();
let mut poll_count = 0;
let max_polls = 45; // 45 * 10s = 7.5 min
info!("MinerU: 开始轮询任务结果 (Batch ID: {})...", batch_id);
let mut full_zip_url = String::new();
loop {
poll_count += 1;
if poll_count > max_polls {
return Err(anyhow::anyhow!("MinerU 结构化解析超时 (Bibcode: {})", bibcode));
}
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
let mut status_req = client.get(format!("{}/extract-results/batch/{}", base_url, batch_id));
if !config.mineru_api_key.is_empty() {
status_req = status_req.header("Authorization", format!("Bearer {}", config.mineru_api_key));
}
let status_res = status_req.send().await?;
let status_text = status_res.text().await?;
let result_data: BatchResultResponse = serde_json::from_str(&status_text)?;
if let Some(data) = result_data.data {
if let Some(file_result) = data.extract_result.first() {
match file_result.state.as_str() {
"done" => {
info!("MinerU: 解析成功！");
full_zip_url = file_result.full_zip_url.clone().unwrap_or_default();
break;
}
"error" | "failed" => {
let err_msg = file_result.err_msg.clone().unwrap_or_default();
return Err(anyhow::anyhow!("MinerU 批量解析任务失败: {}", err_msg));
}
other => {
info!("MinerU 任务处理中... 当前状态: {}", other);
}
}
} else {
return Err(anyhow::anyhow!("MinerU 轮询响应中未发现文件解析任务结果"));
}
} else {
return Err(anyhow::anyhow!("MinerU 轮询响应数据为空"));
}
}
if full_zip_url.is_empty() {
return Err(anyhow::anyhow!("MinerU 转换成功但未返回结果 ZIP 下载 URL"));
}
// 4. 下载并解压 ZIP
info!("MinerU: 正在下载最终提取压缩包: {}", full_zip_url);
let zip_bytes = client.get(&full_zip_url).send().await?.bytes().await?;
let reader = std::io::Cursor::new(zip_bytes);
let mut archive = zip::ZipArchive::new(reader)?;
let mut markdown = String::new();
let mut image_buffers = std::collections::HashMap::new();
for i in 0..archive.len() {
let mut file = archive.by_index(i)?;
let name = file.name().to_string();
if name.ends_with(".md") {
let mut md_content = String::new();
std::io::Read::read_to_string(&mut file, &mut md_content)?;
markdown = md_content;
} else if file.is_file() {
let lower = name.to_lowercase();
if lower.ends_with(".png") || lower.ends_with(".jpg") || lower.ends_with(".jpeg") || lower.ends_with(".gif") || lower.ends_with(".svg") {
let mut buf = Vec::new();
std::io::copy(&mut file, &mut buf)?;
let file_basename = Path::new(&name)
.file_name()
.and_then(|f| f.to_str())
.unwrap_or(&name)
.to_string();
image_buffers.insert(file_basename, buf);
}
}
}
if markdown.is_empty() {
return Err(anyhow::anyhow!("解析后的压缩包中未发现核心 Markdown 文档"));
}
// [DEBUG] 保存 MinerU 原始解析结果，便于调试管线后处理逻辑
{
let mineru_dir = config.library_dir.join("MinerU");
let _ = fs::create_dir_all(&mineru_dir);
let raw_path = mineru_dir.join(format!("{}.md", bibcode));
if let Err(e) = fs::write(&raw_path, &markdown) {
warn!("保存 MinerU 原始 Markdown 失败: {} ({})", raw_path.display(), e);
}
}
// 5. 上传图片并重写链接
if !image_buffers.is_empty() {
let local_img_dir = config.library_dir.join("images").join(bibcode);
let _ = fs::create_dir_all(&local_img_dir);
if qiniu_client.is_configured() {
info!("MinerU 批量模式解析出 {} 张本地插图。准备上传至七牛云...", image_buffers.len());
for (img_name, img_bytes) in image_buffers {
let local_path = local_img_dir.join(&img_name);
let _ = fs::write(&local_path, &img_bytes);
let qiniu_name = format!("{}/{}", bibcode, img_name);
match qiniu_client.upload_buffer(img_bytes, &qiniu_name).await {
Ok(qiniu_url) => {
let escaped_img_name = regex::escape(&img_name);
let link_re = Regex::new(&format!(r"\((?:[^)]*/)?{}\)", escaped_img_name)).unwrap();
markdown = link_re.replace_all(&markdown, |_: &regex::Captures| {
format!("({})", qiniu_url)
}).to_string();
}
Err(e) => warn!("上传图片至七牛云失败 {}: {}", img_name, e),
}
}
} else {
warn!("未检测到七牛云配置，解析出的图片将保存在本地 images 目录下");
for (img_name, img_bytes) in image_buffers {
let local_path = local_img_dir.join(&img_name);
let _ = fs::write(&local_path, &img_bytes);
let escaped_img_name = regex::escape(&img_name);
let link_re = Regex::new(&format!(r"\((?:[^)]*/)?{}\)", escaped_img_name)).unwrap();
let replacement_link = format!("(images/{}/{})", bibcode, img_name);
markdown = link_re.replace_all(&markdown, replacement_link.as_str()).to_string();
}
}
}
let markdown = convert_html_tables_to_markdown(&markdown);
let markdown = cut_acknowledgments_and_references(&markdown);
Ok(markdown)
}
pub async fn parse_pdf_via_mineru(
pdf_path: &Path,
qiniu_client: &QiniuClient,
config: &Config
) -> anyhow::Result<String> {
let bibcode = pdf_path.file_stem()
.and_then(|f| f.to_str())
.unwrap_or("paper")
.to_string();
let batch_id = submit_pdf_to_mineru(pdf_path, config).await?;
poll_and_extract_mineru(&batch_id, &bibcode, qiniu_client, config).await
}

270
src/services/rag.rs Normal file
View File

@ -0,0 +1,270 @@
// src/services/rag.rs
//
// RAG (Retrieval-Augmented Generation) 问答编排模块。
// 职责：
// 1. Ingest: 将文献 Markdown 切片 -> 向量化 -> 写入 SQLite vec0 + content 双表
// 2. Retrieve: 向量化用户提问 -> 相似度检索 -> 返回 Top-K 片段
// 3. Complete: 组装检索上下文 -> 调用 LLM 生成最终回答
use sqlx::SqlitePool;
use tracing::{info, warn, error};
use crate::clients::llm::EmbeddingClient;
use crate::services::chunker::{chunk_markdown, TextChunk};
/// RAG 检索结果条目
#[derive(Debug, Clone, serde::Serialize)]
pub struct RetrievalResult {
/// 来源文献 bibcode
pub bibcode: String,
/// 原文段落索引
pub paragraph_index: i64,
/// 匹配到的文本内容
pub content: String,
/// 向量距离（越小越相关）
pub distance: f64,
}
/// RAG 问答最终结果
#[derive(Debug, Clone, serde::Serialize)]
pub struct RagAnswer {
/// LLM 生成的回答
pub answer: String,
/// 检索到的参考来源
pub sources: Vec<RetrievalResult>,
}
// ─────────────────────── Ingest ───────────────────────
/// 将一篇文献的 Markdown 内容切片、向量化并写入数据库
///
/// 流程：
/// 1. 使用 chunker 对 Markdown 文本进行安全切片
/// 2. 逐片调用 EmbeddingClient 获取向量
/// 3. 在事务中同步插入 paper_chunks_content (文本) 和 vec_paper_chunks (向量)
pub async fn ingest_paper(
pool: &SqlitePool,
embedding_client: &EmbeddingClient,
bibcode: &str,
markdown_text: &str,
chunk_size: Option<usize>,
) -> anyhow::Result<usize> {
let chunks: Vec<TextChunk> = chunk_markdown(markdown_text, chunk_size);
if chunks.is_empty() {
warn!("文献 {} 切片后无有效内容，跳过向量化", bibcode);
return Ok(0);
}
info!(
"文献 {} 共切片 {} 块，开始向量化写入...",
bibcode,
chunks.len()
);
// 先清除该文献的旧切片（重新解析场景）
// 因为 vec0 虚拟表没有 FK，需要先查出旧的 rowid 再手动删除
let old_rowids: Vec<(i64,)> = sqlx::query_as(
"SELECT rowid FROM paper_chunks_content WHERE bibcode = ?"
)
.bind(bibcode)
.fetch_all(pool)
.await?;
if !old_rowids.is_empty() {
info!("清除文献 {} 的 {} 条旧切片", bibcode, old_rowids.len());
for (rid,) in &old_rowids {
sqlx::query("DELETE FROM vec_paper_chunks WHERE rowid = ?")
.bind(rid)
.execute(pool)
.await?;
}
sqlx::query("DELETE FROM paper_chunks_content WHERE bibcode = ?")
.bind(bibcode)
.execute(pool)
.await?;
}
let mut ingested_count = 0;
for chunk in &chunks {
// 获取向量
let embedding = match embedding_client.create_embedding(&chunk.content).await {
Ok(vec) => vec,
Err(e) => {
error!(
"文献 {} 第 {} 块向量化失败: {}",
bibcode, chunk.paragraph_index, e
);
continue;
}
};
// 插入 content 表获得 rowid
let result = sqlx::query(
"INSERT INTO paper_chunks_content (bibcode, paragraph_index, content) VALUES (?, ?, ?)"
)
.bind(bibcode)
.bind(chunk.paragraph_index as i64)
.bind(&chunk.content)
.execute(pool)
.await?;
let rowid = result.last_insert_rowid();
// 将 f32 向量转换为字节切片，插入 vec0 虚拟表
let embedding_bytes = embedding_to_bytes(&embedding);
sqlx::query(
"INSERT INTO vec_paper_chunks (rowid, embedding) VALUES (?, ?)"
)
.bind(rowid)
.bind(&embedding_bytes)
.execute(pool)
.await?;
ingested_count += 1;
}
info!(
"文献 {} 向量化完成，成功写入 {}/{} 块",
bibcode, ingested_count, chunks.len()
);
Ok(ingested_count)
}
// ─────────────────────── Retrieve ───────────────────────
/// 向量化用户提问并检索 Top-K 相似片段
pub async fn retrieve(
pool: &SqlitePool,
embedding_client: &EmbeddingClient,
question: &str,
top_k: usize,
) -> anyhow::Result<Vec<RetrievalResult>> {
// 1. 向量化用户提问
let query_embedding = embedding_client.create_embedding(question).await?;
let query_bytes = embedding_to_bytes(&query_embedding);
// 2. 执行相似度检索
let rows: Vec<(String, i64, String, f64)> = sqlx::query_as(
"SELECT c.bibcode, c.paragraph_index, c.content, v.distance \
FROM vec_paper_chunks v \
INNER JOIN paper_chunks_content c ON v.rowid = c.rowid \
WHERE v.embedding MATCH ? AND k = ? \
ORDER BY v.distance"
)
.bind(&query_bytes)
.bind(top_k as i64)
.fetch_all(pool)
.await?;
let results: Vec<RetrievalResult> = rows
.into_iter()
.map(|(bibcode, paragraph_index, content, distance)| RetrievalResult {
bibcode,
paragraph_index,
content,
distance,
})
.collect();
info!("RAG 检索完成，返回 {} 条结果", results.len());
Ok(results)
}
// ─────────────────────── Complete ───────────────────────
/// 执行完整的 RAG 问答流程：检索 + LLM 生成
pub async fn ask(
pool: &SqlitePool,
embedding_client: &EmbeddingClient,
llm_client: &crate::clients::llm::LlmClient,
question: &str,
top_k: usize,
) -> anyhow::Result<RagAnswer> {
// 1. 检索相关片段
let sources = retrieve(pool, embedding_client, question, top_k).await?;
if sources.is_empty() {
return Ok(RagAnswer {
answer: "未找到与该问题相关的文献内容。请先导入相关文献的 Markdown 版本。".to_string(),
sources: vec![],
});
}
// 2. 组装上下文
let context = sources
.iter()
.enumerate()
.map(|(i, r)| {
format!(
"[来源 {}: {} §{}]\n{}",
i + 1,
r.bibcode,
r.paragraph_index,
r.content
)
})
.collect::<Vec<_>>()
.join("\n\n---\n\n");
// 3. 构建 System Prompt
let system_prompt = "你是一位专业的天体物理学研究助手。\
请根据以下从用户文献库中检索到的参考片段，用中文准确回答用户的问题。\
回答时请标注引用来源（使用 [来源 N] 格式），以便用户溯源核查。\
如果检索到的内容不足以回答问题，请如实说明并建议用户补充相关文献。\
注意保持科学术语的准确性。";
let user_content = format!(
"## 检索到的参考片段\n\n{}\n\n---\n\n## 用户问题\n\n{}",
context, question
);
// 4. 调用 LLM
let answer = llm_client
.chat_completion(system_prompt, &user_content)
.await?;
Ok(RagAnswer { answer, sources })
}
// ─────────────────────── Helpers ───────────────────────
/// 将 f32 向量转换为字节切片（小端序），用于 sqlite-vec 的二进制格式
fn embedding_to_bytes(embedding: &[f32]) -> Vec<u8> {
let mut bytes = Vec::with_capacity(embedding.len() * 4);
for &val in embedding {
bytes.extend_from_slice(&val.to_le_bytes());
}
bytes
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_embedding_to_bytes_roundtrip() {
let original = vec![1.0f32, 2.5, -3.14, 0.0];
let bytes = embedding_to_bytes(&original);
assert_eq!(bytes.len(), 16); // 4 floats × 4 bytes
// 反转验证
let recovered: Vec<f32> = bytes
.chunks_exact(4)
.map(|chunk| f32::from_le_bytes(chunk.try_into().unwrap()))
.collect();
assert_eq!(original.len(), recovered.len());
for (a, b) in original.iter().zip(recovered.iter()) {
assert!((a - b).abs() < f32::EPSILON);
}
}
#[test]
fn test_embedding_to_bytes_empty() {
let bytes = embedding_to_bytes(&[]);
assert!(bytes.is_empty());
}
}

381
src/services/target.rs Normal file
View File

@ -0,0 +1,381 @@
// src/services/target.rs
//
// 天体目标识别与 CDS SIMBAD/Sesame 信息缓存模块。
// 职责：
// 1. 使用 IAU 天体命名正则从文本中提取天体标识符
// 2. 通过 CDS Sesame Name Resolver 查询标准化天体属性
// 3. 在本地 paper_targets 表中缓存查询结果，避免重复外部请求
use regex::Regex;
use reqwest::Client;
use sqlx::SqlitePool;
use tracing::{info, warn, error};
use std::sync::OnceLock;
/// 天体目标的标准化属性信息
#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
pub struct TargetInfo {
pub target_name: String,
pub ra: Option<String>,
pub dec: Option<String>,
pub parallax: Option<f64>,
pub spectral_type: Option<String>,
pub v_magnitude: Option<f64>,
pub aliases: Vec<String>,
}
/// 从文本中提取可能的天体标识符
///
/// 支持的星表命名规范（基于 IAU 标准）:
/// - NGC / IC / Messier: `NGC 1234`, `IC 4567`, `M 31`
/// - Henry Draper: `HD 209458`
/// - Hipparcos: `HIP 12345`
/// - Gaia: `Gaia DR3 12345678`
/// - SDSS: `SDSS J123456.78+012345.6`
/// - White Dwarf: `WD 1234+567` 或 `WD J1234+5678`
/// - Variable Stars: `V* GD 358`, `GD 358`
/// - 2MASS / WISE: `2MASS J12345678+1234567`
/// - Bayer/Flamsteed: `Alpha Cen`, `61 Cyg`
/// - General catalogue: `TYC 1234-567-1`, `KIC 12345678`
pub fn extract_targets(text: &str) -> Vec<String> {
static PATTERNS: OnceLock<Vec<Regex>> = OnceLock::new();
let patterns = PATTERNS.get_or_init(|| {
// 编译所有正则，失败则 panic（仅在首次初始化时）
[
// NGC / IC 天体
r"(?i)\b(NGC|IC)\s*\d{1,5}\b",
// Messier 天体
r"(?i)\bM\s*\d{1,3}\b",
// Henry Draper 星表
r"(?i)\bHD\s*\d{3,6}\b",
// Hipparcos 星表
r"(?i)\bHIP\s*\d{3,6}\b",
// Gaia 星表
r"(?i)\bGaia\s+DR[23]\s+\d+\b",
// SDSS 天体（带坐标编号的名称）
r"(?i)\bSDSS\s*J\d{6}[\.\d]*[+-]\d{4,6}[\.\d]*\b",
// 白矮星编号
r"(?i)\bWD\s*J?\d{4}[+-]\d{2,4}\b",
// GD 白矮星系列
r"(?i)\bGD\s*\d{1,4}\b",
// 2MASS 红外源
r"(?i)\b2MASS\s*J\d{8}[+-]\d{7}\b",
// WISE 红外源
r"(?i)\bWISE\s*J?\d{4,6}[\.\d]*[+-]\d{4,6}[\.\d]*\b",
// TYC (Tycho) 星表
r"(?i)\bTYC\s*\d{1,4}-\d{1,5}-\d\b",
// KIC (Kepler Input Catalog)
r"(?i)\bKIC\s*\d{5,9}\b",
// TIC (TESS Input Catalog)
r"(?i)\bTIC\s*\d{5,10}\b",
// PSR (Pulsar)
r"(?i)\bPSR\s*[BJ]?\d{4}[+-]\d{2,4}\b",
// 通用 V* 变星标记
r"(?i)\bV\*\s+\w+\s+\w+\b",
]
.iter()
.map(|p| Regex::new(p).expect("Invalid target regex pattern"))
.collect()
});
let mut results = Vec::new();
let mut seen = std::collections::HashSet::new();
for re in patterns {
for mat in re.find_iter(text) {
let name = normalize_target_name(mat.as_str());
if seen.insert(name.clone()) {
results.push(name);
}
}
}
results
}
/// 标准化天体名称：统一大小写并规范空白格式（如 "NGC6752" / "ngc 6752" 标准化为 "NGC 6752"）
fn normalize_target_name(raw: &str) -> String {
let raw_trimmed = raw.trim();
// 匹配类似 NGC 6752, NGC6752, M31, M 31 等常见星表格式
static RE_NORM: OnceLock<Regex> = OnceLock::new();
let re_norm = RE_NORM.get_or_init(|| {
Regex::new(r"(?i)^(NGC|IC|M|HD|HIP|Gaia|WD|GD|TYC|KIC|TIC|PSR)\s*(\d+.*)$").unwrap()
});
if let Some(caps) = re_norm.captures(raw_trimmed) {
let prefix = caps.get(1).unwrap().as_str().to_uppercase();
let number = caps.get(2).unwrap().as_str().trim();
// 针对 "Gaia" 等特殊前缀处理大小写
let prefix_formatted = if prefix == "GAIA" {
"Gaia".to_string()
} else {
prefix
};
format!("{} {}", prefix_formatted, number)
} else {
// 如果不能识别为常见格式，降级为原样去除多余空白
raw_trimmed
.split_whitespace()
.collect::<Vec<&str>>()
.join(" ")
}
}
/// 通过 CDS Sesame Name Resolver 查询天体的标准化信息
///
/// Sesame 文档: https://cds.unistra.fr/cgi-bin/nph-sesame
/// 返回包含坐标 (RA/Dec)、光谱类型等的结构化数据
pub async fn query_sesame(target_name: &str, client: &Client) -> anyhow::Result<TargetInfo> {
let url = format!(
"https://cds.unistra.fr/cgi-bin/nph-sesame/-ox/SNV?{}",
urlencoding::encode(target_name)
);
info!("正在查询 CDS Sesame: {}", target_name);
let response = client
.get(&url)
.header("User-Agent", "AstroResearch/0.1 (academic research tool)")
.send()
.await?;
if !response.status().is_success() {
return Err(anyhow::anyhow!(
"Sesame 查询失败: HTTP {}",
response.status()
));
}
let xml_text = response.text().await?;
parse_sesame_xml(&xml_text, target_name)
}
/// 解析 Sesame XML 响应
fn parse_sesame_xml(xml: &str, original_name: &str) -> anyhow::Result<TargetInfo> {
// Sesame 返回的 XML 结构比较简单，我们用正则提取关键字段
// 避免引入完整的 XML 反序列化依赖
let ra = extract_xml_value(xml, "jradeg");
let dec = extract_xml_value(xml, "jdedeg");
let spectral_type = extract_xml_value(xml, "spType")
.or_else(|| extract_xml_value(xml, "sp"));
let v_mag = extract_xml_value(xml, "Vmag")
.and_then(|v| v.parse::<f64>().ok());
let parallax = extract_xml_value(xml, "plx")
.and_then(|v| v.parse::<f64>().ok());
// 提取别名列表
let mut aliases = Vec::new();
let alias_re = Regex::new(r"<alias>([^<]+)</alias>").unwrap();
for cap in alias_re.captures_iter(xml) {
if let Some(alias) = cap.get(1) {
aliases.push(alias.as_str().trim().to_string());
}
}
// 如果连坐标都查不到，说明 Sesame 无法识别这个天体
if ra.is_none() && dec.is_none() && aliases.is_empty() {
warn!("Sesame 未能识别天体: {}", original_name);
}
Ok(TargetInfo {
target_name: original_name.to_string(),
ra,
dec,
parallax,
spectral_type,
v_magnitude: v_mag,
aliases,
})
}
/// 从 XML 文本中提取指定标签的文本内容
fn extract_xml_value(xml: &str, tag: &str) -> Option<String> {
let pattern = format!(r"<{tag}>([^<]+)</{tag}>");
Regex::new(&pattern)
.ok()?
.captures(xml)?
.get(1)
.map(|m| m.as_str().trim().to_string())
}
/// 查询天体信息，优先使用本地缓存
///
/// 流程：
/// 1. 在 `paper_targets` 表中查找本地缓存
/// 2. 若 miss，调用 Sesame API 并写入缓存
/// 3. 返回查询结果
pub async fn query_target_cached(
pool: &SqlitePool,
target_name: &str,
bibcode: Option<&str>,
client: &Client,
) -> anyhow::Result<TargetInfo> {
// 1. 查本地缓存
let row = sqlx::query_as::<_, (String, Option<String>, Option<String>, Option<f64>, Option<String>, Option<f64>, Option<String>)>(
"SELECT target_name, ra, dec, parallax, spectral_type, v_magnitude, aliases FROM paper_targets WHERE target_name = ? LIMIT 1"
)
.bind(target_name)
.fetch_optional(pool)
.await?;
if let Some((name, ra, dec, parallax, spectral_type, v_magnitude, aliases_json)) = row {
info!("天体 {} 命中本地缓存", target_name);
let aliases: Vec<String> = aliases_json
.and_then(|s| serde_json::from_str(&s).ok())
.unwrap_or_default();
return Ok(TargetInfo {
target_name: name,
ra,
dec,
parallax,
spectral_type,
v_magnitude,
aliases,
});
}
// 2. 缓存 Miss，调用 Sesame
info!("天体 {} 本地缓存未命中，查询 CDS Sesame...", target_name);
// 速率节流：50ms 间隔
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let info = query_sesame(target_name, client).await?;
// 3. 写入缓存
if let Some(bib) = bibcode {
let aliases_json = serde_json::to_string(&info.aliases).unwrap_or_else(|_| "[]".to_string());
if let Err(e) = sqlx::query(
"INSERT OR IGNORE INTO paper_targets (bibcode, target_name, ra, dec, parallax, spectral_type, v_magnitude, aliases) VALUES (?, ?, ?, ?, ?, ?, ?, ?)"
)
.bind(bib)
.bind(&info.target_name)
.bind(&info.ra)
.bind(&info.dec)
.bind(info.parallax)
.bind(&info.spectral_type)
.bind(info.v_magnitude)
.bind(&aliases_json)
.execute(pool)
.await {
error!("天体信息写入缓存失败: {}", e);
}
}
Ok(info)
}
/// 批量提取文本中的天体并逐一查询缓存
pub async fn extract_and_cache_targets(
pool: &SqlitePool,
text: &str,
bibcode: &str,
client: &Client,
) -> Vec<TargetInfo> {
let names = extract_targets(text);
let mut results = Vec::new();
for name in names {
match query_target_cached(pool, &name, Some(bibcode), client).await {
Ok(info) => results.push(info),
Err(e) => warn!("天体 {} 查询失败: {}", name, e),
}
}
results
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_extract_ngc() {
let text = "We observed NGC 1234 and NGC 5678 in the survey.";
let targets = extract_targets(text);
assert!(targets.contains(&"NGC 1234".to_string()));
assert!(targets.contains(&"NGC 5678".to_string()));
}
#[test]
fn test_extract_messier() {
let text = "M 31 is the Andromeda Galaxy.";
let targets = extract_targets(text);
assert!(targets.contains(&"M 31".to_string()));
}
#[test]
fn test_extract_hd() {
let text = "The star HD 209458 hosts a transiting planet.";
let targets = extract_targets(text);
assert!(targets.contains(&"HD 209458".to_string()));
}
#[test]
fn test_extract_gd() {
let text = "GD 358 is a well-known pulsating white dwarf.";
let targets = extract_targets(text);
assert!(targets.contains(&"GD 358".to_string()));
}
#[test]
fn test_extract_wd() {
let text = "WD 1145+017 shows transit dips.";
let targets = extract_targets(text);
assert!(targets.contains(&"WD 1145+017".to_string()));
}
#[test]
fn test_extract_gaia() {
let text = "Gaia DR3 1234567890 provides precise astrometry.";
let targets = extract_targets(text);
assert!(targets.contains(&"Gaia DR3 1234567890".to_string()));
}
#[test]
fn test_extract_kic() {
let text = "KIC 8462852 (Tabby's Star) shows unusual dimming.";
let targets = extract_targets(text);
assert!(targets.contains(&"KIC 8462852".to_string()));
}
#[test]
fn test_no_duplicates() {
let text = "NGC 1234 was observed. We also studied NGC 1234 again.";
let targets = extract_targets(text);
let ngc_count = targets.iter().filter(|t| *t == "NGC 1234").count();
assert_eq!(ngc_count, 1, "Should not have duplicates");
}
#[test]
fn test_empty_text() {
let targets = extract_targets("");
assert!(targets.is_empty());
}
#[test]
fn test_parse_sesame_xml_basic() {
let xml = r#"
<Sesame>
<Target>
<Resolver name="S=Simbad">
<jradeg>100.123</jradeg>
<jdedeg>-20.456</jdedeg>
<spType>DA1</spType>
<Vmag>13.65</Vmag>
<alias>GD 358</alias>
<alias>WD 1636+092</alias>
</Resolver>
</Target>
</Sesame>
"#;
let info = parse_sesame_xml(xml, "GD 358").unwrap();
assert_eq!(info.target_name, "GD 358");
assert_eq!(info.ra.as_deref(), Some("100.123"));
assert_eq!(info.dec.as_deref(), Some("-20.456"));
assert_eq!(info.spectral_type.as_deref(), Some("DA1"));
assert_eq!(info.v_magnitude, Some(13.65));
assert_eq!(info.aliases.len(), 2);
}
}

75
src/services/translation.rs
View File

@ -37,7 +37,8 @@ impl Dictionary {
let parts: Vec<&str> = line.split('\t').collect(); let parts: Vec<&str> = line.split('\t').collect();
if parts.len() >= 2 { if parts.len() >= 2 {
let english = parts[0].trim().to_lowercase(); let english = parts[0].trim().to_lowercase();
let chinese = parts[1].trim().to_string(); let chinese_raw = parts[1].trim();
let chinese = clean_chinese_translation(chinese_raw);
if !english.is_empty() && !chinese.is_empty() { if !english.is_empty() && !chinese.is_empty() {
self.terms.insert(english, chinese); self.terms.insert(english, chinese);
count += 1; count += 1;
@ -64,12 +65,16 @@ impl Dictionary {
let words: Vec<&str> = clean_text.split_whitespace().collect(); let words: Vec<&str> = clean_text.split_whitespace().collect();
let mut matched = HashSet::new(); let mut matched = HashSet::new();
let mut results = Vec::new(); let mut results = Vec::new();
let mut matched_indices = HashSet::new();
// 天文学词条跨度最大限制（一般多词短语不超过 6 个英文单词） // 天文学词条跨度最大限制（一般多词短语不超过 6 个英文单词）
let max_span = 6; let max_span = 6;
let n = words.len(); let n = words.len();
for i in 0..n { for i in 0..n {
if matched_indices.contains(&i) {
continue;
}
for len in (1..=max_span).rev() { for len in (1..=max_span).rev() {
if i + len <= n { if i + len <= n {
let phrase_slice = &words[i..i + len]; let phrase_slice = &words[i..i + len];
@ -82,8 +87,12 @@ impl Dictionary {
let original_phrase = &words[i..i + len].join(" "); let original_phrase = &words[i..i + len].join(" ");
results.push((original_phrase.clone(), chinese.clone())); results.push((original_phrase.clone(), chinese.clone()));
matched.insert(phrase); matched.insert(phrase.clone());
} }
for k in i..i + len {
matched_indices.insert(k);
}
break; // 匹配到当前位置的最长词，标记后跳出对当前i的其他长度匹配尝试
} }
} }
} }
@ -95,6 +104,53 @@ impl Dictionary {
} }
} }
// 辅助清理中文译名中的“全称/缩写”等说明性前缀，只保留实际译名
fn clean_chinese_translation(raw: &str) -> String {
let raw_trimmed = raw.trim();
if raw_trimmed.starts_with("1、") || raw_trimmed.starts_with("1. ") || raw_trimmed.starts_with("1.") {
let mut parts = Vec::new();
for part in raw_trimmed.split(|c| c == ';' || c == '；') {
let part_trimmed = part.trim();
if let Some(pos) = part_trimmed.rfind('。') {
let clean = part_trimmed[pos + '。'.len_utf8()..].trim().to_string();
if !clean.is_empty() {
parts.push(clean);
}
} else {
let mut clean = part_trimmed.to_string();
if let Some(pos) = clean.find('、') {
clean = clean[pos + '、'.len_utf8()..].trim().to_string();
} else if let Some(pos) = clean.find('.') {
if clean[..pos].chars().all(|c| c.is_digit(10)) {
clean = clean[pos + 1..].trim().to_string();
}
}
if clean.contains("全称：") || clean.contains("缩写：") {
if let Some(colon_pos) = clean.find('：') {
clean = clean[colon_pos + '：'.len_utf8()..].trim().to_string();
}
}
if !clean.is_empty() {
parts.push(clean);
}
}
}
parts.join("; ")
} else {
if let Some(pos) = raw_trimmed.rfind('。') {
raw_trimmed[pos + '。'.len_utf8()..].trim().to_string()
} else {
let mut clean = raw_trimmed.to_string();
if clean.contains("全称：") || clean.contains("缩写：") {
if let Some(colon_pos) = clean.find('：') {
clean = clean[colon_pos + '：'.len_utf8()..].trim().to_string();
}
}
clean
}
}
}
// 提取文献专业天文词对照提示词，调用 LLM 大模型进行保留公式的高精度学术翻译 // 提取文献专业天文词对照提示词，调用 LLM 大模型进行保留公式的高精度学术翻译
pub async fn translate_markdown( pub async fn translate_markdown(
markdown_content: &str, markdown_content: &str,
@ -121,7 +177,8 @@ pub async fn translate_markdown(
要求：\n\ 要求：\n\
1. 翻译风格必须专业、准确、符合天文学学术规范。\n\ 1. 翻译风格必须专业、准确、符合天文学学术规范。\n\
2. **务必完整保留所有的 LaTeX 数学公式（如 $...$ 或 $$...$$）和 Markdown 排版格式（如标题、粗体、列表、图片链接等），不要翻译公式内的字符。**\n\ 2. **务必完整保留所有的 LaTeX 数学公式（如 $...$ 或 $$...$$）和 Markdown 排版格式（如标题、粗体、列表、图片链接等），不要翻译公式内的字符。**\n\
3. 保持译文段落结构与原文一一对应。{}\n\ 3. 保持译文段落结构与原文一一对应。\n\
4. 如果文献中包含以 <table>、<tr> 等包裹的原生 HTML 表格结构，请完整保留所有 HTML 表格标签和结构属性（包括 colspan、rowspan 等），仅翻译单元格内部的英文文本。{}\n\
请开始你的翻译工作：", 请开始你的翻译工作：",
terms_instruction terms_instruction
); );
@ -160,10 +217,22 @@ mod tests {
assert!(phrases.contains(&"active galactic nucleus".to_string())); assert!(phrases.contains(&"active galactic nucleus".to_string()));
assert!(phrases.contains(&"black hole".to_string())); assert!(phrases.contains(&"black hole".to_string()));
// 验证已经匹配了更长名词的子词 (如 'galactic nucleus' 和 'nucleus') 被成功过滤去掉了，不重复提取
assert!(!phrases.contains(&"galactic nucleus".to_string()));
assert!(!phrases.contains(&"nucleus".to_string()));
// 验证最长的在前面 // 验证最长的在前面
assert_eq!(matched[0].0, "active galactic nucleus"); assert_eq!(matched[0].0, "active galactic nucleus");
} }
#[test]
fn test_clean_chinese_translation_helper() {
assert_eq!(clean_chinese_translation("全称：Transiting Exoplanet Survey Satellite。凌星系外行星巡天卫星"), "凌星系外行星巡天卫星");
assert_eq!(clean_chinese_translation("1、全称：Anglo-Australian Observatory。英澳天文台; 2、全称：Australian Astronomical Observatory。澳大利亚天文台"), "英澳天文台; 澳大利亚天文台");
assert_eq!(clean_chinese_translation("缩写：TESS。凌星系外行星巡天卫星"), "凌星系外行星巡天卫星");
assert_eq!(clean_chinese_translation("黑洞"), "黑洞");
}
#[test] #[test]
fn test_load_from_file() -> anyhow::Result<()> { fn test_load_from_file() -> anyhow::Result<()> {
let mut path = std::env::temp_dir(); let mut path = std::env::temp_dir();

60
tests/live_llm_test.rs
View File

@ -1,60 +0,0 @@
// tests/live_llm_test.rs
use astroresearch::Config;
use astroresearch::clients::llm::{LlmClient, EmbeddingClient};
#[tokio::test]
async fn test_live_llm_and_embedding() -> anyhow::Result<()> {
let config = Config::from_env();
println!("================= 开始大模型与向量模型真实网络集成测试 =================");
// 1. 测试 LlmClient
if config.llm_api_key.is_empty() {
println!("警告: 未在环境配置中检测到 LLM_API_KEY，跳过 LlmClient 集成测试。");
} else {
println!("测试大模型: {} (API Base: {})", config.llm_model, config.llm_api_base);
let llm = LlmClient::new(
config.llm_api_key.clone(),
config.llm_api_base.clone(),
config.llm_model.clone(),
);
match llm.chat_completion("You are a helpful assistant.", "Say Hello!").await {
Ok(reply) => {
println!("LlmClient 响应成功: {}", reply.trim());
assert!(!reply.trim().is_empty(), "错误: 大模型返回了空响应");
}
Err(e) => {
panic!("LlmClient 接口调用失败: {}", e);
}
}
}
// 2. 测试 EmbeddingClient
if config.embedding_api_key.is_empty() {
println!("警告: 未在环境配置中检测到 EMBEDDING_API_KEY，跳过 EmbeddingClient 集成测试。");
} else {
println!("测试向量模型: {} (API Base: {})", config.embedding_model, config.embedding_api_base);
let embedding_client = EmbeddingClient::new(
config.embedding_api_key.clone(),
config.embedding_api_base.clone(),
config.embedding_model.clone(),
);
let test_text = "active galactic nucleus";
match embedding_client.create_embedding(test_text).await {
Ok(vector) => {
println!("EmbeddingClient 响应成功！向量维度: {}", vector.len());
assert!(!vector.is_empty(), "错误: 向量数据为空");
let preview_len = std::cmp::min(5, vector.len());
println!("前 {} 个向量数值样例: {:?}", preview_len, &vector[..preview_len]);
}
Err(e) => {
panic!("EmbeddingClient 接口调用失败: {}", e);
}
}
}
println!("================= 大模型与向量模型真实网络集成测试完成 =================");
Ok(())
}

65
tests/live_search_test.rs
View File

@ -1,65 +0,0 @@
// tests/live_search_test.rs
use astroresearch::Config;
use astroresearch::clients::ads::AdsClient;
use astroresearch::clients::arxiv::ArxivClient;
#[tokio::test]
async fn test_live_search_comparisons() -> anyhow::Result<()> {
// 载入配置以获取 ADS_API_KEY
let config = Config::from_env();
if config.ads_api_key.is_empty() {
println!("警告: 未在环境配置中检测到 ADS_API_KEY，跳过 ADS 集成测试。");
}
let ads = AdsClient::new(config.ads_api_key.clone());
let arxiv = ArxivClient::new();
println!("================= 开始真实检索逻辑集成测试 =================");
// ----------------------------------------
// 测试 1: 比较 "OR" 与 "AND" 逻辑的数据量差异
// ----------------------------------------
let query_or = "\"hot subdwarf\" OR Gaia";
let query_and = "\"hot subdwarf\" AND Gaia";
if !config.ads_api_key.is_empty() {
let count_ads_or = ads.get_total_count(query_or).await?;
let count_ads_and = ads.get_total_count(query_and).await?;
println!("NASA ADS 平台:");
println!(" - (OR) \"{}\" 匹配数: {} 篇", query_or, count_ads_or);
println!(" - (AND) \"{}\" 匹配数: {} 篇", query_and, count_ads_and);
assert!(count_ads_or > count_ads_and, "错误: OR 结果应该多于 AND");
}
let count_arxiv_or = arxiv.get_total_count(query_or).await?;
let count_arxiv_and = arxiv.get_total_count(query_and).await?;
println!("arXiv 平台:");
println!(" - (OR) \"{}\" 匹配数: {} 篇", query_or, count_arxiv_or);
println!(" - (AND) \"{}\" 匹配数: {} 篇", query_and, count_arxiv_and);
assert!(count_arxiv_or > count_arxiv_and, "错误: arXiv 的 OR 结果应该多于 AND");
// ----------------------------------------
// 测试 2: 比较基础词组与含有 "NOT" 排除条件的数据量差异
// ----------------------------------------
let query_base = "\"hot subdwarf\"";
let query_not = "\"hot subdwarf\" NOT \"white dwarf\"";
if !config.ads_api_key.is_empty() {
let count_ads_base = ads.get_total_count(query_base).await?;
let count_ads_not = ads.get_total_count(query_not).await?;
println!("NASA ADS 平台:");
println!(" - (基础) \"{}\" 匹配数: {} 篇", query_base, count_ads_base);
println!(" - (排除) \"{}\" 匹配数: {} 篇", query_not, count_ads_not);
assert!(count_ads_base >= count_ads_not, "错误: 基础结果应该大于或等于排除后的结果");
}
let count_arxiv_base = arxiv.get_total_count(query_base).await?;
let count_arxiv_not = arxiv.get_total_count(query_not).await?;
println!("arXiv 平台:");
println!(" - (基础) \"{}\" 匹配数: {} 篇", query_base, count_arxiv_base);
println!(" - (排除) \"{}\" 匹配数: {} 篇", query_not, count_arxiv_not);
assert!(count_arxiv_base >= count_arxiv_not, "错误: arXiv 基础结果应该大于或等于排除后的结果");
println!("================= 真实检索逻辑集成测试全部通过 =================");
Ok(())
}