diff --git a/Cargo.lock b/Cargo.lock index b34c759..c467bd3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -147,6 +147,7 @@ dependencies = [ "matchit", "memchr", "mime", + "multer", "percent-encoding", "pin-project-lite", "rustversion", @@ -1553,6 +1554,23 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + [[package]] name = "native-tls" version = "0.2.18" diff --git a/Cargo.toml b/Cargo.toml index c8e3c53..f328aa4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ name = "astroresearch" version = "0.1.0" edition = "2021" +default-run = "astroresearch" [lib] path = "src/lib.rs" @@ -11,12 +12,14 @@ name = "astroresearch" path = "src/main.rs" [[bin]] -name = "test_qiniu" -path = "scratch/test_qiniu.rs" +name = "health_check" +path = "src/bin/health_check.rs" + + [dependencies] tokio = { version = "1", features = ["full"] } -axum = { version = "0.7", features = ["macros"] } +axum = { version = "0.7", features = ["macros", "multipart"] } tower-http = { version = "0.5", features = ["cors", "fs", "trace"] } sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "sqlite", "chrono", "json"] } serde = { version = "1.0", features = ["derive"] } diff --git a/README.md b/README.md index 2e2f735..0b2537c 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,14 @@ AstroResearch 是一个基于 **Rust (Axum)** 后端与 **React (Vite + TypeScri AstroResearch 为天文领域的学者与研究人员提供一站式的文献管理与智能阅读解决方案，核心功能包括： -- 🌌 **统一学术检索**：一键跨源检索 NASA ADS 与 arXiv 数据库，支持去重元数据卡片式展示。 -- 📥 **智能双通道下载**：模拟浏览器请求头与延迟，自动绕过出版商防爬墙，官方 HTML 优先并支持 ar5iv 备用兜底。 +- 🌌 **统一学术检索**：一键跨源检索 NASA ADS 与 arXiv 数据库，支持去重元数据卡片式展示、高级组合条件检索与多种排序方式。 +- 📥 **多通道文献同步与防爬绕过**：支持智能后台下载（官方 HTML / ar5iv 回退）、网页端本地 PDF/HTML 手动离线上传，以及浏览器书签脚本一键直推同步（无惧 Cloudflare 等高强度 WAF 拦截）。 +- 🏷️ **下载错误诊断与无资源标记**：自动记录每篇文献 PDF/HTML 下载失败的具体原因（数据库 `error:` 前缀字段），支持一键标记"无有效全文资源"并从批量任务中排除。 - 📝 **结构化文献解析**：解析 HTML 或调用 MinerU (PDF 降级解析) 输出标准 GFM Markdown，对 LaTeX 公式实施占位符保护。 - 🗣️ **大模型双语翻译**：基于本地天文学词汇库 (Trie 树最长匹配) 构建翻译 Glossary，指导大模型进行公式级精准中英翻译。 - 🪐 **引文网络星系图**：基于 HTML5 Canvas 的高性能力导向拓扑渲染，双击节点支持引文深度探索。 - ✍️ **划词高亮与笔记**：在双语阅读器中自由划词、多色高亮并记录学术心得，数据与文献双向绑定。 +- 🩺 **馆藏健康度检查**：内置诊断与修复工具，检测数据库与物理文件的不一致性并支持一键自动修复。 --- @@ -57,7 +59,18 @@ cp .env.example .env cd .. cargo run --release ``` - 运行后直接访问 `http://localhost:8000` 即可使用，此时所有 React 网页和后台 API 均由 Rust 进程统一分发托管，无需额外启动 Vite。 + 运行后直接访问 `http://localhost:8000` 即可使用，此时所有 React 网页和后台 API 均由 Rust 进程统一分发托管，无需额外启动 Vite。 + +### 2.3 馆藏文献健康度检查与修复 (Health Check) +系统提供内置的健康度校验脚本，可用于排查与自动修复数据库状态和物理磁盘文件的不一致问题： +- **只读扫描模式**：检测损坏文件、丢失文件、报错记录和孤立 Markdown，不改动任何数据。 + ```bash + cargo run --bin health_check + ``` +- **自动修复模式**：物理清理磁盘损坏/无源文件，将失效路径重置为 `NULL`（安全保留 `error:` 报错诊断日志）。 + ```bash + cargo run --bin health_check -- --fix + ``` --- @@ -75,7 +88,55 @@ cp .env.example .env --- -## 4. 目录组件 README 交叉引用 (Component READMEs) +## 4. 项目目录结构 (Project Structure) -- 💻 **前端 React 控制台**：查看 [dashboard/README.md](dashboard/README.md) -- ⚙️ **后端 Rust 源码**：参见 [src/README.md](src/README.md) +``` +AstroResearch/ +├── src/ +│ ├── main.rs # Axum 服务入口：路由注册、中间件、静态资源托管 +│ ├── lib.rs # 库入口：Config 配置结构体与环境变量加载 +│ ├── api/ # API 层（模块化拆分） +│ │ ├── mod.rs # AppState / StandardPaper 定义 + handlers 兼容命名空间 +│ │ ├── helpers.rs # 共享工具函数：格式转换、数据库读写、路径校验 +│ │ ├── papers.rs # 文献相关：检索、下载、上传、解析、翻译、引文、导出 +│ │ ├── notes.rs # 笔记 CRUD：创建、查询、删除 +│ │ └── sync.rs # 批量同步：元数据同步、资源同步、查询管理 +│ ├── bin/ +│ │ └── health_check.rs # 独立二进制：馆藏健康度诊断与修复工具 +│ ├── clients/ +│ │ ├── ads.rs # NASA ADS API 客户端 +│ │ ├── arxiv.rs # arXiv Atom XML API 客户端 +│ │ └── qiniu.rs # 七牛云对象存储客户端 +│ └── services/ +│ ├── batch/ # 批量同步引擎（模块化拆分） +│ │ ├── mod.rs # 公共导出 +│ │ ├── meta.rs # 元数据大批量采集 (MetaSync) +│ │ └── asset.rs # 物理资源批量处理 (AssetSync) +│ ├── download.rs # 文献下载器：反爬伪装、多级回退、错误记录 +│ ├── parser.rs # HTML/PDF → Markdown 解析器 +│ ├── translation.rs # LLM 翻译器 + Trie 词典 +│ ├── query_parser.rs # 高级检索语法解析 +│ └── logging.rs # 日志系统：控制台美化 + 滚动文件 +├── dashboard/ +│ └── src/ +│ ├── App.tsx # 全局状态管理与布局 +│ ├── types.ts # TypeScript 类型定义 +│ ├── components/ +│ │ ├── CitationGalaxyCanvas.tsx # Canvas 力导向引文星系图 +│ │ └── CustomSelect.tsx # 可复用下拉选择组件 +│ └── features/ +│ ├── search/SearchPanel.tsx # 跨源检索面板 +│ ├── library/LibraryPanel.tsx # 馆藏管理面板 +│ ├── reader/ReaderPanel.tsx # 双语对照阅读器 +│ ├── citation/CitationPanel.tsx # 引文图谱面板 +│ ├── sync/SyncPanel.tsx # 批量同步控制台 +│ └── settings/SettingsPanel.tsx # 系统设置 +├── migrations/ # SQLite 数据库迁移脚本 +├── library/ # 本地文献物理存储目录 +│ ├── PDF/ # 下载的 PDF 文件 +│ ├── HTML/ # 下载的 HTML 文件 +│ ├── Markdown/ # 解析后的 Markdown 文件 +│ └── Translation/ # 翻译后的中文 Markdown 文件 +├── docs/ # 技术文档 +└── dictionary.txt # 天文学双语名词词典 +``` diff --git a/bin/obscura b/bin/obscura new file mode 100755 index 0000000..2f53321 Binary files /dev/null and b/bin/obscura differ diff --git a/bin/obscura-worker b/bin/obscura-worker new file mode 100755 index 0000000..0e7a04e Binary files /dev/null and b/bin/obscura-worker differ diff --git a/dashboard/index.html b/dashboard/index.html index 984e31d..2b705d3 100644 --- a/dashboard/index.html +++ b/dashboard/index.html @@ -4,7 +4,7 @@ - dashboard + AstroResearch 天文科研辅助系统
diff --git a/dashboard/src/App.tsx b/dashboard/src/App.tsx index 4b4f06d..5f244a0 100644 --- a/dashboard/src/App.tsx +++ b/dashboard/src/App.tsx @@ -11,7 +11,14 @@ import { SyncPanel } from './features/sync/SyncPanel'; import type { StandardPaper, CitationNetwork, NoteRecord } from './types'; export default function App() { - const [activeTab, setActiveTab] = useState<'search' | 'library' | 'reader' | 'citation' | 'sync'>('search'); + const [activeTab, setActiveTab] = useState<'search' | 'library' | 'reader' | 'citation' | 'sync'>(() => { + const saved = localStorage.getItem('astro_active_tab'); + return (saved as any) || 'search'; + }); + + useEffect(() => { + localStorage.setItem('astro_active_tab', activeTab); + }, [activeTab]); // 全局对话框弹窗状态 const [dialog, setDialog] = useState<{ @@ -83,6 +90,7 @@ export default function App() { // 下载进度状态 const [downloadingBibcodes, setDownloadingBibcodes] = useState>({}); + const [uploadingBibcode, setUploadingBibcode] = useState(null); // 1. 初始化时加载本地文献 useEffect(() => { @@ -174,6 +182,83 @@ export default function App() { } }; + // 3b. 手动上传文献文件以绕过反爬/人机验证 + const handleManualUpload = async (bibcode: string, type: 'pdf' | 'html', file: File) => { + setUploadingBibcode(bibcode); + const formData = new FormData(); + formData.append('bibcode', bibcode); + formData.append('type', type); + formData.append('file', file); + + try { + const res = await axios.post('/api/upload', formData, { + headers: { + 'Content-Type': 'multipart/form-data', + }, + }); + // 更新前端库及检索列表状态 + setSearchResults(prev => prev.map(p => p.bibcode === bibcode ? res.data : p)); + setLibrary(prev => { + if (prev.some(p => p.bibcode === bibcode)) { + return prev.map(p => p.bibcode === bibcode ? res.data : p); + } else { + return [res.data, ...prev]; + } + }); + if (selectedPaper?.bibcode === bibcode) { + setSelectedPaper(res.data); + } + showAlert('手动文献文件上传导入成功！', '上传成功'); + } catch (e: any) { + console.error('手动文件上传失败', e); + const errMsg = e.response?.data || '请确保上传的是合法且完整的文件。'; + showAlert(`文件上传失败: ${errMsg}`, '上传出错'); + } finally { + setUploadingBibcode(null); + } + }; + + // 3c. 手动标记文献为“无有效全文资源” + const handleMarkNoResource = async (bibcode: string, clear = false) => { + const performMark = async () => { + try { + const res = await axios.post('/api/no_resource', { bibcode, clear }); + // 更新前端库及检索列表状态 + setSearchResults(prev => prev.map(p => p.bibcode === bibcode ? res.data : p)); + setLibrary(prev => { + if (prev.some(p => p.bibcode === bibcode)) { + return prev.map(p => p.bibcode === bibcode ? res.data : p); + } else { + return [res.data, ...prev]; + } + }); + if (selectedPaper?.bibcode === bibcode) { + setSelectedPaper(res.data); + } + showAlert( + clear + ? '已成功清除“无全文资源”标记，该文献已被重新允许重载！' + : '已成功将文献标记为“无全文资源”，后续批量任务将自动跳过！', + '标记更新' + ); + } catch (e: any) { + console.error('标记更新失败', e); + const errMsg = e.response?.data || '请稍后重试。'; + showAlert(`标记失败: ${errMsg}`, '操作出错'); + } + }; + + if (clear) { + performMark(); + } else { + showConfirm( + '确认将此文献标记为“无有效全文资源”吗？\n标记后，未来的批量下载/解析重试任务将自动跳过此文献。', + performMark, + '标记无有效全文资源' + ); + } + }; + // 4. 文献解析成 Markdown (优先 HTML, 其次 PDF MinerU) const handleParse = async (bibcode: string, force = false) => { setParsing(true); @@ -440,7 +525,7 @@ export default function App() { if (dialog.onCancel) dialog.onCancel(); setDialog(null); }} - className="fixed inset-0 z-50 flex items-center justify-center bg-slate-900/40 backdrop-blur-xs transition-all" + className="fixed inset-0 z-[60] flex items-center justify-center bg-slate-900/40 backdrop-blur-xs transition-all" >
e.stopPropagation()} @@ -538,6 +623,7 @@ export default function App() { + ) : ( + + )} +
+ )} + {/* 底部操作：整合所有动作（阅读、图谱、下载） */} diff --git a/dashboard/src/components/CustomSelect.tsx b/dashboard/src/components/CustomSelect.tsx new file mode 100644 index 0000000..ef52630 --- /dev/null +++ b/dashboard/src/components/CustomSelect.tsx @@ -0,0 +1,88 @@ +import { useState, useRef, useEffect } from 'react'; +import { ChevronDown } from 'lucide-react'; + +export interface SelectOption { + value: string | number; + label: React.ReactNode; +} + +interface CustomSelectProps { + value: string | number; + onChange: (value: any) => void; + options: SelectOption[]; + className?: string; + disabled?: boolean; +} + +export function CustomSelect({ + value, + onChange, + options, + className = '', + disabled = false, +}: CustomSelectProps) { + const [isOpen, setIsOpen] = useState(false); + const containerRef = useRef(null); + + // Close when clicking outside + useEffect(() => { + function handleClickOutside(event: MouseEvent) { + if (containerRef.current && !containerRef.current.contains(event.target as Node)) { + setIsOpen(false); + } + } + if (isOpen) { + document.addEventListener('mousedown', handleClickOutside); + } + return () => { + document.removeEventListener('mousedown', handleClickOutside); + }; + }, [isOpen]); + + const selectedOption = options.find(opt => opt.value === value) || options[0]; + + return ( +
+ + + {isOpen && !disabled && ( +
+ {options.map(option => { + const isSelected = option.value === value; + return ( + + ); + })} +
+ )} +
+ ); +} diff --git a/dashboard/src/features/library/LibraryPanel.tsx b/dashboard/src/features/library/LibraryPanel.tsx index 05d94ea..12faf85 100644 --- a/dashboard/src/features/library/LibraryPanel.tsx +++ b/dashboard/src/features/library/LibraryPanel.tsx @@ -1,12 +1,13 @@ // dashboard/src/features/library/LibraryPanel.tsx -import { useState } from 'react'; -import { Library, RotateCw, Search, SlidersHorizontal } from 'lucide-react'; +import { useState, useCallback } from 'react'; +import { Library, RotateCw, Search, SlidersHorizontal, X, CheckCircle, AlertTriangle } from 'lucide-react'; import type { StandardPaper } from '../../types'; import { getDoctypeBadge } from '../search/SearchPanel'; +import { CustomSelect } from '../../components/CustomSelect'; interface LibraryPanelProps { library: StandardPaper[]; - fetchLibrary: () => void; + fetchLibrary: () => Promise; setActiveTab: (tab: 'search' | 'library' | 'reader' | 'citation' | 'sync') => void; onShowDetail: (paper: StandardPaper) => void; } @@ -18,15 +19,29 @@ export function LibraryPanel({ onShowDetail, }: LibraryPanelProps) { const [searchTerm, setSearchTerm] = useState(''); - const [filterStatus, setFilterStatus] = useState<'all' | 'downloaded' | 'undownloaded' | 'parsed' | 'translated'>('all'); + const [filterStatus, setFilterStatus] = useState<'all' | 'downloaded' | 'undownloaded' | 'download_failed' | 'no_resource' | 'parsed' | 'translated'>('all'); + const [filterDoctype, setFilterDoctype] = useState('all'); const [sortBy, setSortBy] = useState<'created' | 'yearDesc' | 'yearAsc' | 'citations' | 'title'>('created'); + // 重新同步状态反馈 + const [syncing, setSyncing] = useState(false); + const [syncFeedback, setSyncFeedback] = useState<{ type: 'success' | 'error'; message: string } | null>(null); + // 高级元数据细化筛选状态 const [showAdvanced, setShowAdvanced] = useState(false); const [filterAuthor, setFilterAuthor] = useState(''); const [filterYear, setFilterYear] = useState(''); const [filterJournal, setFilterJournal] = useState(''); + // 各种状态的文献总数量 + const countAll = library.length; + const countDownloaded = library.filter(p => p.is_downloaded).length; + const countUndownloaded = library.filter(p => !p.is_downloaded && !p.pdf_error && !p.html_error).length; + const countDownloadFailed = library.filter(p => !p.is_downloaded && (p.pdf_error || p.html_error) && !(p.pdf_error === 'no_resource' && p.html_error === 'no_resource')).length; + const countNoResource = library.filter(p => !p.is_downloaded && p.pdf_error === 'no_resource' && p.html_error === 'no_resource').length; + const countParsed = library.filter(p => p.has_markdown).length; + const countTranslated = library.filter(p => p.has_translation).length; + // 本地检索与筛选过滤 const filteredLibrary = library.filter(paper => { // 1. 关键词全局检索 (标题、作者、摘要、Bibcode、arXiv ID、DOI) @@ -45,11 +60,43 @@ export function LibraryPanel({ // 2. 离线状态筛选 if (filterStatus === 'downloaded' && !paper.is_downloaded) return false; - if (filterStatus === 'undownloaded' && paper.is_downloaded) return false; + if (filterStatus === 'undownloaded' && (paper.is_downloaded || paper.pdf_error || paper.html_error)) return false; + if (filterStatus === 'download_failed') { + if (paper.is_downloaded) return false; + if (!paper.pdf_error && !paper.html_error) return false; + if (paper.pdf_error === 'no_resource' && paper.html_error === 'no_resource') return false; + } + if (filterStatus === 'no_resource') { + if (paper.is_downloaded) return false; + if (!(paper.pdf_error === 'no_resource' && paper.html_error === 'no_resource')) return false; + } if (filterStatus === 'parsed' && !paper.has_markdown) return false; if (filterStatus === 'translated' && !paper.has_translation) return false; - // 3. 高级元数据细化筛选 + // 3. 文献类型筛选 + if (filterDoctype !== 'all') { + const docVal = (paper.doctype || 'article').toLowerCase(); + if (filterDoctype === 'proceedings') { + if (docVal !== 'proceedings' && docVal !== 'inproceedings') return false; + } else if (filterDoctype === 'thesis') { + if (docVal !== 'phdthesis' && docVal !== 'mastersthesis') return false; + } else if (filterDoctype === 'catalog') { + if (docVal !== 'catalog' && docVal !== 'dataset') return false; + } else if (filterDoctype === 'book') { + if (docVal !== 'book' && docVal !== 'inbook') return false; + } else if (filterDoctype === 'other') { + const standardTypes = [ + 'article', 'eprint', 'proceedings', 'inproceedings', 'proposal', + 'phdthesis', 'mastersthesis', 'abstract', 'catalog', 'dataset', + 'software', 'circular', 'book', 'inbook', 'techreport' + ]; + if (standardTypes.includes(docVal)) return false; + } else { + if (docVal !== filterDoctype) return false; + } + } + + // 4. 高级元数据细化筛选 if (filterAuthor.trim()) { const authorQuery = filterAuthor.toLowerCase().trim(); const matchAuthor = paper.authors.some(a => a.toLowerCase().includes(authorQuery)); @@ -68,8 +115,32 @@ export function LibraryPanel({ return true; }); + // 记录每个 bibcode 在原始 library 数组中的索引，作为“导入时间倒序”的绝对依据 + const originalIndices = new Map(); + library.forEach((paper, index) => { + originalIndices.set(paper.bibcode, index); + }); + + const getStatusScore = (paper: StandardPaper) => { + if (paper.has_translation) return 4; + if (paper.has_markdown) return 3; + if (paper.is_downloaded) return 2; + return 1; + }; + // 本地复合排序 const sortedLibrary = [...filteredLibrary].sort((a, b) => { + if (sortBy === 'created') { + const scoreA = getStatusScore(a); + const scoreB = getStatusScore(b); + if (scoreA !== scoreB) { + return scoreB - scoreA; // 状态高（已翻译 > 已解析 > 已下载 > 其他）的排在前面 + } + // 状态相同时，按“导入时间倒序”排序（即原始数组中的索引从小到大） + const idxA = originalIndices.get(a.bibcode) ?? 99999; + const idxB = originalIndices.get(b.bibcode) ?? 99999; + return idxA - idxB; + } if (sortBy === 'yearDesc') { return (parseInt(b.year) || 0) - (parseInt(a.year) || 0); } @@ -82,10 +153,24 @@ export function LibraryPanel({ if (sortBy === 'title') { return a.title.localeCompare(b.title); } - // 'created' (默认): 沿用后端传回的创建时间倒序 return 0; }); + const handleResync = useCallback(async () => { + setSyncing(true); + setSyncFeedback(null); + try { + await fetchLibrary(); + const count = library.length; + setSyncFeedback({ type: 'success', message: `馆藏数据已刷新，共 ${count} 篇文献` }); + } catch { + setSyncFeedback({ type: 'error', message: '同步失败，请检查后端服务连接' }); + } finally { + setSyncing(false); + setTimeout(() => setSyncFeedback(null), 3000); + } + }, [fetchLibrary, library.length]); + return (
@@ -94,13 +179,29 @@ export function LibraryPanel({

查看和整理已同步离线保存的本地文献资源

+ {/* 同步结果反馈条 */} + {syncFeedback && ( +
+ {syncFeedback.type === 'success' + ? + : } + {syncFeedback.message} +
+ )} + {/* 搜索、筛选与排序工具栏 */} {library.length > 0 && (
@@ -115,41 +216,80 @@ export function LibraryPanel({ value={searchTerm} onChange={e => setSearchTerm(e.target.value)} placeholder="搜索文献标题、作者、摘要、Bibcode、arXiv ID 或 DOI..." - className="w-full pl-9 pr-3 py-2 rounded-lg bg-slate-50 border border-slate-250 text-slate-900 placeholder-slate-400 focus:outline-none focus:border-sky-500 focus:bg-white transition-all text-xs font-medium" + className="w-full pl-9 pr-8 py-2 rounded-lg bg-slate-50 border border-slate-250 text-slate-900 placeholder-slate-400 focus:outline-none focus:border-sky-500 focus:bg-white transition-all text-xs font-medium" /> + {searchTerm && ( + + )}
{/* 状态过滤 */} -
+
- + onChange={val => setFilterStatus(val)} + className="w-full" + options={[ + { value: 'all', label: `全部文献 (${countAll})` }, + { value: 'downloaded', label: `已下载 (${countDownloaded})` }, + { value: 'parsed', label: `已解析 (${countParsed})` }, + { value: 'translated', label: `已翻译 (${countTranslated})` }, + { value: 'undownloaded', label: `未下载 (${countUndownloaded})` }, + { value: 'download_failed', label: `下载失败 (${countDownloadFailed})` }, + { value: 'no_resource', label: `无资源 (${countNoResource})` }, + ]} + /> +
+ + {/* 文献类型筛选 */} +
+ + setFilterDoctype(val)} + className="w-full" + options={[ + { value: 'all', label: '全部类型' }, + { value: 'article', label: '期刊文章' }, + { value: 'eprint', label: '预印本' }, + { value: 'proceedings', label: '会议论文/集' }, + { value: 'proposal', label: '观测提案' }, + { value: 'thesis', label: '学位论文' }, + { value: 'abstract', label: '会议摘要' }, + { value: 'catalog', label: '星表数据' }, + { value: 'software', label: '软件代码' }, + { value: 'book', label: '专著/图书章节' }, + { value: 'circular', label: '天文电报' }, + { value: 'techreport', label: '技术报告' }, + { value: 'other', label: '其他文献/杂项' }, + ]} + />
{/* 排序方式 */} -
+
- + onChange={val => setSortBy(val)} + className="w-full" + options={[ + { value: 'created', label: '默认 (导入时间)' }, + { value: 'yearDesc', label: '发表年份 (新 → 旧)' }, + { value: 'yearAsc', label: '发表年份 (旧 → 新)' }, + { value: 'citations', label: '被引用数 (高 → 低)' }, + { value: 'title', label: '文献标题 (A-Z)' }, + ]} + />
{/* 高级元数据细化筛选触发器 */} @@ -233,6 +373,7 @@ export function LibraryPanel({ onClick={() => { setSearchTerm(''); setFilterStatus('all'); + setFilterDoctype('all'); setFilterAuthor(''); setFilterYear(''); setFilterJournal(''); @@ -243,31 +384,52 @@ export function LibraryPanel({
) : ( -
- {sortedLibrary.map(paper => { - return ( -
onShowDetail(paper)} - className="console-panel p-5 rounded-xl border border-slate-200 hover:border-sky-350 bg-white flex flex-col justify-between relative overflow-hidden group transition-all cursor-pointer shadow-sm" - > +
+
+ 已筛选出 {sortedLibrary.length} 篇文献 / 共 {library.length} 篇 +
+
+ {sortedLibrary.map(paper => { + return ( +
onShowDetail(paper)} + className="console-panel p-5 rounded-xl border border-slate-200 hover:border-sky-350 bg-white flex flex-col justify-between relative overflow-hidden group transition-all cursor-pointer shadow-sm" + > {/* 状态角标 */} -
+
{paper.has_translation ? '已翻译' : paper.has_markdown ? '已解析' : paper.is_downloaded ? '已下载' - : '未下载'} + : (paper.pdf_error === 'no_resource' && paper.html_error === 'no_resource') + ? '无资源' + : (paper.pdf_error || paper.html_error) + ? '下载失败' + : '未下载'}
@@ -304,6 +466,7 @@ export function LibraryPanel({
); })} +
)}
diff --git a/dashboard/src/features/search/SearchPanel.tsx b/dashboard/src/features/search/SearchPanel.tsx index 582b577..b56d615 100644 --- a/dashboard/src/features/search/SearchPanel.tsx +++ b/dashboard/src/features/search/SearchPanel.tsx @@ -2,6 +2,7 @@ import React from 'react'; import { Search, Loader, CheckCircle, Copy, Download, ChevronLeft, ChevronRight, SlidersHorizontal } from 'lucide-react'; import type { StandardPaper } from '../../types'; +import { CustomSelect } from '../../components/CustomSelect'; export const getDoctypeBadge = (doctype: string) => { const typeMap: Record = { @@ -12,9 +13,19 @@ export const getDoctypeBadge = (doctype: string) => { proposal: { label: '观测提案', style: 'bg-rose-50 text-rose-700 border-rose-200' }, abstract: { label: '会议摘要', style: 'bg-slate-50 text-slate-700 border-slate-200' }, catalog: { label: '星表数据', style: 'bg-indigo-50 text-indigo-700 border-indigo-200' }, + dataset: { label: '星表数据', style: 'bg-indigo-50 text-indigo-700 border-indigo-200' }, software: { label: '软件代码', style: 'bg-teal-50 text-teal-700 border-teal-200' }, phdthesis: { label: '博士论文', style: 'bg-cyan-50 text-cyan-700 border-cyan-200' }, mastersthesis: { label: '硕士论文', style: 'bg-cyan-50 text-cyan-700 border-cyan-200' }, + circular: { label: '天文电报', style: 'bg-orange-50 text-orange-700 border-orange-200' }, + inbook: { label: '图书章节', style: 'bg-emerald-50 text-emerald-700 border-emerald-200' }, + book: { label: '学术专著', style: 'bg-emerald-50 text-emerald-700 border-emerald-200' }, + editorial: { label: '期刊社论', style: 'bg-slate-50 text-slate-700 border-slate-200' }, + erratum: { label: '勘误说明', style: 'bg-red-50 text-red-700 border-red-200' }, + misc: { label: '其他文献', style: 'bg-slate-50 text-slate-700 border-slate-200' }, + newsletter: { label: '简报新闻', style: 'bg-slate-50 text-slate-700 border-slate-200' }, + techreport: { label: '技术报告', style: 'bg-cyan-50 text-cyan-700 border-cyan-200' }, + obituary: { label: '讣告', style: 'bg-slate-50 text-slate-700 border-slate-200' }, }; const val = doctype ? doctype.toLowerCase() : 'article'; const match = typeMap[val] || { label: '文献', style: 'bg-slate-50 text-slate-700 border-slate-200' }; @@ -199,30 +210,32 @@ export function SearchPanel({ {rules.map((rule, idx) => (
{idx > 0 ? ( - + onChange={val => handleRuleChange(idx, 'op', val)} + className="w-24" + options={[ + { value: 'AND', label: '并且 (AND)' }, + { value: 'OR', label: '或者 (OR)' }, + { value: 'NOT', label: '排除 (NOT)' }, + ]} + /> ) : (
条件过滤
)} - + onChange={val => handleRuleChange(idx, 'field', val)} + className="w-32" + options={[ + { value: 'all', label: '任意字段' }, + { value: 'title', label: '标题名称' }, + { value: 'author', label: '作者名称' }, + { value: 'abs', label: '摘要内容' }, + { value: 'year', label: '年份范围' }, + ]} + />
排序: - + onChange={val => handleSortChange(val)} + options={[ + { value: 'relevance', label: '相关度' }, + { value: 'date_desc', label: '发表日期 (由新到旧)' }, + { value: 'date_asc', label: '发表日期 (由旧到新)' }, + { value: 'citations_desc', label: '被引频次 (从高到低)' }, + ]} + />
每页显示: - + onChange={val => handleRowsChange(Number(val))} + options={[ + { value: 10, label: '10 条' }, + { value: 15, label: '15 条' }, + { value: 30, label: '30 条' }, + { value: 50, label: '50 条' }, + ]} + />
{exportingList.length > 0 && ( @@ -401,14 +414,33 @@ export function SearchPanel({ 已下载 ) : ( - +
+ {(paper.pdf_error || paper.html_error) && ( + paper.pdf_error === 'no_resource' && paper.html_error === 'no_resource' ? ( + + ⚠️ 无资源 + + ) : ( + + ⚠️ 下载失败 + + ) + )} + +
)}
@@ -808,6 +840,69 @@ export function SyncPanel() {
)}
+ + {/* 浏览器快捷直推书签工具 */} +
+
+

+ + 浏览器快捷直推书签导入工具 +

+

+ 无需手动保存和拖拽上传！在您自己真实的浏览器上突破 Cloudflare/WAF 人机验证或在学校内网成功打开 PDF/HTML 页面后，点击此书签即可将文献直接推送同步到本地 AstroResearch 数据库与文献馆中。 +

+
+ +
+
+ 第一步：安装书签到您的浏览器 +

+ 将下方的按钮直接拖拽到浏览器的书签栏中；或者新建一个浏览器书签，将其 URL 设为复制的 JavaScript 代码（点击右侧按钮复制）： +

+ +
+ {/* 可直接拖拽的链接按钮 */} + { + if (el) { + el.setAttribute( + 'href', + `javascript:(async function(){try{let defaultBib='';try{const res=await fetch('http://localhost:8000/api/active_bibcode');if(res.ok){const data=await res.json();if(data&&data.bibcode)defaultBib=data.bibcode;}}catch(e){}const b=prompt('请输入文献的 Bibcode / doi / arxiv_id :',defaultBib);if(!b||!b.trim())return;const bib=b.trim();if(window.location.protocol==='file:'){alert('[ERR] 浏览器安全策略限制：书签脚本无法直接读取本地磁盘文件 (file://)。\\n\\n提示：对于本地 PDF/HTML 文件，请直接在 AstroResearch 的文献详情页点击“上传 PDF/HTML”按钮导入。');return;}let blob,type='html',ext='.html';const isPDF=document.contentType==='application/pdf'||window.location.pathname.toLowerCase().endsWith('.pdf')||document.title.toLowerCase().endsWith('.pdf');if(isPDF){try{const res=await fetch(window.location.href);if(!res.ok)throw new Error('HTTP '+res.status);blob=await res.blob();type='pdf';ext='.pdf';}catch(err){alert('[ERR] 无法读取该 PDF 数据。\\n(错误: '+err.message+')');return;}}else{blob=new Blob([document.documentElement.outerHTML],{type:'text/html'});}const fd=new FormData();fd.append('bibcode',bib);fd.append('type',type);fd.append('file',blob,bib+ext);const r=await fetch('http://localhost:8000/api/upload',{method:'POST',body:fd});if(r.ok){const d=await r.json();alert('[OK] '+(d.title||bib));}else{const t=await r.text();alert('[ERR '+r.status+'] '+t);}}catch(e){alert('[FAIL] '+e.message);}})();void(0);` + ); + } + }} + className="px-4 py-2 bg-gradient-to-r from-sky-500 to-indigo-500 hover:from-sky-600 hover:to-indigo-600 text-white rounded-lg text-xs font-bold transition-all shadow-sm cursor-move flex items-center gap-1.5" + title="拖拽我到您的书签栏中" + onClick={(e) => e.preventDefault()} + > + {/* 浏览器拖拽时读取全部 textContent，sr-only span 的文字成为书签名 */} + 导入AstroResearch + + + + +
+
+ +
+ 第二步：如何使用书签直推？ +
    +
  1. 点击详情弹窗内的 BIBCODE / DOI / arXiv ID 链接；
    2. +
  2. 页面跳转至文献原始网页后，直接点击该浏览器书签；
    3. +
  3. 书签脚本会自动读取并自动填充好 Bibcode，您只需点击确认，文件即可秒级自动上传录入至系统！
    4. +
+
+
+
); } diff --git a/dashboard/src/index.css b/dashboard/src/index.css index b9c9a27..a0a6525 100644 --- a/dashboard/src/index.css +++ b/dashboard/src/index.css @@ -94,3 +94,41 @@ body { color: #0f172a; } +/* Premium clean console select dropdown styling */ +.select-console { + display: inline-block; + background-color: #ffffff; + border: 1px solid #cbd5e1; + border-radius: 0.5rem; /* 8px */ + padding: 0.5rem 1.75rem 0.5rem 0.625rem; /* padding-right leaves space for custom arrow */ + color: #334155; + font-size: 0.75rem; /* text-xs */ + font-weight: 500; + cursor: pointer; + outline: none; + transition: all 0.2s ease; + appearance: none; + background-image: url("data:image/svg+xml,%3csvg xmlns='http://www.w3.org/2000/svg' fill='none' viewBox='0 0 20 20'%3e%3cpath stroke='%2364748b' stroke-linecap='round' stroke-linejoin='round' stroke-width='1.5' d='M6 8l4 4 4-4'/%3e%3c/svg%3e"); + background-position: right 0.5rem center; + background-repeat: no-repeat; + background-size: 1.25rem; +} + +.select-console:hover:not(:disabled) { + border-color: #94a3b8; + color: #0f172a; + box-shadow: 0 1px 2px 0 rgba(0, 0, 0, 0.05); +} + +.select-console:focus { + border-color: #0284c7; + box-shadow: 0 0 0 1px #0284c7; +} + +.select-console:disabled { + background-color: #f1f5f9; + color: #94a3b8; + cursor: not-allowed; +} + + diff --git a/dashboard/src/types.ts b/dashboard/src/types.ts index e450d50..52ea7db 100644 --- a/dashboard/src/types.ts +++ b/dashboard/src/types.ts @@ -16,6 +16,8 @@ export interface StandardPaper { has_markdown: boolean; has_translation: boolean; doctype: string; + pdf_error?: string; + html_error?: string; } export interface CitationNetwork { diff --git a/docs/api.md b/docs/api.md index a55ccb0..18d86a3 100644 --- a/docs/api.md +++ b/docs/api.md @@ -25,6 +25,9 @@ export interface StandardPaper { is_downloaded: boolean; has_markdown: boolean; has_translation: boolean; + doctype: string; + pdf_error?: string; // PDF 下载失败诊断信息（如存在） + html_error?: string; // HTML 下载失败诊断信息（如存在） } // 笔记记录 @@ -37,8 +40,35 @@ export interface NoteRecord { selected_text: string; created_at: string; } + +// 引文网络 +export interface CitationNetwork { + bibcode: string; + title: string; + citation_count: number; + reference_count: number; + references: string[]; + citations: string[]; + citation_counts?: Record; +} + +// 已保存的同步检索条件 +export interface SavedSyncQuery { + id: number; + query: string; + source: string; + limit_count: number; + last_run: string; +} ``` +### 1.1 错误诊断字段说明 + +`pdf_error` 和 `html_error` 字段用于传递文献下载失败的具体原因： +- 当数据库中对应的 `pdf_path` 或 `html_path` 以 `error:` 前缀存储时，前端会自动提取前缀后的内容作为诊断信息。 +- 特殊值 `no_resource`：表示用户手动标记了该文献为"无有效全文资源"，后续批量下载任务将自动跳过此文献。 +- 其他值：为系统自动检测到的下载失败原因（如网络超时、Cloudflare 拦截、404 等）。 + --- ## 2. 接口分模块详述 (API Endpoints) @@ -94,7 +124,7 @@ export interface NoteRecord { #### 2.2.1 获取馆藏文献列表 - **Endpoint**: `GET /api/library` -- **Description**: 查询本地 SQLite 数据库中已收藏入库的所有文献列表，后端会自动**实时感应物理文件是否存在**来修正 `is_downloaded` / `has_markdown` 等布尔状态。 +- **Description**: 查询本地 SQLite 数据库中已收藏入库的所有文献列表，后端会自动**实时感应物理文件是否存在**来修正 `is_downloaded` / `has_markdown` 等布尔状态。同时读取并返回 `pdf_error` / `html_error` 诊断字段。 - **Response Schema (`Vec`)**: - HTTP `200 OK` - **cURL 示例**: @@ -104,7 +134,7 @@ export interface NoteRecord { #### 2.2.2 触发并行文献下载 - **Endpoint**: `POST /api/download` -- **Description**: 触发后台线程拉取文献的 PDF 及 HTML。如果是 arXiv 来源优先官方 HTML 兜底 ar5iv，并支持强制更新。 +- **Description**: 触发后台线程拉取文献的 PDF 及 HTML。如果是 arXiv 来源优先官方 HTML 兜底 ar5iv，并支持强制更新。下载失败时会在数据库中以 `error:` 前缀记录具体原因。 - **Request Body**: ```json { @@ -112,7 +142,7 @@ export interface NoteRecord { "force": false } ``` -- **Response Schema (`StandardPaper`)**: Returns the updated paper structure with `is_downloaded: true`. +- **Response Schema (`StandardPaper`)**: Returns the updated paper structure with `is_downloaded: true` (on success) or `pdf_error`/`html_error` populated (on failure). - **cURL 示例**: ```bash curl -X POST "http://localhost:8000/api/download" \ @@ -120,7 +150,49 @@ export interface NoteRecord { -d '{"bibcode": "2024arXiv241011663H", "force": true}' ``` -#### 2.2.3 触发文献结构化解析 +#### 2.2.3 手动上传文献物理文件 +- **Endpoint**: `POST /api/upload` +- **Description**: 手动上传用户离线下载的 HTML 或 PDF 物理文件，以便系统进行结构化解析和双语翻译。此接口常用于前端手动上传或浏览器书签直推同步，支持绕过防爬与验证码限制。上传时会自动进行文件格式校验（PDF 校验 `%PDF` 文件头），并支持通过 DOI 或 arXiv ID 自动匹配 Bibcode。 +- **Request Body (Multipart Form Data)**: + - `bibcode` (string, required): 文献唯一标识符（Bibcode）、DOI 或 arXiv ID。 + - `type` (string, required): 文件类别，取值为 `pdf` | `html`。 + - `file` (file binary, required): 上传的 PDF 或 HTML 文件。 +- **Response Schema (`StandardPaper`)**: 返回已更新下载状态（`is_downloaded: true`）的文献标准化元数据。 +- **cURL 示例**: + ```bash + curl -X POST "http://localhost:8000/api/upload" \ + -F "bibcode=2024arXiv241011663H" \ + -F "type=pdf" \ + -F "file=@/path/to/downloaded.pdf" + ``` + +#### 2.2.4 标记/取消"无有效全文资源" +- **Endpoint**: `POST /api/no_resource` +- **Description**: 将文献标记为"无有效全文资源"或清除该标记。标记后，后续批量下载/解析任务将自动跳过此文献。此操作会将数据库中的 `pdf_path` 和 `html_path` 设置为（或清除）`error:no_resource`。 +- **Request Body**: + ```json + { + "bibcode": "2024arXiv241011663H", + "clear": false + } + ``` + - `bibcode` (string, required): 文献唯一标识符。 + - `clear` (boolean, optional): 设为 `true` 清除标记（恢复自动下载），默认 `false`（标记无资源）。 +- **Response Schema (`StandardPaper`)**: 返回更新后的文献标准化元数据。 +- **cURL 示例**: + ```bash + # 标记为无资源 + curl -X POST "http://localhost:8000/api/no_resource" \ + -H "Content-Type: application/json" \ + -d '{"bibcode": "2024arXiv241011663H"}' + + # 清除标记（恢复自动下载） + curl -X POST "http://localhost:8000/api/no_resource" \ + -H "Content-Type: application/json" \ + -d '{"bibcode": "2024arXiv241011663H", "clear": true}' + ``` + +#### 2.2.5 触发文献结构化解析 - **Endpoint**: `POST /api/parse` - **Description**: 将本地下载的 HTML/PDF 清洗为 Markdown。支持 `force` 强制重新执行。 - **Request Body**: @@ -205,7 +277,8 @@ export interface NoteRecord { "citation_count": 12, "reference_count": 48, "references": ["bibcode1", "bibcode2"], - "citations": ["bibcode3", "bibcode4"] + "citations": ["bibcode3", "bibcode4"], + "citation_counts": { "bibcode3": 5, "bibcode4": 120 } } ``` - **cURL 示例**: @@ -226,7 +299,7 @@ export interface NoteRecord { "bibcode": "2024arXiv241011663H", "paragraph_index": 12, "note_text": "这是一个重要的物理模型", - "highlight_color": "yellow", // 'yellow' | 'green' | 'blue' | 'pink' + "highlight_color": "yellow", "selected_text": "the standard model of galaxy formation" } ``` @@ -291,7 +364,7 @@ export interface NoteRecord { #### 2.6.2 启动后台元数据同步 - **Endpoint**: `POST /api/sync/meta/run` -- **Description**: 后台异步启动对指定关键词的文献元数据的大批量增量检索与同步入库。 +- **Description**: 后台异步启动对指定关键词的文献元数据的大批量增量检索与同步入库。若当前已有同步任务在运行中，将返回 `409 Conflict`。 - **Request Body**: ```json { @@ -300,7 +373,7 @@ export interface NoteRecord { "limit": 200 } ``` -- **Response Schema**: Returns HTTP `200 OK` (plain text success message). +- **Response Schema**: Returns HTTP `200 OK` (plain text success message) 或 `409 Conflict` (已有任务运行)。 - **cURL 示例**: ```bash curl -X POST "http://localhost:8000/api/sync/meta/run" \ @@ -328,15 +401,21 @@ export interface NoteRecord { #### 2.6.4 启动后台文献资源批量下载/解析 - **Endpoint**: `POST /api/sync/asset/run` -- **Description**: 后台异步启动文献物理资源 (PDF/HTML) 的批量下载及结构化 Markdown 转换任务。 +- **Description**: 后台异步启动文献物理资源 (PDF/HTML) 的批量下载及结构化 Markdown 转换任务。支持按文献 Bibcode 列表或按状态范围筛选处理目标。 - **Request Body**: ```json { - "action": "all", // "all" (下载并解析) | "download" (仅下载) | "parse" (仅解析) - "scope": "undownloaded" // "all" (全部) | "undownloaded" (仅未下载) | "unparsed" (仅未解析) + "action": "all", + "scope": "undownloaded", + "sort_order": "default", + "limit_count": 50 } ``` -- **Response Schema**: Returns HTTP `200 OK` (plain text success message). + - `action` (string): `"all"` (下载并解析) | `"download"` (仅下载) | `"parse"` (仅解析) | `"translate"` (仅翻译)。 + - `scope` (string): `"all"` (全部) | `"undownloaded"` (仅未下载) | `"unparsed"` (仅未解析)。 + - `sort_order` (string, optional): `"default"` | `"pub_year_desc"` | `"created_at_desc"`。 + - `limit_count` (number, optional): 批量处理上限，默认处理全部匹配文献。 +- **Response Schema**: Returns HTTP `200 OK` (plain text success message) 或 `409 Conflict` (已有任务运行)。 - **cURL 示例**: ```bash curl -X POST "http://localhost:8000/api/sync/asset/run" \ @@ -355,7 +434,7 @@ export interface NoteRecord { #### 2.6.6 查询批量处理任务状态与日志 - **Endpoint**: `GET /api/sync/asset/status` -- **Description**: 获取当前后台批量下载与解析任务的状态、总匹配文献数、已下载数、已解析数、当前处理的 Bibcode，以及实时流转的终端日志（最多保留最新 1000 行）。 +- **Description**: 获取当前后台批量下载与解析任务的状态、总匹配文献数、已下载数、已解析数、失败数、当前处理的 Bibcode，以及实时流转的终端日志（最多保留最新 100 条）。 - **Response Schema**: ```json { @@ -363,6 +442,8 @@ export interface NoteRecord { "total": 12, "downloaded": 12, "parsed": 12, + "download_failed": 0, + "parse_failed": 0, "current_bibcode": "2020A&A...635A..38C", "logs": [ "[INFO] 批量处理任务初始化成功", @@ -377,14 +458,102 @@ export interface NoteRecord { curl "http://localhost:8000/api/sync/asset/status" ``` +#### 2.6.7 获取已保存的同步检索条件 +- **Endpoint**: `GET /api/sync/queries` +- **Description**: 获取用户保存的所有同步检索条件列表，用于快速重新同步。 +- **Response Schema (`Vec`)**: + - HTTP `200 OK` +- **cURL 示例**: + ```bash + curl "http://localhost:8000/api/sync/queries" + ``` + +#### 2.6.8 删除已保存的同步检索条件 +- **Endpoint**: `DELETE /api/sync/queries/:id` +- **Description**: 删除指定 ID 的已保存同步检索条件。 +- **Path Parameters**: + - `id` (number, required): 同步检索条件的唯一自增 ID。 +- **Response Schema**: Returns HTTP `200 OK` (plain text success message). +- **cURL 示例**: + ```bash + curl -X DELETE "http://localhost:8000/api/sync/queries/1" + ``` + --- -## 3. 常见 HTTP 状态码与异常处理 (Error Codes) +### 2.7 活跃文献追踪 (Active Bibcode Tracking) + +#### 2.7.1 获取当前活跃文献 +- **Endpoint**: `GET /api/active_bibcode` +- **Description**: 获取当前用户正在查看/操作的文献 Bibcode。前端在用户点击文献外部链接（如 ADS、DOI、arXiv）时自动上报。 +- **Response Schema**: + ```json + { + "bibcode": "2024arXiv241011663H" + } + ``` + 若无活跃文献，`bibcode` 为 `null`。 +- **cURL 示例**: + ```bash + curl "http://localhost:8000/api/active_bibcode" + ``` + +#### 2.7.2 设置当前活跃文献 +- **Endpoint**: `POST /api/active_bibcode` +- **Description**: 设置当前正在查看的文献 Bibcode，用于浏览器书签直推等场景。 +- **Request Body**: + ```json + { + "bibcode": "2024arXiv241011663H" + } + ``` +- **Response Schema**: Returns HTTP `200 OK`. +- **cURL 示例**: + ```bash + curl -X POST "http://localhost:8000/api/active_bibcode" \ + -H "Content-Type: application/json" \ + -d '{"bibcode": "2024arXiv241011663H"}' + ``` + +--- + +## 3. 完整路由表 (Route Summary) + +| 方法 | 路径 | 说明 | +|:---|:---|:---| +| `GET` | `/api/search` | 跨源文献统一搜索 | +| `POST` | `/api/download` | 触发文献下载 | +| `POST` | `/api/upload` | 手动上传文献文件 | +| `POST` | `/api/no_resource` | 标记/取消"无有效全文资源" | +| `POST` | `/api/parse` | 触发文献结构化解析 | +| `POST` | `/api/translate` | 触发 LLM 对照翻译 | +| `GET` | `/api/citations` | 查询引文拓扑网络 | +| `GET` | `/api/paper` | 获取文献阅读详情 | +| `GET` | `/api/library` | 获取馆藏文献列表 | +| `POST` | `/api/export` | 批量 BibTeX 导出 | +| `POST` | `/api/notes` | 创建笔记与高亮 | +| `GET` | `/api/notes` | 获取文献笔记列表 | +| `DELETE` | `/api/notes` | 删除笔记 | +| `GET` | `/api/sync/meta/count` | 预估元数据同步总量 | +| `POST` | `/api/sync/meta/run` | 启动元数据同步 | +| `GET` | `/api/sync/meta/status` | 查询元数据同步状态 | +| `POST` | `/api/sync/asset/run` | 启动资源批量处理 | +| `POST` | `/api/sync/asset/stop` | 停止资源批量处理 | +| `GET` | `/api/sync/asset/status` | 查询资源处理状态 | +| `GET` | `/api/sync/queries` | 获取已保存检索条件 | +| `DELETE` | `/api/sync/queries/:id` | 删除已保存检索条件 | +| `GET` | `/api/active_bibcode` | 获取当前活跃文献 | +| `POST` | `/api/active_bibcode` | 设置当前活跃文献 | + +--- + +## 4. 常见 HTTP 状态码与异常处理 (Error Codes) 系统基于标准的 HTTP Status Codes 返回错误原因，响应的 Response Body 中通常为纯文本提示（String）： | 状态码 | 错误类型 | 触发常见场景及原因说明 | | :--- | :--- | :--- | -| **`400 Bad Request`** | 业务请求不合规 | - 文献未下载/解析却直接调用 `translate`。
- 未在 `.env` 中提供 `ADS_API_KEY` 时调用 `export`。 | +| **`400 Bad Request`** | 业务请求不合规 | - 文献未下载/解析却直接调用 `translate`。
- 上传文件格式不合法（如 PDF 文件头校验失败）。
- 缺少必需参数（如 `bibcode` 为空）。 | | **`404 Not Found`** | 资源未找到 | - 数据库中没有该 Bibcode 的收藏记录。 | +| **`409 Conflict`** | 状态冲突 | - 已有批量同步任务在后台运行中，重复启动时触发。 | | **`500 Internal Error`**| 服务器内部错误 | - 第三方 LLM / ADS 接口通信超时或返回异常。
- 本地磁盘 IO 失败（如写入文件权限受阻）。
- 数据库查询异常。 | diff --git a/docs/architecture.md b/docs/architecture.md index 0ba664b..8075e2a 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1,6 +1,6 @@ # AstroResearch Architecture / 架构设计 -AstroResearch 是一个集成了天文学文献检索、双通道下载、结构化解析、中英学术对比翻译以及引文星系图谱的天文科研辅助系统。 +AstroResearch 是一个集成了天文学文献检索、多通道下载（含防爬绕过与手动上传）、下载错误诊断、结构化解析、中英学术对比翻译、引文星系图谱以及馆藏健康度诊断的天文科研辅助系统。 ## 1. 整体架构 (Overall Architecture) @@ -12,17 +12,29 @@ graph TD UI[仪表盘 UI / ReaderPanel] Canvas[引文 Canvas 拓扑图] API_Client[Axum API 客户端] + CustomSelect[CustomSelect 可复用组件] end subgraph Backend ["Rust Axum 后端 (Port 8000)"] Router[Axum 路由与中间件] - Handlers[业务处理器 api/handlers.rs] - Sync[同步器 services/batch_sync.rs] - Parser[解析器 services/parser.rs] - Downloader[下载器 services/download.rs] - Translator[翻译器 services/translation.rs] - Qiniu[七牛云客户端 clients/qiniu.rs] - Logging[日志记录器 services/logging.rs] + + subgraph API ["API 层 (模块化)"] + Helpers[helpers.rs 格式转换与数据库工具] + Papers[papers.rs 文献检索/下载/上传/解析/翻译/引文/导出] + Notes[notes.rs 笔记 CRUD] + Sync[sync.rs 批量同步控制] + end + + subgraph Services ["服务层"] + Batch[batch/ 批量同步引擎] + BatchMeta[batch/meta.rs 元数据采集] + BatchAsset[batch/asset.rs 资源处理] + Parser[parser.rs HTML/PDF 解析] + Downloader[download.rs 多通道下载器] + Translator[translation.rs LLM 翻译器] + Logging[logging.rs 日志系统] + end + DB[("SQLite / astro_research.db")] end @@ -36,28 +48,31 @@ graph TD UI -->|用户操作| API_Client API_Client -->|RESTful APIs| Router - Router --> Handlers + Router --> API - Handlers -->|查询/保存元数据| DB - Handlers -->|文献下载/解析/翻译| Handlers - Handlers -->|批量操作| Sync + Papers -->|查询/保存元数据| DB + Papers -->|文献下载| Downloader + Papers -->|文件上传| Papers + Papers -->|正文解析| Parser + Papers -->|学术翻译| Translator + Sync -->|批量操作| Batch - Sync -->|元数据同步| ADS - Sync -->|元数据同步| arXiv - Sync -->|批量文件下载| Downloader - Sync -->|批量正文解析| Parser - Sync -->|写库记录| DB + BatchMeta -->|元数据同步| ADS + BatchMeta -->|元数据同步| arXiv + BatchAsset -->|批量文件下载| Downloader + BatchAsset -->|批量正文解析| Parser + BatchAsset -->|批量翻译| Translator + Batch -->|写库记录| DB Downloader -->|代理请求| ADS Downloader -->|直连或 ar5iv| arXiv Parser -->|图文降级解析| MinerU - Parser -->|托管插图| Qiniu - Qiniu -->|上传图片| QiniuCDN + Parser -->|托管插图| QiniuCDN Translator -->|天文术语翻译| LLM - Canvas -->|引文网络请求| Handlers + Canvas -->|引文网络请求| Papers ``` --- @@ -66,12 +81,12 @@ graph TD ### 2.1 文献下载流程 (Download Flow) -本流程实现了文献的双通道流式下载，支持多级回退以及安全反爬防线绕过，其详细步骤与交互如下： +本流程实现了文献的多通道流式下载，支持多级回退、错误诊断记录以及安全反爬防线绕过： ```mermaid sequenceDiagram participant U as 用户 (React 前端) - participant H as 处理器 (handlers.rs) + participant H as 处理器 (papers.rs) participant D as 下载器 (download.rs) participant DB as 本地数据库 (SQLite) @@ -101,9 +116,15 @@ sequenceDiagram D->>D: 6e. CrossRef 兜底：请求 CrossRef API 获取 PDF URL 并直连下载 end - D-->>H: 7. 返回下载好的本地物理 PDF & HTML 路径 - H->>DB: 8. 更新 pdf_path & html_path 记录 - H-->>U: 9. 返回最新文献状态 (is_downloaded: true) + alt 下载成功 + D-->>H: 7a. 返回下载好的本地物理 PDF & HTML 路径 + H->>DB: 8a. 更新 pdf_path & html_path 记录 + H-->>U: 9a. 返回最新文献状态 (is_downloaded: true) + else 下载失败 + D-->>H: 7b. 返回失败原因 + H->>DB: 8b. 以 error: 前缀记录诊断信息 + H-->>U: 9b. 返回文献状态 (pdf_error / html_error 已填充) + end ``` #### 详细下载说明： @@ -112,17 +133,43 @@ sequenceDiagram 3. **内容完整性校验**： - 对 PDF 严格校验前四个字节（必须是 `%PDF`）以及尾部检索（必须包含 `%%EOF` 终止符），排查登录墙、错误页伪装成 PDF 导致下载坏文件的问题。 - 对 HTML 文本利用 `detect_anti_bot` 流水线过滤 "cloudflare"、"captcha"、"robot check" 等拦截特征。 +4. **错误诊断记录**：下载失败时，系统会将具体的失败原因（如 "Cloudflare 拦截"、"404 Not Found" 等）以 `error:` 前缀存入数据库的 `pdf_path` / `html_path` 字段。前端通过 `pdf_error` / `html_error` 字段读取并向用户展示。 --- -### 2.2 文献解析流程 (Parse Flow) +### 2.2 手动上传流程 (Upload Flow) -本流程负责将本地下载的 HTML 或 PDF 转换为高保真的 Markdown。其详细步骤与交互如下： +当自动下载受防爬或人机验证阻碍时，用户可手动上传文献文件： + +```mermaid +sequenceDiagram + participant U as 用户 (React 前端 / 浏览器书签) + participant H as 处理器 (papers.rs) + participant DB as 本地数据库 (SQLite) + participant FS as 本地文件系统 + + U->>H: 1. 上传文件 (POST /api/upload, Multipart: bibcode + type + file) + H->>H: 2. 解析 Multipart 字段 + alt bibcode 未直接匹配数据库 + H->>DB: 3a. 尝试通过 DOI 匹配 + H->>DB: 3b. 尝试通过 arXiv ID 匹配（自动去除版本号） + end + H->>H: 4. 校验文件格式 (PDF 校验 %PDF 文件头) + H->>FS: 5. 写入物理文件 (library/PDF/ 或 library/HTML/) + H->>DB: 6. 更新 pdf_path / html_path，清除 error: 诊断记录 + H-->>U: 7. 返回更新后的文献元数据 (is_downloaded: true) +``` + +--- + +### 2.3 文献解析流程 (Parse Flow) + +本流程负责将本地下载的 HTML 或 PDF 转换为高保真的 Markdown： ```mermaid sequenceDiagram participant U as 用户 (React 前端) - participant H as 处理器 (handlers.rs) + participant H as 处理器 (papers.rs) participant P as 解析器 (parser.rs) participant M as MinerU (PDF解析服务) participant Q as 七牛云 (对象存储) @@ -164,22 +211,18 @@ sequenceDiagram H-->>U: 11. 返回标准 Markdown 内容渲染展示 ``` -#### 详细解析说明： -1. **HTML 转换为 Markdown 保护公式**：由于 MathJax/LaTeX 在 Markdown 转换中极易被当成普通字符进行转义（例如 `_` 倾斜或 `\` 换行失效），解析器在 HTML 解析前，通过正则将 `$` / `$$` 或 `\(` / `\[` 中的内容全部替换为特定的 UUID 占位符，转换为标准 Markdown 之后，再反向替换恢复公式，确保 LaTeX 渲染无损。 -2. **PDF 复杂排版降级与大文件直传**：遇到无法直接提取 HTML 的老文献时，调用 MinerU 进行布局分析与公式提取。为避免在上传大型 PDF 时触发 API 网关的 `413 Payload Too Large` 错误，系统弃用了传统的 Multipart 表单直接 POST 请求，转而采用**两阶段直传机制**：先请求预签名上传 URL，随后使用 HTTP `PUT` 直接流式传输二进制数据至存储服务，最后通过后台任务轮询 `extract-results` 获取转换完毕的 ZIP 并自动托管插图至七牛云。 - --- -### 2.3 智能对照翻译流程 (Translation Flow) +### 2.4 智能对照翻译流程 (Translation Flow) -本流程实现了基于天文学专属词汇表的 LLM 专业对比翻译，其详细步骤与交互如下： +本流程实现了基于天文学专属词汇表的 LLM 专业对比翻译： ```mermaid sequenceDiagram participant U as 用户 (React 前端) - participant H as 处理器 (handlers.rs) + participant H as 处理器 (papers.rs) participant T as 翻译器 (translation.rs) - participant D as 天文词典 (dictionary.rs) + participant D as 天文词典 (Trie 树) participant L as 大模型 (LLM API) participant DB as 本地数据库 (SQLite) @@ -196,7 +239,7 @@ sequenceDiagram T->>D: 7. 加载本地 dictionary.txt 并初始化 Trie 树结构 T->>D: 8. 执行英文 Markdown 文本分词匹配 D->>D: 9a. 进行前缀匹配检索 - D->>D: 9b. 遵循“最长匹配优先”原则，过滤子词去重 + D->>D: 9b. 遵循"最长匹配优先"原则，过滤子词去重 D-->>T: 10. 返回该篇文献提取出的天文学名词对照 (Glossary) loop 针对英文 Markdown 进行段落分块 (Token 长度控制) @@ -211,37 +254,54 @@ sequenceDiagram H-->>U: 16. 返回翻译后 Markdown 渲染展示 ``` -#### 详细步骤说明： -1. **分级翻译缓存机制**： - - 第一级缓存：若未开启 `force` 且本地物理磁盘已存在对应翻译文件，直接读取并返回，避免不必要的 LLM API 调用消耗。 - - 第二级缓存：必须先完成英文 Markdown 的结构化解析，否则接口返回 `400` 错误，引导用户先进行正文解析。 -2. **基于 Trie 树的天文学名词提取**： - - 字典类 `Dictionary` 会加载包含数十万词条的本地天文词表 `dictionary.txt`。 - - 为防止短词覆盖长词（如 `Hertzsprung` 覆盖 `Hertzsprung-Russell diagram`），分词匹配采用 Trie 树的最长前缀匹配。若匹配到长词，自动忽略其包含的子词。 - - 最终只保留文献中真实出现的名词并去重，以 JSON 的形式构建为专有提示词（Glossary）注入 LLM 提示中。 -3. **LLM 强约束 Prompt 设计**： - - 在向大模型发送请求时，利用 System Prompt 声明其“天文学专业翻译家”的角色。 - - 强制约定格式要求：所有的 LaTeX 公式（`$` / `$$`）必须原封不动保留，Markdown 的标题（`#`）、列表（`-`）、加粗（`**`）等语法严禁破坏，使前端可以无缝解析双语结构并左右对齐渲染。 - --- ## 3. 核心模块说明 -- **[src/api/handlers.rs](../src/api/handlers.rs)**: - - 处理 Axum API 路由分发与业务逻辑，包括统一检索、笔记管理、划词高亮及翻译。 -- **[src/services/batch_sync.rs](../src/services/batch_sync.rs)**: - - 核心后台大批量文献元数据采集 (`MetaSync`) 与文献物理资源批量处理 (`AssetSync`) 的业务同步引擎。 -- **[src/services/download.rs](../src/services/download.rs)**: - - 包含浏览器头伪装与请求延迟控制。 - - 处理 ADS Link Gateway 路由重定向追踪与 `validate.perfdrive.com` 防护解码绕过。 - - 实现官方 `arxiv.org/html` 优先及 `ar5iv` 兜底，自动去除版本号后缀。 -- **[src/services/parser.rs](../src/services/parser.rs)**: - - 实现 HTML 语法树向 GFM Markdown 的逆向转换，使用占位符保护机制防止 MathJax/LaTeX 公式被误解析。 - - 统一相对图表链接，并集成 MinerU PDF 解析。 -- **[src/services/translation.rs](../src/services/translation.rs)**: - - 利用本地千万字级别的天文学双语词典对原文进行分词匹配，注入系统提示词让 LLM 实现学术级精细翻译。 -- **[src/services/logging.rs](../src/services/logging.rs)**: - - 全局日志记录系统，基于 `tracing-subscriber` 实现了控制台美化日志输出与基于时间的每日滚动日志文件写出，使用上海时区 (+08:00) 格式化时间。 -- **[dashboard/src/components/CitationGalaxyCanvas.tsx](../dashboard/src/components/CitationGalaxyCanvas.tsx)**: - - 基于原生 HTML5 Canvas 开发的轻量级、高性能力导向图星系物理引擎，用于文献引文网络拓扑结构的可视化渲染。 +### 3.1 API 层 (`src/api/`) +| 模块文件 | 职责 | +|:---|:---| +| **[mod.rs](../src/api/mod.rs)** | 定义全局共享状态 `AppState`（含 `active_bibcode` 追踪）和统一文献格式 `StandardPaper`（含 `pdf_error` / `html_error` 诊断字段），通过 `pub mod handlers` 保持向后兼容命名空间。 | +| **[helpers.rs](../src/api/helpers.rs)** | 共享工具函数：`convert_ads_doc_to_standard`、`convert_arxiv_to_standard`、`save_paper_to_db`、`get_paper_from_db`、`check_paper_paths_in_db`。负责数据库 CRUD 和 `error:` 前缀诊断信息的读取与解析。 | +| **[papers.rs](../src/api/papers.rs)** | 文献相关核心处理器：统一检索 (`search_papers`)、下载 (`download_paper`)、**手动上传 (`upload_paper_file`)**、**无资源标记 (`mark_no_resource`)**、解析 (`parse_paper`)、翻译 (`translate_paper`)、引文拓扑 (`get_citation_network`)、文献详情 (`get_paper_detail`)、馆藏列表 (`get_library`)、BibTeX 导出 (`export_citations`)、**活跃文献追踪 (`get/set_active_bibcode`)**。 | +| **[notes.rs](../src/api/notes.rs)** | 笔记 CRUD 处理器：创建 (`create_note`)、查询 (`get_notes`)、删除 (`delete_note`)。 | +| **[sync.rs](../src/api/sync.rs)** | 批量同步控制处理器：元数据同步启动/状态/计数、资源同步启动/停止/状态、检索条件管理。 | + +### 3.2 服务层 (`src/services/`) + +| 模块文件 | 职责 | +|:---|:---| +| **[batch/mod.rs](../src/services/batch/mod.rs)** | 批量同步引擎公共导出模块。 | +| **[batch/meta.rs](../src/services/batch/meta.rs)** | 元数据大批量采集引擎 (`MetaSync`)：分页检索 ADS/arXiv 并增量入库。 | +| **[batch/asset.rs](../src/services/batch/asset.rs)** | 物理资源批量处理引擎 (`AssetSync`)：后台异步执行下载/解析/翻译流水线，记录 `download_failed` / `parse_failed` 计数，保留最新 100 条日志。 | +| **[download.rs](../src/services/download.rs)** | 多通道下载器：浏览器头伪装与请求延迟控制、ADS Link Gateway 重定向追踪与 `validate.perfdrive.com` 防护解码绕过、官方 `arxiv.org/html` 优先及 `ar5iv` 兜底、**下载失败时以 `error:` 前缀记录诊断信息至数据库**。 | +| **[parser.rs](../src/services/parser.rs)** | HTML 语法树向 GFM Markdown 逆向转换，使用占位符保护 LaTeX 公式；统一图表链接；集成 MinerU PDF 解析。 | +| **[translation.rs](../src/services/translation.rs)** | 基于本地天文双语词典的 Trie 树最长匹配分词，注入 Glossary 系统提示词让 LLM 实现学术级精细翻译。 | +| **[query_parser.rs](../src/services/query_parser.rs)** | 高级检索语法解析器，将前端组合条件（AND/OR/NOT + 字段限定）转换为 ADS API 查询语法。 | +| **[logging.rs](../src/services/logging.rs)** | 全局日志记录系统，基于 `tracing-subscriber` 实现控制台美化日志输出与基于时间的每日滚动日志文件写出，使用上海时区 (+08:00) 格式化时间。 | + +### 3.3 客户端层 (`src/clients/`) + +| 模块文件 | 职责 | +|:---|:---| +| **[ads.rs](../src/clients/ads.rs)** | NASA ADS API 客户端：文献检索、元数据获取、BibTeX 导出。 | +| **[arxiv.rs](../src/clients/arxiv.rs)** | arXiv Atom XML API 客户端：解析 XML Feed 提取文献元数据。 | +| **[qiniu.rs](../src/clients/qiniu.rs)** | 七牛云对象存储客户端：PDF 插图上传与 CDN 外链生成。 | + +### 3.4 独立工具 (`src/bin/`) + +| 文件 | 职责 | +|:---|:---| +| **[health_check.rs](../src/bin/health_check.rs)** | 馆藏健康度诊断与修复工具：检测损坏文件、丢失文件、`error:` 报错记录和孤立 Markdown；`--fix` 模式自动清理并重置数据库状态。 | + +### 3.5 前端核心组件 (`dashboard/src/`) + +| 组件文件 | 职责 | +|:---|:---| +| **[App.tsx](../dashboard/src/App.tsx)** | 全局状态管理：Tab 持久化、手动上传处理、无资源标记、活跃文献追踪、详情弹窗（含错误诊断和上传区）。 | +| **[components/CustomSelect.tsx](../dashboard/src/components/CustomSelect.tsx)** | 可复用下拉选择组件：统一视觉风格、点击外部关闭、选中高亮。 | +| **[components/CitationGalaxyCanvas.tsx](../dashboard/src/components/CitationGalaxyCanvas.tsx)** | 基于 HTML5 Canvas 的自研力导向引文星系图谱引擎：节点排斥力、中心引力、拖拽阻尼、双击多层级衍生。 | +| **[features/library/LibraryPanel.tsx](../dashboard/src/features/library/LibraryPanel.tsx)** | 馆藏管理面板：同步反馈、下载失败/无资源状态筛选、文献类型筛选（13 种）、状态优先排序。 | +| **[features/search/SearchPanel.tsx](../dashboard/src/features/search/SearchPanel.tsx)** | 跨源检索面板：高级组合条件、排序分页、下载失败状态提示、文献类型徽章（16 种）。 | +| **[features/sync/SyncPanel.tsx](../dashboard/src/features/sync/SyncPanel.tsx)** | 批量同步控制台：乐观 UI 更新、容器内日志自动滚动。 | diff --git a/docs/contributing.md b/docs/contributing.md index b58700d..544fad0 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -30,21 +30,43 @@ --- -## 2. 编码规范 (Coding Style Guidelines) +## 2. 项目结构约定 (Project Structure Conventions) + +### 后端模块化架构 + +后端代码已从单文件架构重构为模块化架构： + +- **API 层** (`src/api/`)：按职责拆分为 `papers.rs`（文献相关）、`notes.rs`（笔记相关）、`sync.rs`（同步相关）、`helpers.rs`（共享工具），通过 `mod.rs` 统一暴露 `AppState`、`StandardPaper` 和向后兼容的 `handlers` 命名空间。 +- **服务层** (`src/services/`)：批量同步引擎从单文件 `batch_sync.rs` 拆分为 `batch/mod.rs` + `meta.rs` + `asset.rs`，同时通过 `pub mod batch_sync` 保持路径兼容。 +- **独立工具** (`src/bin/`)：`health_check.rs` 作为独立二进制程序，可直接运行。 + +### 前端组件化架构 + +- **可复用组件** (`components/`)：`CustomSelect` 替代所有原生 ``。 +- **新增 API 字段**：后端 `StandardPaper` 新增字段时，必须同步更新 `dashboard/src/types.ts` 中的 `StandardPaper` 接口。 --- -## 3. 测试与验证 (Testing) +## 4. 测试与验证 (Testing) ### 运行后端单元测试 系统为各个下载、解析、词典分词、接口提取等模块设计了健全的测试。运行测试命令： @@ -58,3 +80,18 @@ cd dashboard npm run build # 运行 TypeScript 类型检查及 Vite 打包编译 ``` 确保无编译 Error 或 Warn 警告后方可提交 PR。 + +### 运行健康检查工具 +提交前建议对本地馆藏运行健康检查，确保功能正常： +```bash +cargo run --bin health_check +``` + +--- + +## 5. 数据库迁移 (Database Migrations) + +添加新的数据库字段或表时，需在 `migrations/` 目录下创建新的迁移脚本： +1. 文件命名格式：`YYYYMMDDHHMMSS_description.sql` +2. 迁移脚本会在 `cargo run` 启动时自动执行 +3. 新增字段需同时在后端 `StandardPaper` 结构体和前端 `types.ts` 中同步更新 diff --git a/docs/database.md b/docs/database.md index b9de37e..0b13dbc 100644 --- a/docs/database.md +++ b/docs/database.md @@ -1,6 +1,6 @@ # AstroResearch Database Schema / 数据库设计 -AstroResearch 使用轻量级、零配置的 **SQLite** 数据库作为持久化存储。数据库文件默认保存在项目根目录下的 `astro_research.db`，由 Rust 中的 `sqlx` 驱动管理并自动执行迁移。 +AstroResearch 使用轻量级、零配置的 **SQLite** 数据库作为持久化存储。数据库文件默认保存在项目根目录下的 `astro_research.db`（可通过 `.env` 中的 `DATABASE_URL` 配置），由 Rust 中的 `sqlx` 驱动管理并自动执行迁移。 --- @@ -20,10 +20,11 @@ erDiagram text arxiv_id integer citation_count integer reference_count - text pdf_path - text html_path - text markdown_path - text translation_path + text doctype "文献类型" + text pdf_path "PDF 物理路径 或 error:诊断" + text html_path "HTML 物理路径 或 error:诊断" + text markdown_path "Markdown 物理路径" + text translation_path "翻译文件物理路径" datetime created_at } @@ -42,6 +43,16 @@ erDiagram text target_bibcode PK } + SYNC_QUERIES { + integer id PK + text query "检索关键词" + text source "数据源" + integer limit_count "拉取上限" + datetime last_run "最近运行时间" + datetime created_at "创建时间" + UNIQUE_query_source_limit "唯一去重约束" + } + PAPERS ||--o{ NOTES : "has" PAPERS ||--o{ CITATIONS_REFERENCES : "cites / cited_by" ``` @@ -52,6 +63,9 @@ erDiagram ### 2.1 papers 表 (文献元数据) 存储文献的核心元数据和本地物理存储路径。 +- **特殊字段说明**： + - `pdf_path` / `html_path`：正常情况下存储相对路径（如 `library/PDF/2024arXiv.pdf`）。当下载失败时，会以 `error:` 前缀存储诊断信息（如 `error:Cloudflare 拦截`）。特殊值 `error:no_resource` 表示用户手动标记了"无有效全文资源"。 + - `doctype`：文献类型标识，如 `article`、`eprint`、`proceedings`、`phdthesis`、`catalog`、`software`、`circular`、`book` 等。 - **索引**： - `idx_papers_doi` -> 基于 `doi` - `idx_papers_arxiv_id` -> 基于 `arxiv_id` @@ -69,9 +83,33 @@ erDiagram - **索引**： - `idx_notes_bibcode` -> 优化单篇文献的笔记列表查询。 +### 2.4 sync_queries 表 (同步检索条件) +存储用户保存的批量同步检索条件，支持快速重新同步。 +- **唯一约束**：`UNIQUE(query, source, limit_count)` 确保相同条件的检索不会重复保存。 + --- ## 3. 数据库迁移说明 迁移脚本存放在 `migrations/` 下，服务启动时（`src/main.rs`）会自动调用 `sqlx::migrate!().run(&pool).await` 自动部署： -1. `20260608000000_init.sql`：初始化 `papers` 与 `citations_references` 结构。 -2. `20260608000001_notes.sql`：添加 `notes` 笔记高亮表，并为关联建立级联删除。 + +| 迁移文件 | 说明 | +|:---|:---| +| `20260608000000_init.sql` | 初始化 `papers` 与 `citations_references` 结构。 | +| `20260608000001_notes.sql` | 添加 `notes` 笔记高亮表，并为关联建立级联删除。 | +| `20260608000002_add_doctype.sql` | 为 `papers` 表新增 `doctype` 文献类型字段。 | +| `20260608000003_sync_features.sql` | 添加 `sync_queries` 同步检索条件表，支持唯一去重。 | + +--- + +## 4. 错误诊断存储约定 + +系统使用 `papers` 表的 `pdf_path` 和 `html_path` 字段的双重语义来同时存储正常路径和错误诊断： + +| 字段值模式 | 含义 | 前端展示 | +|:---|:---|:---| +| `NULL` | 尚未尝试下载 | 琥珀色"未下载"角标 | +| `library/PDF/xxx.pdf` | 下载成功，正常物理路径 | 蓝色"已下载"角标 | +| `error:具体原因` | 下载失败，原因为前缀后的文本 | 红色"下载失败"角标，悬浮显示原因 | +| `error:no_resource` | 用户手动标记为无有效全文资源 | 灰色"无资源"角标 | + +`health_check` 工具在 `--fix` 模式下会清理损坏文件并将路径重置为 `NULL`，但**不会**清除 `error:` 前缀的记录（以保留诊断线索）。 diff --git a/docs/deployment.md b/docs/deployment.md index 460efce..f979a69 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -32,6 +32,13 @@ cargo build --release ``` 编译产物位于 `target/release/astroresearch`。 +### 步骤 3（可选）：编译健康检查工具 +如需在目标服务器上运行馆藏健康度诊断与修复： +```bash +cargo build --release --bin health_check +``` +编译产物位于 `target/release/health_check`。 + --- ## 3. 服务部署与启动 (Running in Production) @@ -44,3 +51,39 @@ cargo build --release ./astroresearch ``` 5. 进程将默认在后台启动并监听 `http://localhost:8000` 端口。你可以通过 Nginx 将此端口反向代理到公网 80/443 端口。 + +--- + +## 4. 环境变量配置 (Environment Variables) + +| 变量名 | 必需 | 默认值 | 说明 | +|:---|:---|:---|:---| +| `DATABASE_URL` | 否 | `sqlite://library/astro_research.db` | SQLite 数据库连接 URL | +| `ADS_API_KEY` | 是 | - | NASA ADS API 访问 Token | +| `LLM_API_KEY` | 是 | - | 大语言模型 API Key | +| `LLM_API_BASE` | 否 | `https://api.openai.com/v1` | 大语言模型 API 基础地址 | +| `LLM_MODEL` | 否 | `gpt-4o-mini` | 翻译大模型名称 | +| `QINIU_AK` | 否 | - | 七牛云 Access Key | +| `QINIU_SK` | 否 | - | 七牛云 Secret Key | +| `QINIU_BUCKET` | 否 | - | 七牛云存储空间名 | +| `QINIU_DOMAIN` | 否 | - | 七牛云外链 CDN 域名 | +| `MINERU_API_URL` | 否 | - | MinerU PDF 解析远程 API 地址 | +| `MINERU_API_KEY` | 否 | - | MinerU API Token | +| `LIBRARY_DIR` | 否 | `./library` | 本地文献馆藏根目录 | +| `PORT` | 否 | `8000` | 后端服务监听端口 | + +--- + +## 5. 健康检查与维护 (Health Check) + +部署后可定期运行健康检查工具排查馆藏一致性问题： + +```bash +# 只读扫描（不修改任何数据） +./health_check + +# 自动修复（清理损坏文件、重置无效路径） +./health_check -- --fix +``` + +详见 [排障指南 §4.2](troubleshooting.md#42-馆藏文献健康度检查工具-health_check)。 diff --git a/docs/design.md b/docs/design.md index 53ffe75..1987fe7 100644 --- a/docs/design.md +++ b/docs/design.md @@ -1,6 +1,6 @@ # AstroResearch Design Systems / 设计系统与交互体验 -AstroResearch 的前端界面设计坚持“未来科技感与学术沉浸”的理念，结合了现代网页设计的高级质感。 +AstroResearch 的前端界面设计坚持"未来科技感与学术沉浸"的理念，结合了现代网页设计的高级质感。 --- @@ -15,6 +15,40 @@ AstroResearch 前端目前重构并统一为**高对比度浅色纯中文学术 | **主背景** | 纯净冷灰白 (`#f1f5f9`) | 深石板灰/接近纯黑 (`#0f172a`) | 控制台按钮 (`.btn-console` / `.btn-console-primary`) | 扁平极简实边框设计 | | **卡片/容器** | 纯白背景 (`#ffffff`)，实线灰色边框 (`#e2e8f0`) | 辅助灰 (`#64748b`) | 指示灯：深宝石绿 (就绪) / 灰石色 (未解析) | 微卡片投影效果 | +### 1.2 文献状态色彩编码 + +馆藏面板中的文献卡片使用统一的角标色彩编码来区分不同状态： + +| 状态 | 背景色 | 文本色 | 含义 | +|:---|:---|:---|:---| +| **已翻译** | 翡翠绿 `bg-emerald-50` | `text-emerald-700` | 文献已解析并完成中英对照翻译 | +| **已解析** | 天蓝 `bg-sky-50` | `text-sky-700` | 文献正文已解析为 Markdown | +| **已下载** | 靛蓝 `bg-indigo-50` | `text-indigo-700` | PDF/HTML 物理文件已下载 | +| **未下载** | 琥珀 `bg-amber-50` | `text-amber-700` | 尚未开始下载 | +| **下载失败** | 玫瑰红 `bg-rose-50` | `text-rose-700` | 自动下载失败，悬浮显示原因 | +| **无资源** | 石板灰 `bg-slate-100` | `text-slate-600` | 用户手动标记为无全文资源 | + +### 1.3 文献类型徽章 + +检索与馆藏面板中的文献类型徽章采用差异化色彩编码，当前支持 16 种文献类型： + +| 类型 | 中文标签 | 色彩 | +|:---|:---|:---| +| article | 期刊文章 | 天蓝 | +| eprint | 预印本 | 紫色 | +| proceedings / inproceedings | 会议论文/集 | 橙色 | +| proposal | 观测提案 | 玫瑰红 | +| abstract | 会议摘要 | 石板灰 | +| catalog / dataset | 星表数据 | 靛蓝 | +| software | 软件代码 | 青绿 | +| phdthesis / mastersthesis | 博士/硕士论文 | 青色 | +| circular | 天文电报 | 橙色 | +| book / inbook | 学术专著/图书章节 | 翡翠绿 | +| editorial | 期刊社论 | 石板灰 | +| erratum | 勘误说明 | 红色 | +| techreport | 技术报告 | 青色 | +| 其他 | 其他文献 | 默认灰 | + --- ## 2. 核心交互组件 (Key Interactive Components) @@ -42,3 +76,24 @@ graph LR - **结构化排版**：中英文双栏段落基准对齐，完美融合 `rehype-katex` 数学公式渲染和 `html2md` 图片嵌入。 - **划词标注与高亮**：鼠标选中阅读器任意段落词句，即刻浮现气泡菜单（支持 4 种高亮配色）。 - **浮动词汇浮屠**：检测到英文正文中含有天文学专业词汇时，自动显示下划线，悬浮可阅读中文释义对照。 + +### 2.3 CustomSelect 可复用下拉组件 +- **替代原生 `` 元素，以实现统一的视觉风格和交互体验。 +- **说明**：所有下拉框（状态筛选、文献类型筛选、排序方式、检索条件组合等）均已统一使用 `CustomSelect` 组件，支持点击外部自动关闭、选中项高亮等交互。 + +### 5.2 同步面板日志自动滚动干扰全页浏览 +- **原因**：早期版本中日志自动滚动使用 `scrollIntoView` 会影响整个页面。 +- **解决机制**：当前版本改为仅滚动日志容器内部元素，并增加了 80px 的阈值判断，只有当用户处于接近底部的阅读位置时才自动滚动追踪最新日志。 diff --git a/recovered_handlers.rs b/recovered_handlers.rs deleted file mode 100644 index 688d1c6..0000000 --- a/recovered_handlers.rs +++ /dev/null @@ -1,1271 +0,0 @@ -// src/handlers.rs -use axum::{ - extract::{Query, State}, - http::StatusCode, - Json, -}; -use serde::{Deserialize, Serialize}; -use std::sync::Arc; -use std::fs; -use tracing::{info, warn, error}; -use sqlx::{SqlitePool, Row}; - -use crate::config::Config; -use crate::translation::Dictionary; -use crate::qiniu::QiniuClient; -use crate::ads::{AdsClient, AdsPaperDoc}; -use crate::arxiv::{ArxivClient, ArxivPaper}; -use crate::download::Downloader; - -// 全局共享的 Axum 应用上下文状态 -pub struct AppState { - pub config: Config, - pub db: SqlitePool, - pub dict: Dictionary, - pub qiniu: QiniuClient, - pub ads: AdsClient, - pub arxiv: ArxivClient, - pub downloader: Downloader, - pub harvest_status: Arc>, - pub process_status: Arc>, -} - -// 检索请求参数 -#[derive(Debug, Deserialize)] -pub struct SearchParams { - pub q: String, - pub source: Option, // "all" | "ads" | "arxiv" - pub rows: Option, - pub start: Option, // 分页起始偏移量 - pub sort: Option, // 排序字段，例如 "date_desc", "citations_desc", "relevance" -} - -// 统一标准化的文献格式，用于向前端传输 -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct StandardPaper { - pub bibcode: String, - pub title: String, - pub authors: Vec, - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - let doi = doc.doi.as_ref() - .and_then(|v: &Vec| v.first()) - .cloned() - .unwrap_or_default(); - - let mut arxiv_id = String::new(); - if let Some(identifiers) = &doc.identifier { - for id in identifiers { - if id.starts_with("arXiv:") { - arxiv_id = id.replace("arXiv:", "").trim().to_string(); - break; - } - } - } - if arxiv_id.is_empty() { - if doc.bibcode.starts_with("arXiv") { - arxiv_id = doc.bibcode.replace("arXiv", "").trim().to_string(); - } - } - - StandardPaper { - bibcode: doc.bibcode.clone(), - title, - authors, - year: doc.year.clone().unwrap_or_default(), - pub_journal: doc.pub_journal.clone().unwrap_or_default(), - keywords, - abstract_text: doc.abstract_text.clone().unwrap_or_default(), - doi, - arxiv_id, - citation_count: doc.citation_count.unwrap_or(0), - reference_count: doc.reference_count.unwrap_or(0), - is_downloaded: false, - has_markdown: false, - has_translation: false, - } -} - -pub(crate) fn convert_arxiv_to_standard(doc: &ArxivPaper) -> StandardPaper { - StandardPaper { - bibcode: doc.id.clone(), - title: doc.title.clone(), - authors: doc.authors.clone(), - year: doc.year.clone(), - pub_journal: "arXiv Preprint".to_string(), - keywords: Vec::new(), - abstract_text: doc.abstract_text.clone(), - doi: doc.doi.clone().unwrap_or_default(), - arxiv_id: doc.id.clone(), - citation_count: 0, - reference_count: 0, - is_downloaded: false, - has_markdown: false, - has_translation: false, - } -} - -pub(crate) async fn save_paper_to_db(db: &SqlitePool, p: &StandardPaper) -> anyhow::Result<()> { - let authors_json = serde_json::to_string(&p.authors)?; - let keywords_json = serde_json::to_string(&p.keywords)?; - - // 1. 如果存在 arxiv_id，检查是否有已存在的相同 arxiv_id 记录以防 duplicate - if !p.arxiv_id.is_empty() { - let existing_opt: Option<(String, Option, Option, Option, Option)> = sqlx::query_as( - "SELECT bibcode, pdf_path, html_path, markdown_path, translation_path FROM papers WHERE arxiv_id = ?" - ) - .bind(&p.arxiv_id) - .fetch_optional(db) - .await?; - - if let Some((existing_bibcode, _pdf, _html, _md, _tr)) = existing_opt { - if existing_bibcode != p.bibcode { - // 发现不同 bibcode 标识的同一篇文献记录，需要进行合并合并 - // 如果已存在的记录使用的是临时 arXiv ID 作为 bibcode，且新记录使用的是正式 ADS bibcode，我们升级 bibcode 主键 - let is_existing_temp = existing_bibcode == p.arxiv_id; - let is_new_formal = p.bibcode != p.arxiv_id; - - if is_existing_temp && is_new_formal { - info!("发现相同 arXiv ID 的 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - .await?; - - // 运行迁移 - sqlx::migrate!("./migrations") - .run(&pool) - .await?; - - let paper = StandardPaper { - bibcode: "2026A&A...123..456X".to_string(), - title: "A Test Title".to_string(), - authors: vec!["Author A".to_string()], - year: "2026".to_string(), - pub_journal: "Astronomy & Astrophysics".to_string(), - keywords: vec!["Keyword 1".to_string()], - abstract_text: "This is abstract".to_string(), - doi: "10.1000/test.doi".to_string(), - arxiv_id: "".to_string(), - citation_count: 5, - reference_count: 10, - is_downloaded: false, - has_markdown: false, - has_translation: false, - }; - - // 保存 - save_paper_to_db(&pool, &paper).await?; - - // 读取 - let retrieved = get_paper_from_db(&pool, std::path::Path::new(""), "2026A&A...123..456X").await?; - assert_eq!(retrieved.title, paper.title); - assert_eq!(retrieved.authors, paper.authors); - assert_eq!(retrieved.keywords, paper.keywords); - - // 检查路径状态（初始为 None） - let paths = check_paper_paths_in_db(&pool, std::path::Path::new(""), "2026A&A...123..456X").await?; - assert!(paths.is_some()); - let (pdf, html, md, tr) = paths.unwrap(); - assert!(pdf.is_none()); - assert!(html.is_none()); - assert!(md.is_none()); - assert!(tr.is_none()); - - Ok(()) - } -} - - diff --git a/scratch/__pycache__/analyze_failures.cpython-312.pyc b/scratch/__pycache__/analyze_failures.cpython-312.pyc deleted file mode 100644 index fa56418..0000000 Binary files a/scratch/__pycache__/analyze_failures.cpython-312.pyc and /dev/null differ diff --git a/scratch/failed_results.txt b/scratch/failed_results.txt deleted file mode 100644 index a250c00..0000000 --- a/scratch/failed_results.txt +++ /dev/null @@ -1,321 +0,0 @@ -=== 403 FORBIDDEN === -### Monthly Notices of the Royal Astronomical Society (共 26 篇) -- `2010MNRAS.401.1080A` -- `2010MNRAS.401.1850K` -- `2010MNRAS.406.2701K` -- `2011MNRAS.412..487K` -- `2011MNRAS.415.3042K` -- `2012MNRAS.419..452Z` -- `2012MNRAS.421.3238K` -- `2013MNRAS.431..240O` -- `2013MNRAS.436.1408Q` -- `2014MNRAS.445.4247K` -- `2015MNRAS.451.3986K` -- `2015MNRAS.453.1879K` -- `2016MNRAS.457..723K` -- `2016MNRAS.459.4343K` -- `2017MNRAS.467.3963K` -- `2018MNRAS.481.2721B` -- `2019MNRAS.482..758S` -- `2019MNRAS.485.4330K` -- `2019MNRAS.489.1556B` -- `2019MNRAS.490.1283K` -- `2020MNRAS.493.5162R` -- `2021MNRAS.508..560K` -- `2022MNRAS.516.1509K` -- `2023MNRAS.525..183S` -- `2023MNRAS.525.1342R` -- `2026MNRAS.548ag689.` -### American Astronomical Society Meeting Abstracts #242 (共 7 篇) -- `2023AAS...24220105B` -- `2023AAS...24230301D` -- `2023AAS...24230501K` -- `2023AAS...24230502S` -- `2023AAS...24230503L` -- `2023AAS...24233706D` -- `2023AAS...24240003S` -### International Conference on Binaries: in celebration of Ron Webbink's 65th Birthday (共 4 篇) -- `2010AIPC.1314...67G` -- `2010AIPC.1314...73W` -- `2010AIPC.1314...85H` -- `2010AIPC.1314...91S` -### 17th European White Dwarf Workshop (共 4 篇) -- `2010AIPC.1273..243S` -- `2010AIPC.1273..255L` -- `2010AIPC.1273..259B` -- `2010AIPC.1273..263G` -### American Astronomical Society Meeting Abstracts #245 (共 3 篇) -- `2025AAS...24540303B` -- `2025AAS...24540312T` -- `2025AAS...24540313K` -### American Astronomical Society Meeting Abstracts #237 (共 3 篇) -- `2021AAS...23714004P` -- `2021AAS...23734904W` -- `2021AAS...23755001C` -### Journal of Physics Conference Series (共 3 篇) -- `2009JPhCS.172a2015H` -- `2016JPhCS.728g2023Z` -- `2019JPhCS1380a2095S` -### The Astrophysical Journal (共 2 篇) -- `2011ApJ...737L..27R` -- `2026ApJ...997...58W` -### Binary Systems, their Evolution and Environments (共 2 篇) -- `2014bsee.confE..25H` -- `2014bsee.confP..25S` -### Publications of the Astronomical Society of the Pacific (共 2 篇) -- `1998PASP..110..906H` -- `2001PASP..113..490W` -### IUE Proposal (共 2 篇) -- `1987iue..prop.2806L` -- `1993iue..prop.4613J` -### Annual Review of Astronomy and Astrophysics (共 1 篇) -- `2009ARA&A..47..211H` -### Stellar Pulsation: Challenges for Theory and Observation (共 1 篇) -- `2009AIPC.1170..585C` -### Ph.D. Thesis (共 1 篇) -- `2015PhDT.......315A` -### Ultraviolet observations of Quasars (共 1 篇) -- `1980ESASP.157..323R` -### American Astronomical Society Meeting Abstracts #244 (共 1 篇) -- `2024AAS...24430302K` -### EAS2023, European Astronomical Society Annual Meeting (共 1 篇) -- `2023eas..conf..491U` -### Future Directions in Ultraviolet Spectroscopy: A Conference Inspired by the Accomplishments of the Far Ultraviolet Spectroscopic Explorer Mission (共 1 篇) -- `2009AIPC.1135..148C` -### American Astronomical Society Meeting Abstracts #234 (共 1 篇) -- `2019AAS...23432204B` -### Astronomicheskii Zhurnal (共 1 篇) -- `1995AZh....72..879E` -### American Astronomical Society Meeting Abstracts #233 (共 1 篇) -- `2019AAS...23346403W` -### American Astronomical Society Meeting Abstracts #241 (共 1 篇) -- `2023AAS...24130225Z` -### The Astronomical Journal (共 1 篇) -- `2021AJ....161..193L` -### American Astronomical Society Meeting Abstracts #227 (共 1 篇) -- `2016AAS...22740404B` -### Astronomische Nachrichten (共 1 篇) -- `2001AN....322..271S` -### Thirteenth Marcel Grossmann Meeting: On Recent Developments in Theoretical and Experimental General Relativity, Astrophysics and Relativistic Field Theories (共 1 篇) -- `2015mgm..conf.2459M` - -=== 404 OR MISSING PDF MAGIC === -### VizieR Online Data Catalog (共 16 篇) -- `1996yCat.3137....0K` -- `2016yCat..74573396P` -- `2023yCat..74910874N` -- `2023yCat..74952844S` -- `2024yCat..19280020B` -- `2024yCat..19420109L` -- `2024yCat..22710021L` -- `2024yCat..22710057X` -- `2024yCat..36840118U` -- `2024yCat..36910223V` -- `2024yCat..36930121H` -- `2024yCat..36930245W` -- `2024yCat..36930268R` -- `2025yCat..36900368G` -- `2025yCat..36970098B` -- `2025yCat..37050248L` -### American Astronomical Society Meeting Abstracts (共 12 篇) -- `1992AAS...181.5003H` -- `1994AAS...185.8005L` -- `1995AAS...187.8202M` -- `2000AAS...197.8302C` -- `2001AAS...199.0615S` -- `2004AAS...20517003S` -- `2006AAS...20915101W` -- `2007AAS...211.0320P` -- `2007AAS...211.0333W` -- `2007AAS...211.6006C` -- `2007AAS...21110422K` -- `2026AAS...24730801S` -### The Astrophysical Journal (共 9 篇) -- `1997ApJ...485..843L` -- `1997ApJ...487L..81B` -- `1997ApJ...491..172S` -- `1998ApJ...493..440G` -- `1998ApJ...494L..75B` -- `2000ApJ...530..441B` -- `2011ApJ...733..100L` -- `2011ApJ...734...59G` -- `2025ApJ...989..177C` -### HST Proposal (共 8 篇) -- `1994hst..prop.5305L` -- `2012hst..prop12954B` -- `2013hst..prop13290G` -- `2014hst..prop13800J` -- `2017hst..prop15284N` -- `2022hst..prop17072D` -- `2024hst..prop17697D` -- `2024hst..prop17799N` -### XMM-Newton Proposal (共 7 篇) -- `2009xmm..prop..182M` -- `2010xmm..prop...51M` -- `2010xmm..prop...57L` -- `2011xmm..prop..162L` -- `2019xmm..prop...66M` -- `2020xmm..prop..117M` -- `2021xmm..prop..123M` -### IUE Proposal (共 7 篇) -- `1980iue..prop..596D` -- `1981iue..prop..889W` -- `1981iue..prop..907D` -- `1986iue..prop.2435D` -- `1988iue..prop.3231H` -- `1991iue..prop.4189D` -- `1994iue..prop.4925T` -### Ph.D. Thesis (共 6 篇) -- `1991PhDT........10S` -- `1991PhDT.......346M` -- `1994PhDT.......261M` -- `2007PhDT.......230R` -- `2013PhDT.......509A` -- `2022PhDT........21F` -### The Astronomical Journal (共 6 篇) -- `2008AJ....136..946M` -- `2023AJ....165..142K` -- `2023AJ....165..148H` -- `2025AJ....170..199O` -- `2026AJ....171..165C` -- `2026AJ....171..217B` -### NOAO Proposal (共 5 篇) -- `1999noao.prop...31W` -- `2002noao.prop..318S` -- `2010noao.prop..372W` -- `2011noao.prop..191V` -- `2012noao.prop..214B` -### White Dwarfs (共 4 篇) -- `1995LNP...443..221S` -- `1995LNP...443..271T` -- `1995LNP...443..272U` -- `2003ASIB..105...99G` -### American Astronomical Society Meeting Abstracts #233 (共 4 篇) -- `2019AAS...23331402W` -- `2019AAS...23336016C` -- `2019AAS...23342201R` -- `2019AAS...23346406D` -### IAU General Assembly (共 4 篇) -- `2015IAUGA..2224007H` -- `2015IAUGA..2233490G` -- `2015IAUGA..2235533G` -- `2015IAUGA..2254919C` -### Research Notes of the American Astronomical Society (共 3 篇) -- `2019RNAAS...3...81B` -- `2023RNAAS...7..255K` -- `2025RNAAS...9..227Z` -### EAS2023, European Astronomical Society Annual Meeting (共 3 篇) -- `2023eas..conf..202T` -- `2023eas..conf..553G` -- `2023eas..conf.2257V` -### American Astronomical Society Meeting Abstracts #221 (共 3 篇) -- `2013AAS...22111605P` -- `2013AAS...22114217B` -- `2013AAS...22144305B` -### American Astronomical Society Meeting Abstracts #227 (共 3 篇) -- `2016AAS...22714405V` -- `2016AAS...22734412B` -- `2016AAS...22734514C` -### Odessa Astronomical Publications (共 3 篇) -- `2001OAP....14...82P` -- `2001OAP....14...87P` -- `2005OAP....18..135V` -### EAS2024, European Astronomical Society Annual Meeting (共 2 篇) -- `2024eas..conf.1492A` -- `2024eas..conf.2481P` -### Nature (共 2 篇) -- `1978Natur.275..385H` -- `1979Natur.279..305H` -### Publications of the Astronomical Society of the Pacific (共 2 篇) -- `1998PASP..110.1315G` -- `2001PASP..113..944W` -### American Astronomical Society Meeting Abstracts #215 (共 2 篇) -- `2010AAS...21541929W` -- `2010AAS...21545206C` -### New Quests in Stellar Astrophysics. II. Ultraviolet Properties of Evolved Stellar Populations (共 2 篇) -- `2009ASSP....7...59H` -- `2009ASSP....7..191N` -### American Astronomical Society Meeting Abstracts #223 (共 2 篇) -- `2014AAS...22315615V` -- `2014AAS...22315625R` -### Astronomy Reports (共 2 篇) -- `1995ARep...39..785E` -- `1997ARep...41..802M` -### The Atmospheres of Early-Type Stars (共 2 篇) -- `1992LNP...401..257D` -- `1992LNP...401..264T` -### American Astronomical Society Meeting Abstracts #198 (共 2 篇) -- `2001AAS...198.4906W` -- `2001AAS...198.4907S` -### Hot Stars in the Galactic Halo (共 2 篇) -- `1994hsgh.conf..182D` -- `1994hsgh.conf..341D` -### American Astronomical Society Meeting Abstracts #219 (共 2 篇) -- `2012AAS...21915325L` -- `2012AAS...21940803B` -### Magnetic Stars (共 1 篇) -- `2011mast.conf..415H` -### The First Year of IUE (共 1 篇) -- `1979IUE1.symp..363D` -### Stellar Atmospheres - Beyond Classical Models (共 1 篇) -- `1991ASIC..341..341W` -### Fourth European IUE Conference (共 1 篇) -- `1984ESASP.218..273H` -### Exploring the Universe with the IUE Satellite (共 1 篇) -- `1987ASSL..129..355V` -### Planetary and Proto-Planetary Nebulae: From IRAS to ISO (共 1 篇) -- `1987ASSL..135..137H` -### Acta Astronomica Sinica (共 1 篇) -- `2020AcASn..61...19M` -### American Astronomical Society Meeting Abstracts #231 (共 1 篇) -- `2018AAS...23114603H` -### Swift and the Surprising Sky: The First Seven Years of Swift. Online at: http://www.brera.inaf.it/docM/OAB/Research/SWIFT/Swift7/?p=program (共 1 篇) -- `2011sssf.confE..42M` -### Stellar Magnetic Fields (共 1 篇) -- `1997smf..proc..122E` -### FUSE Proposal (共 1 篇) -- `2003fuse.prop.D165L` -### Optical Complex Systems: OCS11 (共 1 篇) -- `2011SPIE.8172E..0UV` -### American Astronomical Society Meeting Abstracts #224 (共 1 篇) -- `2014AAS...22421903B` -### The Impact of Asteroseismology across Stellar Astrophysics (共 1 篇) -- `2011iasa.confE...2H` -### Progress in Astronomy (共 1 篇) -- `2008PABei..26..126Y` -### American Astronomical Society Meeting Abstracts #218 (共 1 篇) -- `2011AAS...21812207C` -### NASA Conference Publication (共 1 篇) -- `1981NASCP2171..349K` -### American Astronomical Society Meeting Abstracts #194 (共 1 篇) -- `1999AAS...194.6702H` -### Cataclysmic Variables and Low-Mass X-ray Binaries (共 1 篇) -- `1985ASSL..113...15B` -### Magellanic Clouds and Other Dwarf Galaxies (共 1 篇) -- `1998mcdg.proc..229A` -### Memoires of the Societe Royale des Sciences de Liege (共 1 篇) -- `1975MSRSL...9..247G` -### IAU Colloquium 53: White Dwarfs and Variable Degenerate Stars (共 1 篇) -- `1979wdvd.coll..107O` -### Astrofizika (共 1 篇) -- `1990Afz....33..199S` -### Structure and Evolution of Active Galactic Nuclei (共 1 篇) -- `1986ASSL..121..317K` -### Keck Observatory Archive ESI (共 1 篇) -- `2013koa..prop...89F` -### Astronomy Letters (共 1 篇) -- `2020AstL...46..601A` -### Astronomische Nachrichten (共 1 篇) -- `2007AN....328..708G` -### Pisma v Astronomicheskii Zhurnal (共 1 篇) -- `1990PAZh...16.1095T` -### NASA Special Publication (共 1 篇) -- `1982NASSP.456..147L` -### Variable Stars, the Galactic halo and Galaxy Formation (共 1 篇) -- `2010vsgh.conf..161G` -### GALEX Proposal (共 1 篇) -- `2004galx.prop...60W` -### Visual Double Stars : Formation, Dynamics and Evolutionary Tracks (共 1 篇) -- `1997ASSL..223..209U` -### Peremennye Zvezdy (共 1 篇) -- `2018PZ.....38....2D` diff --git a/scratch/format_failed.py b/scratch/format_failed.py deleted file mode 100644 index 3f4092d..0000000 --- a/scratch/format_failed.py +++ /dev/null @@ -1,67 +0,0 @@ -import re -import sqlite3 - -log_path = "/home/fmq/program/AstroResearch/logs/astro_research.log.2026-06-10" -db_path = "/home/fmq/program/AstroResearch/library/astro_research.db" - -conn = sqlite3.connect(db_path) -cursor = conn.cursor() -bibcode_to_pub = {} -cursor.execute("SELECT bibcode, pub FROM papers") -for row in cursor.fetchall(): - bibcode_to_pub[row[0]] = row[1] - -bibcode_logs = {} -current_bibcode = None - -with open(log_path, "r", encoding="utf-8") as f: - for line in f: - m = re.search(r"开始处理文献:\s*(\S+)", line) - if m: - current_bibcode = m.group(1) - bibcode_logs.setdefault(current_bibcode, []) - bibcode_logs[current_bibcode].append(line) - continue - m = re.search(r"\[下载\] 开始 (PDF|HTML) 下载:\s*(\S+)", line) - if m: - current_bibcode = m.group(2) - bibcode_logs.setdefault(current_bibcode, []) - bibcode_logs[current_bibcode].append(line) - continue - if current_bibcode: - bibcode_logs[current_bibcode].append(line) - -failed_papers = {} -for bibcode, logs in bibcode_logs.items(): - log_text = "".join(logs) - if "下载失败（PDF 和 HTML 均下载失败）" in log_text: - failed_papers[bibcode] = log_text - -err_403 = {} -err_404_magic = {} - -for bibcode, log_text in failed_papers.items(): - pub = bibcode_to_pub.get(bibcode, "Unknown") - has_403 = "403" in log_text or "Forbidden" in log_text or "Cloudflare" in log_text or "验证码" in log_text - has_404 = "404" in log_text or "not found" in log_text.lower() or "魔数" in log_text or "不是有效的 PDF" in log_text or "过小" in log_text or "损坏或不完整" in log_text - - if has_403: - err_403[bibcode] = pub - elif has_404: - err_404_magic[bibcode] = pub - -def format_group(err_dict): - grouped = {} - for b, p in err_dict.items(): - grouped.setdefault(p, []).append(b) - output = [] - for pub, bibs in sorted(grouped.items(), key=lambda x: len(x[1]), reverse=True): - output.append(f"### {pub} (共 {len(bibs)} 篇)") - for bib in sorted(bibs): - output.append(f"- `{bib}`") - return "\n".join(output) - -print("=== 403 FORBIDDEN ===") -print(format_group(err_403)) -print("\n=== 404 OR MISSING PDF MAGIC ===") -print(format_group(err_404_magic)) diff --git a/scratch/list_buckets.py b/scratch/list_buckets.py deleted file mode 100644 index e4f8740..0000000 --- a/scratch/list_buckets.py +++ /dev/null @@ -1,15 +0,0 @@ -import qiniu - -ak = "vf63aPF-QIFbyzULtHaSx9JgiVSS3zRuy0zmBACE" -sk = "JlQvHevHSAgilNYaH0UxQoX68rb4m9VflpaXtYL1" - -auth = qiniu.Auth(ak, sk) -bucket_manager = qiniu.BucketManager(auth) - -print("Listing buckets...") -try: - buckets, info = bucket_manager.buckets() - print("Buckets:", buckets) - print("Info:", info) -except Exception as e: - print("Error listing buckets:", e) diff --git a/scratch/parse_errors.py b/scratch/parse_errors.py deleted file mode 100644 index 3c37ac2..0000000 --- a/scratch/parse_errors.py +++ /dev/null @@ -1,76 +0,0 @@ -import re -import sqlite3 - -log_path = "/home/fmq/program/AstroResearch/logs/astro_research.log.2026-06-10" -db_path = "/home/fmq/program/AstroResearch/library/astro_research.db" - -conn = sqlite3.connect(db_path) -cursor = conn.cursor() -bibcode_to_pub = {} -cursor.execute("SELECT bibcode, pub FROM papers") -for row in cursor.fetchall(): - bibcode_to_pub[row[0]] = row[1] - -bibcode_logs = {} -current_bibcode = None - -with open(log_path, "r", encoding="utf-8") as f: - for line in f: - m = re.search(r"开始处理文献:\s*(\S+)", line) - if m: - current_bibcode = m.group(1) - bibcode_logs.setdefault(current_bibcode, []) - bibcode_logs[current_bibcode].append(line) - continue - m = re.search(r"\[下载\] 开始 (PDF|HTML) 下载:\s*(\S+)", line) - if m: - current_bibcode = m.group(2) - bibcode_logs.setdefault(current_bibcode, []) - bibcode_logs[current_bibcode].append(line) - continue - if current_bibcode: - bibcode_logs[current_bibcode].append(line) - -failed_papers = {} -for bibcode, logs in bibcode_logs.items(): - log_text = "".join(logs) - if "下载失败（PDF 和 HTML 均下载失败）" in log_text: - failed_papers[bibcode] = log_text - -print("Total failed papers:", len(failed_papers)) - -err_403 = {} -err_404_magic = {} - -for bibcode, log_text in failed_papers.items(): - pub = bibcode_to_pub.get(bibcode, "Unknown") - - # We want to identify the primary reason for failure. - # If the log text contains "403 Forbidden" or "Cloudflare", then it's a 403 block. - # Otherwise, if it has "404 Not Found" or "缺少 %PDF 魔数" or "响应不是有效的 PDF", it's 404/Magic. - # Let's check: - has_403 = "403" in log_text or "Forbidden" in log_text or "Cloudflare" in log_text or "验证码" in log_text - has_404 = "404" in log_text or "not found" in log_text.lower() or "魔数" in log_text or "不是有效的 PDF" in log_text or "过小" in log_text or "损坏或不完整" in log_text - - if has_403: - err_403[bibcode] = pub - elif has_404: - err_404_magic[bibcode] = pub - -print(f"Failed via 403 count: {len(err_403)}") -print(f"Failed via 404/Magic count: {len(err_404_magic)}") - -# Print them grouped -print("\n--- 403 Grouped ---") -grouped_403 = {} -for b, p in err_403.items(): - grouped_403.setdefault(p, []).append(b) -for pub, bibs in sorted(grouped_403.items(), key=lambda x: len(x[1]), reverse=True): - print(f"出版社/期刊: {pub} (共 {len(bibs)} 篇): {', '.join(bibs)}") - -print("\n--- 404/Magic Grouped ---") -grouped_404_magic = {} -for b, p in err_404_magic.items(): - grouped_404_magic.setdefault(p, []).append(b) -for pub, bibs in sorted(grouped_404_magic.items(), key=lambda x: len(x[1]), reverse=True): - print(f"出版社/期刊: {pub} (共 {len(bibs)} 篇): {', '.join(bibs)}") diff --git a/scratch/test_qiniu.py b/scratch/test_qiniu.py deleted file mode 100644 index 9238d20..0000000 --- a/scratch/test_qiniu.py +++ /dev/null @@ -1,44 +0,0 @@ -import base64 -import hmac -import hashlib -import json -import time -import requests - -def urlsafe_base64_encode(data): - if isinstance(data, str): - data = data.encode('utf-8') - ret = base64.urlsafe_b64encode(data) - # base64url standard replaces padding '=' with nothing - return ret.decode('utf-8').rstrip('=') - -def generate_token(ak, sk, bucket, key, scope_key=True): - deadline = int(time.time()) + 3600 - scope = f"{bucket}:{key}" if scope_key else bucket - policy = { - "scope": scope, - "deadline": deadline - } - policy_str = json.dumps(policy, separators=(',', ':')) - encoded_policy = urlsafe_base64_encode(policy_str) - - # hmac-sha1 - hashed = hmac.new(sk.encode('utf-8'), encoded_policy.encode('utf-8'), hashlib.sha1) - encoded_signature = urlsafe_base64_encode(hashed.digest()) - - return f"{ak}:{encoded_signature}:{encoded_policy}" - -ak = "vf63aPF-QIFbyzULtHaSx9JgiVSS3zRuy0zmBACE" -sk = "JlQvHevHSAgilNYaH0UxQoX68rb4m9VflpaXtYL1" -bucket = "fmqi-img" -key = "astroresearch/test_hello.txt" - -token = generate_token(ak, sk, bucket, key, scope_key=True) -print(f"Generated python token: {token}") - -# Let's try uploading -files = {'file': ('test.txt', b'hello python')} -data = {'token': token, 'key': key} -res = requests.post("https://up-z1.qiniup.com", data=data, files=files) -print(f"Python upload status: {res.status_code}") -print(f"Python upload response: {res.text}") diff --git a/scratch/test_qiniu.rs b/scratch/test_qiniu.rs deleted file mode 100644 index d9f3061..0000000 --- a/scratch/test_qiniu.rs +++ /dev/null @@ -1,68 +0,0 @@ -use sha1::Sha1; -use hmac::{Hmac, Mac}; -use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD}; -use reqwest::multipart; - -type HmacSha1 = Hmac; - -fn generate_token(ak: &str, sk: &str, bucket: &str, key: &str, scope_key: bool) -> String { - let deadline = chrono::Utc::now().timestamp() + 3600; - - let scope = if scope_key { - format!("{}:{}", bucket, key) - } else { - bucket.to_string() - }; - - let policy = serde_json::json!({ - "scope": scope, - "deadline": deadline - }); - - let policy_str = policy.to_string(); - let encoded_policy = URL_SAFE_NO_PAD.encode(policy_str.as_bytes()); - - let mut mac = HmacSha1::new_from_slice(sk.as_bytes()).unwrap(); - mac.update(encoded_policy.as_bytes()); - let signature = mac.finalize().into_bytes(); - let encoded_signature = URL_SAFE_NO_PAD.encode(&signature); - - format!("{}:{}:{}", ak, encoded_signature, encoded_policy) -} - -#[tokio::main] -async fn main() -> Result<(), Box> { - let ak = "vf63aPF-QIFbyzULtHaSx9JgiVSS3zRuy0zmBACE".to_string(); - let sk = "JlQvHevHSAgilNYaH0UxQoX68rb4m9VflpaXtYL1".to_string(); - let bucket = "fmqi-img".to_string(); - let domain = "http://qnimg.asfmq.cn".to_string(); - - let client = reqwest::Client::new(); - let dummy_data = b"hello world qiniu test".to_vec(); - let filename = "test_hello.txt"; - let key = "astroresearch/test_hello.txt"; - - println!("Testing with scoped key token (bucket:key)..."); - let token1 = generate_token(&ak, &sk, &bucket, key, true); - let form1 = multipart::Form::new() - .text("token", token1) - .text("key", key) - .part("file", multipart::Part::bytes(dummy_data.clone()).file_name(filename)); - - let res1 = client.post("https://up-z1.qiniup.com").multipart(form1).send().await?; - println!("Status (scoped key): {}", res1.status()); - println!("Response: {}", res1.text().await?); - - println!("\nTesting with bucket-only scoped token..."); - let token2 = generate_token(&ak, &sk, &bucket, key, false); - let form2 = multipart::Form::new() - .text("token", token2) - .text("key", key) - .part("file", multipart::Part::bytes(dummy_data).file_name(filename)); - - let res2 = client.post("https://up-z1.qiniup.com").multipart(form2).send().await?; - println!("Status (bucket-only): {}", res2.status()); - println!("Response: {}", res2.text().await?); - - Ok(()) -} diff --git a/scratch/test_qiniu_sdk.py b/scratch/test_qiniu_sdk.py deleted file mode 100644 index b2ecb2f..0000000 --- a/scratch/test_qiniu_sdk.py +++ /dev/null @@ -1,15 +0,0 @@ -import qiniu - -ak = "vf63aPF-QIFbyzULtHaSx9JgiVSS3zRuy0zmBACE" -sk = "JlQvHevHSAgilNYaH0UxQoX68rb4m9VflpaXtYL1" -bucket = "fmqi-img" -key = "astroresearch/test_hello.txt" - -auth = qiniu.Auth(ak, sk) -token = auth.upload_token(bucket, key, 3600) -print("SDK Generated Token:", token) - -# Upload using Qiniu SDK put_data -ret, info = qiniu.put_data(token, key, b"hello SDK") -print("Ret:", ret) -print("Info:", info) diff --git a/src/api/handlers.rs b/src/api/handlers.rs deleted file mode 100644 index e279a0c..0000000 --- a/src/api/handlers.rs +++ /dev/null @@ -1,1338 +0,0 @@ -// src/handlers.rs -use axum::{ - extract::{Query, State}, - http::StatusCode, - Json, -}; -use serde::{Deserialize, Serialize}; -use std::sync::Arc; -use std::fs; -use tracing::{info, warn, error}; -use sqlx::{SqlitePool, Row}; - -use crate::Config; -use crate::services::translation::Dictionary; -use crate::clients::qiniu::QiniuClient; -use crate::clients::ads::{AdsClient, AdsPaperDoc}; -use crate::clients::arxiv::{ArxivClient, ArxivPaper}; -use crate::services::download::Downloader; - -// 全局共享的 Axum 应用上下文状态 -pub struct AppState { - pub config: Config, - pub db: SqlitePool, - pub dict: Dictionary, - pub qiniu: QiniuClient, - pub ads: AdsClient, - pub arxiv: ArxivClient, - pub downloader: Downloader, - pub harvest_status: Arc>, - pub process_status: Arc>, -} - -// 检索请求参数 -#[derive(Debug, Deserialize)] -pub struct SearchParams { - pub q: String, - pub source: Option, // "all" | "ads" | "arxiv" - pub rows: Option, - pub start: Option, // 分页起始偏移量 - pub sort: Option, // 排序字段 -} - -// 统一标准化的文献格式，用于向前端传输 -#[derive(Debug, Serialize, Deserialize, Clone)] -pub struct StandardPaper { - pub bibcode: String, - pub title: String, - pub authors: Vec, - pub year: String, - pub pub_journal: String, - pub keywords: Vec, - pub abstract_text: String, - pub doi: String, - pub arxiv_id: String, - pub citation_count: i32, - pub reference_count: i32, - pub is_downloaded: bool, - pub has_markdown: bool, - pub has_translation: bool, - pub doctype: String, -} - -// ── GET /api/search ── -// 统一检索接口，合并去重 ADS 和 arXiv 数据 -pub async fn search_papers( - State(state): State>, - Query(params): Query, -) -> Result>, (StatusCode, String)> { - let source = params.source.unwrap_or_else(|| "all".to_string()); - let rows = params.rows.unwrap_or(10); - let start = params.start.unwrap_or(0); - let sort = params.sort.as_deref().unwrap_or("relevance"); - let mut results = Vec::new(); - - // 1. 检索 NASA ADS - if source == "all" || source == "ads" { - if !state.config.ads_api_key.is_empty() { - match state.ads.search(¶ms.q, start, rows, sort).await { - Ok(docs) => { - for doc in docs { - let paper = convert_ads_doc_to_standard(&doc); - - // 入库 SQLite - if let Err(e) = save_paper_to_db(&state.db, &paper).await { - warn!("保存 ADS 文献至数据库失败: {}", e); - } - - // 保存引用/参考文献关联拓扑 - if let Some(refs) = doc.reference { - for ref_bib in refs { - let _ = sqlx::query("INSERT OR IGNORE INTO citations_references (source_bibcode, target_bibcode) VALUES (?, ?)") - .bind(&paper.bibcode) - .bind(&ref_bib) - .execute(&state.db) - .await; - } - } - if let Some(cits) = doc.citation { - for cit_bib in cits { - let _ = sqlx::query("INSERT OR IGNORE INTO citations_references (source_bibcode, target_bibcode) VALUES (?, ?)") - .bind(&cit_bib) - .bind(&paper.bibcode) - .execute(&state.db) - .await; - } - } - - results.push(paper); - } - } - Err(e) => { - error!("ADS 检索执行失败: {}", e); - } - } - } else { - warn!("ADS_API_KEY 未配置，跳过 ADS 检索。"); - } - } - - // 2. 检索 arXiv - if source == "all" || source == "arxiv" { - match state.arxiv.search(¶ms.q, start, rows, sort).await { - Ok(papers) => { - for p in papers { - let paper = convert_arxiv_to_standard(&p); - - // 入库 SQLite (使用 arXiv ID 暂作主键以作记录) - if let Err(e) = save_paper_to_db(&state.db, &paper).await { - warn!("保存 arXiv 文献至数据库失败: {}", e); - } - - results.push(paper); - } - } - Err(e) => { - error!("arXiv 检索执行失败: {}", e); - } - } - } - - // 对两端获取的数据进行去重合并，增加对相同 arxiv_id 的判断 - let mut unique_results: Vec = Vec::new(); - for r in results { - if !unique_results.iter().any(|u| u.bibcode == r.bibcode || (!u.doi.is_empty() && u.doi == r.doi) || (!u.arxiv_id.is_empty() && u.arxiv_id == r.arxiv_id)) { - let mut final_paper = r.clone(); - // 实时检查 SQLite 校验下载及解析缓存状态并验证物理文件存在 - if let Ok(Some((pdf, html, md, tr))) = check_paper_paths_in_db(&state.db, &state.config.library_dir, &r.bibcode).await { - final_paper.is_downloaded = pdf.is_some() || html.is_some(); - final_paper.has_markdown = md.is_some(); - final_paper.has_translation = tr.is_some(); - } - unique_results.push(final_paper); - } - } - - Ok(Json(unique_results)) -} - -// ── POST /api/download ── -#[derive(Deserialize)] -pub struct DownloadRequest { - pub bibcode: String, - pub force: Option, // 强制重新下载，即使已存在本地文件 -} - -// 一键双格式并行下载文献 (PDF + HTML)，支持 force=true 强制重新下载 -pub async fn download_paper( - State(state): State>, - Json(req): Json, -) -> Result, (StatusCode, String)> { - let force = req.force.unwrap_or(false); - info!("接收到文献下载指令，标识符: {}, 强制重下: {}", req.bibcode, force); - - let paper = get_paper_from_db(&state.db, &state.config.library_dir, &req.bibcode) - .await - .map_err(|e| (StatusCode::NOT_FOUND, format!("未找到该文献记录: {}", e)))?; - - // force=true 时清除旧路径记录，强制重新下载 - if force { - sqlx::query("UPDATE papers SET pdf_path = NULL, html_path = NULL WHERE bibcode = ?") - .bind(&req.bibcode) - .execute(&state.db) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("重置下载状态失败: {}", e)))?; - } - - // 下载策略： - // 1. 如有 arXiv ID，优先走 arXiv 直连（绕过出版商防护墙，成功率高） - // 2. 否则走 ADS 网关多级回退（PUB_PDF → EPRINT_PDF → CrossRef） - // 3. 若 ADS 路径 PDF/HTML 均失败但有 arXiv ID，再尝试 arXiv 作为兜底 - let (pdf_path, html_path) = if !paper.arxiv_id.is_empty() { - info!("[下载] 优先使用 arXiv 通道: {}", paper.arxiv_id); - state.downloader.download_arxiv_direct(&paper.arxiv_id, &state.config.library_dir).await - } else { - let doi_opt = if !paper.doi.is_empty() { Some(paper.doi.as_str()) } else { None }; - let result = state.downloader.download_paper(&req.bibcode, doi_opt, &state.config.library_dir).await; - result - }; - - if pdf_path.is_none() && html_path.is_none() { - error!("文献 {} PDF 和 HTML 均下载失败，无可用物理文件格式", req.bibcode); - return Err((StatusCode::INTERNAL_SERVER_ERROR, "PDF 和 HTML 均下载失败，请检查网络".to_string())); - } - - let pdf_rel = pdf_path.map(|p| p.strip_prefix(&state.config.library_dir).unwrap_or(&p).to_string_lossy().to_string()); - let html_rel = html_path.map(|p| p.strip_prefix(&state.config.library_dir).unwrap_or(&p).to_string_lossy().to_string()); - - // 回写存储路径至数据库 - sqlx::query("UPDATE papers SET pdf_path = ?, html_path = ? WHERE bibcode = ?") - .bind(pdf_rel) - .bind(html_rel) - .bind(&req.bibcode) - .execute(&state.db) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("更新数据库失败: {}", e)))?; - - let mut updated_paper = paper; - updated_paper.is_downloaded = true; - - Ok(Json(updated_paper)) -} - -// ── POST /api/parse ── -#[derive(Deserialize)] -pub struct ParseRequest { - pub bibcode: String, - pub force: Option, -} - -#[derive(Serialize)] -pub struct ParseResponse { - pub markdown: String, -} - -// 将 HTML / PDF 转换为标准英文 Markdown 文本（HTML 优先，PDF 调用 MinerU 远程 API 且上传七牛云） -pub async fn parse_paper( - State(state): State>, - Json(req): Json, -) -> Result, (StatusCode, String)> { - info!("接收到文献结构化解析指令: {} (强制重新解析: {:?})", req.bibcode, req.force); - - let (pdf_opt, html_opt, md_opt, _) = check_paper_paths_in_db(&state.db, &state.config.library_dir, &req.bibcode) - .await - .map_err(|e| (StatusCode::NOT_FOUND, format!("获取文献路径失败: {}", e)))? - .ok_or((StatusCode::NOT_FOUND, "该文献未注册在数据库中".to_string()))?; - - let force = req.force.unwrap_or(false); - - // 如果先前已经解析成功过且非强制重新解析，直读 Markdown 文件返回 - if !force { - if let Some(md_rel) = md_opt { - let md_abs = state.config.library_dir.join(&md_rel); - if md_abs.exists() { - if let Ok(content) = fs::read_to_string(&md_abs) { - return Ok(Json(ParseResponse { markdown: content })); - } - } - } - } - - let mut parsed_markdown = String::new(); - let mut relative_md_path = String::new(); - - // 查询该文献的元数据以生成 Markdown YAML 头部信息 - let paper = get_paper_from_db(&state.db, &state.config.library_dir, &req.bibcode) - .await - .map_err(|e| (StatusCode::NOT_FOUND, format!("未找到该文献元数据记录: {}", e)))?; - - // 策略 1：HTML 优先解析 - if let Some(html_rel) = html_opt { - let html_abs = state.config.library_dir.join(&html_rel); - if html_abs.exists() { - match crate::services::parser::html_to_markdown(&html_abs) { - Ok(md) => { - let front_matter = format!( - "---\ntitle: {}\nauthor: [{}]\npublisher: {}\nsource: \"https://ui.adsabs.harvard.edu/abs/{}/abstract\"\ndate: \"{}\"\ntags: \"{}\"\n---\n\n", - serde_json::to_string(&paper.title).unwrap_or_else(|_| format!("\"{}\"", paper.title)), - paper.authors.iter().map(|a| format!("\"{}\"", a)).collect::>().join(", "), - serde_json::to_string(&paper.pub_journal).unwrap_or_else(|_| format!("\"{}\"", paper.pub_journal)), - paper.bibcode, - paper.year, - paper.keywords.join(",") - ); - parsed_markdown = format!("{}{}", front_matter, md); - let md_filename = format!("{}_en.md", req.bibcode); - let md_dest = state.config.library_dir.join("Markdown").join(&md_filename); - fs::create_dir_all(md_dest.parent().unwrap()).unwrap_or_default(); - if fs::write(&md_dest, &parsed_markdown).is_ok() { - relative_md_path = format!("Markdown/{}", md_filename); - } - } - Err(e) => { - warn!("HTML 转换为 Markdown 失败 {}: {}。将自动降级使用 PDF 结构化解析。", req.bibcode, e); - } - } - } - } - - // 策略 2：回退至 PDF 远程 MinerU 解析 - if parsed_markdown.is_empty() { - if let Some(pdf_rel) = pdf_opt { - let pdf_abs = state.config.library_dir.join(&pdf_rel); - if pdf_abs.exists() { - match crate::services::parser::parse_pdf_via_mineru(&pdf_abs, &state.qiniu, &state.config).await { - Ok(md) => { - let front_matter = format!( - "---\ntitle: {}\nauthor: [{}]\npublisher: {}\nsource: \"https://ui.adsabs.harvard.edu/abs/{}/abstract\"\ndate: \"{}\"\ntags: \"{}\"\n---\n\n", - serde_json::to_string(&paper.title).unwrap_or_else(|_| format!("\"{}\"", paper.title)), - paper.authors.iter().map(|a| format!("\"{}\"", a)).collect::>().join(", "), - serde_json::to_string(&paper.pub_journal).unwrap_or_else(|_| format!("\"{}\"", paper.pub_journal)), - paper.bibcode, - paper.year, - paper.keywords.join(",") - ); - parsed_markdown = format!("{}{}", front_matter, md); - let md_filename = format!("{}_en.md", req.bibcode); - let md_dest = state.config.library_dir.join("Markdown").join(&md_filename); - fs::create_dir_all(md_dest.parent().unwrap()).unwrap_or_default(); - if fs::write(&md_dest, &parsed_markdown).is_ok() { - relative_md_path = format!("Markdown/{}", md_filename); - } - } - Err(e) => { - error!("PDF layout 远程 MinerU 解析失败: {}", e); - return Err((StatusCode::INTERNAL_SERVER_ERROR, format!("PDF 结构解析失败: {}", e))); - } - } - } else { - error!("文献 {} 解析失败：本地 PDF 文件 {:?} 丢失", req.bibcode, pdf_abs); - return Err((StatusCode::NOT_FOUND, "本地 PDF 文件未找到".to_string())); - } - } else { - error!("文献 {} 解析失败：请先下载该文献的 HTML 或 PDF 文件", req.bibcode); - return Err((StatusCode::BAD_REQUEST, "请先下载该文献的 HTML 或 PDF 文件".to_string())); - } - } - - // 更新本地解析路径至 SQLite 数据库 - if !relative_md_path.is_empty() { - let _ = sqlx::query("UPDATE papers SET markdown_path = ? WHERE bibcode = ?") - .bind(&relative_md_path) - .bind(&req.bibcode) - .execute(&state.db) - .await; - } - - Ok(Json(ParseResponse { markdown: parsed_markdown })) -} - -// ── POST /api/translate ── -#[derive(Deserialize)] -pub struct TranslateRequest { - pub bibcode: String, - pub force: Option, -} - -#[derive(Serialize)] -pub struct TranslateResponse { - pub translation: String, -} - -// 文献中英双栏对比翻译接口（包含词表注入与本地物理缓存） -pub async fn translate_paper( - State(state): State>, - Json(req): Json, -) -> Result, (StatusCode, String)> { - let force = req.force.unwrap_or(false); - info!("接收到对比翻译请求: 文献={}, 强制重译={}", req.bibcode, force); - - let (_, _, md_opt, tr_opt) = check_paper_paths_in_db(&state.db, &state.config.library_dir, &req.bibcode) - .await - .map_err(|e| (StatusCode::NOT_FOUND, format!("查询文献路径失败: {}", e)))? - .ok_or((StatusCode::NOT_FOUND, "该文献未注册在数据库中".to_string()))?; - - // 若本地已存在翻译物理文件且未指明强制重译，直读本地缓存返回 - if !force { - if let Some(tr_rel) = tr_opt { - let tr_abs = state.config.library_dir.join(&tr_rel); - if tr_abs.exists() { - if let Ok(content) = fs::read_to_string(&tr_abs) { - return Ok(Json(TranslateResponse { translation: content })); - } - } - } - } - - // 检查英文解析文件是否存在 - let md_rel = match md_opt { - Some(rel) => rel, - None => { - error!("文献 {} 翻译失败：文献未完成解析，缺少英文 Markdown 路径", req.bibcode); - return Err((StatusCode::BAD_REQUEST, "文献必须先完成解析方可翻译".to_string())); - } - }; - let md_abs = state.config.library_dir.join(&md_rel); - if !md_abs.exists() { - error!("文献 {} 翻译失败：解析的英文 Markdown 文件 {:?} 不存在", req.bibcode, md_abs); - return Err((StatusCode::BAD_REQUEST, "解析 Markdown 文件丢失".to_string())); - } - - let english_markdown = fs::read_to_string(&md_abs) - .map_err(|e| { - error!("文献 {} 翻译失败：读取解析内容失败: {}", req.bibcode, e); - (StatusCode::INTERNAL_SERVER_ERROR, format!("读取解析内容失败: {}", e)) - })?; - - // 调用 LLM 翻译服务并注入对照词表 - let translated_markdown = crate::services::translation::translate_markdown(&english_markdown, &state.dict, &state.config) - .await - .map_err(|e| { - error!("文献 {} 翻译失败：调用 LLM 翻译发生错误: {}", req.bibcode, e); - (StatusCode::INTERNAL_SERVER_ERROR, format!("调用 LLM 翻译失败: {}", e)) - })?; - - // 翻译结果物理写入本地 - let tr_filename = format!("{}_zh.md", req.bibcode); - let tr_dest = state.config.library_dir.join("Translation").join(&tr_filename); - fs::create_dir_all(tr_dest.parent().unwrap()).unwrap_or_default(); - - fs::write(&tr_dest, &translated_markdown) - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("写入翻译文件失败: {}", e)))?; - - let relative_tr_path = format!("Translation/{}", tr_filename); - - // 缓存路径更新入库 - sqlx::query("UPDATE papers SET translation_path = ? WHERE bibcode = ?") - .bind(&relative_tr_path) - .bind(&req.bibcode) - .execute(&state.db) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("更新数据库翻译状态失败: {}", e)))?; - - Ok(Json(TranslateResponse { translation: translated_markdown })) -} - -// ── GET /api/citations ── -#[derive(Debug, Serialize)] -pub struct CitationsResponse { - pub bibcode: String, - pub title: String, - pub citation_count: i32, - pub reference_count: i32, - pub references: Vec, // 该文献参考文献 bibcode 数组 - pub citations: Vec, // 引用该文献的 bibcode 数组 - pub citation_counts: std::collections::HashMap, // 相关文献与被引数映射 -} - -// 从 SQLite 查询引用关联，生成引用星系关系树 -pub async fn get_citation_network( - State(state): State>, - Query(params): Query, -) -> Result, (StatusCode, String)> { - let paper = match get_paper_from_db(&state.db, &state.config.library_dir, ¶ms.bibcode).await { - Ok(p) => p, - Err(_) => { - // 如果本地数据库查不到，尝试从 ADS 在线 API 动态获取 - if !state.config.ads_api_key.is_empty() { - match state.ads.search(&format!("bibcode:{}", params.bibcode), 0, 1, "relevance").await { - Ok(docs) => { - if let Some(doc) = docs.first() { - let standard_paper = convert_ads_doc_to_standard(doc); - // 保存至数据库缓存，并保存引用关联 - let _ = save_paper_to_db(&state.db, &standard_paper).await; - if let Some(refs) = &doc.reference { - for ref_bib in refs { - let _ = sqlx::query("INSERT OR IGNORE INTO citations_references (source_bibcode, target_bibcode) VALUES (?, ?)") - .bind(&standard_paper.bibcode) - .bind(ref_bib) - .execute(&state.db) - .await; - } - } - if let Some(cits) = &doc.citation { - for cit_bib in cits { - let _ = sqlx::query("INSERT OR IGNORE INTO citations_references (source_bibcode, target_bibcode) VALUES (?, ?)") - .bind(cit_bib) - .bind(&standard_paper.bibcode) - .execute(&state.db) - .await; - } - } - standard_paper - } else { - return Err((StatusCode::NOT_FOUND, format!("在本地库及 ADS 中均未找到该文献: {}", params.bibcode))); - } - } - Err(e) => { - return Err((StatusCode::INTERNAL_SERVER_ERROR, format!("在线检索文献元数据失败: {}", e))); - } - } - } else { - return Err((StatusCode::NOT_FOUND, format!("本地数据库未收录该文献，且未配置 ADS_API_KEY，无法在线加载: {}", params.bibcode))); - } - } - }; - - // 加载引用的文献 - let refs_rows = sqlx::query("SELECT target_bibcode FROM citations_references WHERE source_bibcode = ?") - .bind(¶ms.bibcode) - .fetch_all(&state.db) - .await - .unwrap_or_default(); - let references: Vec = refs_rows.iter().map(|row| row.get(0)).collect(); - - // 加载被引用的文献 - let cits_rows = sqlx::query("SELECT source_bibcode FROM citations_references WHERE target_bibcode = ?") - .bind(¶ms.bibcode) - .fetch_all(&state.db) - .await - .unwrap_or_default(); - let citations: Vec = cits_rows.iter().map(|row| row.get(0)).collect(); - - // 加载关联文献的被引数量 (从 SQLite papers 表获取) - let mut citation_counts = std::collections::HashMap::new(); - let mut all_related = references.clone(); - all_related.extend(citations.clone()); - for bib in all_related { - let count_opt: Option = sqlx::query_scalar("SELECT citation_count FROM papers WHERE bibcode = ?") - .bind(&bib) - .fetch_optional(&state.db) - .await - .unwrap_or_default(); - if let Some(c) = count_opt { - citation_counts.insert(bib, c); - } - } - - Ok(Json(CitationsResponse { - bibcode: paper.bibcode, - title: paper.title, - citation_count: paper.citation_count, - reference_count: paper.reference_count, - references, - citations, - citation_counts, - })) -} - -// ── GET /api/paper ── -#[derive(Serialize)] -pub struct PaperDetailResponse { - pub paper: StandardPaper, - pub english_content: Option, - pub translation_content: Option, -} - -// 获取文献标准详情和中英双语内容文件数据 -pub async fn get_paper_detail( - State(state): State>, - Query(params): Query, -) -> Result, (StatusCode, String)> { - let paper = get_paper_from_db(&state.db, &state.config.library_dir, ¶ms.bibcode) - .await - .map_err(|e| (StatusCode::NOT_FOUND, format!("未找到该文献数据: {}", e)))?; - - let (_, _, md_opt, tr_opt) = check_paper_paths_in_db(&state.db, &state.config.library_dir, ¶ms.bibcode) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? - .unwrap_or_default(); - - let english_content = md_opt.and_then(|rel| fs::read_to_string(state.config.library_dir.join(rel)).ok()); - let translation_content = tr_opt.and_then(|rel| fs::read_to_string(state.config.library_dir.join(rel)).ok()); - - Ok(Json(PaperDetailResponse { - paper, - english_content, - translation_content, - })) -} - -// ── GET /api/library ── -// 获取本地图书馆中文献列表 -pub async fn get_library( - State(state): State>, -) -> Result>, (StatusCode, String)> { - let rows = sqlx::query("SELECT bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, pdf_path, html_path, markdown_path, translation_path, doctype FROM papers ORDER BY created_at DESC") - .fetch_all(&state.db) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("访问本地数据库失败: {}", e)))?; - - let mut list = Vec::new(); - for r in rows { - let pdf_path: Option = r.get(11); - let html_path: Option = r.get(12); - let markdown_path: Option = r.get(13); - let translation_path: Option = r.get(14); - let doctype_val: Option = r.get(15); - - let authors_str: Option = r.get(2); - let authors: Vec = authors_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default(); - let keywords_str: Option = r.get(5); - let keywords: Vec = keywords_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default(); - - list.push(StandardPaper { - bibcode: r.get(0), - title: r.get(1), - authors, - year: r.get(3), - pub_journal: r.get(4), - keywords, - abstract_text: r.get(6), - doi: r.get(7), - arxiv_id: r.get(8), - citation_count: r.get(9), - reference_count: r.get(10), - is_downloaded: pdf_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false) - || html_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false), - has_markdown: markdown_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false), - has_translation: translation_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false), - doctype: doctype_val.unwrap_or_else(|| "article".to_string()), - }); - } - - Ok(Json(list)) -} - -// ── POST /api/export ── -#[derive(Deserialize)] -pub struct ExportRequest { - pub bibcodes: Vec, -} - -#[derive(Serialize)] -pub struct ExportResponse { - pub bibtex: String, -} - -// 批量请求 ADS 接口，获取选中 Bibcode 的标准 BibTeX 引文段落 -pub async fn export_citations( - State(state): State>, - Json(req): Json, -) -> Result, (StatusCode, String)> { - if state.config.ads_api_key.is_empty() { - return Err((StatusCode::BAD_REQUEST, "ADS API key 未在 .env 中配置，无法使用该接口".to_string())); - } - - let bibtex = state.ads.export_bibtex(req.bibcodes).await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("批量引文导出失败: {}", e)))?; - - Ok(Json(ExportResponse { bibtex })) -} - - -// ── POST/GET/DELETE /api/notes ── 笔记/高亮系统 - -#[derive(Debug, Serialize, Deserialize)] -pub struct NoteRecord { - pub id: i64, - pub bibcode: String, - pub paragraph_index: i64, - pub note_text: String, - pub highlight_color: String, - pub selected_text: String, - pub created_at: String, -} - -#[derive(Deserialize)] -pub struct CreateNoteRequest { - pub bibcode: String, - pub paragraph_index: i64, - pub note_text: Option, - pub highlight_color: Option, - pub selected_text: Option, -} - -#[derive(Deserialize)] -pub struct DeleteNoteParams { - pub id: i64, -} - -#[derive(Deserialize)] -pub struct GetNotesParams { - pub bibcode: String, -} - -// 创建笔记 -pub async fn create_note( - State(state): State>, - Json(req): Json, -) -> Result, (StatusCode, String)> { - let note_text = req.note_text.unwrap_or_default(); - let highlight_color = req.highlight_color.unwrap_or_else(|| "yellow".to_string()); - let selected_text = req.selected_text.unwrap_or_default(); - - let row = sqlx::query( - "INSERT INTO notes (bibcode, paragraph_index, note_text, highlight_color, selected_text) VALUES (?, ?, ?, ?, ?) RETURNING id, bibcode, paragraph_index, note_text, highlight_color, selected_text, created_at" - ) - .bind(&req.bibcode) - .bind(req.paragraph_index) - .bind(¬e_text) - .bind(&highlight_color) - .bind(&selected_text) - .fetch_one(&state.db) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("保存笔记失败: {}", e)))?; - - Ok(Json(NoteRecord { - id: row.get(0), - bibcode: row.get(1), - paragraph_index: row.get(2), - note_text: row.get(3), - highlight_color: row.get(4), - selected_text: row.get(5), - created_at: row.get(6), - })) -} - -// 查询某篇文献的全部笔记 -pub async fn get_notes( - State(state): State>, - Query(params): Query, -) -> Result>, (StatusCode, String)> { - let rows = sqlx::query( - "SELECT id, bibcode, paragraph_index, note_text, highlight_color, selected_text, created_at FROM notes WHERE bibcode = ? ORDER BY paragraph_index, created_at" - ) - .bind(¶ms.bibcode) - .fetch_all(&state.db) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("查询笔记失败: {}", e)))?; - - let notes: Vec = rows.iter().map(|r| NoteRecord { - id: r.get(0), - bibcode: r.get(1), - paragraph_index: r.get(2), - note_text: r.get(3), - highlight_color: r.get(4), - selected_text: r.get(5), - created_at: r.get(6), - }).collect(); - - Ok(Json(notes)) -} - -// 删除指定 id 的笔记 -pub async fn delete_note( - State(state): State>, - Query(params): Query, -) -> Result { - sqlx::query("DELETE FROM notes WHERE id = ?") - .bind(params.id) - .execute(&state.db) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("删除笔记失败: {}", e)))?; - - Ok(StatusCode::NO_CONTENT) -} - - -// ── 辅助数据库处理函数 ── - -pub(crate) fn convert_ads_doc_to_standard(doc: &AdsPaperDoc) -> StandardPaper { - let title = doc.title.as_ref() - .and_then(|v: &Vec| v.first()) - .cloned() - .unwrap_or_else(|| doc.bibcode.clone()); - let authors = doc.author.clone().unwrap_or_default(); - let keywords = doc.keyword.clone().unwrap_or_default(); - let doi = doc.doi.as_ref() - .and_then(|v: &Vec| v.first()) - .cloned() - .unwrap_or_default(); - - let mut arxiv_id = String::new(); - if let Some(identifiers) = &doc.identifier { - for id in identifiers { - if id.starts_with("arXiv:") { - arxiv_id = id.replace("arXiv:", "").trim().to_string(); - break; - } - } - } - if arxiv_id.is_empty() { - if doc.bibcode.starts_with("arXiv") { - arxiv_id = doc.bibcode.replace("arXiv", "").trim().to_string(); - } - } - - StandardPaper { - bibcode: doc.bibcode.clone(), - title, - authors, - year: doc.year.clone().unwrap_or_default(), - pub_journal: doc.pub_journal.clone().unwrap_or_default(), - keywords, - abstract_text: doc.abstract_text.clone().unwrap_or_default(), - doi, - arxiv_id, - citation_count: doc.citation_count.unwrap_or(0), - reference_count: doc.reference_count.unwrap_or(0), - is_downloaded: false, - has_markdown: false, - has_translation: false, - doctype: doc.doctype.clone().unwrap_or_else(|| "article".to_string()), - } -} - -pub(crate) fn convert_arxiv_to_standard(doc: &ArxivPaper) -> StandardPaper { - StandardPaper { - bibcode: doc.id.clone(), - title: doc.title.clone(), - authors: doc.authors.clone(), - year: doc.year.clone(), - pub_journal: "arXiv Preprint".to_string(), - keywords: Vec::new(), - abstract_text: doc.abstract_text.clone(), - doi: doc.doi.clone().unwrap_or_default(), - arxiv_id: doc.id.clone(), - citation_count: 0, - reference_count: 0, - is_downloaded: false, - has_markdown: false, - has_translation: false, - doctype: "eprint".to_string(), - } -} - -pub(crate) async fn save_paper_to_db(db: &SqlitePool, p: &StandardPaper) -> anyhow::Result<()> { - let authors_json = serde_json::to_string(&p.authors)?; - let keywords_json = serde_json::to_string(&p.keywords)?; - - // 1. 如果存在 arxiv_id，检查是否有已存在的相同 arxiv_id 记录以防 duplicate - if !p.arxiv_id.is_empty() { - let existing_opt: Option<(String, Option, Option, Option, Option)> = sqlx::query_as( - "SELECT bibcode, pdf_path, html_path, markdown_path, translation_path FROM papers WHERE arxiv_id = ?" - ) - .bind(&p.arxiv_id) - .fetch_optional(db) - .await?; - - if let Some((existing_bibcode, _pdf, _html, _md, _tr)) = existing_opt { - if existing_bibcode != p.bibcode { - // 发现不同 bibcode 标识的同一篇文献记录，需要进行合并合并 - // 如果已存在的记录使用的是临时 arXiv ID 作为 bibcode，且新记录使用的是正式 ADS bibcode，我们升级 bibcode 主键 - let is_existing_temp = existing_bibcode == p.arxiv_id; - let is_new_formal = p.bibcode != p.arxiv_id; - - if is_existing_temp && is_new_formal { - info!("发现相同 arXiv ID 的文献，将临时主键 {} 升级为正式 ADS Bibcode: {}", existing_bibcode, p.bibcode); - sqlx::query( - "UPDATE papers SET bibcode = ?, title = ?, authors = ?, year = ?, pub = ?, keywords = ?, abstract = ?, doi = ?, citation_count = ?, reference_count = ?, doctype = ? WHERE bibcode = ?" - ) - .bind(&p.bibcode) - .bind(&p.title) - .bind(&authors_json) - .bind(&p.year) - .bind(&p.pub_journal) - .bind(&keywords_json) - .bind(&p.abstract_text) - .bind(&p.doi) - .bind(p.citation_count) - .bind(p.reference_count) - .bind(&p.doctype) - .bind(&existing_bibcode) - .execute(db) - .await?; - - return Ok(()); - } else { - // 如果已存在的是正式 ADS bibcode，而新插入的是临时 arXiv ID，直接忽略或更新元数据而不更改主键 - info!("发现相同 arXiv ID 的文献 {} 已存在正式记录，忽略临时 arXiv 插入", existing_bibcode); - return Ok(()); - } - } - } - } - - // 2. 正常插入/冲突更新 - sqlx::query( - "INSERT INTO papers (bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, doctype) \ - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) \ - ON CONFLICT(bibcode) DO UPDATE SET \ - title=excluded.title, \ - authors=excluded.authors, \ - pub=excluded.pub, \ - keywords=excluded.keywords, \ - abstract=excluded.abstract, \ - doi=excluded.doi, \ - arxiv_id=excluded.arxiv_id, \ - citation_count=excluded.citation_count, \ - reference_count=excluded.reference_count, \ - doctype=excluded.doctype" - ) - .bind(&p.bibcode) - .bind(&p.title) - .bind(authors_json) - .bind(&p.year) - .bind(&p.pub_journal) - .bind(keywords_json) - .bind(&p.abstract_text) - .bind(&p.doi) - .bind(&p.arxiv_id) - .bind(p.citation_count) - .bind(p.reference_count) - .bind(&p.doctype) - .execute(db) - .await?; - - Ok(()) -} - -async fn get_paper_from_db(db: &SqlitePool, library_dir: &std::path::Path, bibcode: &str) -> anyhow::Result { - let r = sqlx::query("SELECT bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, pdf_path, html_path, markdown_path, translation_path, doctype FROM papers WHERE bibcode = ?") - .bind(bibcode) - .fetch_one(db) - .await?; - - let pdf_path: Option = r.get(11); - let html_path: Option = r.get(12); - let markdown_path: Option = r.get(13); - let translation_path: Option = r.get(14); - let doctype_val: Option = r.get(15); - - let authors_str: Option = r.get(2); - let authors: Vec = authors_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default(); - let keywords_str: Option = r.get(5); - let keywords: Vec = keywords_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default(); - - let is_pdf_exist = pdf_path.as_ref().map(|p| library_dir.join(p).exists()).unwrap_or(false); - let is_html_exist = html_path.as_ref().map(|p| library_dir.join(p).exists()).unwrap_or(false); - let is_md_exist = markdown_path.as_ref().map(|p| library_dir.join(p).exists()).unwrap_or(false); - let is_tr_exist = translation_path.as_ref().map(|p| library_dir.join(p).exists()).unwrap_or(false); - - Ok(StandardPaper { - bibcode: r.get(0), - title: r.get(1), - authors, - year: r.get(3), - pub_journal: r.get(4), - keywords, - abstract_text: r.get(6), - doi: r.get(7), - arxiv_id: r.get(8), - citation_count: r.get(9), - reference_count: r.get(10), - is_downloaded: is_pdf_exist || is_html_exist, - has_markdown: is_md_exist, - has_translation: is_tr_exist, - doctype: doctype_val.unwrap_or_else(|| "article".to_string()), - }) -} - -async fn check_paper_paths_in_db( - db: &SqlitePool, - library_dir: &std::path::Path, - bibcode: &str -) -> anyhow::Result, Option, Option, Option)>> { - let r_opt = sqlx::query("SELECT pdf_path, html_path, markdown_path, translation_path FROM papers WHERE bibcode = ?") - .bind(bibcode) - .fetch_optional(db) - .await?; - - if let Some(r) = r_opt { - let pdf: Option = r.get(0); - let html: Option = r.get(1); - let md: Option = r.get(2); - let tr: Option = r.get(3); - - let pdf_res = pdf.filter(|p| library_dir.join(p).exists()); - let html_res = html.filter(|p| library_dir.join(p).exists()); - let md_res = md.filter(|p| library_dir.join(p).exists()); - let tr_res = tr.filter(|p| library_dir.join(p).exists()); - - Ok(Some((pdf_res, html_res, md_res, tr_res))) - } else { - Ok(None) - } -} - -// ── POST /api/sync/meta/run ── -#[derive(Debug, Deserialize)] -pub struct MetaSyncRunRequest { - pub q: String, - pub source: String, // "all" | "ads" | "arxiv" - pub limit: i32, -} - -pub async fn run_meta_sync( - State(state): State>, - Json(req): Json, -) -> Result { - // 检查是否已在进行同步任务 - { - let status = state.harvest_status.lock().await; - if status.active { - return Err((StatusCode::CONFLICT, "当前已有文献批量同步任务在后台运行中，请勿重复启动".to_string())); - } - } - - crate::services::batch_sync::MetaSync::start_harvest( - state.db.clone(), - Arc::new(state.ads.clone()), - Arc::new(state.arxiv.clone()), - req.q, - req.source, - req.limit, - state.harvest_status.clone(), - ); - - Ok(StatusCode::ACCEPTED) -} - -// ── GET /api/sync/meta/count ── -#[derive(Debug, Deserialize)] -pub struct MetaSyncCountRequest { - pub q: String, - pub source: String, // "all" | "ads" | "arxiv" -} - -#[derive(Debug, Serialize)] -pub struct MetaSyncCountResponse { - pub total: i32, -} - -pub async fn get_meta_sync_count( - State(state): State>, - Query(req): Query, -) -> Result, (StatusCode, String)> { - let total = crate::services::batch_sync::MetaSync::get_total_count( - &req.q, - &req.source, - &state.ads, - &state.arxiv, - ) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("获取预估文献数失败: {}", e)))?; - - Ok(Json(MetaSyncCountResponse { total })) -} - -// ── GET /api/sync/meta/status ── -pub async fn get_meta_sync_status( - State(state): State>, -) -> Json { - let status = state.harvest_status.lock().await; - Json(status.clone()) -} - -// ── POST /api/sync/asset/run ── -#[derive(Debug, Deserialize)] -pub struct AssetSyncRunRequest { - pub action: Option, // "download" | "parse" | "all" - pub scope: String, // "all" | "undownloaded" | "unparsed" | "selected" - pub bibcodes: Option>, -} - -pub async fn run_asset_sync( - State(state): State>, - Json(req): Json, -) -> Result { - // 检查是否已经在进行批量处理任务 - { - let status = state.process_status.lock().await; - if status.active { - return Err((StatusCode::CONFLICT, "当前已有文献批量下载或解析任务在后台运行中，请勿重复启动".to_string())); - } - } - - let action_str = req.action.unwrap_or_else(|| "all".to_string()); - let action = match action_str.as_str() { - "download" => crate::services::batch_sync::SyncAction::Download, - "parse" => crate::services::batch_sync::SyncAction::Parse, - "all" | "download_and_parse" => crate::services::batch_sync::SyncAction::All, - _ => return Err((StatusCode::BAD_REQUEST, "不支持的 action 参数值".to_string())), - }; - - let mut target_bibcodes = Vec::new(); - - match req.scope.as_str() { - "selected" => { - if let Some(bibs) = req.bibcodes { - target_bibcodes = bibs; - } else { - return Err((StatusCode::BAD_REQUEST, "选择模式下必须指定 bibcodes 列表".to_string())); - } - } - "all" => { - // 查询馆藏所有文献 - let rows = sqlx::query("SELECT bibcode FROM papers") - .fetch_all(&state.db) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("读取数据库失败: {}", e)))?; - for r in rows { - target_bibcodes.push(r.get(0)); - } - } - "undownloaded" | "all_undownloaded" => { - // 查询所有本地无 PDF/HTML 文件的文献 - let rows = sqlx::query("SELECT bibcode FROM papers WHERE pdf_path IS NULL AND html_path IS NULL") - .fetch_all(&state.db) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("读取数据库失败: {}", e)))?; - for r in rows { - target_bibcodes.push(r.get(0)); - } - } - "unparsed" | "all_unparsed" => { - // 查询所有本地无 Markdown 文件的文献 (或者处于 mineru_batch: 状态的任务) - let rows = sqlx::query("SELECT bibcode FROM papers WHERE markdown_path IS NULL OR markdown_path LIKE 'mineru_batch:%'") - .fetch_all(&state.db) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("读取数据库失败: {}", e)))?; - for r in rows { - target_bibcodes.push(r.get(0)); - } - } - _ => { - return Err((StatusCode::BAD_REQUEST, "不支持的 scope 参数值".to_string())); - } - } - - if target_bibcodes.is_empty() { - return Err((StatusCode::OK, "没有需要处理的文献".to_string())); - } - - // 启动后台处理 - crate::services::batch_sync::AssetSync::start_process( - state.db.clone(), - state.config.clone(), - Arc::new(state.downloader.clone()), - Arc::new(state.qiniu.clone()), - action, - target_bibcodes, - state.process_status.clone(), - ); - - Ok(StatusCode::ACCEPTED) -} - -// ── POST /api/sync/asset/stop ── -pub async fn stop_asset_sync( - State(state): State>, -) -> StatusCode { - let mut status = state.process_status.lock().await; - if status.active { - status.active = false; - status.add_log("用户手动终止了批量处理任务。".to_string()); - } - StatusCode::OK -} - -// ── GET /api/sync/queries ── -#[derive(Debug, Serialize, Deserialize)] -pub struct SavedSyncQuery { - pub id: i64, - pub query: String, - pub source: String, - pub limit_count: i32, - pub last_run: String, -} - -pub async fn get_sync_queries( - State(state): State>, -) -> Result>, (StatusCode, String)> { - let rows = sqlx::query("SELECT id, query, source, limit_count, datetime(last_run, 'localtime') FROM sync_queries ORDER BY last_run DESC") - .fetch_all(&state.db) - .await - .map_err(|e| { - error!("获取已存同步检索配置失败: {}", e); - (StatusCode::INTERNAL_SERVER_ERROR, format!("获取已存同步检索配置失败: {}", e)) - })?; - - let mut list = Vec::new(); - for r in rows { - list.push(SavedSyncQuery { - id: r.get(0), - query: r.get(1), - source: r.get(2), - limit_count: r.get(3), - last_run: r.get(4), - }); - } - - Ok(Json(list)) -} - -// ── DELETE /api/sync/queries/:id ── -pub async fn delete_sync_query( - State(state): State>, - axum::extract::Path(id): axum::extract::Path, -) -> Result { - sqlx::query("DELETE FROM sync_queries WHERE id = ?") - .bind(id) - .execute(&state.db) - .await - .map_err(|e| { - error!("删除同步检索配置失败: {}", e); - (StatusCode::INTERNAL_SERVER_ERROR, format!("删除同步检索配置失败: {}", e)) - })?; - - Ok(StatusCode::OK) -} - -// ── GET /api/sync/asset/status ── -pub async fn get_asset_sync_status( - State(state): State>, -) -> Json { - let status = state.process_status.lock().await; - Json(status.clone()) -} - -#[cfg(test)] -mod tests { - use super::*; - use sqlx::sqlite::SqlitePoolOptions; - use crate::clients::ads::AdsPaperDoc; - use crate::clients::arxiv::ArxivPaper; - - #[test] - fn test_convert_ads_doc_to_standard() { - let doc = AdsPaperDoc { - bibcode: "2026A&A...123..456X".to_string(), - title: Some(vec!["A Test Title".to_string()]), - author: Some(vec!["Author A".to_string(), "Author B".to_string()]), - year: Some("2026".to_string()), - pub_journal: Some("Astronomy & Astrophysics".to_string()), - keyword: Some(vec!["Keyword 1".to_string()]), - abstract_text: Some("This is abstract".to_string()), - doi: Some(vec!["10.1000/test.doi".to_string()]), - citation_count: Some(5), - reference_count: Some(10), - reference: None, - citation: None, - identifier: None, - doctype: Some("article".to_string()), - }; - - let paper = convert_ads_doc_to_standard(&doc); - assert_eq!(paper.bibcode, "2026A&A...123..456X"); - assert_eq!(paper.title, "A Test Title"); - assert_eq!(paper.authors, vec!["Author A", "Author B"]); - assert_eq!(paper.year, "2026"); - assert_eq!(paper.pub_journal, "Astronomy & Astrophysics"); - assert_eq!(paper.keywords, vec!["Keyword 1"]); - assert_eq!(paper.abstract_text, "This is abstract"); - assert_eq!(paper.doi, "10.1000/test.doi"); - assert_eq!(paper.arxiv_id, ""); - assert_eq!(paper.citation_count, 5); - assert_eq!(paper.reference_count, 10); - } - - #[test] - fn test_convert_ads_doc_to_standard_with_arxiv_identifier() { - let doc = AdsPaperDoc { - bibcode: "2026MNRAS.530.1234A".to_string(), - title: Some(vec!["Another Test Title".to_string()]), - author: Some(vec!["Author A".to_string()]), - year: Some("2026".to_string()), - pub_journal: Some("MNRAS".to_string()), - keyword: None, - abstract_text: None, - doi: None, - citation_count: None, - reference_count: None, - reference: None, - citation: None, - identifier: Some(vec!["2026MNRAS.530.1234A".to_string(), "arXiv:2606.12345".to_string()]), - doctype: Some("article".to_string()), - }; - - let paper = convert_ads_doc_to_standard(&doc); - assert_eq!(paper.bibcode, "2026MNRAS.530.1234A"); - assert_eq!(paper.arxiv_id, "2606.12345"); - } - - #[test] - fn test_convert_arxiv_to_standard() { - let doc = ArxivPaper { - id: "2606.12345".to_string(), - title: "Arxiv Title".to_string(), - authors: vec!["Author C".to_string()], - year: "2026".to_string(), - abstract_text: "Arxiv abstract".to_string(), - doi: Some("10.1000/arxiv.doi".to_string()), - pdf_url: "https://arxiv.org/pdf/2606.12345.pdf".to_string(), - }; - - let paper = convert_arxiv_to_standard(&doc); - assert_eq!(paper.bibcode, "2606.12345"); - assert_eq!(paper.title, "Arxiv Title"); - assert_eq!(paper.authors, vec!["Author C"]); - assert_eq!(paper.year, "2026"); - assert_eq!(paper.pub_journal, "arXiv Preprint"); - assert_eq!(paper.doi, "10.1000/arxiv.doi"); - assert_eq!(paper.arxiv_id, "2606.12345"); - } - - #[tokio::test] - async fn test_db_operations() -> anyhow::Result<()> { - let pool = SqlitePoolOptions::new() - .max_connections(1) - .connect("sqlite::memory:") - .await?; - - // 运行迁移 - sqlx::migrate!("./migrations") - .run(&pool) - .await?; - - let paper = StandardPaper { - bibcode: "2026A&A...123..456X".to_string(), - title: "A Test Title".to_string(), - authors: vec!["Author A".to_string()], - year: "2026".to_string(), - pub_journal: "Astronomy & Astrophysics".to_string(), - keywords: vec!["Keyword 1".to_string()], - abstract_text: "This is abstract".to_string(), - doi: "10.1000/test.doi".to_string(), - arxiv_id: "".to_string(), - citation_count: 5, - reference_count: 10, - is_downloaded: false, - has_markdown: false, - has_translation: false, - doctype: "article".to_string(), - }; - - // 保存 - save_paper_to_db(&pool, &paper).await?; - - // 读取 - let retrieved = get_paper_from_db(&pool, std::path::Path::new(""), "2026A&A...123..456X").await?; - assert_eq!(retrieved.title, paper.title); - assert_eq!(retrieved.authors, paper.authors); - assert_eq!(retrieved.keywords, paper.keywords); - - // 检查路径状态（初始为 None） - let paths = check_paper_paths_in_db(&pool, std::path::Path::new(""), "2026A&A...123..456X").await?; - assert!(paths.is_some()); - let (pdf, html, md, tr) = paths.unwrap(); - assert!(pdf.is_none()); - assert!(html.is_none()); - assert!(md.is_none()); - assert!(tr.is_none()); - - Ok(()) - } -} - diff --git a/src/api/helpers.rs b/src/api/helpers.rs new file mode 100644 index 0000000..656051d --- /dev/null +++ b/src/api/helpers.rs @@ -0,0 +1,373 @@ +// src/api/helpers.rs +use sqlx::{SqlitePool, Row}; +use tracing::info; +use crate::clients::ads::AdsPaperDoc; +use crate::clients::arxiv::ArxivPaper; +use super::StandardPaper; + +pub fn convert_ads_doc_to_standard(doc: &AdsPaperDoc) -> StandardPaper { + let title = doc.title.as_ref() + .and_then(|v: &Vec| v.first()) + .cloned() + .unwrap_or_else(|| doc.bibcode.clone()); + let authors = doc.author.clone().unwrap_or_default(); + let keywords = doc.keyword.clone().unwrap_or_default(); + let doi = doc.doi.as_ref() + .and_then(|v: &Vec| v.first()) + .cloned() + .unwrap_or_default(); + + let mut arxiv_id = String::new(); + if let Some(identifiers) = &doc.identifier { + for id in identifiers { + if id.starts_with("arXiv:") { + arxiv_id = id.replace("arXiv:", "").trim().to_string(); + break; + } + } + } + if arxiv_id.is_empty() { + if doc.bibcode.starts_with("arXiv") { + arxiv_id = doc.bibcode.replace("arXiv", "").trim().to_string(); + } + } + + StandardPaper { + bibcode: doc.bibcode.clone(), + title, + authors, + year: doc.year.clone().unwrap_or_default(), + pub_journal: doc.pub_journal.clone().unwrap_or_default(), + keywords, + abstract_text: doc.abstract_text.clone().unwrap_or_default(), + doi, + arxiv_id, + citation_count: doc.citation_count.unwrap_or(0), + reference_count: doc.reference_count.unwrap_or(0), + is_downloaded: false, + has_markdown: false, + has_translation: false, + doctype: doc.doctype.clone().unwrap_or_else(|| "article".to_string()), + pdf_error: None, + html_error: None, + } +} + +pub fn convert_arxiv_to_standard(doc: &ArxivPaper) -> StandardPaper { + StandardPaper { + bibcode: doc.id.clone(), + title: doc.title.clone(), + authors: doc.authors.clone(), + year: doc.year.clone(), + pub_journal: "arXiv Preprint".to_string(), + keywords: Vec::new(), + abstract_text: doc.abstract_text.clone(), + doi: doc.doi.clone().unwrap_or_default(), + arxiv_id: doc.id.clone(), + citation_count: 0, + reference_count: 0, + is_downloaded: false, + has_markdown: false, + has_translation: false, + doctype: "eprint".to_string(), + pdf_error: None, + html_error: None, + } +} + +pub async fn save_paper_to_db(db: &SqlitePool, p: &StandardPaper) -> anyhow::Result<()> { + let authors_json = serde_json::to_string(&p.authors)?; + let keywords_json = serde_json::to_string(&p.keywords)?; + + // 1. 如果存在 arxiv_id，检查是否有已存在的相同 arxiv_id 记录以防 duplicate + if !p.arxiv_id.is_empty() { + let existing_opt: Option<(String, Option, Option, Option, Option)> = sqlx::query_as( + "SELECT bibcode, pdf_path, html_path, markdown_path, translation_path FROM papers WHERE arxiv_id = ?" + ) + .bind(&p.arxiv_id) + .fetch_optional(db) + .await?; + + if let Some((existing_bibcode, _pdf, _html, _md, _tr)) = existing_opt { + if existing_bibcode != p.bibcode { + // 发现不同 bibcode 标识的同一篇文献记录，需要进行合并 + // 如果已存在的记录使用的是临时 arXiv ID 作为 bibcode，且新记录使用的是正式 ADS bibcode，我们升级 bibcode 主键 + let is_existing_temp = existing_bibcode == p.arxiv_id; + let is_new_formal = p.bibcode != p.arxiv_id; + + if is_existing_temp && is_new_formal { + info!("发现相同 arXiv ID 的文献，将临时主键 {} 升级为正式 ADS Bibcode: {}", existing_bibcode, p.bibcode); + sqlx::query( + "UPDATE papers SET bibcode = ?, title = ?, authors = ?, year = ?, pub = ?, keywords = ?, abstract = ?, doi = ?, citation_count = ?, reference_count = ?, doctype = ? WHERE bibcode = ?" + ) + .bind(&p.bibcode) + .bind(&p.title) + .bind(&authors_json) + .bind(&p.year) + .bind(&p.pub_journal) + .bind(&keywords_json) + .bind(&p.abstract_text) + .bind(&p.doi) + .bind(p.citation_count) + .bind(p.reference_count) + .bind(&p.doctype) + .bind(&existing_bibcode) + .execute(db) + .await?; + + return Ok(()); + } else { + // 如果已存在的是正式 ADS bibcode，而新插入的是临时 arXiv ID，直接忽略或更新元数据而不更改主键 + info!("发现相同 arXiv ID 的文献 {} 已存在正式记录，忽略临时 arXiv 插入", existing_bibcode); + return Ok(()); + } + } + } + } + + // 2. 正常插入/冲突更新 + sqlx::query( + "INSERT INTO papers (bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, doctype) \ + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) \ + ON CONFLICT(bibcode) DO UPDATE SET \ + title=excluded.title, \ + authors=excluded.authors, \ + pub=excluded.pub, \ + keywords=excluded.keywords, \ + abstract=excluded.abstract, \ + doi=excluded.doi, \ + arxiv_id=excluded.arxiv_id, \ + citation_count=excluded.citation_count, \ + reference_count=excluded.reference_count, \ + doctype=excluded.doctype" + ) + .bind(&p.bibcode) + .bind(&p.title) + .bind(authors_json) + .bind(&p.year) + .bind(&p.pub_journal) + .bind(keywords_json) + .bind(&p.abstract_text) + .bind(&p.doi) + .bind(&p.arxiv_id) + .bind(p.citation_count) + .bind(p.reference_count) + .bind(&p.doctype) + .execute(db) + .await?; + + Ok(()) +} + +pub async fn get_paper_from_db(db: &SqlitePool, library_dir: &std::path::Path, bibcode: &str) -> anyhow::Result { + let r = sqlx::query("SELECT bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, pdf_path, html_path, markdown_path, translation_path, doctype FROM papers WHERE bibcode = ?") + .bind(bibcode) + .fetch_one(db) + .await?; + + let pdf_path: Option = r.get(11); + let html_path: Option = r.get(12); + let markdown_path: Option = r.get(13); + let translation_path: Option = r.get(14); + let doctype_val: Option = r.get(15); + + let authors_str: Option = r.get(2); + let authors: Vec = authors_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default(); + let keywords_str: Option = r.get(5); + let keywords: Vec = keywords_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default(); + + let is_pdf_exist = pdf_path.as_ref().map(|p| library_dir.join(p).exists()).unwrap_or(false); + let is_html_exist = html_path.as_ref().map(|p| library_dir.join(p).exists()).unwrap_or(false); + let is_md_exist = markdown_path.as_ref().map(|p| library_dir.join(p).exists()).unwrap_or(false); + let is_tr_exist = translation_path.as_ref().map(|p| library_dir.join(p).exists()).unwrap_or(false); + + let pdf_error = pdf_path.as_ref() + .filter(|p| p.starts_with("error:")) + .map(|p| p["error:".len()..].trim().to_string()); + let html_error = html_path.as_ref() + .filter(|p| p.starts_with("error:")) + .map(|p| p["error:".len()..].trim().to_string()); + + Ok(StandardPaper { + bibcode: r.get(0), + title: r.get(1), + authors, + year: r.get(3), + pub_journal: r.get(4), + keywords, + abstract_text: r.get(6), + doi: r.get(7), + arxiv_id: r.get(8), + citation_count: r.get(9), + reference_count: r.get(10), + is_downloaded: is_pdf_exist || is_html_exist, + has_markdown: is_md_exist, + has_translation: is_tr_exist, + doctype: doctype_val.unwrap_or_else(|| "article".to_string()), + pdf_error, + html_error, + }) +} + +pub async fn check_paper_paths_in_db( + db: &SqlitePool, + library_dir: &std::path::Path, + bibcode: &str +) -> anyhow::Result, Option, Option, Option)>> { + let r_opt = sqlx::query("SELECT pdf_path, html_path, markdown_path, translation_path FROM papers WHERE bibcode = ?") + .bind(bibcode) + .fetch_optional(db) + .await?; + + if let Some(r) = r_opt { + let pdf: Option = r.get(0); + let html: Option = r.get(1); + let md: Option = r.get(2); + let tr: Option = r.get(3); + + let pdf_res = pdf.filter(|p| library_dir.join(p).exists()); + let html_res = html.filter(|p| library_dir.join(p).exists()); + let md_res = md.filter(|p| library_dir.join(p).exists()); + let tr_res = tr.filter(|p| library_dir.join(p).exists()); + + Ok(Some((pdf_res, html_res, md_res, tr_res))) + } else { + Ok(None) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use sqlx::sqlite::SqlitePoolOptions; + + #[test] + fn test_convert_ads_doc_to_standard() { + let doc = AdsPaperDoc { + bibcode: "2026A&A...123..456X".to_string(), + title: Some(vec!["A Test Title".to_string()]), + author: Some(vec!["Author A".to_string(), "Author B".to_string()]), + year: Some("2026".to_string()), + pub_journal: Some("Astronomy & Astrophysics".to_string()), + keyword: Some(vec!["Keyword 1".to_string()]), + abstract_text: Some("This is abstract".to_string()), + doi: Some(vec!["10.1000/test.doi".to_string()]), + citation_count: Some(5), + reference_count: Some(10), + reference: None, + citation: None, + identifier: None, + doctype: Some("article".to_string()), + }; + + let paper = convert_ads_doc_to_standard(&doc); + assert_eq!(paper.bibcode, "2026A&A...123..456X"); + assert_eq!(paper.title, "A Test Title"); + assert_eq!(paper.authors, vec!["Author A", "Author B"]); + assert_eq!(paper.year, "2026"); + assert_eq!(paper.pub_journal, "Astronomy & Astrophysics"); + assert_eq!(paper.keywords, vec!["Keyword 1"]); + assert_eq!(paper.abstract_text, "This is abstract"); + assert_eq!(paper.doi, "10.1000/test.doi"); + assert_eq!(paper.arxiv_id, ""); + assert_eq!(paper.citation_count, 5); + assert_eq!(paper.reference_count, 10); + } + + #[test] + fn test_convert_ads_doc_to_standard_with_arxiv_identifier() { + let doc = AdsPaperDoc { + bibcode: "2026MNRAS.530.1234A".to_string(), + title: Some(vec!["Another Test Title".to_string()]), + author: Some(vec!["Author A".to_string()]), + year: Some("2026".to_string()), + pub_journal: Some("MNRAS".to_string()), + keyword: None, + abstract_text: None, + doi: None, + citation_count: None, + reference_count: None, + reference: None, + citation: None, + identifier: Some(vec!["2026MNRAS.530.1234A".to_string(), "arXiv:2606.12345".to_string()]), + doctype: Some("article".to_string()), + }; + + let paper = convert_ads_doc_to_standard(&doc); + assert_eq!(paper.bibcode, "2026MNRAS.530.1234A"); + assert_eq!(paper.arxiv_id, "2606.12345"); + } + + #[test] + fn test_convert_arxiv_to_standard() { + let doc = ArxivPaper { + id: "2606.12345".to_string(), + title: "Arxiv Title".to_string(), + authors: vec!["Author C".to_string()], + year: "2026".to_string(), + abstract_text: "Arxiv abstract".to_string(), + doi: Some("10.1000/arxiv.doi".to_string()), + pdf_url: "https://arxiv.org/pdf/2606.12345.pdf".to_string(), + }; + + let paper = convert_arxiv_to_standard(&doc); + assert_eq!(paper.bibcode, "2606.12345"); + assert_eq!(paper.title, "Arxiv Title"); + assert_eq!(paper.authors, vec!["Author C"]); + assert_eq!(paper.year, "2026"); + assert_eq!(paper.pub_journal, "arXiv Preprint"); + assert_eq!(paper.doi, "10.1000/arxiv.doi"); + assert_eq!(paper.arxiv_id, "2606.12345"); + } + + #[tokio::test] + async fn test_db_operations() -> anyhow::Result<()> { + let pool = SqlitePoolOptions::new() + .max_connections(1) + .connect("sqlite::memory:") + .await?; + + // 运行迁移 + sqlx::migrate!("./migrations") + .run(&pool) + .await?; + + let paper = StandardPaper { + bibcode: "2026A&A...123..456X".to_string(), + title: "A Test Title".to_string(), + authors: vec!["Author A".to_string()], + year: "2026".to_string(), + pub_journal: "Astronomy & Astrophysics".to_string(), + keywords: vec!["Keyword 1".to_string()], + abstract_text: "This is abstract".to_string(), + doi: "10.1000/test.doi".to_string(), + arxiv_id: "".to_string(), + citation_count: 5, + reference_count: 10, + is_downloaded: false, + has_markdown: false, + has_translation: false, + doctype: "article".to_string(), + }; + + // 保存 + save_paper_to_db(&pool, &paper).await?; + + // 读取 + let retrieved = get_paper_from_db(&pool, std::path::Path::new(""), "2026A&A...123..456X").await?; + assert_eq!(retrieved.title, paper.title); + assert_eq!(retrieved.authors, paper.authors); + assert_eq!(retrieved.keywords, paper.keywords); + + // 检查路径状态（初始为 None） + let paths = check_paper_paths_in_db(&pool, std::path::Path::new(""), "2026A&A...123..456X").await?; + assert!(paths.is_some()); + let (pdf, html, md, tr) = paths.unwrap(); + assert!(pdf.is_none()); + assert!(html.is_none()); + assert!(md.is_none()); + assert!(tr.is_none()); + + Ok(()) + } +} diff --git a/src/api/mod.rs b/src/api/mod.rs index c3d4495..496b841 100644 --- a/src/api/mod.rs +++ b/src/api/mod.rs @@ -1 +1,78 @@ -pub mod handlers; +// src/api/mod.rs +use std::sync::Arc; +use serde::{Deserialize, Serialize}; +use sqlx::SqlitePool; +use crate::Config; +use crate::services::translation::Dictionary; +use crate::clients::qiniu::QiniuClient; +use crate::clients::ads::AdsClient; +use crate::clients::arxiv::ArxivClient; +use crate::services::download::Downloader; + +// 全局共享的 Axum 应用上下文状态 +pub struct AppState { + pub config: Config, + pub db: SqlitePool, + pub dict: Dictionary, + pub qiniu: QiniuClient, + pub ads: AdsClient, + pub arxiv: ArxivClient, + pub downloader: Downloader, + pub harvest_status: Arc>, + pub process_status: Arc>, + pub active_bibcode: Arc>>, +} + +// 统一标准化的文献格式，用于向前端传输 +#[derive(Debug, Serialize, Deserialize, Clone)] +pub struct StandardPaper { + pub bibcode: String, + pub title: String, + pub authors: Vec, + pub year: String, + pub pub_journal: String, + pub keywords: Vec, + pub abstract_text: String, + pub doi: String, + pub arxiv_id: String, + pub citation_count: i32, + pub reference_count: i32, + pub is_downloaded: bool, + pub has_markdown: bool, + pub has_translation: bool, + pub doctype: String, + pub pdf_error: Option, + pub html_error: Option, +} + +pub mod helpers; +pub mod papers; +pub mod notes; +pub mod sync; + +// 提供兼容的 handlers 命名空间，避免修改 main.rs / batch_sync.rs 里的导入 +pub mod handlers { + pub use super::helpers::{ + convert_ads_doc_to_standard, convert_arxiv_to_standard, save_paper_to_db, + get_paper_from_db, check_paper_paths_in_db, + }; + pub use super::papers::{ + search_papers, download_paper, parse_paper, translate_paper, + get_citation_network, get_paper_detail, get_library, export_citations, + upload_paper_file, mark_no_resource, get_active_bibcode, set_active_bibcode, + SearchParams, DownloadRequest, ParseRequest, ParseResponse, + TranslateRequest, TranslateResponse, CitationsResponse, PaperDetailResponse, + ExportRequest, ExportResponse, MarkNoResourceRequest, + }; + pub use super::notes::{ + create_note, get_notes, delete_note, + NoteRecord, CreateNoteRequest, DeleteNoteParams, GetNotesParams, + }; + pub use super::sync::{ + run_meta_sync, get_meta_sync_count, get_meta_sync_status, + run_asset_sync, stop_asset_sync, get_sync_queries, delete_sync_query, + get_asset_sync_status, MetaSyncRunRequest, MetaSyncCountRequest, + MetaSyncCountResponse, AssetSyncRunRequest, SavedSyncQuery, + }; + pub use super::{AppState, StandardPaper}; +} diff --git a/src/api/notes.rs b/src/api/notes.rs new file mode 100644 index 0000000..1103fc8 --- /dev/null +++ b/src/api/notes.rs @@ -0,0 +1,113 @@ +// src/api/notes.rs +use axum::{ + extract::{Query, State}, + http::StatusCode, + Json, +}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use sqlx::Row; + +use super::AppState; + +#[derive(Debug, Serialize, Deserialize)] +pub struct NoteRecord { + pub id: i64, + pub bibcode: String, + pub paragraph_index: i64, + pub note_text: String, + pub highlight_color: String, + pub selected_text: String, + pub created_at: String, +} + +#[derive(Deserialize)] +pub struct CreateNoteRequest { + pub bibcode: String, + pub paragraph_index: i64, + pub note_text: Option, + pub highlight_color: Option, + pub selected_text: Option, +} + +#[derive(Deserialize)] +pub struct DeleteNoteParams { + pub id: i64, +} + +#[derive(Deserialize)] +pub struct GetNotesParams { + pub bibcode: String, +} + +// 创建笔记 +pub async fn create_note( + State(state): State>, + Json(req): Json, +) -> Result, (StatusCode, String)> { + let note_text = req.note_text.unwrap_or_default(); + let highlight_color = req.highlight_color.unwrap_or_else(|| "yellow".to_string()); + let selected_text = req.selected_text.unwrap_or_default(); + + let row = sqlx::query( + "INSERT INTO notes (bibcode, paragraph_index, note_text, highlight_color, selected_text) VALUES (?, ?, ?, ?, ?) RETURNING id, bibcode, paragraph_index, note_text, highlight_color, selected_text, created_at" + ) + .bind(&req.bibcode) + .bind(req.paragraph_index) + .bind(¬e_text) + .bind(&highlight_color) + .bind(&selected_text) + .fetch_one(&state.db) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("保存笔记失败: {}", e)))?; + + Ok(Json(NoteRecord { + id: row.get(0), + bibcode: row.get(1), + paragraph_index: row.get(2), + note_text: row.get(3), + highlight_color: row.get(4), + selected_text: row.get(5), + created_at: row.get(6), + })) +} + +// 查询某篇文献的全部笔记 +pub async fn get_notes( + State(state): State>, + Query(params): Query, +) -> Result>, (StatusCode, String)> { + let rows = sqlx::query( + "SELECT id, bibcode, paragraph_index, note_text, highlight_color, selected_text, created_at FROM notes WHERE bibcode = ? ORDER BY paragraph_index, created_at" + ) + .bind(¶ms.bibcode) + .fetch_all(&state.db) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("查询笔记失败: {}", e)))?; + + let notes: Vec = rows.iter().map(|r| NoteRecord { + id: r.get(0), + bibcode: r.get(1), + paragraph_index: r.get(2), + note_text: r.get(3), + highlight_color: r.get(4), + selected_text: r.get(5), + created_at: r.get(6), + }).collect(); + + Ok(Json(notes)) +} + +// 删除指定 id 的笔记 +pub async fn delete_note( + State(state): State>, + Query(params): Query, +) -> Result { + sqlx::query("DELETE FROM notes WHERE id = ?") + .bind(params.id) + .execute(&state.db) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("删除笔记失败: {}", e)))?; + + Ok(StatusCode::NO_CONTENT) +} diff --git a/src/api/papers.rs b/src/api/papers.rs new file mode 100644 index 0000000..47fc072 --- /dev/null +++ b/src/api/papers.rs @@ -0,0 +1,855 @@ +// src/api/papers.rs +use axum::{ + extract::{Query, State}, + http::StatusCode, + Json, +}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use std::fs; +use tracing::{info, warn, error}; +use sqlx::Row; + +use super::{AppState, StandardPaper}; +use super::helpers::{ + convert_ads_doc_to_standard, convert_arxiv_to_standard, save_paper_to_db, + get_paper_from_db, check_paper_paths_in_db, +}; + +// 检索请求参数 +#[derive(Debug, Deserialize)] +pub struct SearchParams { + pub q: String, + pub source: Option, // "all" | "ads" | "arxiv" + pub rows: Option, + pub start: Option, // 分页起始偏移量 + pub sort: Option, // 排序字段 +} + +// ── GET /api/search ── +// 统一检索接口，合并去重 ADS 和 arXiv 数据 +pub async fn search_papers( + State(state): State>, + Query(params): Query, +) -> Result>, (StatusCode, String)> { + let source = params.source.unwrap_or_else(|| "all".to_string()); + let rows = params.rows.unwrap_or(10); + let start = params.start.unwrap_or(0); + let sort = params.sort.as_deref().unwrap_or("relevance"); + let mut results = Vec::new(); + + // 1. 检索 NASA ADS + if source == "all" || source == "ads" { + if !state.config.ads_api_key.is_empty() { + match state.ads.search(¶ms.q, start, rows, sort).await { + Ok(docs) => { + for doc in docs { + let paper = convert_ads_doc_to_standard(&doc); + + // 入库 SQLite + if let Err(e) = save_paper_to_db(&state.db, &paper).await { + warn!("保存 ADS 文献至数据库失败: {}", e); + } + + // 保存引用/参考文献关联拓扑 + if let Some(refs) = doc.reference { + for ref_bib in refs { + let _ = sqlx::query("INSERT OR IGNORE INTO citations_references (source_bibcode, target_bibcode) VALUES (?, ?)") + .bind(&paper.bibcode) + .bind(&ref_bib) + .execute(&state.db) + .await; + } + } + if let Some(cits) = doc.citation { + for cit_bib in cits { + let _ = sqlx::query("INSERT OR IGNORE INTO citations_references (source_bibcode, target_bibcode) VALUES (?, ?)") + .bind(&cit_bib) + .bind(&paper.bibcode) + .execute(&state.db) + .await; + } + } + + results.push(paper); + } + } + Err(e) => { + error!("ADS 检索执行失败: {}", e); + } + } + } else { + warn!("ADS_API_KEY 未配置，跳过 ADS 检索。"); + } + } + + // 2. 检索 arXiv + if source == "all" || source == "arxiv" { + match state.arxiv.search(¶ms.q, start, rows, sort).await { + Ok(papers) => { + for p in papers { + let paper = convert_arxiv_to_standard(&p); + + // 入库 SQLite (使用 arXiv ID 暂作主键以作记录) + if let Err(e) = save_paper_to_db(&state.db, &paper).await { + warn!("保存 arXiv 文献至数据库失败: {}", e); + } + + results.push(paper); + } + } + Err(e) => { + error!("arXiv 检索执行失败: {}", e); + } + } + } + + // 对两端获取的数据进行去重合并，增加对相同 arxiv_id 的判断 + let mut unique_results: Vec = Vec::new(); + for r in results { + if !unique_results.iter().any(|u| u.bibcode == r.bibcode || (!u.doi.is_empty() && u.doi == r.doi) || (!u.arxiv_id.is_empty() && u.arxiv_id == r.arxiv_id)) { + let mut final_paper = r.clone(); + // 如果本地数据库存在该文献，直接从数据库读取标准元数据（包括 is_downloaded, has_markdown, pdf_error, html_error 等） + if let Ok(db_paper) = get_paper_from_db(&state.db, &state.config.library_dir, &r.bibcode).await { + final_paper = db_paper; + } + unique_results.push(final_paper); + } + } + + Ok(Json(unique_results)) +} + +// ── POST /api/download ── +#[derive(Deserialize)] +pub struct DownloadRequest { + pub bibcode: String, + pub force: Option, // 强制重新下载，即使已存在本地文件 +} + +// 一键双格式并行下载文献 (PDF + HTML)，支持 force=true 强制重新下载 +pub async fn download_paper( + State(state): State>, + Json(req): Json, +) -> Result, (StatusCode, String)> { + let force = req.force.unwrap_or(false); + info!("接收到文献下载指令，标识符: {}, 强制重下: {}", req.bibcode, force); + + let paper = get_paper_from_db(&state.db, &state.config.library_dir, &req.bibcode) + .await + .map_err(|e| (StatusCode::NOT_FOUND, format!("未找到该文献记录: {}", e)))?; + + // force=true 时清除旧路径记录，强制重新下载 + if force { + sqlx::query("UPDATE papers SET pdf_path = NULL, html_path = NULL WHERE bibcode = ?") + .bind(&req.bibcode) + .execute(&state.db) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("重置下载状态失败: {}", e)))?; + } + + // 下载策略： + // 1. 如有 arXiv ID，优先走 arXiv 直连（绕过出版商防护墙，成功率高） + // 2. 否则走 ADS 网关多级回退（PUB_PDF → EPRINT_PDF → CrossRef） + // 3. 若 ADS 路径 PDF/HTML 均失败但有 arXiv ID，再尝试 arXiv 作为兜底 + let (pdf_res, html_res) = if !paper.arxiv_id.is_empty() { + info!("[下载] 优先使用 arXiv 通道: {}", paper.arxiv_id); + state.downloader.download_arxiv_direct(&paper.arxiv_id, &state.config.library_dir).await + } else { + let doi_opt = if !paper.doi.is_empty() { Some(paper.doi.as_str()) } else { None }; + state.downloader.download_paper(&req.bibcode, doi_opt, &state.config.library_dir).await + }; + + if pdf_res.is_err() && html_res.is_err() { + let pdf_err = pdf_res.as_ref().err().unwrap(); + let html_err = html_res.as_ref().err().unwrap(); + error!("文献 {} PDF 和 HTML 均下载失败，无可用物理文件格式", req.bibcode); + + let pdf_db_err = format!("error: {}", pdf_err); + let html_db_err = format!("error: {}", html_err); + let _ = sqlx::query("UPDATE papers SET pdf_path = ?, html_path = ? WHERE bibcode = ?") + .bind(&pdf_db_err) + .bind(&html_db_err) + .bind(&req.bibcode) + .execute(&state.db) + .await; + + return Err((StatusCode::INTERNAL_SERVER_ERROR, format!("下载失败。PDF: {}, HTML: {}", pdf_err, html_err))); + } + + let pdf_rel = match pdf_res { + Ok(p) => Some(p.strip_prefix(&state.config.library_dir).unwrap_or(&p).to_string_lossy().to_string()), + Err(e) => Some(format!("error: {}", e)), + }; + let html_rel = match html_res { + Ok(p) => Some(p.strip_prefix(&state.config.library_dir).unwrap_or(&p).to_string_lossy().to_string()), + Err(e) => Some(format!("error: {}", e)), + }; + + // 回写存储路径至数据库 + sqlx::query("UPDATE papers SET pdf_path = ?, html_path = ? WHERE bibcode = ?") + .bind(&pdf_rel) + .bind(&html_rel) + .bind(&req.bibcode) + .execute(&state.db) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("更新数据库失败: {}", e)))?; + + let mut updated_paper = paper; + updated_paper.is_downloaded = true; + + Ok(Json(updated_paper)) +} + +// ── POST /api/parse ── +#[derive(Deserialize)] +pub struct ParseRequest { + pub bibcode: String, + pub force: Option, +} + +#[derive(Serialize)] +pub struct ParseResponse { + pub markdown: String, +} + +// 将 HTML / PDF 转换为标准英文 Markdown 文本（HTML 优先，PDF 调用 MinerU 远程 API 且上传七牛云） +pub async fn parse_paper( + State(state): State>, + Json(req): Json, +) -> Result, (StatusCode, String)> { + info!("接收到文献结构化解析指令: {} (强制重新解析: {:?})", req.bibcode, req.force); + + let (pdf_opt, html_opt, md_opt, _) = check_paper_paths_in_db(&state.db, &state.config.library_dir, &req.bibcode) + .await + .map_err(|e| (StatusCode::NOT_FOUND, format!("获取文献路径失败: {}", e)))? + .ok_or((StatusCode::NOT_FOUND, "该文献未注册在数据库中".to_string()))?; + + let force = req.force.unwrap_or(false); + + // 如果先前已经解析成功过且非强制重新解析，直读 Markdown 文件返回 + if !force { + if let Some(md_rel) = md_opt { + let md_abs = state.config.library_dir.join(&md_rel); + if md_abs.exists() { + if let Ok(content) = fs::read_to_string(&md_abs) { + return Ok(Json(ParseResponse { markdown: content })); + } + } + } + } + + let mut parsed_markdown = String::new(); + let mut relative_md_path = String::new(); + + // 查询该文献的元数据以生成 Markdown YAML 头部信息 + let paper = get_paper_from_db(&state.db, &state.config.library_dir, &req.bibcode) + .await + .map_err(|e| (StatusCode::NOT_FOUND, format!("未找到该文献元数据记录: {}", e)))?; + + // 策略 1：HTML 优先解析 + if let Some(html_rel) = html_opt { + let html_abs = state.config.library_dir.join(&html_rel); + if html_abs.exists() { + match crate::services::parser::html_to_markdown(&html_abs) { + Ok(md) => { + let front_matter = format!( + "---\ntitle: {}\nauthor: [{}]\npublisher: {}\nsource: \"https://ui.adsabs.harvard.edu/abs/{}/abstract\"\ndate: \"{}\"\ntags: \"{}\"\n---\n\n", + serde_json::to_string(&paper.title).unwrap_or_else(|_| format!("\"{}\"", paper.title)), + paper.authors.iter().map(|a| format!("\"{}\"", a)).collect::>().join(", "), + serde_json::to_string(&paper.pub_journal).unwrap_or_else(|_| format!("\"{}\"", paper.pub_journal)), + paper.bibcode, + paper.year, + paper.keywords.join(",") + ); + parsed_markdown = format!("{}{}", front_matter, md); + let md_filename = format!("{}_en.md", req.bibcode); + let md_dest = state.config.library_dir.join("Markdown").join(&md_filename); + fs::create_dir_all(md_dest.parent().unwrap()).unwrap_or_default(); + if fs::write(&md_dest, &parsed_markdown).is_ok() { + relative_md_path = format!("Markdown/{}", md_filename); + } + } + Err(e) => { + warn!("HTML 转换为 Markdown 失败 {}: {}。将自动降级使用 PDF 结构化解析。", req.bibcode, e); + } + } + } + } + + // 策略 2：回退至 PDF 远程 MinerU 解析 + if parsed_markdown.is_empty() { + if let Some(pdf_rel) = pdf_opt { + let pdf_abs = state.config.library_dir.join(&pdf_rel); + if pdf_abs.exists() { + match crate::services::parser::parse_pdf_via_mineru(&pdf_abs, &state.qiniu, &state.config).await { + Ok(md) => { + let front_matter = format!( + "---\ntitle: {}\nauthor: [{}]\npublisher: {}\nsource: \"https://ui.adsabs.harvard.edu/abs/{}/abstract\"\ndate: \"{}\"\ntags: \"{}\"\n---\n\n", + serde_json::to_string(&paper.title).unwrap_or_else(|_| format!("\"{}\"", paper.title)), + paper.authors.iter().map(|a| format!("\"{}\"", a)).collect::>().join(", "), + serde_json::to_string(&paper.pub_journal).unwrap_or_else(|_| format!("\"{}\"", paper.pub_journal)), + paper.bibcode, + paper.year, + paper.keywords.join(",") + ); + parsed_markdown = format!("{}{}", front_matter, md); + let md_filename = format!("{}_en.md", req.bibcode); + let md_dest = state.config.library_dir.join("Markdown").join(&md_filename); + fs::create_dir_all(md_dest.parent().unwrap()).unwrap_or_default(); + if fs::write(&md_dest, &parsed_markdown).is_ok() { + relative_md_path = format!("Markdown/{}", md_filename); + } + } + Err(e) => { + error!("PDF layout 远程 MinerU 解析失败: {}", e); + return Err((StatusCode::INTERNAL_SERVER_ERROR, format!("PDF 结构解析失败: {}", e))); + } + } + } else { + error!("文献 {} 解析失败：本地 PDF 文件 {:?} 丢失", req.bibcode, pdf_abs); + return Err((StatusCode::NOT_FOUND, "本地 PDF 文件未找到".to_string())); + } + } else { + error!("文献 {} 解析失败：请先下载该文献的 HTML 或 PDF 文件", req.bibcode); + return Err((StatusCode::BAD_REQUEST, "请先下载该文献的 HTML 或 PDF 文件".to_string())); + } + } + + // 更新本地解析路径至 SQLite 数据库 + if !relative_md_path.is_empty() { + let _ = sqlx::query("UPDATE papers SET markdown_path = ? WHERE bibcode = ?") + .bind(&relative_md_path) + .bind(&req.bibcode) + .execute(&state.db) + .await; + } + + Ok(Json(ParseResponse { markdown: parsed_markdown })) +} + +// ── POST /api/translate ── +#[derive(Deserialize)] +pub struct TranslateRequest { + pub bibcode: String, + pub force: Option, +} + +#[derive(Serialize)] +pub struct TranslateResponse { + pub translation: String, +} + +// 文献中英双栏对比翻译接口（包含词表注入与本地物理缓存） +pub async fn translate_paper( + State(state): State>, + Json(req): Json, +) -> Result, (StatusCode, String)> { + let force = req.force.unwrap_or(false); + info!("接收到对比翻译请求: 文献={}, 强制重译={}", req.bibcode, force); + + let (_, _, md_opt, tr_opt) = check_paper_paths_in_db(&state.db, &state.config.library_dir, &req.bibcode) + .await + .map_err(|e| (StatusCode::NOT_FOUND, format!("查询文献路径失败: {}", e)))? + .ok_or((StatusCode::NOT_FOUND, "该文献未注册在数据库中".to_string()))?; + + // 若本地已存在翻译物理文件且未指明强制重译，直读本地缓存返回 + if !force { + if let Some(tr_rel) = tr_opt { + let tr_abs = state.config.library_dir.join(&tr_rel); + if tr_abs.exists() { + if let Ok(content) = fs::read_to_string(&tr_abs) { + return Ok(Json(TranslateResponse { translation: content })); + } + } + } + } + + // 检查英文解析文件是否存在 + let md_rel = match md_opt { + Some(rel) => rel, + None => { + error!("文献 {} 翻译失败：文献未完成解析，缺少英文 Markdown 路径", req.bibcode); + return Err((StatusCode::BAD_REQUEST, "文献必须先完成解析方可翻译".to_string())); + } + }; + let md_abs = state.config.library_dir.join(&md_rel); + if !md_abs.exists() { + error!("文献 {} 翻译失败：解析的英文 Markdown 文件 {:?} 不存在", req.bibcode, md_abs); + return Err((StatusCode::BAD_REQUEST, "解析 Markdown 文件丢失".to_string())); + } + + let english_markdown = fs::read_to_string(&md_abs) + .map_err(|e| { + error!("文献 {} 翻译失败：读取解析内容失败: {}", req.bibcode, e); + (StatusCode::INTERNAL_SERVER_ERROR, format!("读取解析内容失败: {}", e)) + })?; + + // 调用 LLM 翻译服务并注入对照词表 + let translated_markdown = crate::services::translation::translate_markdown(&english_markdown, &state.dict, &state.config) + .await + .map_err(|e| { + error!("文献 {} 翻译失败：调用 LLM 翻译发生错误: {}", req.bibcode, e); + (StatusCode::INTERNAL_SERVER_ERROR, format!("调用 LLM 翻译失败: {}", e)) + })?; + + // 翻译结果物理写入本地 + let tr_filename = format!("{}_zh.md", req.bibcode); + let tr_dest = state.config.library_dir.join("Translation").join(&tr_filename); + fs::create_dir_all(tr_dest.parent().unwrap()).unwrap_or_default(); + + fs::write(&tr_dest, &translated_markdown) + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("写入翻译文件失败: {}", e)))?; + + let relative_tr_path = format!("Translation/{}", tr_filename); + + // 缓存路径更新入库 + sqlx::query("UPDATE papers SET translation_path = ? WHERE bibcode = ?") + .bind(&relative_tr_path) + .bind(&req.bibcode) + .execute(&state.db) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("更新数据库翻译状态失败: {}", e)))?; + + Ok(Json(TranslateResponse { translation: translated_markdown })) +} + +#[derive(Debug, Serialize)] +pub struct CitationsResponse { + pub bibcode: String, + pub title: String, + pub citation_count: i32, + pub reference_count: i32, + pub references: Vec, // 该文献参考文献 bibcode 数组 + pub citations: Vec, // 引用该文献的 bibcode 数组 + pub citation_counts: std::collections::HashMap, // 相关文献与被引数映射 +} + +// 从 SQLite 查询引用关联，生成引用星系关系树 +pub async fn get_citation_network( + State(state): State>, + Query(params): Query, +) -> Result, (StatusCode, String)> { + let paper = match get_paper_from_db(&state.db, &state.config.library_dir, ¶ms.bibcode).await { + Ok(p) => p, + Err(_) => { + // 如果本地数据库查不到，尝试从 ADS 在线 API 动态获取 + if !state.config.ads_api_key.is_empty() { + match state.ads.search(&format!("bibcode:{}", params.bibcode), 0, 1, "relevance").await { + Ok(docs) => { + if let Some(doc) = docs.first() { + let standard_paper = convert_ads_doc_to_standard(doc); + // 保存至数据库缓存，并保存引用关联 + let _ = save_paper_to_db(&state.db, &standard_paper).await; + if let Some(refs) = &doc.reference { + for ref_bib in refs { + let _ = sqlx::query("INSERT OR IGNORE INTO citations_references (source_bibcode, target_bibcode) VALUES (?, ?)") + .bind(&standard_paper.bibcode) + .bind(ref_bib) + .execute(&state.db) + .await; + } + } + if let Some(cits) = &doc.citation { + for cit_bib in cits { + let _ = sqlx::query("INSERT OR IGNORE INTO citations_references (source_bibcode, target_bibcode) VALUES (?, ?)") + .bind(cit_bib) + .bind(&standard_paper.bibcode) + .execute(&state.db) + .await; + } + } + standard_paper + } else { + return Err((StatusCode::NOT_FOUND, format!("在本地库及 ADS 中均未找到该文献: {}", params.bibcode))); + } + } + Err(e) => { + return Err((StatusCode::INTERNAL_SERVER_ERROR, format!("在线检索文献元数据失败: {}", e))); + } + } + } else { + return Err((StatusCode::NOT_FOUND, format!("本地数据库未收录该文献，且未配置 ADS_API_KEY，无法在线加载: {}", params.bibcode))); + } + } + }; + + // 加载引用的文献 + let refs_rows = sqlx::query("SELECT target_bibcode FROM citations_references WHERE source_bibcode = ?") + .bind(¶ms.bibcode) + .fetch_all(&state.db) + .await + .unwrap_or_default(); + let references: Vec = refs_rows.iter().map(|row| row.get(0)).collect(); + + // 加载被引用的文献 + let cits_rows = sqlx::query("SELECT source_bibcode FROM citations_references WHERE target_bibcode = ?") + .bind(¶ms.bibcode) + .fetch_all(&state.db) + .await + .unwrap_or_default(); + let citations: Vec = cits_rows.iter().map(|row| row.get(0)).collect(); + + // 加载关联文献的被引数量 (从 SQLite papers 表获取) + let mut citation_counts = std::collections::HashMap::new(); + let mut all_related = references.clone(); + all_related.extend(citations.clone()); + for bib in all_related { + let count_opt: Option = sqlx::query_scalar("SELECT citation_count FROM papers WHERE bibcode = ?") + .bind(&bib) + .fetch_optional(&state.db) + .await + .unwrap_or_default(); + if let Some(c) = count_opt { + citation_counts.insert(bib, c); + } + } + + Ok(Json(CitationsResponse { + bibcode: paper.bibcode, + title: paper.title, + citation_count: paper.citation_count, + reference_count: paper.reference_count, + references, + citations, + citation_counts, + })) +} + +// ── GET /api/paper ── +#[derive(Serialize)] +pub struct PaperDetailResponse { + pub paper: StandardPaper, + pub english_content: Option, + pub translation_content: Option, +} + +// 获取文献标准详情和中英双语内容文件数据 +pub async fn get_paper_detail( + State(state): State>, + Query(params): Query, +) -> Result, (StatusCode, String)> { + let paper = get_paper_from_db(&state.db, &state.config.library_dir, ¶ms.bibcode) + .await + .map_err(|e| (StatusCode::NOT_FOUND, format!("未找到该文献数据: {}", e)))?; + + let (_, _, md_opt, tr_opt) = check_paper_paths_in_db(&state.db, &state.config.library_dir, ¶ms.bibcode) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))? + .unwrap_or_default(); + + let english_content = md_opt.and_then(|rel| fs::read_to_string(state.config.library_dir.join(rel)).ok()); + let translation_content = tr_opt.and_then(|rel| fs::read_to_string(state.config.library_dir.join(rel)).ok()); + + Ok(Json(PaperDetailResponse { + paper, + english_content, + translation_content, + })) +} + +// ── GET /api/library ── +// 获取本地图书馆文献列表 +pub async fn get_library( + State(state): State>, +) -> Result>, (StatusCode, String)> { + let rows = sqlx::query("SELECT bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, pdf_path, html_path, markdown_path, translation_path, doctype FROM papers ORDER BY created_at DESC") + .fetch_all(&state.db) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("访问本地数据库失败: {}", e)))?; + + let mut list = Vec::new(); + for r in rows { + let pdf_path: Option = r.get(11); + let html_path: Option = r.get(12); + let markdown_path: Option = r.get(13); + let translation_path: Option = r.get(14); + let doctype_val: Option = r.get(15); + + let authors_str: Option = r.get(2); + let authors: Vec = authors_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default(); + let keywords_str: Option = r.get(5); + let keywords: Vec = keywords_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default(); + + let pdf_error = pdf_path.as_ref() + .filter(|p| p.starts_with("error:")) + .map(|p| p["error:".len()..].trim().to_string()); + let html_error = html_path.as_ref() + .filter(|p| p.starts_with("error:")) + .map(|p| p["error:".len()..].trim().to_string()); + + list.push(StandardPaper { + bibcode: r.get(0), + title: r.get(1), + authors, + year: r.get(3), + pub_journal: r.get(4), + keywords, + abstract_text: r.get(6), + doi: r.get(7), + arxiv_id: r.get(8), + citation_count: r.get(9), + reference_count: r.get(10), + is_downloaded: pdf_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false) + || html_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false), + has_markdown: markdown_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false), + has_translation: translation_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false), + doctype: doctype_val.unwrap_or_else(|| "article".to_string()), + pdf_error, + html_error, + }); + } + + Ok(Json(list)) +} + +// ── POST /api/export ── +#[derive(Deserialize)] +pub struct ExportRequest { + pub bibcodes: Vec, +} + +#[derive(Serialize)] +pub struct ExportResponse { + pub bibtex: String, +} + +// 批量请求 ADS 接口，获取选中 Bibcode 的标准 BibTeX 引文段落 +pub async fn export_citations( + State(state): State>, + Json(req): Json, +) -> Result, (StatusCode, String)> { + if state.config.ads_api_key.is_empty() { + return Err((StatusCode::BAD_REQUEST, "ADS API key 未在 .env 中配置，无法使用该接口".to_string())); + } + + let bibtex = state.ads.export_bibtex(req.bibcodes).await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("批量引文导出失败: {}", e)))?; + + Ok(Json(ExportResponse { bibtex })) +} + +// ── POST /api/upload ── +// 允许手动上传/推入本地 PDF 或 HTML 文献文件 +pub async fn upload_paper_file( + State(state): State>, + mut multipart: axum::extract::Multipart, +) -> Result, (StatusCode, String)> { + let mut bibcode = String::new(); + let mut file_type = String::new(); // "pdf" 或 "html" + let mut file_bytes = Vec::new(); + let mut file_name = String::new(); + + while let Some(field) = multipart.next_field().await.map_err(|e| { + (StatusCode::BAD_REQUEST, format!("解析文件分块失败: {}", e)) + })? { + let name = field.name().unwrap_or("").to_string(); + if name == "bibcode" { + bibcode = field.text().await.unwrap_or_default(); + } else if name == "type" { + file_type = field.text().await.unwrap_or_default(); + } else if name == "file" { + file_name = field.file_name().unwrap_or("").to_string(); + file_bytes = field.bytes().await.map_err(|e| { + (StatusCode::INTERNAL_SERVER_ERROR, format!("读取文件字节流失败: {}", e)) + })?.to_vec(); + } + } + + if bibcode.is_empty() { + return Err((StatusCode::BAD_REQUEST, "缺少 bibcode 参数".to_string())); + } + if file_bytes.is_empty() { + return Err((StatusCode::BAD_REQUEST, "上传文件为空或读取失败".to_string())); + } + + // 尝试将可能的 DOI 或 arXiv ID 解析为真实的 bibcode + let mut resolved_bibcode = bibcode.clone(); + let exists_as_bibcode = sqlx::query("SELECT bibcode FROM papers WHERE bibcode = ?") + .bind(&resolved_bibcode) + .fetch_optional(&state.db) + .await + .unwrap_or(None) + .is_some(); + + if !exists_as_bibcode { + // 尝试匹配 DOI + let clean_doi = resolved_bibcode + .trim_start_matches("doi:") + .trim_start_matches("DOI:") + .trim_start_matches("https://doi.org/") + .trim_start_matches("http://doi.org/") + .trim(); + + if let Some(row) = sqlx::query("SELECT bibcode FROM papers WHERE doi = ? OR doi = ? OR LOWER(doi) = LOWER(?)") + .bind(clean_doi) + .bind(&resolved_bibcode) + .bind(clean_doi) + .fetch_optional(&state.db) + .await + .unwrap_or(None) + { + let found: String = row.get(0); + info!("上传接口：通过 DOI 匹配成功，将 '{}' 解析为 bibcode '{}'", bibcode, found); + resolved_bibcode = found; + } else { + // 尝试匹配 arXiv ID + let clean_arxiv = resolved_bibcode + .trim_start_matches("arxiv:") + .trim_start_matches("arXiv:") + .trim_start_matches("ARXIV:") + .trim(); + // 移除可能存在的版本号后缀（如 2303.12345v1 -> 2303.12345） + let clean_arxiv_no_version = if let Some(pos) = clean_arxiv.find('v') { + if clean_arxiv[pos+1..].chars().all(|c| c.is_ascii_digit()) { + &clean_arxiv[..pos] + } else { + clean_arxiv + } + } else { + clean_arxiv + }; + + if let Some(row) = sqlx::query( + "SELECT bibcode FROM papers WHERE arxiv_id = ? OR arxiv_id = ? OR arxiv_id LIKE ? OR arxiv_id LIKE ?" + ) + .bind(clean_arxiv) + .bind(clean_arxiv_no_version) + .bind(format!("{}%", clean_arxiv_no_version)) + .bind(format!("arXiv:{}%", clean_arxiv_no_version)) + .fetch_optional(&state.db) + .await + .unwrap_or(None) { + let found: String = row.get(0); + info!("上传接口：通过 arXiv ID 匹配成功，将 '{}' 解析为 bibcode '{}'", bibcode, found); + resolved_bibcode = found; + } + } + } + let bibcode = resolved_bibcode; + + // 从数据库读取该文献元数据 + let _paper = get_paper_from_db(&state.db, &state.config.library_dir, &bibcode) + .await + .map_err(|e| (StatusCode::NOT_FOUND, format!("未找到该文献记录: {}", e)))?; + + // 校验并保存文件 + let is_pdf = file_type == "pdf" || file_name.to_lowercase().ends_with(".pdf"); + + let relative_path = if is_pdf { + crate::services::download::validate_pdf_content(&file_bytes).map_err(|e| { + (StatusCode::BAD_REQUEST, format!("PDF 文件内容校验失败: {}", e)) + })?; + let pdf_filename = format!("{}.pdf", bibcode); + let pdf_dest = state.config.library_dir.join("PDF").join(&pdf_filename); + if let Some(parent) = pdf_dest.parent() { + std::fs::create_dir_all(parent).unwrap_or_default(); + } + std::fs::write(&pdf_dest, &file_bytes).map_err(|e| { + (StatusCode::INTERNAL_SERVER_ERROR, format!("无法写入 PDF 文件: {}", e)) + })?; + format!("PDF/{}", pdf_filename) + } else { + let text_content = String::from_utf8(file_bytes).map_err(|_| { + (StatusCode::BAD_REQUEST, "上传的 HTML 文件不是有效的 UTF-8 文本".to_string()) + })?; + crate::services::download::validate_html_content_lenient(&text_content).map_err(|e| { + (StatusCode::BAD_REQUEST, format!("HTML 文件内容校验失败: {}", e)) + })?; + let html_filename = format!("{}.html", bibcode); + let html_dest = state.config.library_dir.join("HTML").join(&html_filename); + if let Some(parent) = html_dest.parent() { + std::fs::create_dir_all(parent).unwrap_or_default(); + } + std::fs::write(&html_dest, &text_content).map_err(|e| { + (StatusCode::INTERNAL_SERVER_ERROR, format!("无法写入 HTML 文件: {}", e)) + })?; + format!("HTML/{}", html_filename) + }; + + // 更新数据库路径状态 + let path_field = if is_pdf { "pdf_path" } else { "html_path" }; + let sql = format!("UPDATE papers SET {} = ? WHERE bibcode = ?", path_field); + sqlx::query(&sql) + .bind(&relative_path) + .bind(&bibcode) + .execute(&state.db) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("更新数据库状态失败: {}", e)))?; + + // 重新获取最新的文献信息以更新前端界面 + let updated_paper = get_paper_from_db(&state.db, &state.config.library_dir, &bibcode) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("重读文献数据失败: {}", e)))?; + + Ok(Json(updated_paper)) +} + +// ── POST /api/no_resource ── +#[derive(Debug, Deserialize)] +pub struct MarkNoResourceRequest { + pub bibcode: String, + pub clear: Option, +} + +pub async fn mark_no_resource( + State(state): State>, + Json(req): Json, +) -> Result, (StatusCode, String)> { + let clear_flag = req.clear.unwrap_or(false); + + if clear_flag { + info!("接收到清除文献无资源标记指令，标识符: {}", req.bibcode); + sqlx::query("UPDATE papers SET pdf_path = NULL, html_path = NULL WHERE bibcode = ?") + .bind(&req.bibcode) + .execute(&state.db) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("清除无资源标记失败: {}", e)))?; + } else { + info!("接收到文献无资源标记指令，标识符: {}", req.bibcode); + sqlx::query("UPDATE papers SET pdf_path = 'error:no_resource', html_path = 'error:no_resource' WHERE bibcode = ?") + .bind(&req.bibcode) + .execute(&state.db) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("更新数据库无资源标记失败: {}", e)))?; + } + + // 重新获取最新的文献信息以更新前端界面 + let updated_paper = get_paper_from_db(&state.db, &state.config.library_dir, &req.bibcode) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("重读文献数据失败: {}", e)))?; + + Ok(Json(updated_paper)) +} + +// ── GET /api/active_bibcode ── +#[derive(Debug, Serialize)] +pub struct ActiveBibcodeResponse { + pub bibcode: Option, +} + +pub async fn get_active_bibcode( + State(state): State>, +) -> Json { + let active = state.active_bibcode.lock().await; + Json(ActiveBibcodeResponse { + bibcode: active.clone(), + }) +} + +// ── POST /api/active_bibcode ── +#[derive(Debug, Deserialize)] +pub struct SetActiveBibcodeRequest { + pub bibcode: Option, +} + +pub async fn set_active_bibcode( + State(state): State>, + Json(req): Json, +) -> StatusCode { + let mut active = state.active_bibcode.lock().await; + *active = req.bibcode; + tracing::debug!("已更新当前活跃文献 Bibcode 标记为: {:?}", *active); + StatusCode::OK +} + diff --git a/src/api/sync.rs b/src/api/sync.rs new file mode 100644 index 0000000..ba13375 --- /dev/null +++ b/src/api/sync.rs @@ -0,0 +1,339 @@ +// src/api/sync.rs +use axum::{ + extract::{Query, State}, + http::StatusCode, + Json, +}; +use serde::{Deserialize, Serialize}; +use std::sync::Arc; +use sqlx::Row; +use tracing::error; + +use super::AppState; + +// ── POST /api/sync/meta/run ── +#[derive(Debug, Deserialize)] +pub struct MetaSyncRunRequest { + pub q: String, + pub source: String, // "all" | "ads" | "arxiv" + pub limit: i32, +} + +pub async fn run_meta_sync( + State(state): State>, + Json(req): Json, +) -> Result { + // 检查并同步初始化任务状态，防止前端轮询竞态条件 + { + let mut status = state.harvest_status.lock().await; + if status.active { + return Err((StatusCode::CONFLICT, "当前已有文献批量同步任务在后台运行中，请勿重复启动".to_string())); + } + status.active = true; + status.query = req.q.clone(); + status.source = req.source.clone(); + status.synced = 0; + status.total = 0; + } + + crate::services::batch_sync::MetaSync::start_harvest( + state.db.clone(), + Arc::new(state.ads.clone()), + Arc::new(state.arxiv.clone()), + req.q, + req.source, + req.limit, + state.harvest_status.clone(), + ); + + Ok(StatusCode::ACCEPTED) +} + +// ── GET /api/sync/meta/count ── +#[derive(Debug, Deserialize)] +pub struct MetaSyncCountRequest { + pub q: String, + pub source: String, // "all" | "ads" | "arxiv" +} + +#[derive(Debug, Serialize)] +pub struct MetaSyncCountResponse { + pub total: i32, +} + +pub async fn get_meta_sync_count( + State(state): State>, + Query(req): Query, +) -> Result, (StatusCode, String)> { + let total = crate::services::batch_sync::MetaSync::get_total_count( + &req.q, + &req.source, + &state.ads, + &state.arxiv, + ) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("获取预估文献数失败: {}", e)))?; + + Ok(Json(MetaSyncCountResponse { total })) +} + +// ── GET /api/sync/meta/status ── +pub async fn get_meta_sync_status( + State(state): State>, +) -> Json { + let status = state.harvest_status.lock().await; + Json(status.clone()) +} + +// ── POST /api/sync/asset/run ── +#[derive(Debug, Deserialize)] +pub struct AssetSyncRunRequest { + pub target_phase: String, // "download" | "parse" | "translate" + pub limit_count: Option, // 批量处理上限，默认 100 + pub sort_order: Option, // 处理顺序: "default" | "pub_year_desc" | "created_at_desc" + pub skip_completed: Option, + pub skip_failed: Option, // 跳过当前失败 ('error:*') + pub skip_preceding_failed: Option, // 跳过前置失败 + pub skip_preceding_uncompleted: Option, // 跳过前置未完成 +} + +struct PaperRecord { + bibcode: String, + pdf_path: Option, + html_path: Option, + markdown_path: Option, + translation_path: Option, + year: String, + created_at: String, +} + +pub async fn run_asset_sync( + State(state): State>, + Json(req): Json, +) -> Result { + // 检查是否已经在进行批量处理任务 + { + let status = state.process_status.lock().await; + if status.active { + return Err((StatusCode::CONFLICT, "当前已有文献批量任务在后台运行中，请勿重复启动".to_string())); + } + } + + let target_phase = req.target_phase.clone(); + let action = match target_phase.as_str() { + "download" => crate::services::batch_sync::SyncAction::Download, + "parse" => crate::services::batch_sync::SyncAction::Parse, + "translate" => crate::services::batch_sync::SyncAction::Translate, + _ => return Err((StatusCode::BAD_REQUEST, "不支持的 target_phase 参数值".to_string())), + }; + + let rows = sqlx::query("SELECT bibcode, pdf_path, html_path, markdown_path, translation_path, year, datetime(created_at, 'localtime') FROM papers") + .fetch_all(&state.db) + .await + .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("读取数据库失败: {}", e)))?; + + let mut records = Vec::new(); + for r in rows { + records.push(PaperRecord { + bibcode: r.get(0), + pdf_path: r.get(1), + html_path: r.get(2), + markdown_path: r.get(3), + translation_path: r.get(4), + year: r.get(5), + created_at: r.get(6), + }); + } + + // 排序 + let sort_order = req.sort_order.clone().unwrap_or_else(|| "default".to_string()); + if sort_order == "pub_year_desc" { + records.sort_by(|a, b| b.year.cmp(&a.year)); + } else if sort_order == "created_at_desc" { + records.sort_by(|a, b| b.created_at.cmp(&a.created_at)); + } + + let skip_completed = req.skip_completed.unwrap_or(false); + let skip_failed = req.skip_failed.unwrap_or(false); + let skip_preceding_failed = req.skip_preceding_failed.unwrap_or(false); + let skip_preceding_uncompleted = req.skip_preceding_uncompleted.unwrap_or(false); + + let is_completed = |path: &Option| -> bool { + if let Some(p) = path { + !p.is_empty() && !p.starts_with("error:") && !p.starts_with("mineru_batch:") + } else { + false + } + }; + + let is_failed = |path: &Option| -> bool { + if let Some(p) = path { + p.starts_with("error:") + } else { + false + } + }; + + let is_no_resource = |path: &Option| -> bool { + if let Some(p) = path { + p.starts_with("error:no_resource") || p.starts_with("error:无资源") || p.starts_with("error:无有效全文") + } else { + false + } + }; + + let mut target_bibcodes = Vec::new(); + for rec in records { + let is_no_resource_paper = is_no_resource(&rec.pdf_path) || is_no_resource(&rec.html_path); + if is_no_resource_paper { + continue; + } + + let download_completed = is_completed(&rec.pdf_path) || is_completed(&rec.html_path); + let download_failed = is_failed(&rec.pdf_path) || is_failed(&rec.html_path); + let download_uncompleted = !download_completed && !download_failed; + + let parse_completed = is_completed(&rec.markdown_path); + let parse_failed = is_failed(&rec.markdown_path); + let parse_uncompleted = !parse_completed && !parse_failed; + + let translate_completed = is_completed(&rec.translation_path); + let translate_failed = is_failed(&rec.translation_path); + + match target_phase.as_str() { + "download" => { + if skip_completed && download_completed { + continue; + } + if skip_failed && download_failed { + continue; + } + } + "parse" => { + if skip_completed && parse_completed { + continue; + } + if skip_failed && parse_failed { + continue; + } + if skip_preceding_failed && download_failed { + continue; + } + if skip_preceding_uncompleted && download_uncompleted { + continue; + } + } + "translate" => { + if skip_completed && translate_completed { + continue; + } + if skip_failed && translate_failed { + continue; + } + if skip_preceding_failed && parse_failed { + continue; + } + if skip_preceding_uncompleted && parse_uncompleted { + continue; + } + } + _ => {} + } + target_bibcodes.push(rec.bibcode); + } + + let limit = req.limit_count.unwrap_or(100) as usize; + if target_bibcodes.len() > limit { + target_bibcodes.truncate(limit); + } + + if target_bibcodes.is_empty() { + return Err((StatusCode::OK, "没有需要处理的文献".to_string())); + } + + // 启动后台处理 + crate::services::batch_sync::AssetSync::start_process( + state.db.clone(), + state.config.clone(), + Arc::new(state.downloader.clone()), + Arc::new(state.qiniu.clone()), + Arc::new(state.dict.clone()), + action, + target_bibcodes, + state.process_status.clone(), + ); + + Ok(StatusCode::ACCEPTED) +} + +// ── POST /api/sync/asset/stop ── +pub async fn stop_asset_sync( + State(state): State>, +) -> StatusCode { + let mut status = state.process_status.lock().await; + if status.active { + status.active = false; + status.add_log("用户手动终止了批量处理任务。".to_string()); + } + StatusCode::OK +} + +// ── GET /api/sync/queries ── +#[derive(Debug, Serialize, Deserialize)] +pub struct SavedSyncQuery { + pub id: i64, + pub query: String, + pub source: String, + pub limit_count: i32, + pub last_run: String, +} + +pub async fn get_sync_queries( + State(state): State>, +) -> Result>, (StatusCode, String)> { + let rows = sqlx::query("SELECT id, query, source, limit_count, datetime(last_run, 'localtime') FROM sync_queries ORDER BY last_run DESC") + .fetch_all(&state.db) + .await + .map_err(|e| { + error!("获取已存同步检索配置失败: {}", e); + (StatusCode::INTERNAL_SERVER_ERROR, format!("获取已存同步检索配置失败: {}", e)) + })?; + + let mut list = Vec::new(); + for r in rows { + list.push(SavedSyncQuery { + id: r.get(0), + query: r.get(1), + source: r.get(2), + limit_count: r.get(3), + last_run: r.get(4), + }); + } + + Ok(Json(list)) +} + +// ── DELETE /api/sync/queries/:id ── +pub async fn delete_sync_query( + State(state): State>, + axum::extract::Path(id): axum::extract::Path, +) -> Result { + sqlx::query("DELETE FROM sync_queries WHERE id = ?") + .bind(id) + .execute(&state.db) + .await + .map_err(|e| { + error!("删除同步检索配置失败: {}", e); + (StatusCode::INTERNAL_SERVER_ERROR, format!("删除同步检索配置失败: {}", e)) + })?; + + Ok(StatusCode::OK) +} + +// ── GET /api/sync/asset/status ── +pub async fn get_asset_sync_status( + State(state): State>, +) -> Json { + let status = state.process_status.lock().await; + Json(status.clone()) +} diff --git a/src/bin/health_check.rs b/src/bin/health_check.rs new file mode 100644 index 0000000..572e06b --- /dev/null +++ b/src/bin/health_check.rs @@ -0,0 +1,501 @@ +// src/bin/health_check.rs +use std::fs; +use std::path::{Path, PathBuf}; +use sqlx::{SqlitePool, Row}; +use astroresearch::Config; +use tracing::{error, Level}; +use tracing_subscriber::FmtSubscriber; + +// 检测防爬、验证码、登录墙特征 +fn detect_anti_bot(content: &str) -> Option<&'static str> { + let lower = content.to_lowercase(); + let cf_patterns = [ + ("checking your browser", "Cloudflare WAF 浏览器检查"), + ("please wait while we verify", "Cloudflare WAF 验证"), + ("cf-browser-verification", "Cloudflare WAF 验证特征"), + ("cf_chl_opt", "Cloudflare WAF 特征"), + ("just a moment", "Cloudflare 正在等待提示"), + ("enable javascript and cookies", "Cloudflare JS 挑战"), + ("_cf_chl_tk", "Cloudflare Token 特征"), + ("awswafintegration", "AWS WAF 拦截"), + ("aws waf", "AWS WAF 拦截"), + ("captcha", "人机验证码页面"), + ("recaptcha", "Google reCAPTCHA"), + ("hcaptcha", "hCaptcha 验证"), + ("verify you are human", "人机验证提示"), + ("robot check", "机器人检测"), + ("login required", "出版商登录墙"), + ("please log in", "出版商登录墙"), + ("subscription required", "出版商订阅/付费墙"), + ("access denied", "拒绝访问/付费墙"), + ("you do not have access", "无权访问文献"), + ("purchase this article", "文章付费墙"), + ("sign in to access", "登录以获取访问权限"), + ("radware bot manager captcha", "Radware Bot Manager 验证"), + ("shieldsquare_styles", "ShieldSquare WAF 拦截"), + ]; + + for &(p, desc) in &cf_patterns { + if lower.contains(p) { + return Some(desc); + } + } + None +} + +// 校验 PDF 完整性与是否为虚假内容 +fn validate_pdf_content(bytes: &[u8]) -> Result<(), String> { + if !bytes.starts_with(b"%PDF") { + if bytes.starts_with(b" Result<(), String> { + if let Some(desc) = detect_anti_bot(text) { + return Err(format!("检测到安全拦截或登录限制页：{}", desc)); + } + let lower = text.to_lowercase(); + + // 1. 检查常见的跳转与错误占位特征 + if lower.contains("redirecting") || lower.contains("redirect to") || lower.contains("http-equiv=\"refresh\"") || lower.contains("autoredirecttourl") { + return Err("检测到 HTML 重定向跳转页面，而非真实文献正文".to_string()); + } + + if lower.contains("conversion to html had a fatal error") || lower.contains("no content available") || lower.contains("fatal error and exited abruptly") { + return Err("检测到 ar5iv 转换失败的占位 HTML 页面".to_string()); + } + + if lower.contains("see pages 1-last of") { + return Err("检测到仅包含 PDF 链接 of 占位 HTML 页面".to_string()); + } + + // 2. 网页标题精准黑名单校验（防止正文中提及 NSF/VizieR 导致误伤） + if let Some(start_pos) = lower.find("') { + let title_start = start_pos + tag_end + 1; + if let Some(end_pos) = lower[title_start..].find("") { + let title = &lower[title_start..title_start + end_pos]; + if title.contains("nsf award search") + || title.contains("national science foundation") + || title.contains("vizier") + || title.contains("caltechthesis") + || title.contains("caosp abstract") + || title.contains("asp conference series") + || title.contains("aspbooks") + { + return Err(format!("检测到占位网页标题: \"{}\"，判定为非正本文献", title.trim())); + } + } + } + } + + // 3. 基础字节长度与具体 HTTP 错误特征校验 + if text.len() < 2000 { + let error_patterns = [ + "404 not found", "403 forbidden", "502 bad gateway", + "500 internal server error", "access denied", "site error" + ]; + for kw in &error_patterns { + if lower.contains(kw) { + return Err(format!("HTML 包含错误页面特征（包含: {}）", kw)); + } + } + } + + // 4. 结构启发式校验：如果是小于 50KB 的 HTML，必须包含基本的章节或参考文献结构，否则判定为摘要/存根占位页 + if text.len() < 50000 { + // 匹配 heading 标签或 Markdown 格式的标题，而不是纯文本中的单词 + let has_sections = lower.contains("ltx_title_section") + || lower.contains("class=\"section\"") + || lower.contains("## introduction") + || lower.contains("

introduction") + || lower.contains("

introduction") + || lower.contains("class=\"ltx_section\""); + let has_bib = lower.contains("ltx_bibliography") + || lower.contains("class=\"references\"") + || lower.contains("
    ) { + if let Ok(entries) = fs::read_dir(dir) { + for entry in entries.flatten() { + let path = entry.path(); + if path.is_dir() { + scan_directory(&path, files); + } else { + files.push(path); + } + } + } +} + +#[tokio::main] +async fn main() -> Result<(), Box> { + // 初始化日志 + let subscriber = FmtSubscriber::builder() + .with_max_level(Level::INFO) + .finish(); + tracing::subscriber::set_global_default(subscriber)?; + + let args: Vec = std::env::args().collect(); + let fix = args.contains(&"--fix".to_string()); + + println!("=================================================="); + println!(" AstroResearch 馆藏文献健康度检查工具 "); + println!("=================================================="); + if fix { + println!("⚠️ 警告：检测到 --fix 参数。程序将自动删除坏文件并将数据库路径重置为 NULL 以便下次重新下载。"); + } else { + println!("ℹ️ 提示：当前处于只读扫描模式。如需自动修复坏文件，请附加 '-- --fix' 参数运行。"); + } + println!("--------------------------------------------------"); + + let config = Config::from_env(); + let library_dir = &config.library_dir; + println!("本地文献库目录: {:?}", library_dir); + println!("数据库连接串: {}", config.database_url); + + let db_path = config.database_url.replace("sqlite://", ""); + if !Path::new(&db_path).exists() { + error!("找不到 SQLite 数据库文件: {:?}", db_path); + return Ok(()); + } + + let pool = SqlitePool::connect(&config.database_url).await?; + println!("成功连接数据库。正在准备进行全面健康检查..."); + println!("--------------------------------------------------"); + + // ─── 阶段 1：磁盘物理文件直接扫描（解决孤儿垃圾文件与坏文件） ─── + let mut html_files = Vec::new(); + let mut pdf_files = Vec::new(); + scan_directory(&library_dir.join("HTML"), &mut html_files); + scan_directory(&library_dir.join("PDF"), &mut pdf_files); + + println!("📂 正在扫描物理磁盘文件 (HTML: {} 个, PDF: {} 个)...", html_files.len(), pdf_files.len()); + + let mut disk_html_invalid = 0; + let mut disk_pdf_invalid = 0; + let mut deleted_files = 0; + let mut db_updated_count = 0; + + for path in html_files { + let rel_path = match path.strip_prefix(library_dir) { + Ok(p) => p.to_str().unwrap_or(""), + Err(_) => continue, + }; + + if let Ok(content) = fs::read_to_string(&path) { + if let Err(e) = validate_html_content(&content) { + disk_html_invalid += 1; + println!(" ❌ 发现磁盘上损坏的 HTML 文件: {:?}", rel_path); + println!(" 原因: {}", e); + + if fix { + let _ = fs::remove_file(&path); + deleted_files += 1; + println!(" 🧹 [修复] 已物理删除损坏的文件"); + + // 检索是否有数据库记录并将其重置 + let res = sqlx::query("UPDATE papers SET html_path = NULL WHERE html_path = ? OR html_path = ?") + .bind(rel_path) + .bind(format!("HTML/{}", Path::new(rel_path).file_name().and_then(|f| f.to_str()).unwrap_or(""))) + .execute(&pool) + .await; + if let Ok(r) = res { + if r.rows_affected() > 0 { + db_updated_count += r.rows_affected(); + println!(" ✅ [修复] 数据库对应状态已重置 (受影响行数: {})", r.rows_affected()); + } + } + } + println!("--------------------------------------------------"); + } + } + } + + for path in pdf_files { + let rel_path = match path.strip_prefix(library_dir) { + Ok(p) => p.to_str().unwrap_or(""), + Err(_) => continue, + }; + + if let Ok(bytes) = fs::read(&path) { + if let Err(e) = validate_pdf_content(&bytes) { + disk_pdf_invalid += 1; + println!(" ❌ 发现磁盘上损坏的 PDF 文件: {:?}", rel_path); + println!(" 原因: {}", e); + + if fix { + let _ = fs::remove_file(&path); + deleted_files += 1; + println!(" 🧹 [修复] 已物理删除损坏的文件"); + + let res = sqlx::query("UPDATE papers SET pdf_path = NULL WHERE pdf_path = ? OR pdf_path = ?") + .bind(rel_path) + .bind(format!("PDF/{}", Path::new(rel_path).file_name().and_then(|f| f.to_str()).unwrap_or(""))) + .execute(&pool) + .await; + if let Ok(r) = res { + if r.rows_affected() > 0 { + db_updated_count += r.rows_affected(); + println!(" ✅ [修复] 数据库对应状态已重置 (受影响行数: {})", r.rows_affected()); + } + } + } + println!("--------------------------------------------------"); + } + } + } + + // ─── 阶段 2：数据库记录校验扫描（检测丢失文件、报错记录与孤立 Markdown） ─── + println!("🗄️ 正在校验数据库表记录一致性..."); + let db_rows = sqlx::query( + "SELECT bibcode, pdf_path, html_path, title, markdown_path, doctype FROM papers" + ) + .fetch_all(&pool) + .await?; + + let mut db_pdf_missing = 0; + let mut db_html_missing = 0; + let mut db_pdf_err_text = 0; + let mut db_html_err_text = 0; + let mut db_markdown_missing = 0; + let mut db_markdown_orphaned = 0; + let mut db_skip_type_cleaned = 0; + + for r in db_rows { + let bibcode: String = r.get(0); + let pdf_path_opt: Option = r.get(1); + let html_path_opt: Option = r.get(2); + let title: String = r.get(3); + let markdown_path_opt: Option = r.get(4); + let doctype_opt: Option = r.get(5); + + let mut need_db_fix = false; + let mut pdf_needs_fix = false; + let mut html_needs_fix = false; + let mut markdown_needs_fix = false; + let mut pdf_db_msg = String::new(); + let mut html_db_msg = String::new(); + let mut markdown_db_msg = String::new(); + + let doctype_str = doctype_opt.unwrap_or_else(|| "article".to_string()).to_lowercase(); + let is_skip_type = doctype_str == "proposal" + || doctype_str == "abstract" + || doctype_str == "catalog" + || doctype_str == "dataset" + || doctype_str == "software" + || doctype_str == "circular" + || doctype_str == "newsletter" + || doctype_str == "obituary"; + + if is_skip_type { + let mut has_skip_anomaly = false; + if let Some(ref pdf_p) = pdf_path_opt { + pdf_db_msg = format!("该文献属于跳过类型 [{}]，但包含下载/报错路径记录: {}", doctype_str, pdf_p); + need_db_fix = true; + pdf_needs_fix = true; + has_skip_anomaly = true; + } + if let Some(ref html_p) = html_path_opt { + html_db_msg = format!("该文献属于跳过类型 [{}]，但包含下载/报错路径记录: {}", doctype_str, html_p); + need_db_fix = true; + html_needs_fix = true; + has_skip_anomaly = true; + } + if let Some(ref md_p) = markdown_path_opt { + markdown_db_msg = format!("该文献属于跳过类型 [{}]，但包含解析路径记录: {}", doctype_str, md_p); + need_db_fix = true; + markdown_needs_fix = true; + has_skip_anomaly = true; + } + if has_skip_anomaly { + db_skip_type_cleaned += 1; + } + } else { + if let Some(ref pdf_p) = pdf_path_opt { + if pdf_p.starts_with("error:") { + db_pdf_err_text += 1; + pdf_db_msg = format!("数据库存储了报错字符串: {}", pdf_p); + } else if !library_dir.join(pdf_p).exists() { + db_pdf_missing += 1; + pdf_db_msg = format!("物理 PDF 文件丢失 (路径: {})", pdf_p); + need_db_fix = true; + pdf_needs_fix = true; + } + } + + if let Some(ref html_p) = html_path_opt { + if html_p.starts_with("error:") { + db_html_err_text += 1; + html_db_msg = format!("数据库存储了报错字符串: {}", html_p); + } else if !library_dir.join(html_p).exists() { + db_html_missing += 1; + html_db_msg = format!("物理 HTML 文件丢失 (路径: {})", html_p); + need_db_fix = true; + html_needs_fix = true; + } + } + + if let Some(ref md_p) = markdown_path_opt { + if !library_dir.join(md_p).exists() { + db_markdown_missing += 1; + markdown_db_msg = format!("物理 Markdown 文件丢失 (路径: {})", md_p); + need_db_fix = true; + markdown_needs_fix = true; + } else { + // 如果 Markdown 物理文件存在，但它既没有有效 PDF 也没有有效 HTML + let has_valid_pdf = pdf_path_opt.as_ref() + .map(|p| !p.starts_with("error:") && library_dir.join(p).exists()) + .unwrap_or(false); + let has_valid_html = html_path_opt.as_ref() + .map(|p| !p.starts_with("error:") && library_dir.join(p).exists()) + .unwrap_or(false); + + if !has_valid_pdf && !has_valid_html { + db_markdown_orphaned += 1; + markdown_db_msg = format!("Markdown 存在且完好，但失去有效 PDF/HTML 数据源，判定为孤立的 Markdown (路径: {})", md_p); + need_db_fix = true; + markdown_needs_fix = true; + } + } + } + } + + if need_db_fix { + println!(" ❌ 发现馆藏文献记录损坏/不一致 [{}] 《{}》", bibcode, title); + + if !pdf_db_msg.is_empty() { + if pdf_needs_fix { + println!(" [异常] PDF 状态: {}", pdf_db_msg); + } else { + println!(" [日志] PDF 历史下载失败原因: {}", pdf_db_msg.replace("数据库存储了报错字符串: ", "")); + } + } + if !html_db_msg.is_empty() { + if html_needs_fix { + println!(" [异常] HTML 状态: {}", html_db_msg); + } else { + println!(" [日志] HTML 历史下载失败原因: {}", html_db_msg.replace("数据库存储了报错字符串: ", "")); + } + } + if !markdown_db_msg.is_empty() { + println!(" [异常] Markdown 状态: {}", markdown_db_msg); + } + + if fix { + let mut sql_parts = Vec::new(); + if pdf_needs_fix { + sql_parts.push("pdf_path = NULL"); + if let Some(ref pdf_p) = pdf_path_opt { + if !pdf_p.starts_with("error:") { + let pdf_abs = library_dir.join(pdf_p); + if pdf_abs.exists() { + let _ = fs::remove_file(&pdf_abs); + deleted_files += 1; + println!(" 🧹 [修复] 已物理删除跳过类型或丢失的 PDF 文件"); + } + } + } + } + if html_needs_fix { + sql_parts.push("html_path = NULL"); + if let Some(ref html_p) = html_path_opt { + if !html_p.starts_with("error:") { + let html_abs = library_dir.join(html_p); + if html_abs.exists() { + let _ = fs::remove_file(&html_abs); + deleted_files += 1; + println!(" 🧹 [修复] 已物理删除跳过类型或丢失的 HTML 文件"); + } + } + } + } + if markdown_needs_fix { + sql_parts.push("markdown_path = NULL"); + if let Some(ref md_p) = markdown_path_opt { + let md_abs = library_dir.join(md_p); + if md_abs.exists() { + let _ = fs::remove_file(&md_abs); + deleted_files += 1; + println!(" 🧹 [修复] 已物理删除孤立或跳过类型的 Markdown 文件"); + } + } + } + + if !sql_parts.is_empty() { + let query_str = format!("UPDATE papers SET {} WHERE bibcode = ?", sql_parts.join(", ")); + let res = sqlx::query(&query_str) + .bind(&bibcode) + .execute(&pool) + .await; + if res.is_ok() { + db_updated_count += 1; + println!(" ✅ [修复] 数据库损坏字段已成功重置"); + } + } + } + println!("--------------------------------------------------"); + } + } + + println!("\n=================================================="); + println!(" 全面健康度检测扫描报告 "); + println!("=================================================="); + println!("磁盘物理损坏统计:"); + println!(" - 损坏/假 HTML 文件数: {}", disk_html_invalid); + println!(" - 损坏/假 PDF 文件数: {}", disk_pdf_invalid); + println!("--------------------------------------------------"); + println!("数据库一致性统计:"); + println!(" - 数据库记录下载失败数 (error:): PDF: {}, HTML: {}", db_pdf_err_text, db_html_err_text); + println!(" - 磁盘文件丢失数 (数据库有记录但文件不存在): PDF: {}, HTML: {}, Markdown: {}", db_pdf_missing, db_html_missing, db_markdown_missing); + println!(" - 孤立无源 Markdown 篇数: {}", db_markdown_orphaned); + println!(" - 需跳过类型但包含下载记录篇数 (已清理/待清理): {}", db_skip_type_cleaned); + println!("--------------------------------------------------"); + if fix { + println!("✨ 修复完成！"); + println!(" - 共删除磁盘物理损坏/孤立/跳过类型文件: {} 个", deleted_files); + println!(" - 共重置修复数据库文献字段: {} 处", db_updated_count); + } else { + let total_issues = disk_html_invalid + disk_pdf_invalid + db_pdf_missing + db_html_missing + db_markdown_missing + db_markdown_orphaned + db_skip_type_cleaned; + if total_issues > 0 { + println!("❌ 警告：共检测出 {} 处坏文件、丢失文件或异常数据库记录。", total_issues); + println!("👉 您可以附加 '-- --fix' 执行一键全面修复："); + println!(" cargo run --bin health_check -- --fix"); + } else { + println!("🎉 恭喜！馆藏物理文件及数据库完全健康，未检测到任何损坏或失效记录！"); + } + } + println!("=================================================="); + + Ok(()) +} diff --git a/src/main.rs b/src/main.rs index 79e9e2c..29d1e58 100644 --- a/src/main.rs +++ b/src/main.rs @@ -94,6 +94,7 @@ async fn main() -> anyhow::Result<()> { downloader, harvest_status: Arc::new(tokio::sync::Mutex::new(astroresearch::services::batch_sync::MetaSyncStatus::new())), process_status: Arc::new(tokio::sync::Mutex::new(astroresearch::services::batch_sync::AssetSyncStatus::new())), + active_bibcode: Arc::new(tokio::sync::Mutex::new(None)), }); // 7. 设置 Axum 路由、CORS 头以及 React 仪表盘静态资源托管 @@ -105,6 +106,8 @@ async fn main() -> anyhow::Result<()> { let api_routes = Router::new() .route("/search", get(handlers::search_papers)) .route("/download", post(handlers::download_paper)) + .route("/upload", post(handlers::upload_paper_file)) + .route("/no_resource", post(handlers::mark_no_resource)) .route("/parse", post(handlers::parse_paper)) .route("/translate", post(handlers::translate_paper)) .route("/citations", get(handlers::get_citation_network)) @@ -121,7 +124,8 @@ async fn main() -> anyhow::Result<()> { .route("/sync/asset/stop", post(handlers::stop_asset_sync)) .route("/sync/asset/status", get(handlers::get_asset_sync_status)) .route("/sync/queries", get(handlers::get_sync_queries)) - .route("/sync/queries/:id", axum::routing::delete(handlers::delete_sync_query)); + .route("/sync/queries/:id", axum::routing::delete(handlers::delete_sync_query)) + .route("/active_bibcode", get(handlers::get_active_bibcode).post(handlers::set_active_bibcode)); // 静态文件资源代理托管（当前端打包至 dashboard/dist 后，直接挂载到主域名根路由） let serve_dir = ServeDir::new("dashboard/dist") diff --git a/src/services/batch_sync.rs b/src/services/batch/asset.rs similarity index 75% rename from src/services/batch_sync.rs rename to src/services/batch/asset.rs index f5a9431..1462574 100644 --- a/src/services/batch_sync.rs +++ b/src/services/batch/asset.rs @@ -1,4 +1,4 @@ -// src/services/batch_sync.rs +// src/services/batch/asset.rs use std::sync::Arc; use std::fs; use tokio::sync::Mutex; @@ -7,284 +7,15 @@ use tracing::{info, warn, error}; use sqlx::{SqlitePool, Row}; use crate::Config; -use crate::clients::ads::AdsClient; -use crate::clients::arxiv::ArxivClient; use crate::clients::qiniu::QiniuClient; use crate::services::download::Downloader; -use crate::api::handlers::{convert_ads_doc_to_standard, convert_arxiv_to_standard, save_paper_to_db}; - -// 批量收割进度状态 -#[derive(Debug, Clone, Serialize)] -pub struct MetaSyncStatus { - pub active: bool, - pub query: String, - pub source: String, - pub synced: i32, - pub total: i32, -} - -impl MetaSyncStatus { - pub fn new() -> Self { - MetaSyncStatus { - active: false, - query: String::new(), - source: String::new(), - synced: 0, - total: 0, - } - } -} - -pub struct MetaSync; - -impl MetaSync { - // 预估文献总量 - pub async fn get_total_count( - query: &str, - source: &str, - ads: &AdsClient, - arxiv: &ArxivClient, - ) -> anyhow::Result { - let mut total = 0; - if source == "all" || source == "ads" { - match ads.get_total_count(query).await { - Ok(count) => { - total += count; - info!("ADS 预估文献总量: {} 篇", count); - } - Err(e) => { - warn!("获取 ADS 预估总量失败: {}", e); - } - } - } - if source == "all" || source == "arxiv" { - match arxiv.get_total_count(query).await { - Ok(count) => { - total += count; - info!("arXiv 预估文献总量: {} 篇", count); - } - Err(e) => { - warn!("获取 arXiv 预估总量失败: {}", e); - } - } - } - Ok(total) - } - - // 启动后台收割异步任务 - pub fn start_harvest( - db: SqlitePool, - ads: Arc, - arxiv: Arc, - query: String, - source: String, - limit: i32, - status: Arc>, - ) { - let query_clone = query.clone(); - let source_clone = source.clone(); - - tokio::spawn(async move { - info!("启动后台批量收割任务: 查询词='{}', 源='{}', 上限={}", query_clone, source_clone, limit); - - // 自动将检索配置存入/更新至 sync_queries 数据库表中进行去重和时间更新 - let _ = sqlx::query( - "INSERT INTO sync_queries (query, source, limit_count, last_run) \ - VALUES (?, ?, ?, CURRENT_TIMESTAMP) \ - ON CONFLICT(query, source, limit_count) DO UPDATE SET last_run=excluded.last_run" - ) - .bind(&query_clone) - .bind(&source_clone) - .bind(limit) - .execute(&db) - .await; - - // 1. 并行获取两端预估总量 - let ads_count_fut = { - let ads = ads.clone(); - let query = query_clone.clone(); - let is_active = source_clone == "all" || source_clone == "ads"; - async move { - if is_active { - ads.get_total_count(&query).await.unwrap_or(0) - } else { - 0 - } - } - }; - - let arxiv_count_fut = { - let arxiv = arxiv.clone(); - let query = query_clone.clone(); - let is_active = source_clone == "all" || source_clone == "arxiv"; - async move { - if is_active { - arxiv.get_total_count(&query).await.unwrap_or(0) - } else { - 0 - } - } - }; - - let (ads_total, arxiv_total) = tokio::join!(ads_count_fut, arxiv_count_fut); - let total_count = ads_total + arxiv_total; - - { - let mut s = status.lock().await; - s.active = true; - s.query = query_clone.clone(); - s.source = source_clone.clone(); - s.synced = 0; - s.total = total_count; - } - - // 计算实际需要收割的总上限，并按比例分配或根据实际匹配量上限控制 - let limit_to_harvest = if limit > 0 { std::cmp::min(limit, total_count) } else { total_count }; - - // 共享的 atomic 计数器，以便两端并行同步时独立累加进度 - let synced_counter = Arc::new(std::sync::atomic::AtomicI32::new(0)); - - // 2. 执行并行的同步子任务 - let ads_sync_fut = { - let db = db.clone(); - let ads = ads.clone(); - let query = query_clone.clone(); - let synced_counter = synced_counter.clone(); - let status = status.clone(); - let is_active = source_clone == "all" || source_clone == "ads"; - - // 如果是 all 模式，各平台按比例分摊 limit 额度，或者直接限制自身的最大可用量 - let ads_limit = if source_clone == "all" { - if ads_total == 0 { 0 } else { - let ratio = ads_total as f32 / total_count as f32; - ((limit_to_harvest as f32) * ratio).round() as i32 - } - } else { - limit_to_harvest - }; - - async move { - if !is_active || ads_limit <= 0 { - return; - } - let mut local_synced = 0; - let mut start_offset = 0; - while local_synced < ads_limit { - let chunk_size = std::cmp::min(2000, ads_limit - local_synced); - if chunk_size <= 0 { - break; - } - info!("正在同步 ADS 分批数据: start={}, rows={}", start_offset, chunk_size); - match ads.search(&query, start_offset, chunk_size, "relevance").await { - Ok(docs) => { - if docs.is_empty() { - break; - } - let count = docs.len() as i32; - for doc in docs { - let paper = convert_ads_doc_to_standard(&doc); - let _ = save_paper_to_db(&db, &paper).await; - } - local_synced += count; - start_offset += count; - - // 累加全局进度并更新状态 - let current_global = synced_counter.fetch_add(count, std::sync::atomic::Ordering::SeqCst) + count; - { - let mut s = status.lock().await; - s.synced = current_global; - } - } - Err(e) => { - error!("批量同步 ADS 数据出错: {}", e); - break; - } - } - } - } - }; - - let arxiv_sync_fut = { - let db = db.clone(); - let arxiv = arxiv.clone(); - let query = query_clone.clone(); - let synced_counter = synced_counter.clone(); - let status = status.clone(); - let is_active = source_clone == "all" || source_clone == "arxiv"; - - let arxiv_limit = if source_clone == "all" { - if arxiv_total == 0 { 0 } else { - let ratio = arxiv_total as f32 / total_count as f32; - ((limit_to_harvest as f32) * ratio).round() as i32 - } - } else { - limit_to_harvest - }; - - async move { - if !is_active || arxiv_limit <= 0 { - return; - } - let mut local_synced = 0; - let mut start_offset = 0; - while local_synced < arxiv_limit { - let chunk_size = std::cmp::min(2000, arxiv_limit - local_synced); - if chunk_size <= 0 { - break; - } - info!("正在同步 arXiv 分批数据: start={}, max_results={}", start_offset, chunk_size); - match arxiv.search(&query, start_offset, chunk_size, "relevance").await { - Ok(papers) => { - if papers.is_empty() { - break; - } - let count = papers.len() as i32; - for p in papers { - let paper = convert_arxiv_to_standard(&p); - let _ = save_paper_to_db(&db, &paper).await; - } - local_synced += count; - start_offset += count; - - // 累加全局进度并更新状态 - let current_global = synced_counter.fetch_add(count, std::sync::atomic::Ordering::SeqCst) + count; - { - let mut s = status.lock().await; - s.synced = current_global; - } - } - Err(e) => { - error!("批量同步 arXiv 数据出错: {}", e); - break; - } - } - - // 遵循 arXiv API 3 秒间隔要求 - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; - } - } - }; - - // 使用 tokio::join! 并行驱动两端同步任务 - tokio::join!(ads_sync_fut, arxiv_sync_fut); - - // 4. 收尾并重置状态 - let final_synced = synced_counter.load(std::sync::atomic::Ordering::SeqCst); - { - let mut s = status.lock().await; - s.active = false; - s.synced = final_synced; - info!("后台批量收割任务已结束。共成功同步 {} 篇文献。", final_synced); - } - }); - } -} #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "lowercase")] pub enum SyncAction { Download, Parse, + Translate, All, } @@ -335,6 +66,7 @@ impl AssetSync { config: Config, downloader: Arc, qiniu: Arc, + dict: Arc, action: SyncAction, bibcodes: Vec, status: Arc>, @@ -356,6 +88,7 @@ impl AssetSync { let action_desc = match action { SyncAction::Download => "下载", SyncAction::Parse => "解析", + SyncAction::Translate => "翻译", SyncAction::All => "下载与解析", }; s.add_log(format!("批量{}任务启动，共 {} 篇文献需处理。", action_desc, total)); @@ -383,13 +116,13 @@ impl AssetSync { // 1. 获取文献元数据与当前路径状态 let paper_res = sqlx::query( - "SELECT arxiv_id, doi, pdf_path, html_path, markdown_path, doctype FROM papers WHERE bibcode = ?" + "SELECT arxiv_id, doi, pdf_path, html_path, markdown_path, doctype, translation_path FROM papers WHERE bibcode = ?" ) .bind(&bibcode) .fetch_optional(&db) .await; - let (arxiv_id, doi, mut pdf_path, mut html_path, markdown_path, doctype) = match paper_res { + let (arxiv_id, doi, mut pdf_path, mut html_path, markdown_path, doctype, translation_path) = match paper_res { Ok(Some(row)) => { let arxiv_id: String = row.get(0); let doi: String = row.get(1); @@ -397,7 +130,8 @@ impl AssetSync { let html_path: Option = row.get(3); let markdown_path: Option = row.get(4); let doctype: Option = row.get(5); - (arxiv_id, doi, pdf_path, html_path, markdown_path, doctype) + let translation_path: Option = row.get(6); + (arxiv_id, doi, pdf_path, html_path, markdown_path, doctype, translation_path) } _ => { let mut s = status.lock().await; @@ -406,9 +140,17 @@ impl AssetSync { } }; - // 1b. 检查 doctype，如果是 proposal, abstract, catalog, software 等无数字全文的文件，直接跳过处理 + // 1b. 检查 doctype，如果是 proposal, abstract, catalog, dataset, software, circular 等无数字全文的文件，直接跳过处理 let doctype_str = doctype.unwrap_or_else(|| "article".to_string()).to_lowercase(); - if doctype_str == "proposal" || doctype_str == "abstract" || doctype_str == "catalog" || doctype_str == "software" { + if doctype_str == "proposal" + || doctype_str == "abstract" + || doctype_str == "catalog" + || doctype_str == "dataset" + || doctype_str == "software" + || doctype_str == "circular" + || doctype_str == "newsletter" + || doctype_str == "obituary" + { let mut s = status.lock().await; s.add_log(format!("文献 {} 的类型为 {} (无数字版全文)，跳过下载与解析。", bibcode, doctype_str)); // 同样更新处理进度，防止任务进度条卡住 @@ -434,16 +176,22 @@ impl AssetSync { s.add_log(format!("文献 {} 本地无 PDF/HTML，开始下载...", bibcode)); } - let (downloaded_pdf, downloaded_html) = if !arxiv_id.is_empty() { + let (pdf_res, html_res) = if !arxiv_id.is_empty() { downloader.download_arxiv_direct(&arxiv_id, &config.library_dir).await } else { let doi_opt = if !doi.is_empty() { Some(doi.as_str()) } else { None }; downloader.download_paper(&bibcode, doi_opt, &config.library_dir).await }; - if downloaded_pdf.is_some() || downloaded_html.is_some() { - let pdf_rel = downloaded_pdf.map(|p| p.strip_prefix(&config.library_dir).unwrap_or(&p).to_string_lossy().to_string()); - let html_rel = downloaded_html.map(|p| p.strip_prefix(&config.library_dir).unwrap_or(&p).to_string_lossy().to_string()); + if pdf_res.is_ok() || html_res.is_ok() { + let pdf_rel = match pdf_res { + Ok(p) => Some(p.strip_prefix(&config.library_dir).unwrap_or(&p).to_string_lossy().to_string()), + Err(e) => Some(format!("error: {}", e)), + }; + let html_rel = match html_res { + Ok(p) => Some(p.strip_prefix(&config.library_dir).unwrap_or(&p).to_string_lossy().to_string()), + Err(e) => Some(format!("error: {}", e)), + }; // 更新路径变量与数据库 pdf_path = pdf_rel.clone(); @@ -466,7 +214,24 @@ impl AssetSync { dl_failed_count += 1; let mut s = status.lock().await; s.download_failed = dl_failed_count; - s.add_log(format!("文献 {} 下载失败（PDF 和 HTML 均下载失败）", bibcode)); + + let pdf_err = match pdf_res { + Err(e) => format!("error: {}", e), + _ => "error: 未知错误".to_string(), + }; + let html_err = match html_res { + Err(e) => format!("error: {}", e), + _ => "error: 未知错误".to_string(), + }; + + s.add_log(format!("文献 {} 下载失败。PDF: {}, HTML: {}", bibcode, pdf_err, html_err)); + + let _ = sqlx::query("UPDATE papers SET pdf_path = ?, html_path = ? WHERE bibcode = ?") + .bind(&pdf_err) + .bind(&html_err) + .bind(&bibcode) + .execute(&db) + .await; } // 每次下载尝试后，加入 3-5 秒随机延迟，防爬防封 @@ -723,6 +488,117 @@ impl AssetSync { s.parsed += 1; } } + + // 4. 检查并执行翻译 + if action == SyncAction::Translate { + let is_tr_exist = translation_path.as_ref().map(|p| config.library_dir.join(p).exists() && !p.starts_with("error:")).unwrap_or(false); + if !is_tr_exist { + if let Some(md_rel) = &markdown_path { + if !md_rel.starts_with("error:") { + let md_abs = config.library_dir.join(md_rel); + if md_abs.exists() { + { + let mut s = status.lock().await; + s.add_log(format!("文献 {} 开始调用 LLM 翻译...", bibcode)); + } + + match fs::read_to_string(&md_abs) { + Ok(english_markdown) => { + match crate::services::translation::translate_markdown(&english_markdown, &dict, &config).await { + Ok(translated_markdown) => { + let tr_filename = format!("{}_zh.md", bibcode); + let tr_dest = config.library_dir.join("Translation").join(&tr_filename); + let _ = fs::create_dir_all(tr_dest.parent().unwrap()); + if fs::write(&tr_dest, &translated_markdown).is_ok() { + let relative_tr_path = format!("Translation/{}", tr_filename); + let _ = sqlx::query("UPDATE papers SET translation_path = ? WHERE bibcode = ?") + .bind(&relative_tr_path) + .bind(&bibcode) + .execute(&db) + .await; + + let mut s = status.lock().await; + s.parsed += 1; + s.add_log(format!("文献 {} 翻译成功！", bibcode)); + } else { + let error_msg = "error: 写入翻译文件失败"; + let _ = sqlx::query("UPDATE papers SET translation_path = ? WHERE bibcode = ?") + .bind(error_msg) + .bind(&bibcode) + .execute(&db) + .await; + let mut s = status.lock().await; + s.parse_failed += 1; + s.add_log(format!("文献 {} 翻译文件写入失败。", bibcode)); + } + } + Err(e) => { + let error_msg = format!("error: {}", e); + let _ = sqlx::query("UPDATE papers SET translation_path = ? WHERE bibcode = ?") + .bind(&error_msg) + .bind(&bibcode) + .execute(&db) + .await; + let mut s = status.lock().await; + s.parse_failed += 1; + s.add_log(format!("文献 {} 翻译失败: {}", bibcode, e)); + } + } + } + Err(e) => { + let error_msg = format!("error: 读取英文 Markdown 失败: {}", e); + let _ = sqlx::query("UPDATE papers SET translation_path = ? WHERE bibcode = ?") + .bind(&error_msg) + .bind(&bibcode) + .execute(&db) + .await; + let mut s = status.lock().await; + s.parse_failed += 1; + s.add_log(format!("文献 {} 读取英文 Markdown 失败: {}", bibcode, e)); + } + } + } else { + let error_msg = "error: 英文 Markdown 文件不存在"; + let _ = sqlx::query("UPDATE papers SET translation_path = ? WHERE bibcode = ?") + .bind(error_msg) + .bind(&bibcode) + .execute(&db) + .await; + let mut s = status.lock().await; + s.parse_failed += 1; + s.add_log(format!("文献 {} 英文 Markdown 文件不存在，无法翻译。", bibcode)); + } + } else { + let error_msg = "error: 英文 Markdown 文件处于解析失败状态"; + let _ = sqlx::query("UPDATE papers SET translation_path = ? WHERE bibcode = ?") + .bind(error_msg) + .bind(&bibcode) + .execute(&db) + .await; + let mut s = status.lock().await; + s.parse_failed += 1; + s.add_log(format!("文献 {} 英文 Markdown 解析失败，跳过翻译。", bibcode)); + } + } else { + let error_msg = "error: 尚未解析英文 Markdown 路径为 NULL"; + let _ = sqlx::query("UPDATE papers SET translation_path = ? WHERE bibcode = ?") + .bind(error_msg) + .bind(&bibcode) + .execute(&db) + .await; + let mut s = status.lock().await; + s.parse_failed += 1; + s.add_log(format!("文献 {} 尚未解析英文 Markdown，跳过翻译。", bibcode)); + } + } else { + { + let mut s = status.lock().await; + s.add_log(format!("文献 {} 已存在翻译，跳过。", bibcode)); + } + let mut s = status.lock().await; + s.parsed += 1; + } + } } if !join_handles.is_empty() { @@ -741,6 +617,7 @@ impl AssetSync { let action_desc = match action { SyncAction::Download => "下载", SyncAction::Parse => "解析", + SyncAction::Translate => "翻译", SyncAction::All => "下载与解析", }; s.add_log(format!("批量{}任务顺利完成！", action_desc)); @@ -829,11 +706,13 @@ mod tests { let qiniu = Arc::new(QiniuClient::new("test_access".to_string(), "test_secret".to_string(), "test_bucket".to_string(), "test_domain".to_string())); let status = Arc::new(Mutex::new(AssetSyncStatus::new())); + let dict = Arc::new(crate::services::translation::Dictionary::new()); AssetSync::start_process( pool.clone(), config, downloader, qiniu, + dict, SyncAction::All, vec![bibcode.clone()], status.clone(), @@ -947,11 +826,13 @@ mod tests { let qiniu = Arc::new(QiniuClient::new("test_access".to_string(), "test_secret".to_string(), "test_bucket".to_string(), "test_domain".to_string())); let status = Arc::new(Mutex::new(AssetSyncStatus::new())); + let dict = Arc::new(crate::services::translation::Dictionary::new()); AssetSync::start_process( pool.clone(), config, downloader, qiniu, + dict, SyncAction::All, vec![bib1.clone(), bib2.clone()], status.clone(), diff --git a/src/services/batch/meta.rs b/src/services/batch/meta.rs new file mode 100644 index 0000000..7b7a70c --- /dev/null +++ b/src/services/batch/meta.rs @@ -0,0 +1,273 @@ +// src/services/batch/meta.rs +use std::sync::Arc; +use tokio::sync::Mutex; +use serde::Serialize; +use tracing::{info, warn, error}; +use sqlx::SqlitePool; + +use crate::clients::ads::AdsClient; +use crate::clients::arxiv::ArxivClient; +use crate::api::handlers::{convert_ads_doc_to_standard, convert_arxiv_to_standard, save_paper_to_db}; + +// 批量元数据同步进度状态 +#[derive(Debug, Clone, Serialize)] +pub struct MetaSyncStatus { + pub active: bool, + pub query: String, + pub source: String, + pub synced: i32, + pub total: i32, +} + +impl MetaSyncStatus { + pub fn new() -> Self { + MetaSyncStatus { + active: false, + query: String::new(), + source: String::new(), + synced: 0, + total: 0, + } + } +} + +pub struct MetaSync; + +impl MetaSync { + // 预估文献总量 + pub async fn get_total_count( + query: &str, + source: &str, + ads: &AdsClient, + arxiv: &ArxivClient, + ) -> anyhow::Result { + let mut total = 0; + if source == "all" || source == "ads" { + match ads.get_total_count(query).await { + Ok(count) => { + total += count; + info!("ADS 预估文献总量: {} 篇", count); + } + Err(e) => { + warn!("获取 ADS 预估总量失败: {}", e); + } + } + } + if source == "all" || source == "arxiv" { + match arxiv.get_total_count(query).await { + Ok(count) => { + total += count; + info!("arXiv 预估文献总量: {} 篇", count); + } + Err(e) => { + warn!("获取 arXiv 预估总量失败: {}", e); + } + } + } + Ok(total) + } + + // 启动后台元数据同步异步任务 + pub fn start_harvest( + db: SqlitePool, + ads: Arc, + arxiv: Arc, + query: String, + source: String, + limit: i32, + status: Arc>, + ) { + let query_clone = query.clone(); + let source_clone = source.clone(); + + tokio::spawn(async move { + info!("启动后台批量元数据同步任务: 查询词='{}', 源='{}', 上限={}", query_clone, source_clone, limit); + + // 自动将检索配置存入/更新至 sync_queries 数据库表中进行去重和时间更新 + let _ = sqlx::query( + "INSERT INTO sync_queries (query, source, limit_count, last_run) \ + VALUES (?, ?, ?, CURRENT_TIMESTAMP) \ + ON CONFLICT(query, source, limit_count) DO UPDATE SET last_run=excluded.last_run" + ) + .bind(&query_clone) + .bind(&source_clone) + .bind(limit) + .execute(&db) + .await; + + // 1. 并行获取两端预估总量 + let ads_count_fut = { + let ads = ads.clone(); + let query = query_clone.clone(); + let is_active = source_clone == "all" || source_clone == "ads"; + async move { + if is_active { + ads.get_total_count(&query).await.unwrap_or(0) + } else { + 0 + } + } + }; + + let arxiv_count_fut = { + let arxiv = arxiv.clone(); + let query = query_clone.clone(); + let is_active = source_clone == "all" || source_clone == "arxiv"; + async move { + if is_active { + arxiv.get_total_count(&query).await.unwrap_or(0) + } else { + 0 + } + } + }; + + let (ads_total, arxiv_total) = tokio::join!(ads_count_fut, arxiv_count_fut); + let total_count = ads_total + arxiv_total; + + { + let mut s = status.lock().await; + s.total = total_count; + } + + // 计算实际需要元数据同步的总上限，并按比例分配或根据实际匹配量上限控制 + let limit_to_harvest = if limit > 0 { std::cmp::min(limit, total_count) } else { total_count }; + + // 共享的 atomic 计数器，以便两端并行同步时独立累加进度 + let synced_counter = Arc::new(std::sync::atomic::AtomicI32::new(0)); + + // 2. 执行并行的同步子任务 + let ads_sync_fut = { + let db = db.clone(); + let ads = ads.clone(); + let query = query_clone.clone(); + let synced_counter = synced_counter.clone(); + let status = status.clone(); + let is_active = source_clone == "all" || source_clone == "ads"; + + // 如果是 all 模式，各平台按比例分摊 limit 额度，或者直接限制自身的最大可用量 + let ads_limit = if source_clone == "all" { + if ads_total == 0 { 0 } else { + let ratio = ads_total as f32 / total_count as f32; + ((limit_to_harvest as f32) * ratio).round() as i32 + } + } else { + limit_to_harvest + }; + + async move { + if !is_active || ads_limit <= 0 { + return; + } + let mut local_synced = 0; + let mut start_offset = 0; + while local_synced < ads_limit { + let chunk_size = std::cmp::min(2000, ads_limit - local_synced); + if chunk_size <= 0 { + break; + } + info!("正在同步 ADS 分批数据: start={}, rows={}", start_offset, chunk_size); + match ads.search(&query, start_offset, chunk_size, "relevance").await { + Ok(docs) => { + if docs.is_empty() { + break; + } + let count = docs.len() as i32; + for doc in docs { + let paper = convert_ads_doc_to_standard(&doc); + let _ = save_paper_to_db(&db, &paper).await; + } + local_synced += count; + start_offset += count; + + // 累加全局进度并更新状态 + let current_global = synced_counter.fetch_add(count, std::sync::atomic::Ordering::SeqCst) + count; + { + let mut s = status.lock().await; + s.synced = current_global; + } + } + Err(e) => { + error!("批量同步 ADS 数据出错: {}", e); + break; + } + } + } + } + }; + + let arxiv_sync_fut = { + let db = db.clone(); + let arxiv = arxiv.clone(); + let query = query_clone.clone(); + let synced_counter = synced_counter.clone(); + let status = status.clone(); + let is_active = source_clone == "all" || source_clone == "arxiv"; + + let arxiv_limit = if source_clone == "all" { + if arxiv_total == 0 { 0 } else { + let ratio = arxiv_total as f32 / total_count as f32; + ((limit_to_harvest as f32) * ratio).round() as i32 + } + } else { + limit_to_harvest + }; + + async move { + if !is_active || arxiv_limit <= 0 { + return; + } + let mut local_synced = 0; + let mut start_offset = 0; + while local_synced < arxiv_limit { + let chunk_size = std::cmp::min(2000, arxiv_limit - local_synced); + if chunk_size <= 0 { + break; + } + info!("正在同步 arXiv 分批数据: start={}, max_results={}", start_offset, chunk_size); + match arxiv.search(&query, start_offset, chunk_size, "relevance").await { + Ok(papers) => { + if papers.is_empty() { + break; + } + let count = papers.len() as i32; + for p in papers { + let paper = convert_arxiv_to_standard(&p); + let _ = save_paper_to_db(&db, &paper).await; + } + local_synced += count; + start_offset += count; + + // 累加全局进度并更新状态 + let current_global = synced_counter.fetch_add(count, std::sync::atomic::Ordering::SeqCst) + count; + { + let mut s = status.lock().await; + s.synced = current_global; + } + } + Err(e) => { + error!("批量同步 arXiv 数据出错: {}", e); + break; + } + } + + // 遵循 arXiv API 3 秒间隔要求 + tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + } + } + }; + + // 使用 tokio::join! 并行驱动两端同步任务 + tokio::join!(ads_sync_fut, arxiv_sync_fut); + + // 4. 收尾并重置状态 + let final_synced = synced_counter.load(std::sync::atomic::Ordering::SeqCst); + { + let mut s = status.lock().await; + s.active = false; + s.synced = final_synced; + info!("后台批量元数据同步任务已结束。共成功同步 {} 篇文献。", final_synced); + } + }); + } +} diff --git a/src/services/batch/mod.rs b/src/services/batch/mod.rs new file mode 100644 index 0000000..5fbdbbc --- /dev/null +++ b/src/services/batch/mod.rs @@ -0,0 +1,6 @@ +// src/services/batch/mod.rs +pub mod meta; +pub mod asset; + +pub use meta::{MetaSyncStatus, MetaSync}; +pub use asset::{SyncAction, AssetSyncStatus, AssetSync}; diff --git a/src/services/download.rs b/src/services/download.rs index ffc889a..cf11467 100644 --- a/src/services/download.rs +++ b/src/services/download.rs @@ -90,10 +90,11 @@ fn detect_anti_bot(content: &str, url: Option<&str>) -> Result<()> { "checking your browser", "please wait while we verify", "cf-browser-verification", "cf_chl_opt", "just a moment", "enable javascript and cookies", "_cf_chl_tk", + "awswafintegration", "aws waf", ]; for p in &cf_patterns { if lower.contains(p) { - anyhow::bail!("检测到 Cloudflare 挑战页面（特征: {}）", p); + anyhow::bail!("检测到 Cloudflare 或 AWS WAF 挑战页面（特征: {}）", p); } } @@ -125,7 +126,7 @@ fn detect_anti_bot(content: &str, url: Option<&str>) -> Result<()> { } /// 校验响应字节是否为有效 PDF（魔数 + 最小大小 + EOF 标记） -fn validate_pdf_content(bytes: &[u8]) -> Result<()> { +pub(crate) fn validate_pdf_content(bytes: &[u8]) -> Result<()> { if !bytes.starts_with(b"%PDF") { if bytes.starts_with(b" Result<()> { Ok(()) } -/// 校验 HTML 内容是否为有效文献页（非错误/登录墙） -fn validate_html_content(text: &str) -> Result<()> { +/// 校验 HTML 内容是否为有效文献页（非错误/登录墙/跳转/摘要占位页） +pub(crate) fn validate_html_content(text: &str) -> Result<()> { detect_anti_bot(text, None)?; + let lower = text.to_lowercase(); + + // 1. 检查常见的跳转与错误占位特征 + if lower.contains("redirecting") || lower.contains("redirect to") || lower.contains("http-equiv=\"refresh\"") || lower.contains("autoredirecttourl") { + anyhow::bail!("检测到 HTML 重定向跳转页面，而非真实文献正文"); + } + + if lower.contains("conversion to html had a fatal error") || lower.contains("no content available") || lower.contains("fatal error and exited abruptly") { + anyhow::bail!("检测到 ar5iv 转换失败的占位 HTML 页面"); + } + + if lower.contains("see pages 1-last of") { + anyhow::bail!("检测到仅包含 PDF 链接的占位 HTML 页面"); + } + + // 2. 网页标题精准黑名单校验（防止正文中提及 NSF/VizieR 导致误伤） + if let Some(start_pos) = lower.find("') { + let title_start = start_pos + tag_end + 1; + if let Some(end_pos) = lower[title_start..].find("") { + let title = &lower[title_start..title_start + end_pos]; + if title.contains("nsf award search") + || title.contains("national science foundation") + || title.contains("vizier") + || title.contains("caltechthesis") + || title.contains("caosp abstract") + || title.contains("asp conference series") + || title.contains("aspbooks") + { + anyhow::bail!("检测到占位网页标题: \"{}\"，判定为非正本文献", title.trim()); + } + } + } + } + + // 3. 基础字节长度与具体 HTTP 错误特征校验 if text.len() < 2000 { - let lower = text.to_lowercase(); - for kw in &["error", "404", "not found", "forbidden", "access denied"] { + let error_patterns = [ + "404 not found", "403 forbidden", "502 bad gateway", + "500 internal server error", "access denied", "site error" + ]; + for kw in &error_patterns { if lower.contains(kw) { anyhow::bail!("响应是错误页面（包含: {}）", kw); } } - warn!("HTML 内容较短（{} 字节），可能不完整", text.len()); } + + // 4. 结构启发式校验：如果是小于 50KB 的 HTML，必须包含基本的章节或参考文献结构，否则判定为摘要/存根占位页 + if text.len() < 50000 { + // 匹配 heading 标签或 Markdown 格式的标题，而不是纯文本中的单词 + let has_sections = lower.contains("ltx_title_section") + || lower.contains("class=\"section\"") + || lower.contains("## introduction") + || lower.contains("

    introduction") + || lower.contains("

    introduction") + || lower.contains("class=\"ltx_section\""); + let has_bib = lower.contains("ltx_bibliography") + || lower.contains("class=\"references\"") + || lower.contains("
      Result<()> { + if text.len() < 500 { + anyhow::bail!("上传的 HTML 文件过小（{} 字节），可能是空白或错误页面", text.len()); + } + + let lower = text.to_lowercase(); + + // 仅拒绝明确的重定向占位页（通常 body 极短且没有正文） + let is_redirect = lower.contains("http-equiv=\"refresh\"") + || lower.contains("autoredirecttourl"); + if is_redirect && text.len() < 5000 { + anyhow::bail!("检测到 HTML 重定向跳转页面，而非真实文献正文"); + } + Ok(()) } @@ -191,6 +272,45 @@ impl Downloader { Downloader { client } } + /// 使用 Obscura 作为后备通道进行下载 + async fn download_via_obscura(&self, url: &str, dest_path: &Path, is_pdf: bool) -> Result<()> { + info!("[Obscura 后备通道] 启动下载: {}", url); + + if let Some(parent) = dest_path.parent() { + std::fs::create_dir_all(parent)?; + } + + let mut cmd = tokio::process::Command::new("bin/obscura"); + cmd.arg("fetch").arg(url).arg("--stealth"); + + if is_pdf { + cmd.arg("--dump").arg("original"); + } else { + cmd.arg("--dump").arg("html"); + } + + cmd.arg("--output").arg(dest_path); + + let status = cmd.status().await + .context("启动 Obscura 进程失败，请检查 bin/obscura 是否存在且有执行权限")?; + + if !status.success() { + anyhow::bail!("Obscura 进程退出状态非成功: {:?}", status); + } + + // 校验下载得到的文件 + if is_pdf { + let bytes = tokio::fs::read(dest_path).await?; + validate_pdf_content(&bytes)?; + } else { + let text = tokio::fs::read_to_string(dest_path).await?; + validate_html_content(&text)?; + } + + info!("[Obscura 后备通道] 下载并校验成功: {:?}", dest_path); + Ok(()) + } + // ─── 辅助工具 ────────────────────────────────────────────── /// 请求前随机延迟 500-2000ms（模拟人类浏览间隔，降低反爬触发） @@ -273,38 +393,60 @@ impl Downloader { let main_url = format!("https://iopscience.iop.org/article/{}", doi); let pdf_url = format!("https://iopscience.iop.org/article/{}/pdf", doi); - // 步骤 1：访问文章主页，建立 Cookie 会话 - debug!("[IOP] 预热主页: {}", main_url); - Self::maybe_delay().await; - match self.client.get(&main_url) - .headers(build_chrome_headers(None)) - .send().await - { - Ok(r) => debug!("[IOP] 主页响应: {}", r.status()), - Err(e) => warn!("[IOP] 主页访问失败（继续尝试）: {:?}", e), + let res = async { + // 步骤 1：访问文章主页，建立 Cookie 会话 + debug!("[IOP] 预热主页: {}", main_url); + Self::maybe_delay().await; + match self.client.get(&main_url) + .headers(build_chrome_headers(None)) + .send().await + { + Ok(r) => debug!("[IOP] 主页响应: {}", r.status()), + Err(e) => warn!("[IOP] 主页访问失败（继续尝试）: {:?}", e), + } + + // 步骤 2：携带 Referer 下载 PDF + debug!("[IOP] 下载 PDF: {}", pdf_url); + Self::maybe_delay().await; + let response = self.client.get(&pdf_url) + .headers(build_chrome_headers(Some(&main_url))) + .send().await + .context("IOP PDF 请求失败")?; + + let status = response.status(); + if !status.is_success() { + anyhow::bail!("[IOP] 返回 HTTP {}", status); + } + + self.stream_download(response, dest_path).await?; + + // 步骤 3：校验下载内容 + let bytes = tokio::fs::read(dest_path).await?; + validate_pdf_content(&bytes)?; + Ok(()) + }.await; + + match res { + Ok(()) => { + info!("[IOP] PDF 下载成功: {:?}", dest_path); + Ok(()) + } + Err(e) => { + let err_msg = e.to_string(); + if err_msg.contains("人机验证") + || err_msg.contains("挑战页面") + || err_msg.contains("WAF") + || err_msg.contains("Cloudflare") + || err_msg.contains("HTTP 403") + || err_msg.contains("HTTP 503") + { + warn!("[IOP] 下载触发人机验证或拦截: {}。尝试使用 Obscura 后备通道...", err_msg); + self.download_via_obscura(&pdf_url, dest_path, true).await + } else { + Err(e) + } + } } - - // 步骤 2：携带 Referer 下载 PDF - debug!("[IOP] 下载 PDF: {}", pdf_url); - Self::maybe_delay().await; - let response = self.client.get(&pdf_url) - .headers(build_chrome_headers(Some(&main_url))) - .send().await - .context("IOP PDF 请求失败")?; - - let status = response.status(); - if !status.is_success() { - anyhow::bail!("[IOP] 返回 HTTP {}", status); - } - - self.stream_download(response, dest_path).await?; - - // 步骤 3：校验下载内容 - let bytes = tokio::fs::read(dest_path).await?; - validate_pdf_content(&bytes)?; - - info!("[IOP] PDF 下载成功: {:?}", dest_path); - Ok(()) } /// Springer/Nature HTML 下载（含会话预热） @@ -312,74 +454,140 @@ impl Downloader { let url = format!("https://link.springer.com/article/{}", doi); info!("[Springer] 下载 HTML: {}", url); - Self::maybe_delay().await; - let response = self.client.get(&url) - .headers(build_browser_headers()) - .send().await - .context("Springer HTML 请求失败")?; + let res = async { + Self::maybe_delay().await; + let response = self.client.get(&url) + .headers(build_browser_headers()) + .send().await + .context("Springer HTML 请求失败")?; - let status = response.status(); - if !status.is_success() { - anyhow::bail!("[Springer] 返回 HTTP {}", status); + let status = response.status(); + if !status.is_success() { + anyhow::bail!("[Springer] 返回 HTTP {}", status); + } + + self.stream_download(response, dest_path).await?; + + let text = tokio::fs::read_to_string(dest_path).await + .context("读取 HTML 文件失败")?; + validate_html_content(&text)?; + Ok(()) + }.await; + + match res { + Ok(()) => { + info!("[Springer] HTML 下载成功: {:?}", dest_path); + Ok(()) + } + Err(e) => { + let err_msg = e.to_string(); + if err_msg.contains("人机验证") + || err_msg.contains("挑战页面") + || err_msg.contains("WAF") + || err_msg.contains("Cloudflare") + || err_msg.contains("HTTP 403") + || err_msg.contains("HTTP 503") + { + warn!("[Springer] 下载触发人机验证或拦截: {}。尝试使用 Obscura 后备通道...", err_msg); + self.download_via_obscura(&url, dest_path, false).await + } else { + Err(e) + } + } } - - self.stream_download(response, dest_path).await?; - - let sniff = Self::read_file_header(dest_path).await?; - let text = String::from_utf8_lossy(&sniff); - validate_html_content(&text)?; - - info!("[Springer] HTML 下载成功: {:?}", dest_path); - Ok(()) } - /// 通用 PDF 直链下载（带随机延迟 + 内容校验） + /// 通用 PDF 直链下载（带随机延迟 + 内容校验 + Obscura 后备） async fn download_pdf_direct(&self, url: &str, dest_path: &Path, label: &str) -> Result<()> { info!("[{}] 下载 PDF: {}", label, url); Self::maybe_delay().await; - let response = self.client.get(url) - .headers(build_browser_headers()) - .send().await - .context(format!("[{}] PDF 请求失败", label))?; + let res = async { + let response = self.client.get(url) + .headers(build_browser_headers()) + .send().await + .context(format!("[{}] PDF 请求失败", label))?; - let status = response.status(); - if !status.is_success() { - anyhow::bail!("[{}] 返回 HTTP {}", label, status); + let status = response.status(); + if !status.is_success() { + anyhow::bail!("[{}] 返回 HTTP {}", label, status); + } + + self.stream_download(response, dest_path).await?; + + let bytes = tokio::fs::read(dest_path).await?; + validate_pdf_content(&bytes)?; + Ok(()) + }.await; + + match res { + Ok(()) => { + info!("[{}] PDF 下载成功: {:?}", label, dest_path); + Ok(()) + } + Err(e) => { + let err_msg = e.to_string(); + if err_msg.contains("人机验证") + || err_msg.contains("挑战页面") + || err_msg.contains("WAF") + || err_msg.contains("Cloudflare") + || err_msg.contains("HTTP 403") + || err_msg.contains("HTTP 503") + { + warn!("[{}] 下载触发人机验证或拦截: {}。尝试使用 Obscura 后备通道...", label, err_msg); + self.download_via_obscura(url, dest_path, true).await + } else { + Err(e) + } + } } - - self.stream_download(response, dest_path).await?; - - let bytes = tokio::fs::read(dest_path).await?; - validate_pdf_content(&bytes)?; - - info!("[{}] PDF 下载成功: {:?}", label, dest_path); - Ok(()) } - /// 通用 HTML 直链下载（带随机延迟 + 反爬检测） + /// 通用 HTML 直链下载（带随机延迟 + 反爬检测 + Obscura 后备） async fn download_html_direct(&self, url: &str, dest_path: &Path, label: &str) -> Result<()> { info!("[{}] 下载 HTML: {}", label, url); Self::maybe_delay().await; - let response = self.client.get(url) - .headers(build_browser_headers()) - .send().await - .context(format!("[{}] HTML 请求失败", label))?; + let res = async { + let response = self.client.get(url) + .headers(build_browser_headers()) + .send().await + .context(format!("[{}] HTML 请求失败", label))?; - let status = response.status(); - if !status.is_success() { - anyhow::bail!("[{}] 返回 HTTP {}", label, status); + let status = response.status(); + if !status.is_success() { + anyhow::bail!("[{}] 返回 HTTP {}", label, status); + } + + self.stream_download(response, dest_path).await?; + + let text = tokio::fs::read_to_string(dest_path).await + .context("读取 HTML 文件失败")?; + validate_html_content(&text)?; + Ok(()) + }.await; + + match res { + Ok(()) => { + info!("[{}] HTML 下载成功: {:?}", label, dest_path); + Ok(()) + } + Err(e) => { + let err_msg = e.to_string(); + if err_msg.contains("人机验证") + || err_msg.contains("挑战页面") + || err_msg.contains("WAF") + || err_msg.contains("Cloudflare") + || err_msg.contains("HTTP 403") + || err_msg.contains("HTTP 503") + { + warn!("[{}] 下载触发人机验证或拦截: {}。尝试使用 Obscura 后备通道...", label, err_msg); + self.download_via_obscura(url, dest_path, false).await + } else { + Err(e) + } + } } - - self.stream_download(response, dest_path).await?; - - let sniff = Self::read_file_header(dest_path).await?; - let text = String::from_utf8_lossy(&sniff); - validate_html_content(&text)?; - - info!("[{}] HTML 下载成功: {:?}", label, dest_path); - Ok(()) } // ─── CrossRef 回退通道 ───────────────────────────────────── @@ -418,7 +626,7 @@ impl Downloader { /// HTML 下载优先级： /// 1. 官方 `arxiv.org/html/{id}`（2023-12 起支持，质量与 ar5iv 相同，更稳定） /// 2. ar5iv `ar5iv.labs.arxiv.org/html/{id}`（约 3% 论文转换失败时跳过） - pub async fn download_arxiv_direct(&self, arxiv_id: &str, library_dir: &Path) -> (Option, Option) { + pub async fn download_arxiv_direct(&self, arxiv_id: &str, library_dir: &Path) -> (Result, Result) { // 去除版本号（v1/v2/v3），arxiv.org/html/ 和 ar5iv 均只提供最新渲染版 let clean_id = strip_arxiv_version(arxiv_id); @@ -426,31 +634,36 @@ impl Downloader { let pdf_dest = library_dir.join("PDF").join(format!("{}.pdf", arxiv_id)); let html_dest = library_dir.join("HTML").join(format!("{}.html", arxiv_id)); - let mut pdf_ok = None; - let mut html_ok = None; - // PDF 下载 - match self.download_pdf_direct(&pdf_url, &pdf_dest, "arXiv").await { - Ok(_) => pdf_ok = Some(pdf_dest), - Err(e) => warn!("[arXiv] PDF 下载失败: {:?}", e), - } + let pdf_res = match self.download_pdf_direct(&pdf_url, &pdf_dest, "arXiv").await { + Ok(_) => Ok(pdf_dest), + Err(e) => { + let err_msg = format!("arXiv PDF 下载失败: {}", e); + warn!("{}", err_msg); + Err(err_msg) + } + }; // HTML 下载：官方 arxiv.org/html/ 优先 let official_html_url = format!("https://arxiv.org/html/{}", clean_id); - match self.download_html_direct(&official_html_url, &html_dest, "arXiv-HTML").await { - Ok(_) => html_ok = Some(html_dest.clone()), + let html_res = match self.download_html_direct(&official_html_url, &html_dest, "arXiv-HTML").await { + Ok(_) => Ok(html_dest.clone()), Err(e) => { warn!("[arXiv-HTML] 官方 HTML 下载失败，回退 ar5iv: {:?}", e); // ar5iv 兜底：约 97% 成功率，可能有延迟 let ar5iv_url = format!("https://ar5iv.labs.arxiv.org/html/{}", clean_id); match self.download_html_direct(&ar5iv_url, &html_dest, "ar5iv").await { - Ok(_) => html_ok = Some(html_dest), - Err(e2) => warn!("[ar5iv] HTML 下载也失败: {:?}", e2), + Ok(_) => Ok(html_dest), + Err(e2) => { + let err_msg = format!("arXiv HTML 下载失败 (官方: {}, ar5iv: {})", e, e2); + warn!("{}", err_msg); + Err(err_msg) + } } } - } + }; - (pdf_ok, html_ok) + (pdf_res, html_res) } /// 下载 arXiv HTML：官方 arxiv.org/html/ 优先，ar5iv 兜底 @@ -477,13 +690,16 @@ impl Downloader { /// HTML 回退顺序： /// 1. ADS PUB_HTML 网关（IOP→ 直联 iopscience，arxiv abs → ar5iv） /// 2. ADS EPRINT_HTML 网关（arxiv abs → ar5iv） - pub async fn download_paper(&self, bibcode: &str, doi: Option<&str>, library_dir: &Path) -> (Option, Option) { + pub async fn download_paper(&self, bibcode: &str, doi: Option<&str>, library_dir: &Path) -> (Result, Result) { let base = "https://ui.adsabs.harvard.edu/link_gateway"; let pdf_dest = library_dir.join("PDF").join(format!("{}.pdf", bibcode)); let html_dest = library_dir.join("HTML").join(format!("{}.html", bibcode)); - let mut pdf_ok: Option = None; - let mut html_ok: Option = None; + let mut pdf_res = Err("未尝试任何下载通道".to_string()); + let mut html_res = Err("未尝试任何下载通道".to_string()); + + let mut pdf_errors = Vec::new(); + let mut html_errors = Vec::new(); // ── PDF 下载 ─────────────────────────────────────────── info!("[下载] 开始 PDF 下载: {}", bibcode); @@ -507,11 +723,19 @@ impl Downloader { self.download_pdf_direct(&resolved, &pdf_dest, "PUB_PDF").await }; match result { - Ok(_) => { pdf_ok = Some(pdf_dest.clone()); break 'pdf; } - Err(e) => warn!("[PUB_PDF] 下载失败: {:?}", e), + Ok(_) => { pdf_res = Ok(pdf_dest.clone()); break 'pdf; } + Err(e) => { + let msg = format!("PUB_PDF下载失败: {}", e); + warn!("{}", msg); + pdf_errors.push(msg); + } } } - Err(e) => warn!("[PUB_PDF] 网关解析失败: {:?}", e), + Err(e) => { + let msg = format!("PUB_PDF网关解析失败: {}", e); + warn!("{}", msg); + pdf_errors.push(msg); + } } // 1b. ADS_PDF 网关 (经典 ADS 整合 PDF 直接通道) @@ -519,11 +743,19 @@ impl Downloader { match self.resolve_ads_gateway(&gw).await { Ok(resolved) => { match self.download_pdf_direct(&resolved, &pdf_dest, "ADS_PDF").await { - Ok(_) => { pdf_ok = Some(pdf_dest.clone()); break 'pdf; } - Err(e) => warn!("[ADS_PDF] 下载失败: {:?}", e), + Ok(_) => { pdf_res = Ok(pdf_dest.clone()); break 'pdf; } + Err(e) => { + let msg = format!("ADS_PDF下载失败: {}", e); + warn!("{}", msg); + pdf_errors.push(msg); + } } } - Err(e) => warn!("[ADS_PDF] 网关解析失败: {:?}", e), + Err(e) => { + let msg = format!("ADS_PDF网关解析失败: {}", e); + warn!("{}", msg); + pdf_errors.push(msg); + } } // 1c. ADS EPRINT_PDF 网关 @@ -531,29 +763,49 @@ impl Downloader { match self.resolve_ads_gateway(&gw).await { Ok(resolved) => { match self.download_pdf_direct(&resolved, &pdf_dest, "EPRINT_PDF").await { - Ok(_) => { pdf_ok = Some(pdf_dest.clone()); break 'pdf; } - Err(e) => warn!("[EPRINT_PDF] 下载失败: {:?}", e), + Ok(_) => { pdf_res = Ok(pdf_dest.clone()); break 'pdf; } + Err(e) => { + let msg = format!("EPRINT_PDF下载失败: {}", e); + warn!("{}", msg); + pdf_errors.push(msg); + } } } - Err(e) => warn!("[EPRINT_PDF] 网关解析失败: {:?}", e), + Err(e) => { + let msg = format!("EPRINT_PDF网关解析失败: {}", e); + warn!("{}", msg); + pdf_errors.push(msg); + } } // 1c. CrossRef API 回退（需要 DOI） if let Some(doi_str) = doi { match self.download_crossref_pdf(doi_str, &pdf_dest).await { - Ok(_) => { pdf_ok = Some(pdf_dest.clone()); break 'pdf; } - Err(e) => warn!("[CrossRef] PDF 下载失败: {:?}", e), + Ok(_) => { pdf_res = Ok(pdf_dest.clone()); break 'pdf; } + Err(e) => { + let msg = format!("CrossRef下载失败: {}", e); + warn!("{}", msg); + pdf_errors.push(msg); + } } } // 1d. ADS SCAN 扫描版文献直接合并下载 PDF（主要针对早期/不可下载直接 PDF 的文献） let scan_url = format!("https://articles.adsabs.harvard.edu/cgi-bin/nph-iarticle_query?bibcode={}&db_key=AST&data_type=PDF_HIGH", bibcode); match self.download_pdf_direct(&scan_url, &pdf_dest, "ADS_SCAN").await { - Ok(_) => { pdf_ok = Some(pdf_dest.clone()); } - Err(e) => warn!("[ADS_SCAN] 下载失败: {:?}", e), + Ok(_) => { pdf_res = Ok(pdf_dest.clone()); } + Err(e) => { + let msg = format!("ADS_SCAN下载失败: {}", e); + warn!("{}", msg); + pdf_errors.push(msg); + } } } + if pdf_res.is_err() && !pdf_errors.is_empty() { + pdf_res = Err(pdf_errors.join("; ")); + } + // ── HTML 下载 ────────────────────────────────────────── info!("[下载] 开始 HTML 下载: {}", bibcode); @@ -576,11 +828,19 @@ impl Downloader { self.download_html_direct(&resolved, &html_dest, "PUB_HTML").await }; match result { - Ok(_) => { html_ok = Some(html_dest.clone()); break 'html; } - Err(e) => warn!("[PUB_HTML] 下载失败: {:?}", e), + Ok(_) => { html_res = Ok(html_dest.clone()); break 'html; } + Err(e) => { + let msg = format!("PUB_HTML下载失败: {}", e); + warn!("{}", msg); + html_errors.push(msg); + } } } - Err(e) => warn!("[PUB_HTML] 网关解析失败: {:?}", e), + Err(e) => { + let msg = format!("PUB_HTML网关解析失败: {}", e); + warn!("{}", msg); + html_errors.push(msg); + } } // 2b. ADS EPRINT_HTML 网关（大多数天文论文有 arXiv eprint） @@ -593,15 +853,27 @@ impl Downloader { self.download_html_direct(&resolved, &html_dest, "EPRINT_HTML").await }; match result { - Ok(_) => { html_ok = Some(html_dest.clone()); } - Err(e) => warn!("[EPRINT_HTML] 下载失败: {:?}", e), + Ok(_) => { html_res = Ok(html_dest.clone()); } + Err(e) => { + let msg = format!("EPRINT_HTML下载失败: {}", e); + warn!("{}", msg); + html_errors.push(msg); + } } } - Err(e) => warn!("[EPRINT_HTML] 网关解析失败: {:?}", e), + Err(e) => { + let msg = format!("EPRINT_HTML网关解析失败: {}", e); + warn!("{}", msg); + html_errors.push(msg); + } } } - (pdf_ok, html_ok) + if html_res.is_err() && !html_errors.is_empty() { + html_res = Err(html_errors.join("; ")); + } + + (pdf_res, html_res) } } @@ -736,7 +1008,7 @@ mod tests { let temp_dir = std::env::temp_dir(); let (pdf_path, _html_path) = downloader.download_paper(bibcode, None, &temp_dir).await; - assert!(pdf_path.is_some()); + assert!(pdf_path.is_ok()); let path = pdf_path.unwrap(); assert!(path.exists()); diff --git a/src/services/mod.rs b/src/services/mod.rs index a747361..d49148f 100644 --- a/src/services/mod.rs +++ b/src/services/mod.rs @@ -2,5 +2,9 @@ pub mod download; pub mod parser; pub mod translation; pub mod query_parser; -pub mod batch_sync; +pub mod batch; pub mod logging; + +pub mod batch_sync { + pub use super::batch::*; +} diff --git a/src/services/parser.rs b/src/services/parser.rs index 8637021..33d69cb 100644 --- a/src/services/parser.rs +++ b/src/services/parser.rs @@ -200,9 +200,7 @@ fn postprocess_markdown(text: &str) -> String { } } let mut md = clean_lines.join("\n"); - if md.contains("Keywords") { - println!("DEBUG 0: {:?}", md); - } + let div_re = Regex::new(r"]*>").unwrap(); let span_re = Regex::new(r"]*>").unwrap(); @@ -214,9 +212,7 @@ fn postprocess_markdown(text: &str) -> String { let excessive_newlines = Regex::new(r"\n{4,}").unwrap(); md = excessive_newlines.replace_all(&md, "\n\n\n").to_string(); - if md.contains("Keywords") { - println!("DEBUG 1 (excessive): {:?}", md); - } + // 还原被 html2md 自动转义的标题与引用符号 let unescape_h1 = Regex::new(r"\\#\s+").unwrap();