feat: 手动上传绕防爬、下载错误诊断与健康检查工具;模块化重构 API 与批量同步

后端:
  - 将 handlers.rs (1338行) 拆分为 helpers/papers/notes/sync 四模块
  - 将 batch_sync.rs 拆分为 batch/{mod,meta,asset} 三模块
  - 新增 POST /api/upload 多部件文件上传接口
  - 新增 POST /api/no_resource 标记文献"无全文资源"
  - 新增 GET/POST /api/active_bibcode 追踪活跃文献
  - StandardPaper 结构体扩展 pdf_error / html_error 错误诊断字段
  - download.rs 记录下载失败详情至数据库
  - 新增 health_check 二进制工具,支持只读扫描与 --fix 自动修复
  - 移除 scratch/ 目录、recovered_handlers.rs 及调试日志

  前端:
  - 新建 CustomSelect 可复用组件,替换全部原生 select
  - LibraryPanel:同步按钮反馈动画、下载失败/无资源状态筛选与计数、
    文献类型筛选、状态优先排序、搜索一键清空
  - 详情弹窗:错误诊断展示、手动 PDF/HTML 上传区、无资源标记/恢复
  - SearchPanel:扩展文献类型徽章、下载失败状态提示
  - SyncPanel:同步启动乐观 UI 更新、日志容器内自动滚动
  - Tab 状态 localStorage 持久化、弹窗 z-index 修复
This commit is contained in:
fmq
2026-06-11 22:56:36 +08:00
parent cd6af4f995
commit 8cc2b74abc
43 changed files with 4512 additions and 3879 deletions
+501
View File
@@ -0,0 +1,501 @@
// src/bin/health_check.rs
use std::fs;
use std::path::{Path, PathBuf};
use sqlx::{SqlitePool, Row};
use astroresearch::Config;
use tracing::{error, Level};
use tracing_subscriber::FmtSubscriber;
// 检测防爬、验证码、登录墙特征
fn detect_anti_bot(content: &str) -> Option<&'static str> {
let lower = content.to_lowercase();
let cf_patterns = [
("checking your browser", "Cloudflare WAF 浏览器检查"),
("please wait while we verify", "Cloudflare WAF 验证"),
("cf-browser-verification", "Cloudflare WAF 验证特征"),
("cf_chl_opt", "Cloudflare WAF 特征"),
("just a moment", "Cloudflare 正在等待提示"),
("enable javascript and cookies", "Cloudflare JS 挑战"),
("_cf_chl_tk", "Cloudflare Token 特征"),
("awswafintegration", "AWS WAF 拦截"),
("aws waf", "AWS WAF 拦截"),
("captcha", "人机验证码页面"),
("recaptcha", "Google reCAPTCHA"),
("hcaptcha", "hCaptcha 验证"),
("verify you are human", "人机验证提示"),
("robot check", "机器人检测"),
("login required", "出版商登录墙"),
("please log in", "出版商登录墙"),
("subscription required", "出版商订阅/付费墙"),
("access denied", "拒绝访问/付费墙"),
("you do not have access", "无权访问文献"),
("purchase this article", "文章付费墙"),
("sign in to access", "登录以获取访问权限"),
("radware bot manager captcha", "Radware Bot Manager 验证"),
("shieldsquare_styles", "ShieldSquare WAF 拦截"),
];
for &(p, desc) in &cf_patterns {
if lower.contains(p) {
return Some(desc);
}
}
None
}
// 校验 PDF 完整性与是否为虚假内容
fn validate_pdf_content(bytes: &[u8]) -> Result<(), String> {
if !bytes.starts_with(b"%PDF") {
if bytes.starts_with(b"<!") || bytes.starts_with(b"<html") || bytes.starts_with(b"<HTML") {
let scan_len = std::cmp::min(2048, bytes.len());
let text = String::from_utf8_lossy(&bytes[..scan_len]);
if let Some(desc) = detect_anti_bot(&text) {
return Err(format!("虽然文件后缀是 PDF,但实际内容是 HTML(检测到:{}", desc));
}
return Err("虽然文件后缀是 PDF,但实际内容是 HTML 网页,可能是重定向或拦截页面".to_string());
}
return Err("缺少 %PDF 文件头魔数,文件损坏或并非 PDF".to_string());
}
if bytes.len() < 5000 {
return Err(format!("PDF 文件过小(仅 {} 字节),极可能是错误信息页", bytes.len()));
}
let scan_len = std::cmp::min(1024, bytes.len());
let tail = &bytes[bytes.len() - scan_len..];
if !tail.windows(5).any(|w| w == b"%%EOF") {
return Err("PDF 文件未包含尾部 %%EOF 标记,文件已损坏或不完整".to_string());
}
Ok(())
}
// 校验 HTML 内容有效性(过滤假网页、存根页、跳转页及摘要页)
fn validate_html_content(text: &str) -> Result<(), String> {
if let Some(desc) = detect_anti_bot(text) {
return Err(format!("检测到安全拦截或登录限制页:{}", desc));
}
let lower = text.to_lowercase();
// 1. 检查常见的跳转与错误占位特征
if lower.contains("redirecting") || lower.contains("redirect to") || lower.contains("http-equiv=\"refresh\"") || lower.contains("autoredirecttourl") {
return Err("检测到 HTML 重定向跳转页面,而非真实文献正文".to_string());
}
if lower.contains("conversion to html had a fatal error") || lower.contains("no content available") || lower.contains("fatal error and exited abruptly") {
return Err("检测到 ar5iv 转换失败的占位 HTML 页面".to_string());
}
if lower.contains("see pages 1-last of") {
return Err("检测到仅包含 PDF 链接 of 占位 HTML 页面".to_string());
}
// 2. 网页标题精准黑名单校验(防止正文中提及 NSF/VizieR 导致误伤)
if let Some(start_pos) = lower.find("<title") {
if let Some(tag_end) = lower[start_pos..].find('>') {
let title_start = start_pos + tag_end + 1;
if let Some(end_pos) = lower[title_start..].find("</title>") {
let title = &lower[title_start..title_start + end_pos];
if title.contains("nsf award search")
|| title.contains("national science foundation")
|| title.contains("vizier")
|| title.contains("caltechthesis")
|| title.contains("caosp abstract")
|| title.contains("asp conference series")
|| title.contains("aspbooks")
{
return Err(format!("检测到占位网页标题: \"{}\",判定为非正本文献", title.trim()));
}
}
}
}
// 3. 基础字节长度与具体 HTTP 错误特征校验
if text.len() < 2000 {
let error_patterns = [
"404 not found", "403 forbidden", "502 bad gateway",
"500 internal server error", "access denied", "site error"
];
for kw in &error_patterns {
if lower.contains(kw) {
return Err(format!("HTML 包含错误页面特征(包含: {}", kw));
}
}
}
// 4. 结构启发式校验:如果是小于 50KB 的 HTML,必须包含基本的章节或参考文献结构,否则判定为摘要/存根占位页
if text.len() < 50000 {
// 匹配 heading 标签或 Markdown 格式的标题,而不是纯文本中的单词
let has_sections = lower.contains("ltx_title_section")
|| lower.contains("class=\"section\"")
|| lower.contains("## introduction")
|| lower.contains("<h2>introduction")
|| lower.contains("<h3>introduction")
|| lower.contains("class=\"ltx_section\"");
let has_bib = lower.contains("ltx_bibliography")
|| lower.contains("class=\"references\"")
|| lower.contains("<ol class=\"references\"")
|| lower.contains("<ul class=\"references\"")
|| lower.contains("id=\"bib\"")
|| lower.contains("class=\"ltx_bibliography\"");
if !has_sections && !has_bib {
return Err(format!("HTML 长度偏小({} 字节)且缺少正文章节或参考文献,判定为非正本文献", text.len()));
}
}
Ok(())
}
// 递归扫描目录下的所有物理文件
fn scan_directory(dir: &Path, files: &mut Vec<PathBuf>) {
if let Ok(entries) = fs::read_dir(dir) {
for entry in entries.flatten() {
let path = entry.path();
if path.is_dir() {
scan_directory(&path, files);
} else {
files.push(path);
}
}
}
}
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
// 初始化日志
let subscriber = FmtSubscriber::builder()
.with_max_level(Level::INFO)
.finish();
tracing::subscriber::set_global_default(subscriber)?;
let args: Vec<String> = std::env::args().collect();
let fix = args.contains(&"--fix".to_string());
println!("==================================================");
println!(" AstroResearch 馆藏文献健康度检查工具 ");
println!("==================================================");
if fix {
println!("⚠️ 警告:检测到 --fix 参数。程序将自动删除坏文件并将数据库路径重置为 NULL 以便下次重新下载。");
} else {
println!("ℹ️ 提示:当前处于只读扫描模式。如需自动修复坏文件,请附加 '-- --fix' 参数运行。");
}
println!("--------------------------------------------------");
let config = Config::from_env();
let library_dir = &config.library_dir;
println!("本地文献库目录: {:?}", library_dir);
println!("数据库连接串: {}", config.database_url);
let db_path = config.database_url.replace("sqlite://", "");
if !Path::new(&db_path).exists() {
error!("找不到 SQLite 数据库文件: {:?}", db_path);
return Ok(());
}
let pool = SqlitePool::connect(&config.database_url).await?;
println!("成功连接数据库。正在准备进行全面健康检查...");
println!("--------------------------------------------------");
// ─── 阶段 1:磁盘物理文件直接扫描(解决孤儿垃圾文件与坏文件) ───
let mut html_files = Vec::new();
let mut pdf_files = Vec::new();
scan_directory(&library_dir.join("HTML"), &mut html_files);
scan_directory(&library_dir.join("PDF"), &mut pdf_files);
println!("📂 正在扫描物理磁盘文件 (HTML: {} 个, PDF: {} 个)...", html_files.len(), pdf_files.len());
let mut disk_html_invalid = 0;
let mut disk_pdf_invalid = 0;
let mut deleted_files = 0;
let mut db_updated_count = 0;
for path in html_files {
let rel_path = match path.strip_prefix(library_dir) {
Ok(p) => p.to_str().unwrap_or(""),
Err(_) => continue,
};
if let Ok(content) = fs::read_to_string(&path) {
if let Err(e) = validate_html_content(&content) {
disk_html_invalid += 1;
println!(" ❌ 发现磁盘上损坏的 HTML 文件: {:?}", rel_path);
println!(" 原因: {}", e);
if fix {
let _ = fs::remove_file(&path);
deleted_files += 1;
println!(" 🧹 [修复] 已物理删除损坏的文件");
// 检索是否有数据库记录并将其重置
let res = sqlx::query("UPDATE papers SET html_path = NULL WHERE html_path = ? OR html_path = ?")
.bind(rel_path)
.bind(format!("HTML/{}", Path::new(rel_path).file_name().and_then(|f| f.to_str()).unwrap_or("")))
.execute(&pool)
.await;
if let Ok(r) = res {
if r.rows_affected() > 0 {
db_updated_count += r.rows_affected();
println!(" ✅ [修复] 数据库对应状态已重置 (受影响行数: {})", r.rows_affected());
}
}
}
println!("--------------------------------------------------");
}
}
}
for path in pdf_files {
let rel_path = match path.strip_prefix(library_dir) {
Ok(p) => p.to_str().unwrap_or(""),
Err(_) => continue,
};
if let Ok(bytes) = fs::read(&path) {
if let Err(e) = validate_pdf_content(&bytes) {
disk_pdf_invalid += 1;
println!(" ❌ 发现磁盘上损坏的 PDF 文件: {:?}", rel_path);
println!(" 原因: {}", e);
if fix {
let _ = fs::remove_file(&path);
deleted_files += 1;
println!(" 🧹 [修复] 已物理删除损坏的文件");
let res = sqlx::query("UPDATE papers SET pdf_path = NULL WHERE pdf_path = ? OR pdf_path = ?")
.bind(rel_path)
.bind(format!("PDF/{}", Path::new(rel_path).file_name().and_then(|f| f.to_str()).unwrap_or("")))
.execute(&pool)
.await;
if let Ok(r) = res {
if r.rows_affected() > 0 {
db_updated_count += r.rows_affected();
println!(" ✅ [修复] 数据库对应状态已重置 (受影响行数: {})", r.rows_affected());
}
}
}
println!("--------------------------------------------------");
}
}
}
// ─── 阶段 2:数据库记录校验扫描(检测丢失文件、报错记录与孤立 Markdown) ───
println!("🗄️ 正在校验数据库表记录一致性...");
let db_rows = sqlx::query(
"SELECT bibcode, pdf_path, html_path, title, markdown_path, doctype FROM papers"
)
.fetch_all(&pool)
.await?;
let mut db_pdf_missing = 0;
let mut db_html_missing = 0;
let mut db_pdf_err_text = 0;
let mut db_html_err_text = 0;
let mut db_markdown_missing = 0;
let mut db_markdown_orphaned = 0;
let mut db_skip_type_cleaned = 0;
for r in db_rows {
let bibcode: String = r.get(0);
let pdf_path_opt: Option<String> = r.get(1);
let html_path_opt: Option<String> = r.get(2);
let title: String = r.get(3);
let markdown_path_opt: Option<String> = r.get(4);
let doctype_opt: Option<String> = r.get(5);
let mut need_db_fix = false;
let mut pdf_needs_fix = false;
let mut html_needs_fix = false;
let mut markdown_needs_fix = false;
let mut pdf_db_msg = String::new();
let mut html_db_msg = String::new();
let mut markdown_db_msg = String::new();
let doctype_str = doctype_opt.unwrap_or_else(|| "article".to_string()).to_lowercase();
let is_skip_type = doctype_str == "proposal"
|| doctype_str == "abstract"
|| doctype_str == "catalog"
|| doctype_str == "dataset"
|| doctype_str == "software"
|| doctype_str == "circular"
|| doctype_str == "newsletter"
|| doctype_str == "obituary";
if is_skip_type {
let mut has_skip_anomaly = false;
if let Some(ref pdf_p) = pdf_path_opt {
pdf_db_msg = format!("该文献属于跳过类型 [{}],但包含下载/报错路径记录: {}", doctype_str, pdf_p);
need_db_fix = true;
pdf_needs_fix = true;
has_skip_anomaly = true;
}
if let Some(ref html_p) = html_path_opt {
html_db_msg = format!("该文献属于跳过类型 [{}],但包含下载/报错路径记录: {}", doctype_str, html_p);
need_db_fix = true;
html_needs_fix = true;
has_skip_anomaly = true;
}
if let Some(ref md_p) = markdown_path_opt {
markdown_db_msg = format!("该文献属于跳过类型 [{}],但包含解析路径记录: {}", doctype_str, md_p);
need_db_fix = true;
markdown_needs_fix = true;
has_skip_anomaly = true;
}
if has_skip_anomaly {
db_skip_type_cleaned += 1;
}
} else {
if let Some(ref pdf_p) = pdf_path_opt {
if pdf_p.starts_with("error:") {
db_pdf_err_text += 1;
pdf_db_msg = format!("数据库存储了报错字符串: {}", pdf_p);
} else if !library_dir.join(pdf_p).exists() {
db_pdf_missing += 1;
pdf_db_msg = format!("物理 PDF 文件丢失 (路径: {})", pdf_p);
need_db_fix = true;
pdf_needs_fix = true;
}
}
if let Some(ref html_p) = html_path_opt {
if html_p.starts_with("error:") {
db_html_err_text += 1;
html_db_msg = format!("数据库存储了报错字符串: {}", html_p);
} else if !library_dir.join(html_p).exists() {
db_html_missing += 1;
html_db_msg = format!("物理 HTML 文件丢失 (路径: {})", html_p);
need_db_fix = true;
html_needs_fix = true;
}
}
if let Some(ref md_p) = markdown_path_opt {
if !library_dir.join(md_p).exists() {
db_markdown_missing += 1;
markdown_db_msg = format!("物理 Markdown 文件丢失 (路径: {})", md_p);
need_db_fix = true;
markdown_needs_fix = true;
} else {
// 如果 Markdown 物理文件存在,但它既没有有效 PDF 也没有有效 HTML
let has_valid_pdf = pdf_path_opt.as_ref()
.map(|p| !p.starts_with("error:") && library_dir.join(p).exists())
.unwrap_or(false);
let has_valid_html = html_path_opt.as_ref()
.map(|p| !p.starts_with("error:") && library_dir.join(p).exists())
.unwrap_or(false);
if !has_valid_pdf && !has_valid_html {
db_markdown_orphaned += 1;
markdown_db_msg = format!("Markdown 存在且完好,但失去有效 PDF/HTML 数据源,判定为孤立的 Markdown (路径: {})", md_p);
need_db_fix = true;
markdown_needs_fix = true;
}
}
}
}
if need_db_fix {
println!(" ❌ 发现馆藏文献记录损坏/不一致 [{}] 《{}", bibcode, title);
if !pdf_db_msg.is_empty() {
if pdf_needs_fix {
println!(" [异常] PDF 状态: {}", pdf_db_msg);
} else {
println!(" [日志] PDF 历史下载失败原因: {}", pdf_db_msg.replace("数据库存储了报错字符串: ", ""));
}
}
if !html_db_msg.is_empty() {
if html_needs_fix {
println!(" [异常] HTML 状态: {}", html_db_msg);
} else {
println!(" [日志] HTML 历史下载失败原因: {}", html_db_msg.replace("数据库存储了报错字符串: ", ""));
}
}
if !markdown_db_msg.is_empty() {
println!(" [异常] Markdown 状态: {}", markdown_db_msg);
}
if fix {
let mut sql_parts = Vec::new();
if pdf_needs_fix {
sql_parts.push("pdf_path = NULL");
if let Some(ref pdf_p) = pdf_path_opt {
if !pdf_p.starts_with("error:") {
let pdf_abs = library_dir.join(pdf_p);
if pdf_abs.exists() {
let _ = fs::remove_file(&pdf_abs);
deleted_files += 1;
println!(" 🧹 [修复] 已物理删除跳过类型或丢失的 PDF 文件");
}
}
}
}
if html_needs_fix {
sql_parts.push("html_path = NULL");
if let Some(ref html_p) = html_path_opt {
if !html_p.starts_with("error:") {
let html_abs = library_dir.join(html_p);
if html_abs.exists() {
let _ = fs::remove_file(&html_abs);
deleted_files += 1;
println!(" 🧹 [修复] 已物理删除跳过类型或丢失的 HTML 文件");
}
}
}
}
if markdown_needs_fix {
sql_parts.push("markdown_path = NULL");
if let Some(ref md_p) = markdown_path_opt {
let md_abs = library_dir.join(md_p);
if md_abs.exists() {
let _ = fs::remove_file(&md_abs);
deleted_files += 1;
println!(" 🧹 [修复] 已物理删除孤立或跳过类型的 Markdown 文件");
}
}
}
if !sql_parts.is_empty() {
let query_str = format!("UPDATE papers SET {} WHERE bibcode = ?", sql_parts.join(", "));
let res = sqlx::query(&query_str)
.bind(&bibcode)
.execute(&pool)
.await;
if res.is_ok() {
db_updated_count += 1;
println!(" ✅ [修复] 数据库损坏字段已成功重置");
}
}
}
println!("--------------------------------------------------");
}
}
println!("\n==================================================");
println!(" 全面健康度检测扫描报告 ");
println!("==================================================");
println!("磁盘物理损坏统计:");
println!(" - 损坏/假 HTML 文件数: {}", disk_html_invalid);
println!(" - 损坏/假 PDF 文件数: {}", disk_pdf_invalid);
println!("--------------------------------------------------");
println!("数据库一致性统计:");
println!(" - 数据库记录下载失败数 (error:): PDF: {}, HTML: {}", db_pdf_err_text, db_html_err_text);
println!(" - 磁盘文件丢失数 (数据库有记录但文件不存在): PDF: {}, HTML: {}, Markdown: {}", db_pdf_missing, db_html_missing, db_markdown_missing);
println!(" - 孤立无源 Markdown 篇数: {}", db_markdown_orphaned);
println!(" - 需跳过类型但包含下载记录篇数 (已清理/待清理): {}", db_skip_type_cleaned);
println!("--------------------------------------------------");
if fix {
println!("✨ 修复完成!");
println!(" - 共删除磁盘物理损坏/孤立/跳过类型文件: {}", deleted_files);
println!(" - 共重置修复数据库文献字段: {}", db_updated_count);
} else {
let total_issues = disk_html_invalid + disk_pdf_invalid + db_pdf_missing + db_html_missing + db_markdown_missing + db_markdown_orphaned + db_skip_type_cleaned;
if total_issues > 0 {
println!("❌ 警告:共检测出 {} 处坏文件、丢失文件或异常数据库记录。", total_issues);
println!("👉 您可以附加 '-- --fix' 执行一键全面修复:");
println!(" cargo run --bin health_check -- --fix");
} else {
println!("🎉 恭喜!馆藏物理文件及数据库完全健康,未检测到任何损坏或失效记录!");
}
}
println!("==================================================");
Ok(())
}