feat: 重构 PDF/文献检索同步机制、升级引力图交互与控制台 UI 样式
- [后端/PDF解析] 重构 MinerU PDF 解析流程：引入预签名两阶段直传机制，解决大文件 API 传输限制问题；支持轮询机制与本地 images 备用目录存储。 - [后端/同步与下载] 新增经典 ADS SCAN 扫描件 PDF 和 ADS_PDF 直接通道的下载逻辑；新增常用同步检索配置的持久化存储与去重管理 API。 - [后端/日志] 重构日志系统，支持控制台 pretty 输出与每日滚动文件日志（使用上海 +08:00 时区），引入 HTTP 路由请求链路追踪。 - [前端/引力图] 升级引用星系图 canvas 交互：支持平移拖拽与滚轮缩放，添加引力圈轨道装饰及未导入文献的半透明视觉区分。 - [前端/控制台] 统一重构为扁平高对比度浅色纯中文控制台样式；重新设计文献详情弹窗与状态进度条。 - [数据库] 新增 papers 表的 doctype 字段及 sync_queries 检索配置表。
This commit is contained in:
@@ -20,6 +20,7 @@
|
||||
* **[parser.rs](services/parser.rs)**：文献排版转换与清洗器，支持 MathJax LaTeX 占位符防护及 MinerU 图文 PDF 降级解析。
|
||||
* **[translation.rs](services/translation.rs)**：大模型对比翻译流水线。支持基于天文学对照词表的分词过滤，通过 Trie 树最长匹配机制生成 Glossary 专有名词注入 Prompt。
|
||||
* **[query_parser.rs](services/query_parser.rs)**：解析并标准化学术检索式，为 ADS 和 arXiv 分别生成合规的专有检索语法。
|
||||
* **[logging.rs](services/logging.rs)**：系统日志服务，支持控制台彩色日志输出、每日滚动写入磁盘日志文件，采用自定义上海时区 (+08:00) 格式化时间。
|
||||
|
||||
---
|
||||
|
||||
|
||||
+152
-14
@@ -57,6 +57,7 @@ pub struct StandardPaper {
|
||||
pub is_downloaded: bool,
|
||||
pub has_markdown: bool,
|
||||
pub has_translation: bool,
|
||||
pub doctype: String,
|
||||
}
|
||||
|
||||
// ── GET /api/search ──
|
||||
@@ -197,6 +198,7 @@ pub async fn download_paper(
|
||||
};
|
||||
|
||||
if pdf_path.is_none() && html_path.is_none() {
|
||||
error!("文献 {} PDF 和 HTML 均下载失败，无可用物理文件格式", req.bibcode);
|
||||
return Err((StatusCode::INTERNAL_SERVER_ERROR, "PDF 和 HTML 均下载失败，请检查网络".to_string()));
|
||||
}
|
||||
|
||||
@@ -324,9 +326,11 @@ pub async fn parse_paper(
|
||||
}
|
||||
}
|
||||
} else {
|
||||
error!("文献 {} 解析失败：本地 PDF 文件 {:?} 丢失", req.bibcode, pdf_abs);
|
||||
return Err((StatusCode::NOT_FOUND, "本地 PDF 文件未找到".to_string()));
|
||||
}
|
||||
} else {
|
||||
error!("文献 {} 解析失败：请先下载该文献的 HTML 或 PDF 文件", req.bibcode);
|
||||
return Err((StatusCode::BAD_REQUEST, "请先下载该文献的 HTML 或 PDF 文件".to_string()));
|
||||
}
|
||||
}
|
||||
@@ -381,19 +385,32 @@ pub async fn translate_paper(
|
||||
}
|
||||
|
||||
// 检查英文解析文件是否存在
|
||||
let md_rel = md_opt.ok_or((StatusCode::BAD_REQUEST, "文献必须先完成解析方可翻译".to_string()))?;
|
||||
let md_rel = match md_opt {
|
||||
Some(rel) => rel,
|
||||
None => {
|
||||
error!("文献 {} 翻译失败：文献未完成解析，缺少英文 Markdown 路径", req.bibcode);
|
||||
return Err((StatusCode::BAD_REQUEST, "文献必须先完成解析方可翻译".to_string()));
|
||||
}
|
||||
};
|
||||
let md_abs = state.config.library_dir.join(&md_rel);
|
||||
if !md_abs.exists() {
|
||||
error!("文献 {} 翻译失败：解析的英文 Markdown 文件 {:?} 不存在", req.bibcode, md_abs);
|
||||
return Err((StatusCode::BAD_REQUEST, "解析 Markdown 文件丢失".to_string()));
|
||||
}
|
||||
|
||||
let english_markdown = fs::read_to_string(&md_abs)
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("读取解析内容失败: {}", e)))?;
|
||||
.map_err(|e| {
|
||||
error!("文献 {} 翻译失败：读取解析内容失败: {}", req.bibcode, e);
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, format!("读取解析内容失败: {}", e))
|
||||
})?;
|
||||
|
||||
// 调用 LLM 翻译服务并注入对照词表
|
||||
let translated_markdown = crate::services::translation::translate_markdown(&english_markdown, &state.dict, &state.config)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("调用 LLM 翻译失败: {}", e)))?;
|
||||
.map_err(|e| {
|
||||
error!("文献 {} 翻译失败：调用 LLM 翻译发生错误: {}", req.bibcode, e);
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, format!("调用 LLM 翻译失败: {}", e))
|
||||
})?;
|
||||
|
||||
// 翻译结果物理写入本地
|
||||
let tr_filename = format!("{}_zh.md", req.bibcode);
|
||||
@@ -425,6 +442,7 @@ pub struct CitationsResponse {
|
||||
pub reference_count: i32,
|
||||
pub references: Vec<String>, // 该文献参考文献 bibcode 数组
|
||||
pub citations: Vec<String>, // 引用该文献的 bibcode 数组
|
||||
pub citation_counts: std::collections::HashMap<String, i32>, // 相关文献与被引数映射
|
||||
}
|
||||
|
||||
// 从 SQLite 查询引用关联，生成引用星系关系树
|
||||
@@ -432,9 +450,49 @@ pub async fn get_citation_network(
|
||||
State(state): State<Arc<AppState>>,
|
||||
Query(params): Query<DownloadRequest>,
|
||||
) -> Result<Json<CitationsResponse>, (StatusCode, String)> {
|
||||
let paper = get_paper_from_db(&state.db, &state.config.library_dir, ¶ms.bibcode)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::NOT_FOUND, format!("未找到文献数据: {}", e)))?;
|
||||
let paper = match get_paper_from_db(&state.db, &state.config.library_dir, ¶ms.bibcode).await {
|
||||
Ok(p) => p,
|
||||
Err(_) => {
|
||||
// 如果本地数据库查不到，尝试从 ADS 在线 API 动态获取
|
||||
if !state.config.ads_api_key.is_empty() {
|
||||
match state.ads.search(&format!("bibcode:{}", params.bibcode), 0, 1, "relevance").await {
|
||||
Ok(docs) => {
|
||||
if let Some(doc) = docs.first() {
|
||||
let standard_paper = convert_ads_doc_to_standard(doc);
|
||||
// 保存至数据库缓存，并保存引用关联
|
||||
let _ = save_paper_to_db(&state.db, &standard_paper).await;
|
||||
if let Some(refs) = &doc.reference {
|
||||
for ref_bib in refs {
|
||||
let _ = sqlx::query("INSERT OR IGNORE INTO citations_references (source_bibcode, target_bibcode) VALUES (?, ?)")
|
||||
.bind(&standard_paper.bibcode)
|
||||
.bind(ref_bib)
|
||||
.execute(&state.db)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
if let Some(cits) = &doc.citation {
|
||||
for cit_bib in cits {
|
||||
let _ = sqlx::query("INSERT OR IGNORE INTO citations_references (source_bibcode, target_bibcode) VALUES (?, ?)")
|
||||
.bind(cit_bib)
|
||||
.bind(&standard_paper.bibcode)
|
||||
.execute(&state.db)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
standard_paper
|
||||
} else {
|
||||
return Err((StatusCode::NOT_FOUND, format!("在本地库及 ADS 中均未找到该文献: {}", params.bibcode)));
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
return Err((StatusCode::INTERNAL_SERVER_ERROR, format!("在线检索文献元数据失败: {}", e)));
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return Err((StatusCode::NOT_FOUND, format!("本地数据库未收录该文献，且未配置 ADS_API_KEY，无法在线加载: {}", params.bibcode)));
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
// 加载引用的文献
|
||||
let refs_rows = sqlx::query("SELECT target_bibcode FROM citations_references WHERE source_bibcode = ?")
|
||||
@@ -452,6 +510,21 @@ pub async fn get_citation_network(
|
||||
.unwrap_or_default();
|
||||
let citations: Vec<String> = cits_rows.iter().map(|row| row.get(0)).collect();
|
||||
|
||||
// 加载关联文献的被引数量 (从 SQLite papers 表获取)
|
||||
let mut citation_counts = std::collections::HashMap::new();
|
||||
let mut all_related = references.clone();
|
||||
all_related.extend(citations.clone());
|
||||
for bib in all_related {
|
||||
let count_opt: Option<i32> = sqlx::query_scalar("SELECT citation_count FROM papers WHERE bibcode = ?")
|
||||
.bind(&bib)
|
||||
.fetch_optional(&state.db)
|
||||
.await
|
||||
.unwrap_or_default();
|
||||
if let Some(c) = count_opt {
|
||||
citation_counts.insert(bib, c);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(Json(CitationsResponse {
|
||||
bibcode: paper.bibcode,
|
||||
title: paper.title,
|
||||
@@ -459,6 +532,7 @@ pub async fn get_citation_network(
|
||||
reference_count: paper.reference_count,
|
||||
references,
|
||||
citations,
|
||||
citation_counts,
|
||||
}))
|
||||
}
|
||||
|
||||
@@ -499,7 +573,7 @@ pub async fn get_paper_detail(
|
||||
pub async fn get_library(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Result<Json<Vec<StandardPaper>>, (StatusCode, String)> {
|
||||
let rows = sqlx::query("SELECT bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, pdf_path, html_path, markdown_path, translation_path FROM papers ORDER BY created_at DESC")
|
||||
let rows = sqlx::query("SELECT bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, pdf_path, html_path, markdown_path, translation_path, doctype FROM papers ORDER BY created_at DESC")
|
||||
.fetch_all(&state.db)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("访问本地数据库失败: {}", e)))?;
|
||||
@@ -510,6 +584,7 @@ pub async fn get_library(
|
||||
let html_path: Option<String> = r.get(12);
|
||||
let markdown_path: Option<String> = r.get(13);
|
||||
let translation_path: Option<String> = r.get(14);
|
||||
let doctype_val: Option<String> = r.get(15);
|
||||
|
||||
let authors_str: Option<String> = r.get(2);
|
||||
let authors: Vec<String> = authors_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default();
|
||||
@@ -532,6 +607,7 @@ pub async fn get_library(
|
||||
|| html_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false),
|
||||
has_markdown: markdown_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false),
|
||||
has_translation: translation_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false),
|
||||
doctype: doctype_val.unwrap_or_else(|| "article".to_string()),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -714,6 +790,7 @@ pub(crate) fn convert_ads_doc_to_standard(doc: &AdsPaperDoc) -> StandardPaper {
|
||||
is_downloaded: false,
|
||||
has_markdown: false,
|
||||
has_translation: false,
|
||||
doctype: doc.doctype.clone().unwrap_or_else(|| "article".to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -733,6 +810,7 @@ pub(crate) fn convert_arxiv_to_standard(doc: &ArxivPaper) -> StandardPaper {
|
||||
is_downloaded: false,
|
||||
has_markdown: false,
|
||||
has_translation: false,
|
||||
doctype: "eprint".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -759,7 +837,7 @@ pub(crate) async fn save_paper_to_db(db: &SqlitePool, p: &StandardPaper) -> anyh
|
||||
if is_existing_temp && is_new_formal {
|
||||
info!("发现相同 arXiv ID 的文献，将临时主键 {} 升级为正式 ADS Bibcode: {}", existing_bibcode, p.bibcode);
|
||||
sqlx::query(
|
||||
"UPDATE papers SET bibcode = ?, title = ?, authors = ?, year = ?, pub = ?, keywords = ?, abstract = ?, doi = ?, citation_count = ?, reference_count = ? WHERE bibcode = ?"
|
||||
"UPDATE papers SET bibcode = ?, title = ?, authors = ?, year = ?, pub = ?, keywords = ?, abstract = ?, doi = ?, citation_count = ?, reference_count = ?, doctype = ? WHERE bibcode = ?"
|
||||
)
|
||||
.bind(&p.bibcode)
|
||||
.bind(&p.title)
|
||||
@@ -771,6 +849,7 @@ pub(crate) async fn save_paper_to_db(db: &SqlitePool, p: &StandardPaper) -> anyh
|
||||
.bind(&p.doi)
|
||||
.bind(p.citation_count)
|
||||
.bind(p.reference_count)
|
||||
.bind(&p.doctype)
|
||||
.bind(&existing_bibcode)
|
||||
.execute(db)
|
||||
.await?;
|
||||
@@ -787,8 +866,8 @@ pub(crate) async fn save_paper_to_db(db: &SqlitePool, p: &StandardPaper) -> anyh
|
||||
|
||||
// 2. 正常插入/冲突更新
|
||||
sqlx::query(
|
||||
"INSERT INTO papers (bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count) \
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) \
|
||||
"INSERT INTO papers (bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, doctype) \
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) \
|
||||
ON CONFLICT(bibcode) DO UPDATE SET \
|
||||
title=excluded.title, \
|
||||
authors=excluded.authors, \
|
||||
@@ -798,7 +877,8 @@ pub(crate) async fn save_paper_to_db(db: &SqlitePool, p: &StandardPaper) -> anyh
|
||||
doi=excluded.doi, \
|
||||
arxiv_id=excluded.arxiv_id, \
|
||||
citation_count=excluded.citation_count, \
|
||||
reference_count=excluded.reference_count"
|
||||
reference_count=excluded.reference_count, \
|
||||
doctype=excluded.doctype"
|
||||
)
|
||||
.bind(&p.bibcode)
|
||||
.bind(&p.title)
|
||||
@@ -811,6 +891,7 @@ pub(crate) async fn save_paper_to_db(db: &SqlitePool, p: &StandardPaper) -> anyh
|
||||
.bind(&p.arxiv_id)
|
||||
.bind(p.citation_count)
|
||||
.bind(p.reference_count)
|
||||
.bind(&p.doctype)
|
||||
.execute(db)
|
||||
.await?;
|
||||
|
||||
@@ -818,7 +899,7 @@ pub(crate) async fn save_paper_to_db(db: &SqlitePool, p: &StandardPaper) -> anyh
|
||||
}
|
||||
|
||||
async fn get_paper_from_db(db: &SqlitePool, library_dir: &std::path::Path, bibcode: &str) -> anyhow::Result<StandardPaper> {
|
||||
let r = sqlx::query("SELECT bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, pdf_path, html_path, markdown_path, translation_path FROM papers WHERE bibcode = ?")
|
||||
let r = sqlx::query("SELECT bibcode, title, authors, year, pub, keywords, abstract, doi, arxiv_id, citation_count, reference_count, pdf_path, html_path, markdown_path, translation_path, doctype FROM papers WHERE bibcode = ?")
|
||||
.bind(bibcode)
|
||||
.fetch_one(db)
|
||||
.await?;
|
||||
@@ -827,6 +908,7 @@ async fn get_paper_from_db(db: &SqlitePool, library_dir: &std::path::Path, bibco
|
||||
let html_path: Option<String> = r.get(12);
|
||||
let markdown_path: Option<String> = r.get(13);
|
||||
let translation_path: Option<String> = r.get(14);
|
||||
let doctype_val: Option<String> = r.get(15);
|
||||
|
||||
let authors_str: Option<String> = r.get(2);
|
||||
let authors: Vec<String> = authors_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default();
|
||||
@@ -853,6 +935,7 @@ async fn get_paper_from_db(db: &SqlitePool, library_dir: &std::path::Path, bibco
|
||||
is_downloaded: is_pdf_exist || is_html_exist,
|
||||
has_markdown: is_md_exist,
|
||||
has_translation: is_tr_exist,
|
||||
doctype: doctype_val.unwrap_or_else(|| "article".to_string()),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1011,8 +1094,8 @@ pub async fn run_asset_sync(
|
||||
}
|
||||
}
|
||||
"unparsed" | "all_unparsed" => {
|
||||
// 查询所有本地无 Markdown 文件的文献
|
||||
let rows = sqlx::query("SELECT bibcode FROM papers WHERE markdown_path IS NULL")
|
||||
// 查询所有本地无 Markdown 文件的文献 (或者处于 mineru_batch: 状态的任务)
|
||||
let rows = sqlx::query("SELECT bibcode FROM papers WHERE markdown_path IS NULL OR markdown_path LIKE 'mineru_batch:%'")
|
||||
.fetch_all(&state.db)
|
||||
.await
|
||||
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, format!("读取数据库失败: {}", e)))?;
|
||||
@@ -1055,6 +1138,58 @@ pub async fn stop_asset_sync(
|
||||
StatusCode::OK
|
||||
}
|
||||
|
||||
// ── GET /api/sync/queries ──
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub struct SavedSyncQuery {
|
||||
pub id: i64,
|
||||
pub query: String,
|
||||
pub source: String,
|
||||
pub limit_count: i32,
|
||||
pub last_run: String,
|
||||
}
|
||||
|
||||
pub async fn get_sync_queries(
|
||||
State(state): State<Arc<AppState>>,
|
||||
) -> Result<Json<Vec<SavedSyncQuery>>, (StatusCode, String)> {
|
||||
let rows = sqlx::query("SELECT id, query, source, limit_count, datetime(last_run, 'localtime') FROM sync_queries ORDER BY last_run DESC")
|
||||
.fetch_all(&state.db)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!("获取已存同步检索配置失败: {}", e);
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, format!("获取已存同步检索配置失败: {}", e))
|
||||
})?;
|
||||
|
||||
let mut list = Vec::new();
|
||||
for r in rows {
|
||||
list.push(SavedSyncQuery {
|
||||
id: r.get(0),
|
||||
query: r.get(1),
|
||||
source: r.get(2),
|
||||
limit_count: r.get(3),
|
||||
last_run: r.get(4),
|
||||
});
|
||||
}
|
||||
|
||||
Ok(Json(list))
|
||||
}
|
||||
|
||||
// ── DELETE /api/sync/queries/:id ──
|
||||
pub async fn delete_sync_query(
|
||||
State(state): State<Arc<AppState>>,
|
||||
axum::extract::Path(id): axum::extract::Path<i64>,
|
||||
) -> Result<StatusCode, (StatusCode, String)> {
|
||||
sqlx::query("DELETE FROM sync_queries WHERE id = ?")
|
||||
.bind(id)
|
||||
.execute(&state.db)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
error!("删除同步检索配置失败: {}", e);
|
||||
(StatusCode::INTERNAL_SERVER_ERROR, format!("删除同步检索配置失败: {}", e))
|
||||
})?;
|
||||
|
||||
Ok(StatusCode::OK)
|
||||
}
|
||||
|
||||
// ── GET /api/sync/asset/status ──
|
||||
pub async fn get_asset_sync_status(
|
||||
State(state): State<Arc<AppState>>,
|
||||
@@ -1086,6 +1221,7 @@ mod tests {
|
||||
reference: None,
|
||||
citation: None,
|
||||
identifier: None,
|
||||
doctype: Some("article".to_string()),
|
||||
};
|
||||
|
||||
let paper = convert_ads_doc_to_standard(&doc);
|
||||
@@ -1118,6 +1254,7 @@ mod tests {
|
||||
reference: None,
|
||||
citation: None,
|
||||
identifier: Some(vec!["2026MNRAS.530.1234A".to_string(), "arXiv:2606.12345".to_string()]),
|
||||
doctype: Some("article".to_string()),
|
||||
};
|
||||
|
||||
let paper = convert_ads_doc_to_standard(&doc);
|
||||
@@ -1174,6 +1311,7 @@ mod tests {
|
||||
is_downloaded: false,
|
||||
has_markdown: false,
|
||||
has_translation: false,
|
||||
doctype: "article".to_string(),
|
||||
};
|
||||
|
||||
// 保存
|
||||
|
||||
+5
-2
@@ -20,6 +20,7 @@ pub struct AdsPaperDoc {
|
||||
pub reference: Option<Vec<String>>,
|
||||
pub citation: Option<Vec<String>>,
|
||||
pub identifier: Option<Vec<String>>,
|
||||
pub doctype: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
@@ -69,8 +70,8 @@ impl AdsClient {
|
||||
|
||||
let translated = crate::services::query_parser::to_ads_query(query);
|
||||
|
||||
// fl 声明返回字段，包括 reference 和 citation 引用关系数组及 identifier
|
||||
let fl = "bibcode,title,author,year,pub,keyword,abstract,doi,citation_count,reference_count,reference,citation,identifier";
|
||||
// fl 声明返回字段，包括 reference 和 citation 引用关系数组及 identifier 和 doctype
|
||||
let fl = "bibcode,title,author,year,pub,keyword,abstract,doi,citation_count,reference_count,reference,citation,identifier,doctype";
|
||||
|
||||
let ads_sort = match sort {
|
||||
"date_desc" => "date desc",
|
||||
@@ -120,6 +121,7 @@ impl AdsClient {
|
||||
reference: d.reference,
|
||||
citation: d.citation,
|
||||
identifier: d.identifier,
|
||||
doctype: d.doctype,
|
||||
}
|
||||
}).collect();
|
||||
|
||||
@@ -204,6 +206,7 @@ struct RawDoc {
|
||||
reference: Option<Vec<String>>,
|
||||
citation: Option<Vec<String>>,
|
||||
identifier: Option<Vec<String>>,
|
||||
doctype: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use sha1::Sha1;
|
||||
use hmac::{Hmac, Mac};
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE_NO_PAD};
|
||||
use base64::{Engine as _, engine::general_purpose::URL_SAFE};
|
||||
use reqwest::multipart;
|
||||
use tracing::{info, error};
|
||||
|
||||
@@ -43,7 +43,7 @@ impl QiniuClient {
|
||||
});
|
||||
|
||||
let policy_str = policy.to_string();
|
||||
let encoded_policy = URL_SAFE_NO_PAD.encode(policy_str.as_bytes());
|
||||
let encoded_policy = URL_SAFE.encode(policy_str.as_bytes());
|
||||
|
||||
let mut mac = HmacSha1::new_from_slice(self.secret_key.as_bytes())
|
||||
.expect("HMAC 密钥可接收任意大小");
|
||||
@@ -51,7 +51,7 @@ impl QiniuClient {
|
||||
let result = mac.finalize();
|
||||
let signature = result.into_bytes();
|
||||
|
||||
let encoded_signature = URL_SAFE_NO_PAD.encode(&signature);
|
||||
let encoded_signature = URL_SAFE.encode(&signature);
|
||||
|
||||
format!("{}:{}:{}", self.access_key, encoded_signature, encoded_policy)
|
||||
}
|
||||
@@ -62,9 +62,9 @@ impl QiniuClient {
|
||||
return Err(anyhow::anyhow!("本地 .env 文件中未正确配置七牛云参数"));
|
||||
}
|
||||
|
||||
// 使用毫秒级时间戳防重名覆盖
|
||||
// 使用毫秒级时间戳防重名覆盖，并放置在 astroresearch 虚拟文件夹下
|
||||
let timestamp = chrono::Utc::now().timestamp_millis();
|
||||
let key = format!("astroresearch_{}_{}", timestamp, filename);
|
||||
let key = format!("astroresearch/{}_{}", timestamp, filename);
|
||||
|
||||
let token = self.generate_upload_token(&key);
|
||||
info!("正在上传文献提取图片到七牛云: key='{}'", key);
|
||||
@@ -74,7 +74,7 @@ impl QiniuClient {
|
||||
.text("key", key.clone())
|
||||
.part("file", multipart::Part::bytes(buffer).file_name(filename.to_string()));
|
||||
|
||||
let upload_url = "https://up.qiniu.com";
|
||||
let upload_url = "https://up-z1.qiniup.com";
|
||||
|
||||
let response = self.client.post(upload_url)
|
||||
.multipart(form)
|
||||
|
||||
+6
-8
@@ -21,13 +21,8 @@ use astroresearch::api::handlers::{AppState, self};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
// 1. 初始化日志记录器
|
||||
tracing_subscriber::fmt()
|
||||
.with_env_filter(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info,astroresearch=debug")),
|
||||
)
|
||||
.init();
|
||||
// 1. 初始化日志记录器并保留异步写保护 Guard
|
||||
let _logging_guards = astroresearch::services::logging::init_logging()?;
|
||||
|
||||
info!("正在启动 AstroResearch 天文学文献辅助系统后端服务...");
|
||||
|
||||
@@ -124,7 +119,9 @@ async fn main() -> anyhow::Result<()> {
|
||||
.route("/sync/meta/status", get(handlers::get_meta_sync_status))
|
||||
.route("/sync/asset/run", post(handlers::run_asset_sync))
|
||||
.route("/sync/asset/stop", post(handlers::stop_asset_sync))
|
||||
.route("/sync/asset/status", get(handlers::get_asset_sync_status));
|
||||
.route("/sync/asset/status", get(handlers::get_asset_sync_status))
|
||||
.route("/sync/queries", get(handlers::get_sync_queries))
|
||||
.route("/sync/queries/:id", axum::routing::delete(handlers::delete_sync_query));
|
||||
|
||||
// 静态文件资源代理托管（当前端打包至 dashboard/dist 后，直接挂载到主域名根路由）
|
||||
let serve_dir = ServeDir::new("dashboard/dist")
|
||||
@@ -134,6 +131,7 @@ async fn main() -> anyhow::Result<()> {
|
||||
.nest("/api", api_routes)
|
||||
.fallback_service(serve_dir)
|
||||
.layer(cors)
|
||||
.layer(tower_http::trace::TraceLayer::new_for_http())
|
||||
.with_state(app_state);
|
||||
|
||||
let addr = SocketAddr::from(([0, 0, 0, 0], config.port));
|
||||
|
||||
+205
-65
@@ -87,6 +87,18 @@ impl MetaSync {
|
||||
tokio::spawn(async move {
|
||||
info!("启动后台批量收割任务: 查询词='{}', 源='{}', 上限={}", query_clone, source_clone, limit);
|
||||
|
||||
// 自动将检索配置存入/更新至 sync_queries 数据库表中进行去重和时间更新
|
||||
let _ = sqlx::query(
|
||||
"INSERT INTO sync_queries (query, source, limit_count, last_run) \
|
||||
VALUES (?, ?, ?, CURRENT_TIMESTAMP) \
|
||||
ON CONFLICT(query, source, limit_count) DO UPDATE SET last_run=excluded.last_run"
|
||||
)
|
||||
.bind(&query_clone)
|
||||
.bind(&source_clone)
|
||||
.bind(limit)
|
||||
.execute(&db)
|
||||
.await;
|
||||
|
||||
// 1. 并行获取两端预估总量
|
||||
let ads_count_fut = {
|
||||
let ads = ads.clone();
|
||||
@@ -282,6 +294,8 @@ pub struct AssetSyncStatus {
|
||||
pub total: i32,
|
||||
pub downloaded: i32,
|
||||
pub parsed: i32,
|
||||
pub download_failed: i32,
|
||||
pub parse_failed: i32,
|
||||
pub current_bibcode: String,
|
||||
pub logs: Vec<String>,
|
||||
pub action: Option<SyncAction>,
|
||||
@@ -294,6 +308,8 @@ impl AssetSyncStatus {
|
||||
total: 0,
|
||||
downloaded: 0,
|
||||
parsed: 0,
|
||||
download_failed: 0,
|
||||
parse_failed: 0,
|
||||
current_bibcode: String::new(),
|
||||
logs: Vec::new(),
|
||||
action: None,
|
||||
@@ -331,6 +347,8 @@ impl AssetSync {
|
||||
s.total = total;
|
||||
s.downloaded = 0;
|
||||
s.parsed = 0;
|
||||
s.download_failed = 0;
|
||||
s.parse_failed = 0;
|
||||
s.current_bibcode = String::new();
|
||||
s.logs.clear();
|
||||
s.action = Some(action);
|
||||
@@ -344,7 +362,8 @@ impl AssetSync {
|
||||
}
|
||||
|
||||
let mut dl_count = 0;
|
||||
let mut parse_count = 0;
|
||||
let mut dl_failed_count = 0;
|
||||
let mut join_handles = Vec::new();
|
||||
|
||||
for bibcode in bibcodes {
|
||||
// 每次循环前，检查是否被外部停止了（active 设为 false）
|
||||
@@ -364,20 +383,21 @@ impl AssetSync {
|
||||
|
||||
// 1. 获取文献元数据与当前路径状态
|
||||
let paper_res = sqlx::query(
|
||||
"SELECT arxiv_id, doi, pdf_path, html_path, markdown_path FROM papers WHERE bibcode = ?"
|
||||
"SELECT arxiv_id, doi, pdf_path, html_path, markdown_path, doctype FROM papers WHERE bibcode = ?"
|
||||
)
|
||||
.bind(&bibcode)
|
||||
.fetch_optional(&db)
|
||||
.await;
|
||||
|
||||
let (arxiv_id, doi, mut pdf_path, mut html_path, markdown_path) = match paper_res {
|
||||
let (arxiv_id, doi, mut pdf_path, mut html_path, markdown_path, doctype) = match paper_res {
|
||||
Ok(Some(row)) => {
|
||||
let arxiv_id: String = row.get(0);
|
||||
let doi: String = row.get(1);
|
||||
let pdf_path: Option<String> = row.get(2);
|
||||
let html_path: Option<String> = row.get(3);
|
||||
let markdown_path: Option<String> = row.get(4);
|
||||
(arxiv_id, doi, pdf_path, html_path, markdown_path)
|
||||
let doctype: Option<String> = row.get(5);
|
||||
(arxiv_id, doi, pdf_path, html_path, markdown_path, doctype)
|
||||
}
|
||||
_ => {
|
||||
let mut s = status.lock().await;
|
||||
@@ -386,6 +406,22 @@ impl AssetSync {
|
||||
}
|
||||
};
|
||||
|
||||
// 1b. 检查 doctype，如果是 proposal, abstract, catalog, software 等无数字全文的文件，直接跳过处理
|
||||
let doctype_str = doctype.unwrap_or_else(|| "article".to_string()).to_lowercase();
|
||||
if doctype_str == "proposal" || doctype_str == "abstract" || doctype_str == "catalog" || doctype_str == "software" {
|
||||
let mut s = status.lock().await;
|
||||
s.add_log(format!("文献 {} 的类型为 {} (无数字版全文)，跳过下载与解析。", bibcode, doctype_str));
|
||||
// 同样更新处理进度，防止任务进度条卡住
|
||||
if action == SyncAction::Download || action == SyncAction::All {
|
||||
dl_count += 1;
|
||||
s.downloaded = dl_count;
|
||||
}
|
||||
if action == SyncAction::Parse || action == SyncAction::All {
|
||||
s.parsed += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// 2. 检查并执行下载
|
||||
if action == SyncAction::Download || action == SyncAction::All {
|
||||
let is_pdf_exist = pdf_path.as_ref().map(|p| config.library_dir.join(p).exists()).unwrap_or(false);
|
||||
@@ -427,7 +463,9 @@ impl AssetSync {
|
||||
s.add_log(format!("文献 {} 下载成功！", bibcode));
|
||||
}
|
||||
} else {
|
||||
dl_failed_count += 1;
|
||||
let mut s = status.lock().await;
|
||||
s.download_failed = dl_failed_count;
|
||||
s.add_log(format!("文献 {} 下载失败（PDF 和 HTML 均下载失败）", bibcode));
|
||||
}
|
||||
|
||||
@@ -457,7 +495,6 @@ impl AssetSync {
|
||||
s.add_log(format!("文献 {} 开始进行排版提取与 Markdown 转换...", bibcode));
|
||||
}
|
||||
|
||||
let mut parsed_markdown = String::new();
|
||||
let mut relative_md_path = String::new();
|
||||
|
||||
// 确定源链接
|
||||
@@ -499,7 +536,7 @@ impl AssetSync {
|
||||
year,
|
||||
keywords.join(",")
|
||||
);
|
||||
parsed_markdown = format!("{}{}", front_matter, md);
|
||||
let parsed_markdown = format!("{}{}", front_matter, md);
|
||||
let md_filename = format!("{}.md", bibcode);
|
||||
let md_dest = config.library_dir.join("Markdown").join(&md_filename);
|
||||
let _ = fs::create_dir_all(md_dest.parent().unwrap());
|
||||
@@ -511,74 +548,170 @@ impl AssetSync {
|
||||
}
|
||||
}
|
||||
|
||||
// 策略 2：PDF 回退（远程 MinerU）
|
||||
if parsed_markdown.is_empty() {
|
||||
if let Some(pdf_rel) = &pdf_path {
|
||||
let pdf_abs = config.library_dir.join(pdf_rel);
|
||||
if pdf_abs.exists() {
|
||||
match crate::services::parser::parse_pdf_via_mineru(&pdf_abs, &qiniu, &config).await {
|
||||
Ok(md) => {
|
||||
let paper_meta_res = sqlx::query("SELECT title, authors, pub, year, keywords FROM papers WHERE bibcode = ?")
|
||||
.bind(&bibcode)
|
||||
.fetch_optional(&db)
|
||||
.await;
|
||||
|
||||
if let Ok(Some(meta_row)) = paper_meta_res {
|
||||
let title: String = meta_row.get(0);
|
||||
let authors_json: String = meta_row.get(1);
|
||||
let pub_journal: String = meta_row.get(2);
|
||||
let year: String = meta_row.get(3);
|
||||
let keywords_json: String = meta_row.get(4);
|
||||
|
||||
let authors: Vec<String> = serde_json::from_str(&authors_json).unwrap_or_default();
|
||||
let keywords: Vec<String> = serde_json::from_str(&keywords_json).unwrap_or_default();
|
||||
|
||||
let front_matter = format!(
|
||||
"---\ntitle: {}\nauthor: [{}]\npublisher: {}\nsource: \"{}\"\ndate: \"{}\"\ntags: \"{}\"\n---\n\n",
|
||||
serde_json::to_string(&title).unwrap_or_else(|_| format!("\"{}\"", title)),
|
||||
authors.iter().map(|a| format!("\"{}\"", a)).collect::<Vec<_>>().join(", "),
|
||||
serde_json::to_string(&pub_journal).unwrap_or_else(|_| format!("\"{}\"", pub_journal)),
|
||||
source_url,
|
||||
year,
|
||||
keywords.join(",")
|
||||
);
|
||||
parsed_markdown = format!("{}{}", front_matter, md);
|
||||
let md_filename = format!("{}.md", bibcode);
|
||||
let md_dest = config.library_dir.join("Markdown").join(&md_filename);
|
||||
let _ = fs::create_dir_all(md_dest.parent().unwrap());
|
||||
if fs::write(&md_dest, &parsed_markdown).is_ok() {
|
||||
relative_md_path = format!("Markdown/{}", md_filename);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
let mut s = status.lock().await;
|
||||
s.add_log(format!("PDF 结构解析失败 (MinerU): {}", e));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !relative_md_path.is_empty() {
|
||||
// HTML 解析成功，直接写入数据库并记录成功
|
||||
let _ = sqlx::query("UPDATE papers SET markdown_path = ? WHERE bibcode = ?")
|
||||
.bind(&relative_md_path)
|
||||
.bind(&bibcode)
|
||||
.execute(&db)
|
||||
.await;
|
||||
|
||||
parse_count += 1;
|
||||
{
|
||||
let mut s = status.lock().await;
|
||||
s.parsed = parse_count;
|
||||
s.add_log(format!("文献 {} Markdown 解析成功！", bibcode));
|
||||
s.parsed += 1;
|
||||
s.add_log(format!("文献 {} HTML 本地解析成功！", bibcode));
|
||||
}
|
||||
} else {
|
||||
let mut s = status.lock().await;
|
||||
s.add_log(format!("文献 {} 转换为 Markdown 失败。", bibcode));
|
||||
// HTML 解析失败或无 HTML，执行 PDF 回退（异步非阻塞提交 MinerU）
|
||||
if let Some(pdf_rel) = &pdf_path {
|
||||
let pdf_abs = config.library_dir.join(pdf_rel);
|
||||
if pdf_abs.exists() {
|
||||
// 检查是否已经是 mineru_batch: 状态
|
||||
let existing_batch_id = markdown_path.as_ref()
|
||||
.and_then(|p| p.strip_prefix("mineru_batch:"))
|
||||
.map(|s| s.trim().to_string());
|
||||
|
||||
let db_clone = db.clone();
|
||||
let config_clone = config.clone();
|
||||
let qiniu_clone = qiniu.clone();
|
||||
let status_clone = status.clone();
|
||||
let bibcode_clone = bibcode.clone();
|
||||
let source_url_clone = source_url.clone();
|
||||
|
||||
let mut submitted_ok = true;
|
||||
let batch_id = if let Some(id) = existing_batch_id {
|
||||
{
|
||||
let mut s = status.lock().await;
|
||||
s.add_log(format!("文献 {} 检测到未完成的 MinerU 任务，正在恢复轮询 (Batch ID: {})...", bibcode, id));
|
||||
}
|
||||
id
|
||||
} else {
|
||||
{
|
||||
let mut s = status.lock().await;
|
||||
s.add_log(format!("文献 {} PDF 提交后台解析 (MinerU)...", bibcode));
|
||||
}
|
||||
match crate::services::parser::submit_pdf_to_mineru(&pdf_abs, &config).await {
|
||||
Ok(id) => {
|
||||
// 提交成功，立刻把 batch_id 存入数据库以备断点续跑
|
||||
let marker = format!("mineru_batch:{}", id);
|
||||
let _ = sqlx::query("UPDATE papers SET markdown_path = ? WHERE bibcode = ?")
|
||||
.bind(&marker)
|
||||
.bind(&bibcode)
|
||||
.execute(&db)
|
||||
.await;
|
||||
id
|
||||
}
|
||||
Err(e) => {
|
||||
let mut s = status.lock().await;
|
||||
s.parse_failed += 1;
|
||||
s.add_log(format!("文献 {} PDF 提交 MinerU 失败: {}", bibcode, e));
|
||||
let err_reason = format!("error: {}", e);
|
||||
let _ = sqlx::query("UPDATE papers SET markdown_path = ? WHERE bibcode = ?")
|
||||
.bind(&err_reason)
|
||||
.bind(&bibcode)
|
||||
.execute(&db)
|
||||
.await;
|
||||
submitted_ok = false;
|
||||
String::new()
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if submitted_ok {
|
||||
let handle = tokio::spawn(async move {
|
||||
match crate::services::parser::poll_and_extract_mineru(&batch_id, &bibcode_clone, &qiniu_clone, &config_clone).await {
|
||||
Ok(md) => {
|
||||
let paper_meta_res = sqlx::query("SELECT title, authors, pub, year, keywords FROM papers WHERE bibcode = ?")
|
||||
.bind(&bibcode_clone)
|
||||
.fetch_optional(&db_clone)
|
||||
.await;
|
||||
|
||||
let mut rel_md = String::new();
|
||||
if let Ok(Some(meta_row)) = paper_meta_res {
|
||||
let title: String = meta_row.get(0);
|
||||
let authors_json: String = meta_row.get(1);
|
||||
let pub_journal: String = meta_row.get(2);
|
||||
let year: String = meta_row.get(3);
|
||||
let keywords_json: String = meta_row.get(4);
|
||||
|
||||
let authors: Vec<String> = serde_json::from_str(&authors_json).unwrap_or_default();
|
||||
let keywords: Vec<String> = serde_json::from_str(&keywords_json).unwrap_or_default();
|
||||
|
||||
let front_matter = format!(
|
||||
"---\ntitle: {}\nauthor: [{}]\npublisher: {}\nsource: \"{}\"\ndate: \"{}\"\ntags: \"{}\"\n---\n\n",
|
||||
serde_json::to_string(&title).unwrap_or_else(|_| format!("\"{}\"", title)),
|
||||
authors.iter().map(|a| format!("\"{}\"", a)).collect::<Vec<_>>().join(", "),
|
||||
serde_json::to_string(&pub_journal).unwrap_or_else(|_| format!("\"{}\"", pub_journal)),
|
||||
source_url_clone,
|
||||
year,
|
||||
keywords.join(",")
|
||||
);
|
||||
let parsed_markdown = format!("{}{}", front_matter, md);
|
||||
let md_filename = format!("{}.md", bibcode_clone);
|
||||
let md_dest = config_clone.library_dir.join("Markdown").join(&md_filename);
|
||||
let _ = fs::create_dir_all(md_dest.parent().unwrap());
|
||||
if fs::write(&md_dest, &parsed_markdown).is_ok() {
|
||||
rel_md = format!("Markdown/{}", md_filename);
|
||||
}
|
||||
}
|
||||
|
||||
if !rel_md.is_empty() {
|
||||
let _ = sqlx::query("UPDATE papers SET markdown_path = ? WHERE bibcode = ?")
|
||||
.bind(&rel_md)
|
||||
.bind(&bibcode_clone)
|
||||
.execute(&db_clone)
|
||||
.await;
|
||||
|
||||
let mut s = status_clone.lock().await;
|
||||
s.parsed += 1;
|
||||
s.add_log(format!("文献 {} PDF (MinerU) 解析成功！", bibcode_clone));
|
||||
} else {
|
||||
let mut s = status_clone.lock().await;
|
||||
s.parse_failed += 1;
|
||||
s.add_log(format!("文献 {} PDF 写入 Markdown 失败。", bibcode_clone));
|
||||
let _ = sqlx::query("UPDATE papers SET markdown_path = 'error: PDF 写入 Markdown 失败' WHERE bibcode = ?")
|
||||
.bind(&bibcode_clone)
|
||||
.execute(&db_clone)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
let mut s = status_clone.lock().await;
|
||||
s.parse_failed += 1;
|
||||
s.add_log(format!("文献 {} PDF 结构解析失败 (MinerU): {}", bibcode_clone, e));
|
||||
let err_reason = format!("error: {}", e);
|
||||
let _ = sqlx::query("UPDATE papers SET markdown_path = ? WHERE bibcode = ?")
|
||||
.bind(&err_reason)
|
||||
.bind(&bibcode_clone)
|
||||
.execute(&db_clone)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
});
|
||||
join_handles.push(handle);
|
||||
}
|
||||
} else {
|
||||
let mut s = status.lock().await;
|
||||
s.parse_failed += 1;
|
||||
s.add_log(format!("文献 {} 本地 PDF 文件不存在，无法解析。", bibcode));
|
||||
let _ = sqlx::query("UPDATE papers SET markdown_path = 'error: 本地 PDF 文件不存在' WHERE bibcode = ?")
|
||||
.bind(&bibcode)
|
||||
.execute(&db)
|
||||
.await;
|
||||
}
|
||||
} else {
|
||||
let mut s = status.lock().await;
|
||||
s.parse_failed += 1;
|
||||
s.add_log(format!("文献 {} HTML 转换失败，且无本地 PDF，无法解析。", bibcode));
|
||||
let _ = sqlx::query("UPDATE papers SET markdown_path = 'error: HTML 转换失败且无本地 PDF' WHERE bibcode = ?")
|
||||
.bind(&bibcode)
|
||||
.execute(&db)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let mut s = status.lock().await;
|
||||
s.parse_failed += 1;
|
||||
s.add_log(format!("文献 {} 无本地 PDF/HTML，无法解析，跳过。", bibcode));
|
||||
}
|
||||
} else {
|
||||
@@ -586,15 +719,22 @@ impl AssetSync {
|
||||
let mut s = status.lock().await;
|
||||
s.add_log(format!("文献 {} 已存在解析后的 Markdown，跳过。", bibcode));
|
||||
}
|
||||
parse_count += 1;
|
||||
{
|
||||
let mut s = status.lock().await;
|
||||
s.parsed = parse_count;
|
||||
}
|
||||
let mut s = status.lock().await;
|
||||
s.parsed += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !join_handles.is_empty() {
|
||||
{
|
||||
let mut s = status.lock().await;
|
||||
s.add_log(format!("本地下载与快速解析已完成，正在等待后台共 {} 个 MinerU 异步解析任务结束...", join_handles.len()));
|
||||
}
|
||||
for handle in join_handles {
|
||||
let _ = handle.await;
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let mut s = status.lock().await;
|
||||
s.active = false;
|
||||
|
||||
+48
-11
@@ -14,7 +14,7 @@ use std::path::{Path, PathBuf};
|
||||
use reqwest::header::{HeaderMap, HeaderValue};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use url::Url;
|
||||
use tracing::{info, warn};
|
||||
use tracing::{info, warn, debug};
|
||||
use anyhow::{Context, Result};
|
||||
|
||||
// ─── 浏览器伪装辅助 ────────────────────────────────────────────
|
||||
@@ -43,7 +43,6 @@ fn build_browser_headers() -> HeaderMap {
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
));
|
||||
h.insert("Accept-Language", HeaderValue::from_static("en-US,en;q=0.9,zh-CN;q=0.8,zh;q=0.7"));
|
||||
h.insert("Accept-Encoding", HeaderValue::from_static("gzip, deflate, br"));
|
||||
h.insert("DNT", HeaderValue::from_static("1"));
|
||||
h.insert("Connection", HeaderValue::from_static("keep-alive"));
|
||||
h.insert("Upgrade-Insecure-Requests", HeaderValue::from_static("1"));
|
||||
@@ -64,7 +63,6 @@ fn build_chrome_headers(referer: Option<&str>) -> HeaderMap {
|
||||
"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7",
|
||||
));
|
||||
h.insert("Accept-Language", HeaderValue::from_static("en-US,en;q=0.9"));
|
||||
h.insert("Accept-Encoding", HeaderValue::from_static("gzip, deflate, br, zstd"));
|
||||
h.insert("Sec-Ch-Ua", HeaderValue::from_static(
|
||||
"\"Google Chrome\";v=\"143\", \"Chromium\";v=\"143\", \"Not A(Brand\";v=\"24\"",
|
||||
));
|
||||
@@ -224,7 +222,7 @@ impl Downloader {
|
||||
|
||||
/// 解析 ADS Link Gateway 路由，若遇 perfdrive 防护则提取 ssc 参数绕过
|
||||
async fn resolve_ads_gateway(&self, gateway_url: &str) -> Result<String> {
|
||||
info!("解析 ADS 网关: {}", gateway_url);
|
||||
debug!("解析 ADS 网关: {}", gateway_url);
|
||||
|
||||
// HEAD 请求跟踪重定向（部分出版商阻断 HEAD，自动降级 GET）
|
||||
let response = match self.client.head(gateway_url).send().await {
|
||||
@@ -234,7 +232,7 @@ impl Downloader {
|
||||
};
|
||||
|
||||
let final_url = response.url().as_str().to_string();
|
||||
info!("网关解析结果: {}", final_url);
|
||||
debug!("网关解析结果: {}", final_url);
|
||||
|
||||
// 如重定向至 validate.perfdrive.com，提取 ssc 参数中的真实 URL
|
||||
if final_url.contains("validate.perfdrive.com") {
|
||||
@@ -242,7 +240,7 @@ impl Downloader {
|
||||
if let Some(ssc) = parsed.query_pairs().find(|(k, _)| k == "ssc").map(|(_, v)| v.into_owned()) {
|
||||
if let Ok(decoded) = urlencoding::decode(&ssc) {
|
||||
let real_url = decoded.into_owned();
|
||||
info!("检测到 perfdrive 拦截，解码真实地址: {}", real_url);
|
||||
debug!("检测到 perfdrive 拦截，解码真实地址: {}", real_url);
|
||||
return Ok(real_url);
|
||||
}
|
||||
}
|
||||
@@ -276,18 +274,18 @@ impl Downloader {
|
||||
let pdf_url = format!("https://iopscience.iop.org/article/{}/pdf", doi);
|
||||
|
||||
// 步骤 1：访问文章主页，建立 Cookie 会话
|
||||
info!("[IOP] 预热主页: {}", main_url);
|
||||
debug!("[IOP] 预热主页: {}", main_url);
|
||||
Self::maybe_delay().await;
|
||||
match self.client.get(&main_url)
|
||||
.headers(build_chrome_headers(None))
|
||||
.send().await
|
||||
{
|
||||
Ok(r) => info!("[IOP] 主页响应: {}", r.status()),
|
||||
Ok(r) => debug!("[IOP] 主页响应: {}", r.status()),
|
||||
Err(e) => warn!("[IOP] 主页访问失败（继续尝试）: {:?}", e),
|
||||
}
|
||||
|
||||
// 步骤 2：携带 Referer 下载 PDF
|
||||
info!("[IOP] 下载 PDF: {}", pdf_url);
|
||||
debug!("[IOP] 下载 PDF: {}", pdf_url);
|
||||
Self::maybe_delay().await;
|
||||
let response = self.client.get(&pdf_url)
|
||||
.headers(build_chrome_headers(Some(&main_url)))
|
||||
@@ -516,7 +514,19 @@ impl Downloader {
|
||||
Err(e) => warn!("[PUB_PDF] 网关解析失败: {:?}", e),
|
||||
}
|
||||
|
||||
// 1b. ADS EPRINT_PDF 网关
|
||||
// 1b. ADS_PDF 网关 (经典 ADS 整合 PDF 直接通道)
|
||||
let gw = format!("{}/{}/ADS_PDF", base, bibcode);
|
||||
match self.resolve_ads_gateway(&gw).await {
|
||||
Ok(resolved) => {
|
||||
match self.download_pdf_direct(&resolved, &pdf_dest, "ADS_PDF").await {
|
||||
Ok(_) => { pdf_ok = Some(pdf_dest.clone()); break 'pdf; }
|
||||
Err(e) => warn!("[ADS_PDF] 下载失败: {:?}", e),
|
||||
}
|
||||
}
|
||||
Err(e) => warn!("[ADS_PDF] 网关解析失败: {:?}", e),
|
||||
}
|
||||
|
||||
// 1c. ADS EPRINT_PDF 网关
|
||||
let gw = format!("{}/{}/EPRINT_PDF", base, bibcode);
|
||||
match self.resolve_ads_gateway(&gw).await {
|
||||
Ok(resolved) => {
|
||||
@@ -531,10 +541,17 @@ impl Downloader {
|
||||
// 1c. CrossRef API 回退（需要 DOI）
|
||||
if let Some(doi_str) = doi {
|
||||
match self.download_crossref_pdf(doi_str, &pdf_dest).await {
|
||||
Ok(_) => { pdf_ok = Some(pdf_dest.clone()); }
|
||||
Ok(_) => { pdf_ok = Some(pdf_dest.clone()); break 'pdf; }
|
||||
Err(e) => warn!("[CrossRef] PDF 下载失败: {:?}", e),
|
||||
}
|
||||
}
|
||||
|
||||
// 1d. ADS SCAN 扫描版文献直接合并下载 PDF（主要针对早期/不可下载直接 PDF 的文献）
|
||||
let scan_url = format!("https://articles.adsabs.harvard.edu/cgi-bin/nph-iarticle_query?bibcode={}&db_key=AST&data_type=PDF_HIGH", bibcode);
|
||||
match self.download_pdf_direct(&scan_url, &pdf_dest, "ADS_SCAN").await {
|
||||
Ok(_) => { pdf_ok = Some(pdf_dest.clone()); }
|
||||
Err(e) => warn!("[ADS_SCAN] 下载失败: {:?}", e),
|
||||
}
|
||||
}
|
||||
|
||||
// ── HTML 下载 ──────────────────────────────────────────
|
||||
@@ -710,5 +727,25 @@ mod tests {
|
||||
Some("2101.00001".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore]
|
||||
async fn test_download_scan_pdf() -> anyhow::Result<()> {
|
||||
let downloader = Downloader::new();
|
||||
let bibcode = "2005MNRAS.359..315E";
|
||||
let temp_dir = std::env::temp_dir();
|
||||
|
||||
let (pdf_path, _html_path) = downloader.download_paper(bibcode, None, &temp_dir).await;
|
||||
assert!(pdf_path.is_some());
|
||||
|
||||
let path = pdf_path.unwrap();
|
||||
assert!(path.exists());
|
||||
|
||||
let bytes = std::fs::read(&path)?;
|
||||
assert!(bytes.starts_with(b"%PDF"));
|
||||
|
||||
let _ = std::fs::remove_file(&path);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
// src/services/logging.rs
|
||||
use chrono::{DateTime, FixedOffset, Utc};
|
||||
use std::env;
|
||||
use std::fs;
|
||||
use tracing_appender::non_blocking::WorkerGuard;
|
||||
use tracing_appender::rolling;
|
||||
use tracing_subscriber::fmt::format::Writer;
|
||||
use tracing_subscriber::fmt::time::FormatTime;
|
||||
use tracing_subscriber::{
|
||||
fmt, layer::SubscriberExt, util::SubscriberInitExt,
|
||||
EnvFilter, Layer,
|
||||
};
|
||||
|
||||
pub struct ShanghaiTime;
|
||||
|
||||
impl FormatTime for ShanghaiTime {
|
||||
fn format_time(&self, w: &mut Writer<'_>) -> std::fmt::Result {
|
||||
let now: DateTime<Utc> = Utc::now();
|
||||
let offset = FixedOffset::east_opt(8 * 3600).unwrap();
|
||||
let shanghai_time = now.with_timezone(&offset);
|
||||
write!(w, "{}", shanghai_time.format("%Y-%m-%dT%H:%M:%S%.3f%:z"))
|
||||
}
|
||||
}
|
||||
|
||||
/// 初始化系统全局日志模块，支持控制台输出与每天自动滚动的日志文件
|
||||
pub fn init_logging() -> anyhow::Result<Vec<WorkerGuard>> {
|
||||
let mut guards = Vec::new();
|
||||
|
||||
// 从环境变量中读取配置
|
||||
let log_level = env::var("LOG_LEVEL").unwrap_or_else(|_| "info,astroresearch=debug".to_string());
|
||||
let log_format = env::var("LOG_FORMAT").unwrap_or_else(|_| "pretty".to_string());
|
||||
let log_outputs = env::var("LOG_OUTPUTS").unwrap_or_else(|_| "stdout,file".to_string());
|
||||
|
||||
let env_filter = EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| EnvFilter::new(&log_level));
|
||||
let is_json = log_format.to_lowercase() == "json";
|
||||
|
||||
let mut layers: Vec<Box<dyn Layer<tracing_subscriber::Registry> + Send + Sync>> = Vec::new();
|
||||
|
||||
// 1. 控制台输出层 (stdout)
|
||||
if log_outputs.contains("stdout") {
|
||||
let (non_blocking, guard) = tracing_appender::non_blocking(std::io::stdout());
|
||||
guards.push(guard);
|
||||
|
||||
let fmt_layer = fmt::layer()
|
||||
.with_timer(ShanghaiTime)
|
||||
.with_writer(non_blocking);
|
||||
|
||||
if is_json {
|
||||
layers.push(fmt_layer.json().with_ansi(false).boxed());
|
||||
} else {
|
||||
layers.push(fmt_layer.pretty().with_ansi(true).boxed());
|
||||
}
|
||||
}
|
||||
|
||||
// 2. 每日滚动文件日志层 (file)
|
||||
if log_outputs.contains("file") {
|
||||
let log_dir = env::var("LOG_DIR").unwrap_or_else(|_| "logs".to_string());
|
||||
fs::create_dir_all(&log_dir).unwrap_or(());
|
||||
|
||||
let file_appender = rolling::daily(log_dir, "astro_research.log");
|
||||
let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
|
||||
guards.push(guard);
|
||||
|
||||
let fmt_layer = fmt::layer()
|
||||
.with_timer(ShanghaiTime)
|
||||
.with_writer(non_blocking)
|
||||
.with_ansi(false);
|
||||
|
||||
if is_json {
|
||||
layers.push(fmt_layer.json().boxed());
|
||||
} else {
|
||||
layers.push(fmt_layer.boxed());
|
||||
}
|
||||
}
|
||||
|
||||
// 3. 注册全部日志层
|
||||
tracing_subscriber::registry()
|
||||
.with(layers)
|
||||
.with(env_filter)
|
||||
.init();
|
||||
|
||||
Ok(guards)
|
||||
}
|
||||
@@ -3,3 +3,4 @@ pub mod parser;
|
||||
pub mod translation;
|
||||
pub mod query_parser;
|
||||
pub mod batch_sync;
|
||||
pub mod logging;
|
||||
|
||||
+261
-38
@@ -1,11 +1,9 @@
|
||||
// src/parser.rs
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use serde::Deserialize;
|
||||
use reqwest::multipart;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing::{info, warn};
|
||||
use regex::Regex;
|
||||
use base64::Engine;
|
||||
|
||||
use crate::Config;
|
||||
use crate::clients::qiniu::QiniuClient;
|
||||
@@ -13,7 +11,20 @@ use crate::clients::qiniu::QiniuClient;
|
||||
// 清理 HTML 结构，仅提取正文部分并转换为标准 Markdown
|
||||
pub fn html_to_markdown(html_path: &Path) -> anyhow::Result<String> {
|
||||
info!("正在解析本地 HTML 并提取 Markdown: {:?}", html_path);
|
||||
let html_content = fs::read_to_string(html_path)?;
|
||||
let html_bytes = fs::read(html_path)?;
|
||||
|
||||
// 检查是否为 Gzip 压缩文件 (Gzip 幻数: 0x1f 0x8b)
|
||||
let decompressed_bytes = if html_bytes.starts_with(&[0x1f, 0x8b]) {
|
||||
use std::io::Read;
|
||||
let mut decoder = flate2::read::GzDecoder::new(&html_bytes[..]);
|
||||
let mut buf = Vec::new();
|
||||
decoder.read_to_end(&mut buf)?;
|
||||
buf
|
||||
} else {
|
||||
html_bytes
|
||||
};
|
||||
|
||||
let html_content = String::from_utf8_lossy(&decompressed_bytes).into_owned();
|
||||
|
||||
// 截断页脚及之后的不相关内容以防干扰解析
|
||||
let mut truncated_html = html_content.as_str();
|
||||
@@ -287,10 +298,61 @@ fn strip_html_tags(html: &str) -> String {
|
||||
.replace("'", "'")
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct BatchUploadRequest {
|
||||
files: Vec<PendingFile>,
|
||||
language: String,
|
||||
is_ocr: bool,
|
||||
model_version: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct PendingFile {
|
||||
name: String,
|
||||
data_id: String,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Deserialize)]
|
||||
struct BatchUploadResponse {
|
||||
code: i32,
|
||||
msg: String,
|
||||
data: BatchUploadData,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct BatchUploadData {
|
||||
batch_id: String,
|
||||
file_urls: Vec<String>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Deserialize)]
|
||||
struct BatchResultResponse {
|
||||
code: i32,
|
||||
msg: String,
|
||||
data: Option<BatchResultData>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Deserialize)]
|
||||
struct BatchResultData {
|
||||
batch_id: String,
|
||||
extract_result: Vec<ExtractResult>,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
#[derive(Deserialize)]
|
||||
struct ExtractResult {
|
||||
file_name: String,
|
||||
state: String,
|
||||
full_zip_url: Option<String>,
|
||||
err_msg: Option<String>,
|
||||
}
|
||||
|
||||
// 调用 MinerU 远程接口解析 PDF，并在提取出图片后自动上传至七牛云进行外链替换
|
||||
pub async fn parse_pdf_via_mineru(
|
||||
pub async fn submit_pdf_to_mineru(
|
||||
pdf_path: &Path,
|
||||
qiniu_client: &QiniuClient,
|
||||
config: &Config
|
||||
) -> anyhow::Result<String> {
|
||||
info!("正在请求 MinerU 解析本地 PDF 文献: {:?}", pdf_path);
|
||||
@@ -305,60 +367,221 @@ pub async fn parse_pdf_via_mineru(
|
||||
.unwrap_or("paper.pdf")
|
||||
.to_string();
|
||||
|
||||
let file_part = multipart::Part::bytes(pdf_bytes).file_name(filename);
|
||||
let form = multipart::Form::new()
|
||||
.part("file", file_part);
|
||||
let bibcode = pdf_path.file_stem()
|
||||
.and_then(|f| f.to_str())
|
||||
.unwrap_or("paper")
|
||||
.to_string();
|
||||
|
||||
// 提取 base_url
|
||||
let base_url = config.mineru_api_url
|
||||
.replace("/extract/task", "")
|
||||
.replace("/extract", "")
|
||||
.trim_end_matches('/')
|
||||
.to_string();
|
||||
|
||||
info!("正在发送 PDF 字节流至 MinerU 接口地址: {}", config.mineru_api_url);
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
let mut request = client.post(&config.mineru_api_url).multipart(form);
|
||||
let data_id = uuid::Uuid::new_v4().to_string();
|
||||
|
||||
// 1. 获取预签名上传 URL
|
||||
info!("MinerU: 正在请求批量直传 URL (Bibcode: {})", bibcode);
|
||||
let upload_req = BatchUploadRequest {
|
||||
files: vec![PendingFile {
|
||||
name: filename.clone(),
|
||||
data_id: data_id.clone(),
|
||||
}],
|
||||
language: "en".to_string(),
|
||||
is_ocr: true,
|
||||
model_version: "vlm".to_string(),
|
||||
};
|
||||
|
||||
let mut request = client.post(format!("{}/file-urls/batch/", base_url))
|
||||
.json(&upload_req);
|
||||
|
||||
if !config.mineru_api_key.is_empty() {
|
||||
request = request.header("Authorization", format!("Bearer {}", config.mineru_api_key));
|
||||
}
|
||||
|
||||
let response = request.send().await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(anyhow::anyhow!("MinerU 解析接口返回失败码: {}", response.status()));
|
||||
let status = response.status();
|
||||
let res_text = response.text().await?;
|
||||
if !status.is_success() {
|
||||
return Err(anyhow::anyhow!("请求 MinerU 批量上传 URL 失败 (状态码: {}): {}", status, res_text));
|
||||
}
|
||||
|
||||
// MinerU 远程服务响应 JSON，包含转换出的 markdown 正文和图片映射
|
||||
#[derive(Deserialize)]
|
||||
struct MinerUResponse {
|
||||
markdown: String,
|
||||
images: Option<std::collections::HashMap<String, String>>, // 图片文件名 -> Base64 字符串
|
||||
let upload_res: BatchUploadResponse = serde_json::from_str(&res_text)?;
|
||||
if upload_res.code != 0 {
|
||||
return Err(anyhow::anyhow!("MinerU API 错误: {}", upload_res.msg));
|
||||
}
|
||||
|
||||
let result: MinerUResponse = response.json().await?;
|
||||
let mut markdown = result.markdown;
|
||||
let upload_url = upload_res.data.file_urls.first()
|
||||
.ok_or_else(|| anyhow::anyhow!("MinerU 未返回上传 URL"))?;
|
||||
|
||||
// 上传图片并重写 Markdown 连接地址
|
||||
if let Some(images) = result.images {
|
||||
if qiniu_client.is_configured() {
|
||||
info!("MinerU 成功解析出 {} 张本地插图。正在准备同步至七牛云...", images.len());
|
||||
for (img_name, base64_data) in images {
|
||||
if let Ok(img_bytes) = base64::engine::general_purpose::STANDARD.decode(base64_data) {
|
||||
match qiniu_client.upload_buffer(img_bytes, &img_name).await {
|
||||
Ok(qiniu_url) => {
|
||||
// 使用正则将 Markdown 中的本地临时图地址替换为七牛云 CDN 地址
|
||||
let escaped_img_name = regex::escape(&img_name);
|
||||
let link_re = Regex::new(&format!(r"\(([^)]*?){}\)", escaped_img_name)).unwrap();
|
||||
markdown = link_re.replace_all(&markdown, |_: ®ex::Captures| {
|
||||
format!("({})", qiniu_url)
|
||||
}).to_string();
|
||||
},
|
||||
Err(e) => warn!("上传图片至七牛云失败 {}: {}", img_name, e),
|
||||
// 2. 上传文件 (PUT)
|
||||
info!("MinerU: 正在直接上传 PDF 字节流至对象存储...");
|
||||
let put_res = client.put(upload_url)
|
||||
.body(pdf_bytes)
|
||||
.send()
|
||||
.await?;
|
||||
if !put_res.status().is_success() {
|
||||
return Err(anyhow::anyhow!("上传 PDF 至 MinerU 对象存储直传 URL 失败: {}", put_res.status()));
|
||||
}
|
||||
|
||||
let batch_id = upload_res.data.batch_id;
|
||||
Ok(batch_id)
|
||||
}
|
||||
|
||||
pub async fn poll_and_extract_mineru(
|
||||
batch_id: &str,
|
||||
bibcode: &str,
|
||||
qiniu_client: &QiniuClient,
|
||||
config: &Config
|
||||
) -> anyhow::Result<String> {
|
||||
let client = reqwest::Client::new();
|
||||
let base_url = config.mineru_api_url
|
||||
.replace("/extract/task", "")
|
||||
.replace("/extract", "")
|
||||
.trim_end_matches('/')
|
||||
.to_string();
|
||||
|
||||
let mut poll_count = 0;
|
||||
let max_polls = 45; // 45 * 10s = 7.5 min
|
||||
info!("MinerU: 开始轮询任务结果 (Batch ID: {})...", batch_id);
|
||||
|
||||
let mut full_zip_url = String::new();
|
||||
loop {
|
||||
poll_count += 1;
|
||||
if poll_count > max_polls {
|
||||
return Err(anyhow::anyhow!("MinerU 结构化解析超时 (Bibcode: {})", bibcode));
|
||||
}
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_secs(10)).await;
|
||||
|
||||
let mut status_req = client.get(format!("{}/extract-results/batch/{}", base_url, batch_id));
|
||||
if !config.mineru_api_key.is_empty() {
|
||||
status_req = status_req.header("Authorization", format!("Bearer {}", config.mineru_api_key));
|
||||
}
|
||||
|
||||
let status_res = status_req.send().await?;
|
||||
let status_text = status_res.text().await?;
|
||||
let result_data: BatchResultResponse = serde_json::from_str(&status_text)?;
|
||||
|
||||
if let Some(data) = result_data.data {
|
||||
if let Some(file_result) = data.extract_result.first() {
|
||||
match file_result.state.as_str() {
|
||||
"done" => {
|
||||
info!("MinerU: 解析成功！");
|
||||
full_zip_url = file_result.full_zip_url.clone().unwrap_or_default();
|
||||
break;
|
||||
}
|
||||
"error" | "failed" => {
|
||||
let err_msg = file_result.err_msg.clone().unwrap_or_default();
|
||||
return Err(anyhow::anyhow!("MinerU 批量解析任务失败: {}", err_msg));
|
||||
}
|
||||
other => {
|
||||
info!("MinerU 任务处理中... 当前状态: {}", other);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
return Err(anyhow::anyhow!("MinerU 轮询响应中未发现文件解析任务结果"));
|
||||
}
|
||||
} else {
|
||||
return Err(anyhow::anyhow!("MinerU 轮询响应数据为空"));
|
||||
}
|
||||
}
|
||||
|
||||
if full_zip_url.is_empty() {
|
||||
return Err(anyhow::anyhow!("MinerU 转换成功但未返回结果 ZIP 下载 URL"));
|
||||
}
|
||||
|
||||
// 4. 下载并解压 ZIP
|
||||
info!("MinerU: 正在下载最终提取压缩包: {}", full_zip_url);
|
||||
let zip_bytes = client.get(&full_zip_url).send().await?.bytes().await?;
|
||||
|
||||
let reader = std::io::Cursor::new(zip_bytes);
|
||||
let mut archive = zip::ZipArchive::new(reader)?;
|
||||
|
||||
let mut markdown = String::new();
|
||||
let mut image_buffers = std::collections::HashMap::new();
|
||||
|
||||
for i in 0..archive.len() {
|
||||
let mut file = archive.by_index(i)?;
|
||||
let name = file.name().to_string();
|
||||
|
||||
if name.ends_with(".md") {
|
||||
let mut md_content = String::new();
|
||||
std::io::Read::read_to_string(&mut file, &mut md_content)?;
|
||||
markdown = md_content;
|
||||
} else if file.is_file() {
|
||||
let lower = name.to_lowercase();
|
||||
if lower.ends_with(".png") || lower.ends_with(".jpg") || lower.ends_with(".jpeg") || lower.ends_with(".gif") || lower.ends_with(".svg") {
|
||||
let mut buf = Vec::new();
|
||||
std::io::copy(&mut file, &mut buf)?;
|
||||
let file_basename = Path::new(&name)
|
||||
.file_name()
|
||||
.and_then(|f| f.to_str())
|
||||
.unwrap_or(&name)
|
||||
.to_string();
|
||||
image_buffers.insert(file_basename, buf);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if markdown.is_empty() {
|
||||
return Err(anyhow::anyhow!("解析后的压缩包中未发现核心 Markdown 文档"));
|
||||
}
|
||||
|
||||
// 5. 上传图片并重写链接
|
||||
if !image_buffers.is_empty() {
|
||||
let local_img_dir = config.library_dir.join("images").join(bibcode);
|
||||
let _ = fs::create_dir_all(&local_img_dir);
|
||||
|
||||
if qiniu_client.is_configured() {
|
||||
info!("MinerU 批量模式解析出 {} 张本地插图。准备上传至七牛云...", image_buffers.len());
|
||||
for (img_name, img_bytes) in image_buffers {
|
||||
let local_path = local_img_dir.join(&img_name);
|
||||
let _ = fs::write(&local_path, &img_bytes);
|
||||
|
||||
match qiniu_client.upload_buffer(img_bytes, &img_name).await {
|
||||
Ok(qiniu_url) => {
|
||||
let escaped_img_name = regex::escape(&img_name);
|
||||
let link_re = Regex::new(&format!(r"\(([^)]*?){}\)", escaped_img_name)).unwrap();
|
||||
markdown = link_re.replace_all(&markdown, |_: ®ex::Captures| {
|
||||
format!("({})", qiniu_url)
|
||||
}).to_string();
|
||||
}
|
||||
Err(e) => warn!("上传图片至七牛云失败 {}: {}", img_name, e),
|
||||
}
|
||||
}
|
||||
} else {
|
||||
warn!("未检测到七牛云配置，解析出的图片将保留临时地址，无法在外网或 Obsidian 中直观预览");
|
||||
warn!("未检测到七牛云配置，解析出的图片将保存在本地 images 目录下");
|
||||
for (img_name, img_bytes) in image_buffers {
|
||||
let local_path = local_img_dir.join(&img_name);
|
||||
let _ = fs::write(&local_path, &img_bytes);
|
||||
|
||||
let escaped_img_name = regex::escape(&img_name);
|
||||
let link_re = Regex::new(&format!(r"\(([^)]*?){}\)", escaped_img_name)).unwrap();
|
||||
let replacement_link = format!("(images/{}/{})", bibcode, img_name);
|
||||
markdown = link_re.replace_all(&markdown, replacement_link.as_str()).to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(markdown)
|
||||
}
|
||||
|
||||
pub async fn parse_pdf_via_mineru(
|
||||
pdf_path: &Path,
|
||||
qiniu_client: &QiniuClient,
|
||||
config: &Config
|
||||
) -> anyhow::Result<String> {
|
||||
let bibcode = pdf_path.file_stem()
|
||||
.and_then(|f| f.to_str())
|
||||
.unwrap_or("paper")
|
||||
.to_string();
|
||||
let batch_id = submit_pdf_to_mineru(pdf_path, config).await?;
|
||||
poll_and_extract_mineru(&batch_id, &bibcode, qiniu_client, config).await
|
||||
}
|
||||
|
||||
// 采用栈式解析模型，将 LaTeXML 用 span/div 模拟出的表格容器（ltx_tabular/tbody/thead/tfoot/tr/td/th）还原为真正的 HTML <table> 结构
|
||||
fn replace_latexml_tables(html: &str) -> String {
|
||||
use regex::Regex;
|
||||
|
||||
@@ -129,6 +129,7 @@ pub async fn translate_markdown(
|
||||
);
|
||||
|
||||
info!("正在请求大模型开展中英翻译。所选大模型: {}", config.llm_model);
|
||||
let start_time = std::time::Instant::now();
|
||||
|
||||
let client = reqwest::Client::new();
|
||||
let url = format!("{}/chat/completions", config.llm_api_base);
|
||||
@@ -179,6 +180,8 @@ pub async fn translate_markdown(
|
||||
|
||||
let res_data: LLMResponse = response.json().await?;
|
||||
if let Some(choice) = res_data.choices.first() {
|
||||
let duration = start_time.elapsed();
|
||||
info!("LLM 翻译成功。所选大模型: {}, 耗时: {:?}, 译文字符数: {}", config.llm_model, duration, choice.message.content.len());
|
||||
Ok(choice.message.content.clone())
|
||||
} else {
|
||||
Err(anyhow::anyhow!("大模型返回空翻译选项集"))
|
||||
|
||||
Reference in New Issue
Block a user