fmq/AstroResearch
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: LLM/Embedding 客户端模块化、侧边栏折叠交互、arXiv→ADS 下载回退与前端体验重构

Browse Source 
**后端架构**
  - 抽取翻译服务中内嵌的 LLM HTTP 调用为独立的 LlmClient /
    EmbeddingClient（src/clients/llm.rs），翻译模块改为委托调用，消除
    对 reqwest/serde 的直接耦合
  - Config 新增 EMBEDDING_API_KEY/EMBEDDING_API_BASE/EMBEDDING_MODEL
    三项配置，默认 fallback 至 LLM 对应值，补齐向量嵌入基础设施

  **下载策略优化**
  - arXiv 直连下载失败后自动回退至 ADS 网关 PUB_PDF→EPRINT_PDF→CrossRef
    多级通道，替换此前单路径策略；批量同步同步应用此逻辑
  - PDF/HTML 任一方成功时，失败方的 path 字段不再存储 "error:" 报错字符串，
    改为置 NULL，防止日志污染数据

  **前端交互增强**
  - 侧边栏支持折叠/展开：收起为仅图标模式(w-16)，展开恢复完整模式(w-64)；
    收起后点击 Logo 展开，含流畅 cubic-bezier 过渡动画
  - 阅读面板新增 PDF 内嵌预览：已下载 PDF 时可通过 iframe 切换查看
    /api/files 下的本地文献
  - reader/citation 面板未选文献时展示带图标的空状态引导页，替代空白页
  - 文献详情面板改为固定高度弹性布局(h-[460px])，各区块按比例分配避免
    内容挤压；期刊名过长截断+悬停tooltip；关键词无数据显式占位
  - 全局移除 emoji Unicode，统一替换为 lucide-react 图标组件，
    消除跨平台字体渲染差异

  **反爬检测精细化**
  - 按响应长度分层：>150KB 跳过检测（完整文献），<5KB 才扫描通用 HTTP
    错误关键字，杜绝长文献误触 Cloudflare/503 模式匹配
  - 新增 Radware Bot Manager、ShieldSquare WAF 特征识别

  **健壮性**
  - Obscura 下载校验失败后自动清理硬盘残留坏文件
  - 健康检查工具：文献已有有效 HTML 但 PDF 字段为旧报错时自动判定可修复
  - 上传接口 body limit 提升至 100MB，新增 /api/files 静态文件服务路由
  - StandardPaper 新增 has_pdf/has_html 字段区分格式级下载状态
This commit is contained in:
fmq
2026-06-13 11:11:33 +08:00
parent 2a5b1c0c91
commit 3f1935678b
27 changed files with 1275 additions and 473 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/api/helpers.rs
+8
View File
@@ -45,6 +45,8 @@ pub fn convert_ads_doc_to_standard(doc: &AdsPaperDoc) -> StandardPaper {
citation_count: doc.citation_count.unwrap_or(0),
reference_count: doc.reference_count.unwrap_or(0),
is_downloaded: false,
has_pdf: false,
has_html: false,
has_markdown: false,
has_translation: false,
doctype: doc.doctype.clone().unwrap_or_else(|| "article".to_string()),
@@ -67,6 +69,8 @@ pub fn convert_arxiv_to_standard(doc: &ArxivPaper) -> StandardPaper {
citation_count: 0,
reference_count: 0,
is_downloaded: false,
has_pdf: false,
has_html: false,
has_markdown: false,
has_translation: false,
doctype: "eprint".to_string(),
@@ -201,6 +205,8 @@ pub async fn get_paper_from_db(db: &SqlitePool, library_dir: &std::path::Path, b
citation_count: r.get(9),
reference_count: r.get(10),
is_downloaded: is_pdf_exist || is_html_exist,
has_pdf: is_pdf_exist,
has_html: is_html_exist,
has_markdown: is_md_exist,
has_translation: is_tr_exist,
doctype: doctype_val.unwrap_or_else(|| "article".to_string()),
@@ -345,6 +351,8 @@ mod tests {
citation_count: 5,
reference_count: 10,
is_downloaded: false,
has_pdf: false,
has_html: false,
has_markdown: false,
has_translation: false,
doctype: "article".to_string(),
src/api/mod.rs
+5
View File
@@ -7,6 +7,7 @@ use crate::services::translation::Dictionary;
use crate::clients::qiniu::QiniuClient;
use crate::clients::ads::AdsClient;
use crate::clients::arxiv::ArxivClient;
use crate::clients::llm::{LlmClient, EmbeddingClient};
use crate::services::download::Downloader;
// 全局共享的 Axum 应用上下文状态
@@ -17,6 +18,8 @@ pub struct AppState {
pub qiniu: QiniuClient,
pub ads: AdsClient,
pub arxiv: ArxivClient,
pub llm: LlmClient,
pub embedding: EmbeddingClient,
pub downloader: Downloader,
pub harvest_status: Arc<tokio::sync::Mutex<crate::services::batch_sync::MetaSyncStatus>>,
pub process_status: Arc<tokio::sync::Mutex<crate::services::batch_sync::AssetSyncStatus>>,
@@ -38,6 +41,8 @@ pub struct StandardPaper {
pub citation_count: i32,
pub reference_count: i32,
pub is_downloaded: bool,
pub has_pdf: bool,
pub has_html: bool,
pub has_markdown: bool,
pub has_translation: bool,
pub doctype: String,
src/api/papers.rs
+18 -8
View File
@@ -150,11 +150,17 @@ pub async fn download_paper(
// 下载策略：
// 1. 如有 arXiv ID，优先走 arXiv 直连（绕过出版商防护墙，成功率高）
// 2. 否则走 ADS 网关多级回退（PUB_PDF → EPRINT_PDF → CrossRef）
// 3. 若 ADS 路径 PDF/HTML 均失败但有 arXiv ID，再尝试 arXiv 作为兜底
// 2. 若 arXiv 直连失败，回退走 ADS 网关多级回退（PUB_PDF → EPRINT_PDF → CrossRef）
let (pdf_res, html_res) = if !paper.arxiv_id.is_empty() {
info!("[下载] 优先使用 arXiv 通道: {}", paper.arxiv_id);
state.downloader.download_arxiv_direct(&paper.arxiv_id, &state.config.library_dir).await
let res = state.downloader.download_arxiv_direct(&paper.arxiv_id, &state.config.library_dir).await;
if res.0.is_ok() || res.1.is_ok() {
res
} else {
warn!("[下载] arXiv 通道下载失败，开始回退至 ADS/出版商通道: {}", req.bibcode);
let doi_opt = if !paper.doi.is_empty() { Some(paper.doi.as_str()) } else { None };
state.downloader.download_paper(&req.bibcode, doi_opt, &state.config.library_dir).await
}
} else {
let doi_opt = if !paper.doi.is_empty() { Some(paper.doi.as_str()) } else { None };
state.downloader.download_paper(&req.bibcode, doi_opt, &state.config.library_dir).await
@@ -179,11 +185,11 @@ pub async fn download_paper(
let pdf_rel = match pdf_res {
Ok(p) => Some(p.strip_prefix(&state.config.library_dir).unwrap_or(&p).to_string_lossy().to_string()),
Err(e) => Some(format!("error: {}", e)),
Err(_) => None, // 只要有一方下载成功，失败的一方字段置空（NULL），避免在 path 字段中留存报错日志
};
let html_rel = match html_res {
Ok(p) => Some(p.strip_prefix(&state.config.library_dir).unwrap_or(&p).to_string_lossy().to_string()),
Err(e) => Some(format!("error: {}", e)),
Err(_) => None, // 只要有一方下载成功，失败的一方字段置空（NULL），避免在 path 字段中留存报错日志
};
// 回写存储路径至数据库
@@ -386,7 +392,7 @@ pub async fn translate_paper(
})?;
// 调用 LLM 翻译服务并注入对照词表
let translated_markdown = crate::services::translation::translate_markdown(&english_markdown, &state.dict, &state.config)
let translated_markdown = crate::services::translation::translate_markdown(&english_markdown, &state.dict, &state.llm)
.await
.map_err(|e| {
error!("文献 {} 翻译失败：调用 LLM 翻译发生错误: {}", req.bibcode, e);
@@ -571,6 +577,9 @@ pub async fn get_library(
let keywords_str: Option<String> = r.get(5);
let keywords: Vec<String> = keywords_str.and_then(|s| serde_json::from_str(&s).ok()).unwrap_or_default();
let is_pdf_exist = pdf_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false);
let is_html_exist = html_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false);
let pdf_error = pdf_path.as_ref()
.filter(|p| p.starts_with("error:"))
.map(|p| p["error:".len()..].trim().to_string());
@@ -590,8 +599,9 @@ pub async fn get_library(
arxiv_id: r.get(8),
citation_count: r.get(9),
reference_count: r.get(10),
is_downloaded: pdf_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false)
|| html_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false),
is_downloaded: is_pdf_exist || is_html_exist,
has_pdf: is_pdf_exist,
has_html: is_html_exist,
has_markdown: markdown_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false),
has_translation: translation_path.as_ref().map(|p| state.config.library_dir.join(p).exists()).unwrap_or(false),
doctype: doctype_val.unwrap_or_else(|| "article".to_string()),
src/bin/health_check.rs
+53 -13
View File
@@ -8,8 +8,13 @@ use tracing_subscriber::FmtSubscriber;
// 检测防爬、验证码、登录墙特征
fn detect_anti_bot(content: &str) -> Option<&'static str> {
if content.len() > 150_000 {
return None;
}
let lower = content.to_lowercase();
let cf_patterns = [
// 1. 强特征防爬与 WAF 挑战（任何小于 150KB 的内容都做检测）
let waf_patterns = [
("checking your browser", "Cloudflare WAF 浏览器检查"),
("please wait while we verify", "Cloudflare WAF 验证"),
("cf-browser-verification", "Cloudflare WAF 验证特征"),
@@ -35,11 +40,32 @@ fn detect_anti_bot(content: &str) -> Option<&'static str> {
("shieldsquare_styles", "ShieldSquare WAF 拦截"),
];
for &(p, desc) in &cf_patterns {
for &(p, desc) in &waf_patterns {
if lower.contains(p) {
return Some(desc);
}
}
// 2. 通用 HTTP 错误与 CDN 关键字检测（仅当内容长度小于 5000 字节时检测，避免在正常文献中误判 CDN 脚本等）
if content.len() < 5000 {
let err_patterns = [
("cloudflare", "Cloudflare 错误/防护页面"),
("service temporarily unavailable", "503 服务暂时不可用"),
("503 service", "503 服务异常"),
("502 bad gateway", "502 网关错误"),
("504 gateway timeout", "504 网关超时"),
("403 forbidden", "403 访问被拒绝"),
("404 not found", "404 资源未找到"),
("500 internal server error", "500 服务器错误"),
("site error", "网站错误"),
];
for &(p, desc) in &err_patterns {
if lower.contains(p) {
return Some(desc);
}
}
}
None
}
@@ -342,10 +368,24 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
db_skip_type_cleaned += 1;
}
} else {
let has_valid_pdf = pdf_path_opt.as_ref()
.map(|p| !p.starts_with("error:") && library_dir.join(p).exists())
.unwrap_or(false);
let has_valid_html = html_path_opt.as_ref()
.map(|p| !p.starts_with("error:") && library_dir.join(p).exists())
.unwrap_or(false);
if let Some(ref pdf_p) = pdf_path_opt {
if pdf_p.starts_with("error:") {
db_pdf_err_text += 1;
pdf_db_msg = format!("数据库存储了报错字符串: {}", pdf_p);
if has_valid_html {
db_pdf_err_text += 1;
pdf_db_msg = format!("文献已成功下载 HTML 格式，但 PDF 仍留有报错日志（将清理为 NULL）: {}", pdf_p);
need_db_fix = true;
pdf_needs_fix = true;
} else {
db_pdf_err_text += 1;
pdf_db_msg = format!("数据库存储了报错字符串: {}", pdf_p);
}
} else if !library_dir.join(pdf_p).exists() {
db_pdf_missing += 1;
pdf_db_msg = format!("物理 PDF 文件丢失 (路径: {})", pdf_p);
@@ -356,8 +396,15 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
if let Some(ref html_p) = html_path_opt {
if html_p.starts_with("error:") {
db_html_err_text += 1;
html_db_msg = format!("数据库存储了报错字符串: {}", html_p);
if has_valid_pdf {
db_html_err_text += 1;
html_db_msg = format!("文献已成功下载 PDF 格式，但 HTML 仍留有报错日志（将清理为 NULL）: {}", html_p);
need_db_fix = true;
html_needs_fix = true;
} else {
db_html_err_text += 1;
html_db_msg = format!("数据库存储了报错字符串: {}", html_p);
}
} else if !library_dir.join(html_p).exists() {
db_html_missing += 1;
html_db_msg = format!("物理 HTML 文件丢失 (路径: {})", html_p);
@@ -374,13 +421,6 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
markdown_needs_fix = true;
} else {
// 如果 Markdown 物理文件存在，但它既没有有效 PDF 也没有有效 HTML
let has_valid_pdf = pdf_path_opt.as_ref()
.map(|p| !p.starts_with("error:") && library_dir.join(p).exists())
.unwrap_or(false);
let has_valid_html = html_path_opt.as_ref()
.map(|p| !p.starts_with("error:") && library_dir.join(p).exists())
.unwrap_or(false);
if !has_valid_pdf && !has_valid_html {
db_markdown_orphaned += 1;
markdown_db_msg = format!("Markdown 存在且完好，但失去有效 PDF/HTML 数据源，判定为孤立的 Markdown (路径: {})", md_p);
src/clients/llm.rs
+190
View File
@@ -0,0 +1,190 @@
// src/clients/llm.rs
use serde::Deserialize;
use reqwest::Client;
use tracing::error;
#[derive(Clone, Debug)]
pub struct LlmClient {
api_key: String,
api_base: String,
model: String,
client: Client,
}
impl LlmClient {
pub fn new(api_key: String, api_base: String, model: String) -> Self {
LlmClient {
api_key,
api_base,
model,
client: Client::new(),
}
}
pub fn model(&self) -> &str {
&self.model
}
pub fn api_base(&self) -> &str {
&self.api_base
}
pub fn api_key(&self) -> &str {
&self.api_key
}
pub async fn chat_completion(&self, system_prompt: &str, user_content: &str) -> anyhow::Result<String> {
let url = format!("{}/chat/completions", self.api_base);
let payload = serde_json::json!({
"model": self.model,
"messages": [
{
"role": "system",
"content": system_prompt
},
{
"role": "user",
"content": user_content
}
],
"temperature": 0.3
});
let response = self.client.post(&url)
.header("Authorization", format!("Bearer {}", self.api_key))
.header("Content-Type", "application/json")
.json(&payload)
.send()
.await?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
error!("LLM 接口调用失败: 状态码={}, 报错={}", status, body);
return Err(anyhow::anyhow!("大模型接口返回错误状态: {}", status));
}
#[derive(Deserialize)]
struct Message {
content: String,
}
#[derive(Deserialize)]
struct Choice {
message: Message,
}
#[derive(Deserialize)]
struct LLMResponse {
choices: Vec<Choice>,
}
let res_data: LLMResponse = response.json().await?;
if let Some(choice) = res_data.choices.first() {
Ok(choice.message.content.clone())
} else {
Err(anyhow::anyhow!("大模型返回空翻译选项集"))
}
}
}
#[derive(Clone, Debug)]
pub struct EmbeddingClient {
api_key: String,
api_base: String,
model: String,
client: Client,
}
impl EmbeddingClient {
pub fn new(api_key: String, api_base: String, model: String) -> Self {
EmbeddingClient {
api_key,
api_base,
model,
client: Client::new(),
}
}
pub fn model(&self) -> &str {
&self.model
}
pub fn api_base(&self) -> &str {
&self.api_base
}
pub fn api_key(&self) -> &str {
&self.api_key
}
pub async fn create_embedding(&self, text: &str) -> anyhow::Result<Vec<f32>> {
let url = format!("{}/embeddings", self.api_base);
let payload = serde_json::json!({
"model": self.model,
"input": text,
});
let response = self.client.post(&url)
.header("Authorization", format!("Bearer {}", self.api_key))
.header("Content-Type", "application/json")
.json(&payload)
.send()
.await?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
error!("Embedding 接口调用失败: 状态码={}, 报错={}", status, body);
return Err(anyhow::anyhow!("向量接口返回错误状态: {}", status));
}
#[derive(Deserialize)]
struct EmbeddingData {
embedding: Vec<f32>,
}
#[derive(Deserialize)]
struct EmbeddingResponse {
data: Vec<EmbeddingData>,
}
let res_data: EmbeddingResponse = response.json().await?;
if let Some(data) = res_data.data.first() {
Ok(data.embedding.clone())
} else {
Err(anyhow::anyhow!("向量接口返回空向量数据"))
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_llm_client_initialization() {
let client = LlmClient::new(
"key".to_string(),
"base".to_string(),
"model".to_string(),
);
assert_eq!(client.api_key(), "key");
assert_eq!(client.api_base(), "base");
assert_eq!(client.model(), "model");
}
#[test]
fn test_embedding_client_initialization() {
let client = EmbeddingClient::new(
"key".to_string(),
"base".to_string(),
"model".to_string(),
);
assert_eq!(client.api_key(), "key");
assert_eq!(client.api_base(), "base");
assert_eq!(client.model(), "model");
}
}
src/clients/mod.rs
+1
View File
@@ -1,3 +1,4 @@
pub mod ads;
pub mod arxiv;
pub mod qiniu;
pub mod llm;
src/lib.rs
+13
View File
@@ -10,6 +10,9 @@ pub struct Config {
pub llm_api_key: String, // 大语言模型 API Key
pub llm_api_base: String, // 大语言模型 API 基础地址
pub llm_model: String, // 调用的翻译大模型名称
pub embedding_api_key: String, // 向量模型 API Key
pub embedding_api_base: String,// 向量模型 API 基础地址
pub embedding_model: String, // 向量模型名称
pub qiniu_ak: String, // 七牛云 Access Key
pub qiniu_sk: String, // 七牛云 Secret Key
pub qiniu_bucket: String, // 七牛云存储空间名 (Bucket)
@@ -34,6 +37,13 @@ impl Config {
let llm_model = env::var("LLM_MODEL")
.unwrap_or_else(|_| "gpt-4o-mini".to_string());
let embedding_api_key = env::var("EMBEDDING_API_KEY")
.unwrap_or_else(|_| llm_api_key.clone());
let embedding_api_base = env::var("EMBEDDING_API_BASE")
.unwrap_or_else(|_| llm_api_base.clone());
let embedding_model = env::var("EMBEDDING_MODEL")
.unwrap_or_else(|_| "text-embedding-3-small".to_string());
let qiniu_ak = env::var("QINIU_AK").unwrap_or_default();
let qiniu_sk = env::var("QINIU_SK").unwrap_or_default();
let qiniu_bucket = env::var("QINIU_BUCKET").unwrap_or_default();
@@ -56,6 +66,9 @@ impl Config {
llm_api_key,
llm_api_base,
llm_model,
embedding_api_key,
embedding_api_base,
embedding_model,
qiniu_ak,
qiniu_sk,
qiniu_bucket,
src/main.rs
+15 -1
View File
@@ -16,6 +16,7 @@ use astroresearch::services::translation::Dictionary;
use astroresearch::clients::qiniu::QiniuClient;
use astroresearch::clients::ads::AdsClient;
use astroresearch::clients::arxiv::ArxivClient;
use astroresearch::clients::llm::{LlmClient, EmbeddingClient};
use astroresearch::services::download::Downloader;
use astroresearch::api::handlers::{AppState, self};
@@ -83,6 +84,16 @@ async fn main() -> anyhow::Result<()> {
let ads = AdsClient::new(config.ads_api_key.clone());
let arxiv = ArxivClient::new();
let downloader = Downloader::new();
let llm = LlmClient::new(
config.llm_api_key.clone(),
config.llm_api_base.clone(),
config.llm_model.clone(),
);
let embedding = EmbeddingClient::new(
config.embedding_api_key.clone(),
config.embedding_api_base.clone(),
config.embedding_model.clone(),
);
let app_state = Arc::new(AppState {
config: config.clone(),
@@ -91,6 +102,8 @@ async fn main() -> anyhow::Result<()> {
qiniu,
ads,
arxiv,
llm,
embedding,
downloader,
harvest_status: Arc::new(tokio::sync::Mutex::new(astroresearch::services::batch_sync::MetaSyncStatus::new())),
process_status: Arc::new(tokio::sync::Mutex::new(astroresearch::services::batch_sync::AssetSyncStatus::new())),
@@ -106,7 +119,7 @@ async fn main() -> anyhow::Result<()> {
let api_routes = Router::new()
.route("/search", get(handlers::search_papers))
.route("/download", post(handlers::download_paper))
.route("/upload", post(handlers::upload_paper_file))
.route("/upload", post(handlers::upload_paper_file).layer(axum::extract::DefaultBodyLimit::max(100 * 1024 * 1024)))
.route("/no_resource", post(handlers::mark_no_resource))
.route("/parse", post(handlers::parse_paper))
.route("/translate", post(handlers::translate_paper))
@@ -133,6 +146,7 @@ async fn main() -> anyhow::Result<()> {
let app = Router::new()
.nest("/api", api_routes)
.nest_service("/api/files", ServeDir::new(&config.library_dir))
.fallback_service(serve_dir)
.layer(cors)
.layer(tower_http::trace::TraceLayer::new_for_http())
src/services/batch/asset.rs
+19 -4
View File
@@ -72,6 +72,11 @@ impl AssetSync {
status: Arc<Mutex<AssetSyncStatus>>,
) {
tokio::spawn(async move {
let llm_client = crate::clients::llm::LlmClient::new(
config.llm_api_key.clone(),
config.llm_api_base.clone(),
config.llm_model.clone(),
);
let total = bibcodes.len() as i32;
{
let mut s = status.lock().await;
@@ -177,7 +182,17 @@ impl AssetSync {
}
let (pdf_res, html_res) = if !arxiv_id.is_empty() {
downloader.download_arxiv_direct(&arxiv_id, &config.library_dir).await
let res = downloader.download_arxiv_direct(&arxiv_id, &config.library_dir).await;
if res.0.is_ok() || res.1.is_ok() {
res
} else {
{
let mut s = status.lock().await;
s.add_log(format!("文献 {} arXiv 通道下载失败，回退尝试 ADS/出版商下载...", bibcode));
}
let doi_opt = if !doi.is_empty() { Some(doi.as_str()) } else { None };
downloader.download_paper(&bibcode, doi_opt, &config.library_dir).await
}
} else {
let doi_opt = if !doi.is_empty() { Some(doi.as_str()) } else { None };
downloader.download_paper(&bibcode, doi_opt, &config.library_dir).await
@@ -186,11 +201,11 @@ impl AssetSync {
if pdf_res.is_ok() || html_res.is_ok() {
let pdf_rel = match pdf_res {
Ok(p) => Some(p.strip_prefix(&config.library_dir).unwrap_or(&p).to_string_lossy().to_string()),
Err(e) => Some(format!("error: {}", e)),
Err(_) => None, // 只要有一方下载成功，失败的一方字段置空（NULL），避免在 path 字段中留存报错日志
};
let html_rel = match html_res {
Ok(p) => Some(p.strip_prefix(&config.library_dir).unwrap_or(&p).to_string_lossy().to_string()),
Err(e) => Some(format!("error: {}", e)),
Err(_) => None, // 只要有一方下载成功，失败的一方字段置空（NULL），避免在 path 字段中留存报错日志
};
// 更新路径变量与数据库
@@ -504,7 +519,7 @@ impl AssetSync {
match fs::read_to_string(&md_abs) {
Ok(english_markdown) => {
match crate::services::translation::translate_markdown(&english_markdown, &dict, &config).await {
match crate::services::translation::translate_markdown(&english_markdown, &dict, &llm_client).await {
Ok(translated_markdown) => {
let tr_filename = format!("{}_zh.md", bibcode);
let tr_dest = config.library_dir.join("Translation").join(&tr_filename);
src/services/download.rs
+47 -9
View File
@@ -84,21 +84,29 @@ fn build_chrome_headers(referer: Option<&str>) -> HeaderMap {
/// 统一验证码/反爬虫检测（参考 SearXNG 异常处理机制）
fn detect_anti_bot(content: &str, url: Option<&str>) -> Result<()> {
// 如果页面长度大于 150KB，通常是完整渲染的文献正文，忽略反爬/人机验证特征检测以避免误伤（例如正常页面中嵌有 recaptcha 的 sitekey 配置）
if content.len() > 150_000 {
return Ok(());
}
let lower = content.to_lowercase();
let cf_patterns = [
// 1. 强特征防爬与 WAF 挑战（任何小于 150KB 的内容都做检测）
let waf_patterns = [
"checking your browser", "please wait while we verify",
"cf-browser-verification", "cf_chl_opt", "just a moment",
"enable javascript and cookies", "_cf_chl_tk",
"awswafintegration", "aws waf",
];
for p in &cf_patterns {
for p in &waf_patterns {
if lower.contains(p) {
anyhow::bail!("检测到 Cloudflare 或 AWS WAF 挑战页面（特征: {}）", p);
anyhow::bail!("检测到 Cloudflare 或 AWS WAF 挑战/错误页面（特征: {}）", p);
}
}
let captcha_patterns = ["captcha", "recaptcha", "hcaptcha", "verify you are human", "robot check"];
let captcha_patterns = [
"captcha", "recaptcha", "hcaptcha", "verify you are human", "robot check",
"radware bot manager", "shieldsquare",
];
for p in &captcha_patterns {
if lower.contains(p) {
anyhow::bail!("检测到人机验证页面（包含: {}）", p);
@@ -122,6 +130,20 @@ fn detect_anti_bot(content: &str, url: Option<&str>) -> Result<()> {
}
}
// 2. 通用 HTTP 错误与 CDN 关键字检测（仅当内容长度小于 5000 字节时检测，避免在正常文献中误判 CDN 脚本等）
if content.len() < 5000 {
let err_patterns = [
"cloudflare", "service temporarily unavailable", "503 service",
"502 bad gateway", "504 gateway timeout", "403 forbidden",
"404 not found", "500 internal server error", "site error",
];
for p in &err_patterns {
if lower.contains(p) {
anyhow::bail!("检测到服务错误或防护页面（特征: {}）", p);
}
}
}
Ok(())
}
@@ -348,11 +370,19 @@ impl Downloader {
});
match handle.await {
Ok(res) => {
Ok(Ok(())) => {
info!("[Obscura 进程内后备通道] 下载并校验成功: {:?}", dest_path);
res
Ok(())
}
Ok(Err(e)) => {
warn!("[Obscura 进程内后备通道] 下载或校验失败: {:?}", e);
let _ = std::fs::remove_file(dest_path); // 清理校验失败的残留文件
Err(e)
}
Err(e) => {
let _ = std::fs::remove_file(dest_path); // 清理校验失败的残留文件
anyhow::bail!("进程内 Obscura 执行线程异常退出: {:?}", e)
}
Err(e) => anyhow::bail!("进程内 Obscura 执行线程异常退出: {:?}", e),
}
}
@@ -373,16 +403,23 @@ impl Downloader {
.context("启动 Obscura 进程失败，请检查 bin/obscura 是否存在且有执行权限")?;
if !status.success() {
let _ = tokio::fs::remove_file(dest_path).await; // 清理可能的残留坏文件
anyhow::bail!("Obscura 进程退出状态非成功: {:?}", status);
}
// 校验下载得到的文件
if is_pdf {
let bytes = tokio::fs::read(dest_path).await?;
validate_pdf_content(&bytes)?;
if let Err(e) = validate_pdf_content(&bytes) {
let _ = tokio::fs::remove_file(dest_path).await; // 清理校验失败的残留文件
return Err(e);
}
} else {
let text = tokio::fs::read_to_string(dest_path).await?;
validate_html_content(&text)?;
if let Err(e) = validate_html_content(&text) {
let _ = tokio::fs::remove_file(dest_path).await; // 清理校验失败的残留文件
return Err(e);
}
}
info!("[Obscura 命令行后备通道] 下载并校验成功: {:?}", dest_path);
@@ -604,6 +641,7 @@ impl Downloader {
Ok(())
}
Err(e) => {
let _ = tokio::fs::remove_file(dest_path).await; // 清理直连失败的残留物理文件
let err_msg = e.to_string();
if err_msg.contains("人机验证")
|| err_msg.contains("挑战页面")
src/services/translation.rs
+11 -61
View File
@@ -3,10 +3,7 @@ use std::collections::{HashMap, HashSet};
use std::fs::File;
use std::io::{BufRead, BufReader};
use std::path::Path;
use serde::Deserialize;
use tracing::{info, warn, error};
use crate::Config;
use tracing::{info, warn};
// 天文学专有名词英汉词典匹配管理
#[derive(Clone, Debug)]
@@ -102,9 +99,9 @@ impl Dictionary {
pub async fn translate_markdown(
markdown_content: &str,
dict: &Dictionary,
config: &Config
llm_client: &crate::clients::llm::LlmClient,
) -> anyhow::Result<String> {
if config.llm_api_key.is_empty() {
if llm_client.api_key().is_empty() {
return Err(anyhow::anyhow!("本地配置中缺少 LLM_API_KEY"));
}
@@ -129,63 +126,16 @@ pub async fn translate_markdown(
terms_instruction
);
info!("正在请求大模型开展中英翻译。所选大模型: {}", config.llm_model);
info!("正在请求大模型开展中英翻译。所选大模型: {}", llm_client.model());
let start_time = std::time::Instant::now();
let client = reqwest::Client::new();
let url = format!("{}/chat/completions", config.llm_api_base);
let payload = serde_json::json!({
"model": config.llm_model,
"messages": [
{
"role": "system",
"content": system_prompt
},
{
"role": "user",
"content": markdown_content
}
],
"temperature": 0.3
});
let response = client.post(&url)
.header("Authorization", format!("Bearer {}", config.llm_api_key))
.header("Content-Type", "application/json")
.json(&payload)
.send()
.await?;
if !response.status().is_success() {
let status = response.status();
let body = response.text().await.unwrap_or_default();
error!("LLM 翻译接口调用失败: 状态码={}, 报错={}", status, body);
return Err(anyhow::anyhow!("大模型接口返回错误状态: {}", status));
}
#[derive(Deserialize)]
struct Message {
content: String,
}
#[derive(Deserialize)]
struct Choice {
message: Message,
}
#[derive(Deserialize)]
struct LLMResponse {
choices: Vec<Choice>,
}
let res_data: LLMResponse = response.json().await?;
if let Some(choice) = res_data.choices.first() {
let duration = start_time.elapsed();
info!("LLM 翻译成功。所选大模型: {}, 耗时: {:?}, 译文字符数: {}", config.llm_model, duration, choice.message.content.len());
Ok(choice.message.content.clone())
} else {
Err(anyhow::anyhow!("大模型返回空翻译选项集"))
match llm_client.chat_completion(&system_prompt, markdown_content).await {
Ok(translated) => {
let duration = start_time.elapsed();
info!("LLM 翻译成功。所选大模型: {}, 耗时: {:?}, 译文字符数: {}", llm_client.model(), duration, translated.len());
Ok(translated)
}
Err(e) => Err(e),
}
}