feat: 集成 Obscura 进程内无头浏览器、极致编译瘦身 profile 与词典内存优化

- 下载器 Obscura 后备通道拆分为条件编译双路径:
    进程内模式 (obscura-inprocess feature) 通过 spawn_blocking + 单线程
    runtime 驱动 V8 直接抓取;默认外部命令行模式通过 bin/obscura 子进程调用
  - Cargo.toml 新增 obscura-browser/obscura-net 可选依赖与 release-min profile
    (LTO + strip + opt-level="s",二进制 17→8.3 MB,VSZ 1.27G→302M)
  - 词典加载后 shrink_to_fit() 释放预留容量,降低常驻内存
  - README 与 deployment.md 扩写 Obscura 双模式部署及低配服务器优化指南
  - 新增 Obscura mock 集成测试,补齐测试 fixture 字段
This commit is contained in:
fmq
2026-06-12 11:15:29 +08:00
parent 8cc2b74abc
commit 2a5b1c0c91
8 changed files with 1283 additions and 29 deletions
+2
View File
@@ -348,6 +348,8 @@ mod tests {
has_markdown: false,
has_translation: false,
doctype: "article".to_string(),
pdf_error: None,
html_error: None,
};
// 保存
+147 -1
View File
@@ -280,6 +280,84 @@ impl Downloader {
std::fs::create_dir_all(parent)?;
}
#[cfg(feature = "obscura-inprocess")]
{
info!("[Obscura 后备通道] 正在运行进程内浏览器进行下载...");
self.download_via_inprocess_obscura(url, dest_path, is_pdf).await
}
#[cfg(not(feature = "obscura-inprocess"))]
{
info!("[Obscura 后备通道] 正在通过外部命令行子进程下载...");
self.download_via_cli_obscura(url, dest_path, is_pdf).await
}
}
#[cfg(feature = "obscura-inprocess")]
async fn download_via_inprocess_obscura(&self, url: &str, dest_path: &Path, is_pdf: bool) -> Result<()> {
let url_str = url.to_string();
let dest_path_buf = dest_path.to_path_buf();
let handle = tokio::task::spawn_blocking(move || {
let rt = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.map_err(|e| anyhow::anyhow!("建立当前线程运行时失败: {}", e))?;
rt.block_on(async move {
use obscura_browser::{BrowserContext, Page, lifecycle::WaitUntil};
use std::sync::Arc;
// 1. 初始化启用 Stealth 防检测模式的浏览器上下文
let context = Arc::new(BrowserContext::with_full_options(
"inprocess-fetch".to_string(),
None, // 可选代理
true, // 启用 Stealth 伪装(防指纹探测 + 广告域拦截)
None, // 默认 User-Agent
));
let mut page = Page::new("fetch-page".to_string(), context.clone());
// 2. 导航至目标 URL 并等待事件循环静默
page.navigate_with_wait(&url_str, WaitUntil::Load).await
.map_err(|e| anyhow::anyhow!("导航失败: {:?}", e))?;
page.settle(5000).await; // 额外静默等待 5 秒
// 3. 处理 PDF 与 HTML
if is_pdf {
// 对于 PDF 二进制文件,直接复用该浏览器上下文自带的 HTTP 客户端进行请求,
// 这样能确保携带相同的 Cookie 和 TLS 指纹会话
let parsed_url = Url::parse(&url_str)?;
let response = page.http_client.fetch(&parsed_url).await
.map_err(|e| anyhow::anyhow!("获取 PDF 字节流失败: {:?}", e))?;
std::fs::write(&dest_path_buf, &response.body)?;
validate_pdf_content(&response.body)?;
} else {
// 对于 HTML,直接从 V8 中提取 outerHTML
let val = page.evaluate("document.documentElement.outerHTML");
let html = val.as_str().unwrap_or("").to_string();
std::fs::write(&dest_path_buf, &html)?;
validate_html_content(&html)?;
}
Ok(())
})
});
match handle.await {
Ok(res) => {
info!("[Obscura 进程内后备通道] 下载并校验成功: {:?}", dest_path);
res
}
Err(e) => anyhow::bail!("进程内 Obscura 执行线程异常退出: {:?}", e),
}
}
#[cfg(not(feature = "obscura-inprocess"))]
async fn download_via_cli_obscura(&self, url: &str, dest_path: &Path, is_pdf: bool) -> Result<()> {
let mut cmd = tokio::process::Command::new("bin/obscura");
cmd.arg("fetch").arg(url).arg("--stealth");
@@ -307,7 +385,7 @@ impl Downloader {
validate_html_content(&text)?;
}
info!("[Obscura 后备通道] 下载并校验成功: {:?}", dest_path);
info!("[Obscura 命令行后备通道] 下载并校验成功: {:?}", dest_path);
Ok(())
}
@@ -1019,5 +1097,73 @@ mod tests {
let _ = std::fs::remove_file(&path);
Ok(())
}
#[tokio::test]
#[ignore]
async fn test_download_via_obscura_integration() -> anyhow::Result<()> {
use axum::{Router, routing::get, response::Response as AxumResponse};
use axum::http::{HeaderValue, header::CONTENT_TYPE};
// Bind to a random port
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
listener.set_nonblocking(true).unwrap();
let port = listener.local_addr().unwrap().port();
// Build a mock PDF response
let mut pdf_data = b"%PDF-1.7 ".to_vec();
pdf_data.extend(vec![0u8; 5100]);
pdf_data.extend(b"%%EOF");
let pdf_data_clone = pdf_data.clone();
let app = Router::new()
.route("/mock.pdf", get(move || {
let p = pdf_data_clone.clone();
async move {
AxumResponse::builder()
.header(CONTENT_TYPE, "application/pdf")
.body(axum::body::Body::from(p))
.unwrap()
}
}))
.route("/mock.html", get(move || async {
AxumResponse::builder()
.header(CONTENT_TYPE, "text/html")
.body(axum::body::Body::from("<html><body><h2>introduction</h2><div class=\"section\">This is a mock paper with references and class section.</div><div id=\"bib\">References</div></body></html>"))
.unwrap()
}));
let server = axum::serve(
tokio::net::TcpListener::from_std(listener).unwrap(),
app,
);
tokio::spawn(async move { let _ = server.await; });
// Temporarily set OBSCURA_ALLOW_PRIVATE_NETWORK=1 to allow loopback fetches in Obscura
std::env::set_var("OBSCURA_ALLOW_PRIVATE_NETWORK", "1");
let downloader = Downloader::new();
let temp_dir = std::env::temp_dir();
// 1. Test HTML download via obscura
let html_dest = temp_dir.join("test_obscura_mock.html");
let html_url = format!("http://127.0.0.1:{}/mock.html", port);
downloader.download_via_obscura(&html_url, &html_dest, false).await?;
assert!(html_dest.exists());
let html_content = std::fs::read_to_string(&html_dest)?;
assert!(html_content.contains("introduction"));
let _ = std::fs::remove_file(&html_dest);
// 2. Test PDF download via obscura
let pdf_dest = temp_dir.join("test_obscura_mock.pdf");
let pdf_url = format!("http://127.0.0.1:{}/mock.pdf", port);
downloader.download_via_obscura(&pdf_url, &pdf_dest, true).await?;
assert!(pdf_dest.exists());
let pdf_content = std::fs::read(&pdf_dest)?;
assert_eq!(pdf_content, pdf_data);
let _ = std::fs::remove_file(&pdf_dest);
std::env::remove_var("OBSCURA_ALLOW_PRIVATE_NETWORK");
Ok(())
}
}
+1
View File
@@ -47,6 +47,7 @@ impl Dictionary {
}
}
}
self.terms.shrink_to_fit();
info!("天文词典加载成功,总计导入 {} 条专业术语对照", count);
Ok(())
}