AstroResearch/src/services/parser/mod.rs

// JournalParser trait + 期刊自动检测 + HTML→Markdown 解析管线
mod ar5iv;
mod aanda;
mod iop;
mod generic;
pub mod common;
pub mod pdf;

use std::path::Path;
use tracing::info;

use ar5iv::Ar5ivParser;
use aanda::AandaParser;
use iop::IopParser;
use generic::GenericParser;

/// 期刊解析器 trait —— 每种期刊实现自己特有的预处理规则
trait JournalParser {
    /// 解析器名称（用于日志）
    fn name(&self) -> &str { "未知" }
    /// 从完整 HTML 页面中提取正文区域
    fn extract_body<'a>(&self, html: &'a str) -> &'a str;
    /// 删除页面框架垃圾（导航、面包屑、元数据表格、JS、广告）
    fn clean_chrome(&self, html: &str) -> String;
    /// 清理期刊特有的标记（机构上标、页眉、引用链接格式）
    fn clean_markers(&self, html: &str) -> String;
    /// 将期刊特有的标题标记转换为 Markdown 标题
    fn convert_headings(&self, html: &str) -> String;
    /// 将期刊特有的图注/图片结构转换为 !\[caption\](url)
    fn convert_figures(&self, html: &str, base_origin: &str) -> String;
}

/// 自动检测期刊类型并返回对应的解析器
/// 优先级：ar5iv > IOP > A&A > 通用回退
fn detect_parser(html: &str) -> Box<dyn JournalParser> {
    if Ar5ivParser::detect(html) { return Box::new(Ar5ivParser); }
    if IopParser::detect(html)    { return Box::new(IopParser); }
    if AandaParser::detect(html)  { return Box::new(AandaParser); }
    Box::new(GenericParser)
}

// 将 PDF/MinerU 函数重新导出到 parser 模块层级
pub use pdf::{submit_pdf_to_mineru, poll_and_extract_mineru, parse_pdf_via_mineru};

/// HTML → Markdown 核心解析管线
pub fn html_to_markdown(html_path: &Path) -> anyhow::Result<String> {
    info!("正在解析 HTML → Markdown: {:?}", html_path);
    let html_bytes = std::fs::read(html_path)?;

    // Gzip 解压检测
    let decompressed = if html_bytes.starts_with(&[0x1f, 0x8b]) {
        use std::io::Read;
        let mut decoder = flate2::read::GzDecoder::new(&html_bytes[..]);
        let mut buf = Vec::new();
        decoder.read_to_end(&mut buf)?;
        buf
    } else {
        html_bytes
    };

    let html_content = String::from_utf8_lossy(&decompressed).into_owned();
    let base_origin = common::extract_html_base_origin(&html_content);

    // 截断页脚
    let html = common::truncate_footer(&html_content);

    // 自动检测期刊类型
    let parser = detect_parser(html);
    info!("检测到期刊类型: {}", parser.name());

    // ── 阶段一：期刊特有预处理 ──
    let body = parser.extract_body(html);
    let html = parser.clean_chrome(body);
    let html = parser.clean_markers(&html);

    // ── 阶段二：公共预处理（实体解码 + 公式提取必须在标题转换之前） ──
    let html = common::decode_html_numeric_entities(&html);
    let (html, img_eqs) = common::extract_img_equations(&html);
    let (html, math_formulas) = common::extract_math_blocks(&html);

    // ── 阶段三：标题与图注转换（公式提取之后执行） ──
    let html = parser.convert_headings(&html);
    let html = parser.convert_figures(&html, &base_origin);

    // 其余通用标记转换
    let html = common::convert_common_markup(&html);
    let html = common::convert_img_tags(&html, &base_origin);
    let html = common::replace_latexml_tables(&html);

    // 行内数学公式 → LaTeX 占位符
    let (html, all_formulas) = common::convert_html_math_to_latex(html, math_formulas);

    // ── 阶段四：核心 html2md 转换 ──
    let mut markdown = html2md::parse_html(&html);

    // ── 阶段五：公式占位符逆序还原 ──
    markdown = common::restore_math_placeholders(&markdown, &all_formulas);
    markdown = common::restore_img_equations(&markdown, &img_eqs);

    // ── 阶段六：Markdown 后处理 ──
    let cleaned = common::postprocess_markdown(&markdown);
    Ok(cleaned)
}

// ── 测试 ──────────────────────────────────────────────────────────
#[cfg(test)]
mod tests {
    use super::html_to_markdown;
    use super::common::{postprocess_markdown, convert_html_tables_to_markdown};
    use std::io::Write;

    #[test]
    fn test_convert_html_tables_to_markdown() {
        let raw_md = "Here is a table:\n<table><tr><td>HJD</td><td>Phase</td></tr><tr><td>143.41</td><td>0.42</td></tr></table>\nEnd of table.";
        let cleaned = postprocess_markdown(raw_md);
        assert!(cleaned.contains("| HJD | Phase |"));
        assert!(cleaned.contains("|---|---|"));
        assert!(cleaned.contains("| 143.41 | 0.42 |"));
    }

    #[test]
    fn test_convert_html_tables_with_merged_cells() {
        let raw_md = "Here is a complex table:\n<table border=\"1\"><tr><td colspan=\"2\">Header</td></tr><tr><td>A</td><td>B</td></tr></table>\nEnd.";
        let cleaned = postprocess_markdown(raw_md);
        assert!(cleaned.contains("<table border=\"1\">"));
        assert!(!cleaned.contains("| Header |"));
    }

    #[test]
    fn test_postprocess_markdown() {
        let dirty = "<div>Hello</div> <span class=\"abc\">World</span> <iframe src=\"abc\"></iframe> <astrobj>V391 Peg</astrobj> [] &lt;math&gt;\n\n\n\n\nNew Paragraph";
        let cleaned = postprocess_markdown(dirty);
        assert_eq!(cleaned, "Hello World  V391 Peg  <math>\n\n\nNew Paragraph");

        let dirty_abstract = "###### Abstract\n\n[Abstract]\n\nHot subdwarfs are core helium burning stars.";
        let cleaned_abstract = postprocess_markdown(dirty_abstract);
        assert!(cleaned_abstract.contains("## Abstract\n\nHot subdwarfs are core"));
        assert!(!cleaned_abstract.contains("[Abstract]"));

        let with_ack = "Some content here.\n\n## Acknowledgments\n\nMany thanks to everyone.";
        assert_eq!(postprocess_markdown(with_ack), "Some content here.");

        let with_ref = "Main text content.\n\n## References\n\n1. Author A, 2024";
        assert_eq!(postprocess_markdown(with_ref), "Main text content.");

        let with_bib = "Main text.\n\n{thebibliography*}\n\n84\nAuthor C";
        assert_eq!(postprocess_markdown(with_bib), "Main text.");

        let with_inline_ack = "Main body.\n\nAcknowledgements. This work has been supported by...";
        assert_eq!(postprocess_markdown(with_inline_ack), "Main body.");

        // 图注与图片连行修复
        let merged_fig = r"![alt](https://example.com/img.png) \> \*\*Figure:\*\* This is a caption.";
        let fixed_fig = postprocess_markdown(merged_fig);
        assert!(fixed_fig.contains("img.png)\n\n> **Figure:**"), "got: '{fixed_fig}'");

        // #bib 链接剥离
        let bib_txt = "([1976](#bib.bib16)) and [1984](#bib.bib30).";
        let bib_fixed = postprocess_markdown(bib_txt);
        assert!(!bib_fixed.contains("](#bib"));
        assert!(bib_fixed.contains("(1976)"));

        // 误报检测：正文中 "Reference objects" 标题不应被截断
        let fp = "We compare with reference objects in Sect. 2.\n\n## 2.4 Reference objects\n\nText.";
        let fixed = postprocess_markdown(fp);
        assert!(fixed.contains("## 2.4 Reference objects"));
    }

    #[test]
    fn test_html_to_markdown() -> anyhow::Result<()> {
        let html = r#"<!DOCTYPE html><html><body><div class="ltx_page_main"><h1>Test</h1><p><strong>bold</strong></p></div></body></html>"#;
        let mut path = std::env::temp_dir();
        path.push("t1.html");
        std::fs::write(&path, html)?;
        let md = html_to_markdown(&path)?;
        let _ = std::fs::remove_file(&path);
        assert!(md.contains("Test"));
        assert!(md.contains("**bold**"));
        Ok(())
    }

    #[test]
    fn test_math_and_table() -> anyhow::Result<()> {
        let html = r#"<div class="ltx_page_main"><p><math alttext="\approx" display="inline"><mo>≈</mo></math></p><span class="ltx_tabular"><span class="ltx_tr"><span class="ltx_td">A</span></span></span></div>"#;
        let mut path = std::env::temp_dir();
        path.push("t2.html");
        std::fs::write(&path, html)?;
        let md = html_to_markdown(&path)?;
        let _ = std::fs::remove_file(&path);
        assert!(md.contains(r"$\approx$"));
        assert!(md.contains("| A |"));
        Ok(())
    }

    #[test]
    fn test_math_in_headings() -> anyhow::Result<()> {
        let html = r#"<div class="ltx_page_main"><h2 class="ltx_title_section">H <math alttext="\theta_{eff}" display="inline"><annotation>\theta_{eff}</annotation></math></h2><figcaption>F <math alttext="M_\odot" display="inline"><annotation>M_\odot</annotation></math></figcaption></div>"#;
        let mut path = std::env::temp_dir();
        path.push("t3.html");
        std::fs::write(&path, html)?;
        let md = html_to_markdown(&path)?;
        let _ = std::fs::remove_file(&path);
        assert!(md.contains("## H"));
        assert!(md.contains(r"$\theta_{eff}$"));
        assert!(md.contains(r"$M_\odot$"));
        Ok(())
    }

    #[test]
    fn test_aa_full_html_conversion() -> anyhow::Result<()> {
        let html_path = std::path::Path::new("library/HTML/2026A&A...709A..52F.html");
        if !html_path.exists() { eprintln!("Skip: file not found"); return Ok(()); }
        let md = html_to_markdown(html_path)?;
        assert!(!md.contains("var prefix"));
        assert!(!md.contains("[Home](/"));
        assert!(!md.contains("|       Issue      |"));
        assert!(!md.contains("&#955;"));
        assert!(md.trim_start().starts_with("A&A, 709, A52"));
        assert!(md.contains("## Abstract"));
        assert!(md.contains("## 1. Introduction"));
        assert!(md.contains("### 2.1."));
        assert!(md.contains("*Aims.*"));
        assert!(md.contains("aanda.org/articles/"));
        assert!(!md.contains("## Acknowledgments"));
        Ok(())
    }

    #[test]
    fn test_ar5iv_full_html_conversion() -> anyhow::Result<()> {
        let html_path = std::path::Path::new("library/HTML/1103.1435.html");
        if !html_path.exists() { eprintln!("Skip: file not found"); return Ok(()); }
        let md = html_to_markdown(html_path)?;
        assert!(md.contains("x1.png)\n\n> **Figure:**"));
        assert!(!md.contains("[1]"));
        assert!(!md.contains("class=\"ltx_ref\""));
        assert!(!md.contains("pagerange:"));
        assert!(!md.contains("<sup class="));
        Ok(())
    }

    #[test]
    fn bench_parse_speed() {
        for (label, file) in [
            ("ar5iv", "library/HTML/1103.1435.html"),
            ("A&A", "library/HTML/2026A&A...709A..52F.html"),
            ("small", "library/HTML/0804.1287.html"),
        ] {
            let path = std::path::Path::new(file);
            if !path.exists() { continue; }
            let start = std::time::Instant::now();
            let _ = html_to_markdown(path).unwrap();
            let ms = start.elapsed().as_secs_f64() * 1000.0;
            println!("  {label:>6}: {ms:.0}ms");
        }
    }
}