feat: RAG 文献问答、天体目标识别、解析器模块化与批量管线扩展

核心新增： - RAG 问答系统：Markdown 安全切片器 (LaTeX 保护) + 向量化 + sqlite-vec 检索 + LLM 生成 - 天体目标识别：IAU 标准正则提取 15+ 星表标识符，CDS SIMBAD/Sesame 查询与本地缓存 - 多模态 LLM：chat_completion_with_image 支持图表视觉分析 - CLI Skills Agent (cli.rs)：对外暴露 rag/target/ingest 等 5 个子命令 - 解析器模块化重构：单体 778 行 → 按期刊拆分 (A&A/ar5iv/IOP/Generic/PDF) + common.rs 静态正则工具库管线与 Schema： - AssetSync→AssetBatch 重命名，批量管线新增 embed/target 两个处理阶段 - 新增 paper_chunks_content (RAG 切片) 和 paper_targets (天体缓存) 两张表 - StandardPaper 新增 has_vector 字段，所有查询同步更新前端： - 新增 AI 助手侧边栏 (RAG 问答 + 来源跳转高亮) - 最近浏览文献列表 (localStorage 持久化)、跨面板无缝导航 - SyncPanel 批量阶段扩展为下拉选项，支持向量化/天体识别测试与清理： - 集成测试合并至 ads.rs 和 llm.rs，新增 chunker + target 单元测试 15 个 - 删除旧单体 parser.rs、独立测试文件及过期 scratch 脚本
2026-06-15 13:25:18 +08:00
parent 3f1935678b
commit 22e7e1dcee
45 changed files with 5289 additions and 1411 deletions
@@ -1,151 +0,0 @@
-# scratch/audit_anomalies.py
-import os
-import re
-
-LOG_PATH = "/home/fmq/.gemini/antigravity/brain/4e405818-ae6d-46f6-a14a-d59613a4ee1c/.system_generated/tasks/task-914.log"
-LIBRARY_DIR = "/home/fmq/program/AstroResearch/library"
-
-def parse_flagged_files(log_path):
-    html_files = []
-    pdf_files = []
-    
-    if not os.path.exists(log_path):
-        print(f"Log path not found: {log_path}")
-        return html_files, pdf_files
-        
-    with open(log_path, 'r', encoding='utf-8') as f:
-        content = f.read()
-        
-    # Match lines like: ❌ 发现磁盘上损坏的 HTML 文件: "HTML/2010AIPC.1273..269M.html"
-    html_matches = re.findall(r'发现磁盘上损坏的 HTML 文件:\s*"([^"]+)"', content)
-    pdf_matches = re.findall(r'发现磁盘上损坏的 PDF 文件:\s*"([^"]+)"', content)
-    
-    # Remove duplicates
-    html_files = sorted(list(set(html_matches)))
-    pdf_files = sorted(list(set(pdf_matches)))
-    
-    return html_files, pdf_files
-
-def get_html_title(text):
-    match = re.search(r'<title[^>]*>(.*?)</title>', text, re.IGNORECASE | re.DOTALL)
-    if match:
-        return match.group(1).strip()
-    return "No Title"
-
-def audit_html(file_path):
-    if not os.path.exists(file_path):
-        return "File Missing", 0, ""
-        
-    size = os.path.getsize(file_path)
-    if size == 0:
-        return "Empty File (0 bytes)", size, ""
-        
-    try:
-        with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
-            content = f.read()
-    except Exception as e:
-        return f"Read Error: {e}", size, ""
-        
-    title = get_html_title(content)
-    lower = content.to_lowercase() if hasattr(content, 'to_lowercase') else content.lower()
-    
-    # Determine reason
-    if "just a moment" in lower or "please wait while we verify" in lower:
-        return "Cloudflare Turnstile WAF Block Page", size, title
-    if "radware bot manager" in lower:
-        return "Radware Bot Manager Captcha Page", size, title
-    if "aws waf" in lower or "awswafintegration" in lower:
-        return "AWS WAF Block Page", size, title
-    if "purchase access" in lower or "buy-box" in lower or "subscription required" in lower:
-        return "Publisher Paywall / Purchase Prompt", size, title
-    if "redirecting" in lower or "http-equiv=\"refresh\"" in lower:
-        return "HTML Redirect Page", size, title
-    if "conversion to html had a fatal error" in lower:
-        return "ar5iv Conversion Failed Stub Page", size, title
-    
-    # Check sections/references
-    has_sections = any(x in lower for x in ["ltx_title_section", "class=\"section\"", "## introduction", "<h2>introduction", "<h3>introduction", "class=\"ltx_section\""])
-    has_bib = any(x in lower for x in ["ltx_bibliography", "class=\"references\"", "<ol class=\"references\"", "<ul class=\"references\"", "id=\"bib\""])
-    
-    if size < 50000 and not (has_sections or has_bib):
-        return f"Snippet / Abstract Page (Missing sections/references, size={size}B)", size, title
-        
-    return "Valid HTML Content?", size, title
-
-def audit_pdf(file_path):
-    if not os.path.exists(file_path):
-        return "File Missing", 0, ""
-        
-    size = os.path.getsize(file_path)
-    if size == 0:
-        return "Empty File (0 bytes)", size, ""
-        
-    try:
-        with open(file_path, 'rb') as f:
-            header = f.read(512)
-    except Exception as e:
-        return f"Read Error: {e}", size, ""
-        
-    if not header.startswith(b"%PDF"):
-        if header.startswith(b"<!") or header.startswith(b"<html") or header.startswith(b"<HTML"):
-            # It's an HTML file disguised as a PDF
-            html_text = header.decode('utf-8', errors='ignore')
-            title = get_html_title(html_text)
-            lower = html_text.lower()
-            if "just a moment" in lower or "cloudflare" in lower:
-                return "HTML Disguised as PDF (Cloudflare WAF Block)", size, title
-            if "radware" in lower:
-                return "HTML Disguised as PDF (Radware Captcha)", size, title
-            if "open journal systems" in lower or "pkp_page_article" in lower:
-                return "HTML Disguised as PDF (OJS Viewer Page)", size, title
-            return "HTML Disguised as PDF (Unknown Webpage)", size, title
-        return "Corrupted / Missing %PDF Header Magic Number", size, ""
-        
-    # Check tail EOF
-    try:
-        with open(file_path, 'rb') as f:
-            f.seek(max(0, size - 1024))
-            tail = f.read(1024)
-    except Exception as e:
-        return f"Read Error seeking tail: {e}", size, ""
-        
-    if b"%%EOF" not in tail:
-        return "Corrupted PDF (Missing tail %%EOF marker)", size, ""
-        
-    if size < 5000:
-        return f"PDF Too Small ({size}B, likely error page)", size, ""
-        
-    return "Valid PDF Content?", size, ""
-
-def main():
-    html_files, pdf_files = parse_flagged_files(LOG_PATH)
-    print(f"Parsed {len(html_files)} HTML files and {len(pdf_files)} PDF files from log.")
-    
-    html_results = []
-    for rel_path in html_files:
-        abs_path = os.path.join(LIBRARY_DIR, rel_path)
-        status, size, title = audit_html(abs_path)
-        html_results.append((rel_path, status, size, title))
-        
-    pdf_results = []
-    for rel_path in pdf_files:
-        abs_path = os.path.join(LIBRARY_DIR, rel_path)
-        status, size, title = audit_pdf(abs_path)
-        pdf_results.append((rel_path, status, size, title))
-        
-    print("\n--- HTML AUDIT REPORT ---")
-    print("| File | Audit Status | Size (Bytes) | HTML Title |")
-    print("| --- | --- | --- | --- |")
-    for file, status, size, title in html_results:
-        clean_title = title.replace("|", "\\|").replace("\n", " ")
-        print(f"| {file} | {status} | {size} | {clean_title} |")
-        
-    print("\n--- PDF AUDIT REPORT ---")
-    print("| File | Audit Status | Size (Bytes) | HTML Title (if HTML) |")
-    print("| --- | --- | --- | --- |")
-    for file, status, size, title in pdf_results:
-        clean_title = title.replace("|", "\\|").replace("\n", " ")
-        print(f"| {file} | {status} | {size} | {clean_title} |")
-
-if __name__ == "__main__":
-    main()
@@ -1,38 +0,0 @@
-import os
-import re
-import sys
-
-# Define a regex for emojis
-# This pattern matches common emoji characters
-emoji_pattern = re.compile(
-    "["
-    "\U00010000-\U0010ffff"  # All supplementary Unicode characters (includes most emojis)
-    "\u2600-\u27bf"          # Miscellaneous symbols, dingbats
-    "\u2300-\u23ff"          # Miscellaneous technical
-    "\u2b50"                 # Medium white star
-    "\u2934-\u2935"          # Arrows
-    "\u3297"                 # Congratulation sign in circle
-    "\u3299"                 # Secret sign in circle
-    "]",
-    flags=re.UNICODE
-)
-
-src_dir = "/home/fmq/program/AstroResearch/dashboard/src"
-found = False
-
-for root, dirs, files in os.walk(src_dir):
-    for file in files:
-        if file.endswith(('.tsx', '.ts')):
-            path = os.path.join(root, file)
-            try:
-                with open(path, 'r', encoding='utf-8') as f:
-                    for line_num, line in enumerate(f, 1):
-                        matches = emoji_pattern.findall(line)
-                        if matches:
-                            print(f"{path}:{line_num}: {' '.join(matches)} -> {line.strip()}")
-                            found = True
-            except Exception as e:
-                print(f"Error reading {path}: {e}")
-
-if not found:
-    print("No emojis found.")