From 2c8d0b8f8b97308da80ce80d013837d00ec3d363 Mon Sep 17 00:00:00 2001 From: Asfmq <2696428814@qq.com> Date: Tue, 7 Jul 2026 21:16:46 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20LAMOST=20DR12-14=20=E4=B8=8E=E5=AD=90?= =?UTF-8?q?=E7=89=88=E6=9C=AC=E4=BD=93=E7=B3=BB=E6=8E=A5=E5=85=A5=E3=80=81?= =?UTF-8?q?=E8=A7=82=E6=B5=8B=E5=B1=82=E5=AE=89=E5=85=A8=E5=8A=A0=E5=9B=BA?= =?UTF-8?q?=E4=B8=8E=E5=B9=B6=E5=8F=91=E5=BC=82=E6=AD=A5=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - LAMOST 新增 DR12/13/14 及子版本（v0/v1.0/v1.1/v2.0）维度，Internal 发布标记需登录认证并前端灰显，release×subtype 交叉约束下沉至 capabilities 统一声明 - ObservationFetcher trait 扩展版本/认证/交叉约束能力声明，version 参数贯穿 client→service→API→Agent tool→前端全链路 - 安全：observation cache SQL 全参数绑定 + LIKE 转义、cone_cache_hash 加长度前缀防碰撞、DESI survey/program 白名单防穿越 - 异步化：persist/cached_files_total_size/maybe_persist_tool_result迁移到 tokio::fs；cancelled_runs 与 session_permission_checkers改用 DashMap；auth 读锁优先 + 60s 节流 - Gaia 去 native-tls 改禁用连接池规避 UnexpectedEof，reqwest 移除 native-tls feature - 重构：Source/ProductType from_str 集中解析、download 模块拆分为 try_download_pdf/html、AgentRuntime::init 抽取共享逻辑 - 部署：新增 deploy.sh 一键打包推送脚本、catch-panic 启用 --- CLAUDE.md | 147 ++++++-- Cargo.lock | 168 ++------- Cargo.toml | 5 +- dashboard/index.html | 1 + dashboard/src/App.tsx | 8 +- .../components/agent/SpecialToolRenderers.tsx | 15 +- .../src/components/agent/ToolCallCard.tsx | 7 +- .../observation/ObservationResultCard.tsx | 10 +- dashboard/src/hooks/useObservation.ts | 83 +++-- dashboard/src/pages/ObservationPanel.tsx | 148 ++++++-- dashboard/src/types/index.ts | 6 + dashboard/src/utils/apiError.ts | 5 +- deploy.sh | 86 +++++ docs/deployment.md | 126 ++++--- src/agent/compact.rs | 3 +- src/agent/hooks/builtins.rs | 11 +- src/agent/hooks/mod.rs | 7 +- src/agent/hooks/registry.rs | 2 +- src/agent/runtime/executor/helpers.rs | 1 + src/agent/runtime/executor/mod.rs | 3 +- src/agent/runtime/mod.rs | 153 +++----- src/agent/runtime/session.rs | 14 +- src/agent/runtime/streaming.rs | 5 +- src/agent/runtime/streaming_executor.rs | 17 +- src/agent/skills/creator.rs | 8 +- src/agent/skills/curator.rs | 12 +- src/agent/team/inbox.rs | 34 +- src/agent/team/manager.rs | 18 +- src/agent/team/teammate.rs | 10 +- src/agent/tools/astro/analyze_image.rs | 4 +- src/agent/tools/astro/research/library.rs | 20 +- src/agent/tools/astro/research/mod.rs | 2 +- src/agent/tools/astro/research/note.rs | 1 + src/agent/tools/astro/research/observation.rs | 145 +++++--- src/agent/tools/astro/research/vizier.rs | 73 ++-- src/agent/tools/filesystem/edit.rs | 20 +- src/agent/tools/filesystem/grep.rs | 35 +- src/agent/tools/filesystem/read.rs | 30 +- src/agent/tools/filesystem/write.rs | 13 +- src/agent/tools/persist.rs | 78 ++-- src/agent/tools/team.rs | 7 +- src/agent/tools/todo.rs | 17 +- src/api/agent.rs | 10 +- src/api/auth.rs | 53 ++- src/api/mod.rs | 9 +- src/api/observation.rs | 53 +-- src/api/papers.rs | 5 + src/api/permissions.rs | 44 +-- src/bin/cli.rs | 7 +- src/clients/gaia/mod.rs | 11 +- src/clients/lamost/mod.rs | 23 +- src/clients/llm/chat.rs | 40 +- src/lib.rs | 63 +--- src/main.rs | 22 +- src/services/batch/asset/helpers.rs | 6 +- src/services/batch/asset/process.rs | 5 +- src/services/cds/target.rs | 36 +- src/services/cds/vizier.rs | 19 +- src/services/download/mod.rs | 342 ++++++++--------- src/services/download/strategies.rs | 4 +- src/services/logging.rs | 6 +- src/services/note.rs | 13 +- src/services/observation/cache.rs | 80 ++-- src/services/observation/desi.rs | 167 +++++++-- src/services/observation/dispatch.rs | 124 +++++-- src/services/observation/fetcher.rs | 100 ++++- src/services/observation/gaia.rs | 269 ++++++++++---- src/services/observation/lamost.rs | 348 +++++++++++++++--- src/services/observation/registry.rs | 57 ++- src/services/observation/sdss.rs | 262 +++++++++---- src/services/observation/types.rs | 56 ++- src/services/paper/ingest.rs | 12 + src/services/search.rs | 22 +- src/services/translation.rs | 24 +- src/services/vision.rs | 2 +- 75 files changed, 2482 insertions(+), 1370 deletions(-) create mode 100755 deploy.sh diff --git a/CLAUDE.md b/CLAUDE.md index bf1be51..3003760 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -42,6 +42,9 @@ cargo test --lib # Run a specific test cargo test test_name +# Coverage (requires cargo-llvm-cov) +cargo llvm-cov --fail-under-lines 80 + # Frontend (cd dashboard first) npm run dev # HMR dev server on :5173, proxies /api to :8000 npm run build # TypeScript check + Vite production build → dashboard/dist/ @@ -62,46 +65,103 @@ src/ ├── main.rs # Axum server entry: logging, DB pool, migrations, vec0 table, │ # client/service construction, route registration, AppState assembly ├── lib.rs # Config struct + from_env() loading from .env -├── api/ # HTTP handlers, AppState, StandardPaper type -│ ├── mod.rs # AppState (shared state), StandardPaper, handlers re-exports +├── api/ # HTTP handlers, AppState, re-exports via handlers namespace +│ ├── mod.rs # AppState (shared state), handler re-exports, permission/event types │ ├── agent.rs # SSE chat_agent endpoint, session CRUD, metrics, audit log │ ├── papers.rs # Search, download, parse, translate, embed, citations, library, export +│ ├── catalog.rs # VizieR TAP catalog queries + cone search endpoints +│ ├── observation.rs # Observation data search/download (LAMOST/Gaia/SDSS/DESI) │ ├── notes.rs # Highlight/note CRUD │ ├── sync.rs # Meta-sync and asset-batch endpoints │ ├── targets.rs # Target query/associate/extract, RAG chat, figure chat -│ └── helpers.rs # Shared DB helpers, format conversion, path validation +│ ├── auth.rs # Login/logout/auth-check (token-based, session memory) +│ ├── permissions.rs # Agent tool permission rule management +│ ├── search.rs # Search history endpoint +│ ├── error.rs # Unified API error type (status code + user-safe message) +│ └── helpers.rs # Shared DB helpers, format conversion, path validation (private) ├── agent/ # ReAct-based research agent (LLM-driven tool-use loop) -│ ├── tools/ # AgentTool trait, ToolRegistry, 25+ tool implementations per domain file -│ ├── runtime/ # ReAct loop core: session, context, streaming, executor, token_budget, -│ │ # permission, permission_profile, checkpoint, circuit_breaker, error_recovery, -│ │ # hardline, partitioner, file_cache, system_prompt, finalize, denial_tracker +│ ├── tools/ # AgentTool trait, ToolRegistry, tool implementations per domain +│ │ ├── mod.rs # AgentTool trait, ToolContext (session_id, sse_tx, silent, etc.) +│ │ ├── filesystem/ # read_file, grep_files, write, edit +│ │ ├── astro/ # search_papers, download_paper, parse_paper, rag_search, +│ │ │ # query_target, save_note, analyze_image, research/* +│ │ ├── ask_user.rs # Permission-gated interactive user question +│ │ ├── background.rs# bg_task_run, bg_task_check (async slow tasks) +│ │ ├── compress.rs # Context compression trigger +│ │ ├── memory.rs # save_memory, load_memory +│ │ ├── persist.rs # Data persistence (used by memory/compaction) +│ │ ├── skill.rs # load_skill +│ │ ├── subagent.rs # Context-isolated sub-agent spawner +│ │ ├── team.rs # spawn_teammate, send_teammate_message, etc. +│ │ └── todo.rs # todo_write (task tracking) +│ ├── runtime/ # ReAct loop core +│ │ ├── mod.rs # AgentRuntime: session create/resume → context → ReAct → finalize +│ │ ├── executor/ # Step executor + helpers (tool dispatch, parallel tool calls) +│ │ ├── streaming.rs # SSE AgentStreamEvent types +│ │ ├── streaming_executor.rs # SSE-emitting executor wrapper +│ │ ├── session.rs # Session CRUD, DB persist, agent_config +│ │ ├── token_budget.rs # Soft/hard token limits, tiktoken counting +│ │ ├── permission.rs # Deny > Allow > Ask rule chain, pattern matching +│ │ ├── permission_profile.rs # Named profiles (research, readonly) +│ │ ├── checkpoint.rs # Rewind/retry state snapshots +│ │ ├── circuit_breaker.rs# CompactionCircuitBreaker (cooldown rate limiting) +│ │ ├── error_recovery.rs # Error classification + retry strategies +│ │ ├── hardline.rs # Terminal conditions (too many denials, loops, etc.) +│ │ ├── partitioner.rs # Message window partitioning for compaction +│ │ ├── file_cache.rs # ReadFileState — file content dedup cache +│ │ ├── system_prompt.rs # System prompt assembly (mode, skills, memory, tools) +│ │ ├── finalize.rs # Post-loop cleanup: memory extraction, audit log +│ │ └── denial_tracker.rs # Consecutive denial detection │ ├── compact/ # Context compression (micro/auto/manual layers + collapse) │ ├── memory/ # Persistent memory manager: extraction, dedup, decay, age, guardrails -│ ├── hooks/ # Lifecycle hooks: registry, dispatch, builtins, matcher, traits (PreToolUse/PostToolUse/Stop/etc.) -│ ├── modes/ # Agent session modes: default, deep-research, literature-reader (identity/tools/config presets) +│ ├── hooks/ # Lifecycle hooks: registry, dispatch, builtins, matcher, traits +│ ├── modes/ # Agent session modes (see Agent Session Modes section below) │ ├── skills.rs # SkillRegistry: hot-loads SKILL.md files from skills/ directory -│ ├── subagent.rs # Context-isolated sub-agent runner (subagent tool) +│ ├── subagent.rs # Context-isolated sub-agent runner │ ├── team/ # Multi-agent team: file-based inbox, lead/teammate coordination -│ ├── background.rs# BgNotificationQueue for async slow-task (download/parse) notifications +│ ├── background.rs# BgNotificationQueue for async slow-task notifications │ ├── task_board.rs# Persistent task board (agent_tasks table) │ ├── trajectory.rs# Session trajectory recording for audit/debug │ └── terminal.rs # Escape sequence filter for ANSI-heavy tool outputs -├── clients/ # External API wrappers -│ ├── llm.rs # LlmClient (OpenAI-compatible chat + streaming), EmbeddingClient +├── clients/ # External API wrappers (pure communication, no business logic) +│ ├── llm/ # LlmClient (OpenAI-compatible chat + streaming), EmbeddingClient │ ├── ads.rs # NASA ADS API │ ├── arxiv.rs # arXiv Atom XML API +│ ├── cds/ # CDS services (sesame.rs — Sesame name resolver; vizier.rs — TAP queries) +│ ├── gaia/ # Gaia archive (TAP + DataLink) +│ ├── lamost/ # LAMOST archive (ConeSearch + FITS.gz download) +│ ├── sdss/ # SDSS archive (Data Lab TAP + SAS bulk download) +│ ├── desi/ # DESI archive (Data Lab TAP + HEALPix coadd SAS) +│ ├── vo/ # Virtual Observatory generic client │ └── qiniu.rs # Qiniu cloud storage ├── services/ # Business logic -│ ├── search.rs # Unified cross-source search (ADS + arXiv dedup) -│ ├── download.rs # PDF/HTML download with anti-bot measures and fallback chain -│ ├── parser/ # HTML/PDF → Markdown parsers (A&A, IOP, ar5iv, generic, PDF via MinerU) -│ ├── translation.rs# LLM bilingual translation with Trie-based astronomy glossary -│ ├── rag.rs # Embedding ingest + vector similarity retrieval + LLM answer generation -│ ├── target.rs # Celestial target extraction (IAU name regex) + CDS Sesame lookup -│ ├── chunker.rs # Markdown text chunking for embedding -│ ├── batch/ # Meta-sync (ADS bulk harvest) and asset-batch processing engines -│ ├── query_parser.rs# Advanced search query syntax parser -│ └── logging.rs # Pretty console + rolling file logger +│ ├── search.rs # Unified cross-source search (ADS + arXiv dedup) +│ ├── download.rs # PDF/HTML download with anti-bot measures and fallback chain +│ ├── parser/ # HTML/PDF → Markdown parsers (A&A, IOP, ar5iv, generic, PDF via MinerU) +│ ├── paper/ # Paper lifecycle: model, db (CRUD), ingest (ADS/arXiv→DB), reader (markdown) +│ ├── translation.rs # LLM bilingual translation with Trie-based astronomy glossary +│ ├── rag.rs # Embedding ingest + vector similarity retrieval + LLM answer generation +│ ├── cds/ # CDS business layer: target.rs (Sesame name resolver), vizier.rs (TAP + cache) +│ ├── observation/ # Cross-source observation data (spectra, light curves, photometry, images) +│ │ ├── mod.rs # 双轴正交架构: Source × ProductType → ObservationFetcher → Registry +│ │ ├── types.rs # Source/ProductType/ProductSpec/Artifact/ObservationProduct/Candidate +│ │ ├── fetcher.rs # ObservationFetcher trait + cone search cache template methods +│ │ ├── registry.rs # ObservationRegistry (fetcher registration, capability discovery) +│ │ ├── dispatch.rs # search_observation + download_observation (unified entry points) +│ │ ├── cache.rs # observation_cache table read/write + file staging +│ │ ├── lamost.rs # LamostSpectrumFetcher +│ │ ├── gaia.rs # GaiaSpectrumFetcher + GaiaLightCurveFetcher +│ │ ├── sdss.rs # SdssSpectrumFetcher + SdssApogeeFetcher +│ │ └── desi.rs # DesiSpectrumFetcher +│ ├── batch/ # Meta-sync (ADS bulk harvest) and asset-batch processing engines +│ ├── citation.rs # Citation network graph data (references + citations) +│ ├── session.rs # Agent session listing, metrics, audit log (for API consumption) +│ ├── vision.rs # Image analysis via dedicated vision model +│ ├── note.rs # Highlight/note storage and retrieval +│ ├── pipeline.rs # Synchronous paper processing pipeline (download→parse→translate→embed) +│ ├── chunker.rs # Markdown text chunking for embedding +│ ├── query_parser.rs # Advanced search query syntax parser +│ └── logging.rs # Pretty console + rolling file logger └── bin/ ├── health_check.rs # Library consistency checker and auto-repair ├── cli.rs # CLI interface @@ -113,11 +173,18 @@ src/ All handlers access state via `Arc`. Key fields: - `db: SqlitePool` — SQLite connection pool (5 max connections, foreign keys enforced) -- `llm: LlmClient` / `embedding: EmbeddingClient` — OpenAI-compatible LLM clients +- `llm: LlmClient` / `medium_llm` / `fast_llm` — OpenAI-compatible LLM clients (multi-tier) +- `vision_llm: Option` — dedicated vision model client +- `embedding: EmbeddingClient` — embedding model client - `ads: AdsClient` / `arxiv: ArxivClient` — academic search clients +- `vizier: VizierClient` — CDS VizieR TAP catalog query client +- `lamost` / `gaia` / `sdss` / `desi` — observation data source clients +- `observation_registry: Arc` — cross-source observation fetcher registry - `skill_registry: Arc>` — hot-reloaded agent skills - `memory_manager: Arc` — persistent agent memory (MEMORY.md + decay) -- `cancelled_runs: Arc>>` — agent cancellation tokens +- `cancelled_runs: Arc>` — agent cancellation tokens +- `session_permission_checkers: Arc>` — per-session permission rules +- `pending_questions` / `pending_permissions` — interactive agent-user handshake channels - `harvest_status` / `batch_status` — async batch operation status tracking ### Agent System Design @@ -131,7 +198,7 @@ The agent (`src/agent/`) implements a **ReAct** (Thought → Action → Observat 5. **Tool call ID tracking**: LLM may not return tool_call IDs — `LlmClient` generates UUID fallbacks. `ToolCall` and `ToolResult` SSE events carry matching IDs for precise frontend pairing. 6. **ToolContext** (`tools/mod.rs`): Injected into every tool execution — holds `app_state`, `session_id`, `sse_tx` (for intermediate events), `enable_thinking`, `read_file_state` (file cache for dedup), `silent` (sub-agents skip permission prompts). 7. **Context compression**: Four layers — micro (placeholder replacement), snip (old-message truncation), auto (LLM summarization), aggro_micro (aggressive placeholder). Protected by `CompactionCircuitBreaker`. Transcripts persisted in `agent_messages` table, not filesystem snapshots. -8. **Skills** (`skills.rs`): Two-layer loading — system-reminder lists names (~20 tokens each), LLM calls `load_skill` to inject full SKILL.md content +8. **Skills** (`skills.rs`): Two-layer loading — system-reminder lists names (~20 tokens each), LLM calls `load_skill` to inject full SKILL.md content. Skills are organized into subdirectories: `methodology/`, `plotting/`, `presentation/`. 9. **Sub-agents** (`subagent.rs`): `subagent` tool spawns a context-isolated sub-agent with its own ReAct loop. Sub-agent messages (system/user/assistant/tool) are persisted to `agent_messages` with `agent_name` identifier. Returns final summary + activity log. SSE progress forwarded to parent via ToolContext. 10. **Memory** (`memory/`): File-based persistent memory (MEMORY.md). `MemoryManager` handles extraction from conversation, dedup, recency decay, age-based pruning, and guardrails. Tools: `save_memory`, `load_memory` (auto-injected in system prompt). 11. **Teams** (`team/`): File-based inbox directory per session for lead/teammate message passing @@ -141,17 +208,29 @@ Environment variables for agent tuning: `AGENT_TOKEN_SOFT_LIMIT` (default 80000) ### Database -SQLite via `sqlx::sqlite`. Migrations in `migrations/` are auto-run on startup (`sqlx::migrate!("./migrations")`). Key tables: `papers`, `citations_references`, `notes`, `agent_sessions`, `agent_messages`, `agent_tasks`, `agent_audit_log`, `paper_chunks_content`. Vector embeddings use `sqlite-vec` (`vec_paper_chunks` virtual table, auto-registered before any DB connection). +SQLite via `sqlx::sqlite`. Migrations in `migrations/` are auto-run on startup (`sqlx::migrate!("./migrations")`). Key tables: `papers`, `citations_references`, `notes`, `agent_sessions`, `agent_messages`, `agent_tasks`, `agent_audit_log`, `paper_chunks_content`, `vizier_cache`, `observation_cache`. Vector embeddings use `sqlite-vec` (`vec_paper_chunks` virtual table, auto-registered before any DB connection). The embedding dimension is controlled by `EMBEDDING_DIM` env var (default 1536). On dimension mismatch, the vec table and chunk content are dropped and recreated. +### Observation Data System (`src/services/observation/`) + +A cross-source astronomical observation data subsystem with a **dual-axis orthogonal** architecture: **Source** (LAMOST/Gaia/SDSS/DESI) × **ProductType** (Spectrum/LightCurve/Photometry/Image). Each valid combination implements an `ObservationFetcher`, registered in `ObservationRegistry`. + +New fetchers follow OCP (pure addition, no modification to existing code): + +1. Create `{source}.rs`, implement `ObservationFetcher` trait +2. Add `register!(YourFetcher)` in `registry.rs::default()` +3. Add new variants to `Source`/`ProductType` enums in `types.rs` if needed + +External callers use `dispatch::search_observation()` (metadata only) and `dispatch::download_observation()` (metadata + file download) via `ObservationRequest`. Results are cached in `observation_cache` table with file staging to `library/`. + ### Frontend (dashboard/) React 19 + TypeScript + Vite + Tailwind CSS 4. Organized by technical layer (Type-Based): -- `pages/` — Page-level panel view components (SearchPanel, LibraryPanel, ReaderPanel, CitationPanel, SyncPanel, ResearchAgentPanel, SettingsPanel) +- `pages/` — Page-level panel view components (SearchPanel, LibraryPanel, ReaderPanel, CitationPanel, SyncPanel, ResearchAgentPanel, ObservationPanel, SettingsPanel) - `components/` — Reusable and layout components (sub-folders: agent, reader, sync, layout, dialogs) -- `hooks/` — Global and feature-specific custom stateful React Hooks (e.g., useLibrary, useSearch, useNotes) +- `hooks/` — Global and feature-specific custom stateful React Hooks (e.g., useLibrary, useSearch, useNotes, useObservation) - `types/` — Global TypeScript type definitions (types/index.ts) - `utils/` — Common utility helper functions - `assets/` — Static assets and global stylesheet styles @@ -160,7 +239,7 @@ Dependencies: `react-markdown` + `rehype-katex` + `remark-math` for Markdown/LaT ### Obscura In-Process Browser -The `obscura-inprocess` feature compiles `obscura-browser` and `obscura-net` directly into the binary, eliminating the need for external browser binaries. This is used for bypassing Cloudflare/WAF on PDF download. Enabled via `--features obscura-inprocess`. +The `obscura-inprocess` feature compiles `obscura-browser` and `obscura-net` from `libs/obscura/` directly into the binary, eliminating the need for external browser binaries. This is used for bypassing Cloudflare/WAF on PDF download. Enabled via `--features obscura-inprocess`. ### Astronomy Glossary (dictionary.txt) @@ -235,6 +314,10 @@ Controlled by env vars: - `EXTRACT_MEMORY_THROTTLE_TURNS` (default `3`) — extract every N turns - `EXTRACT_MEMORY_MAX_STEPS` (default `3`) — sub-agent max steps for extraction +### Deployment (`deploy.sh`) + +The `deploy.sh` script builds the project locally (release-min profile, Composer-based) and deploys to a remote server via rsync + SSH. It handles: source compilation, frontend build, remote directory sync, service restart via docker compose, and health check verification. + ## Code Conventions - Use `anyhow` for application errors, `thiserror` for library-style typed errors @@ -243,3 +326,7 @@ Controlled by env vars: - API handlers take `State(Arc)` and return Axum-compatible responses - Agent tools implement `AgentTool` trait; new tools register in `ToolRegistry::new()` - Front-end build is triggered by `build.rs` (auto npm install + build when `dashboard/src/` changes) +- New observation fetchers follow OCP: add module + register — don't modify existing fetchers or dispatch +- External API clients (`src/clients/`) are pure communication; business logic lives in `src/services/` +- Route registration happens in `src/main.rs`; handler re-exports are maintained in `src/api/mod.rs` → `handlers` namespace +- Agent hooks (`src/agent/hooks/`) implement lifecycles: PreToolUse, PostToolUse, Stop, etc. — register in `builtins.rs` diff --git a/Cargo.lock b/Cargo.lock index 7e421bc..c4493be 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -138,6 +138,7 @@ dependencies = [ "base64 0.22.1", "chrono", "clap", + "dashmap", "dotenvy", "flate2", "futures-util", @@ -481,7 +482,7 @@ checksum = "2c5e60b8c8d282c86360cab651ded04ab0335a7b5390c8d34145cbeab8cacf5f" dependencies = [ "bitflags 2.13.0", "btls-sys", - "foreign-types 0.5.0", + "foreign-types", "libc", "openssl-macros", ] @@ -761,16 +762,6 @@ dependencies = [ "url", ] -[[package]] -name = "core-foundation" -version = "0.10.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -900,6 +891,20 @@ dependencies = [ "cmov", ] +[[package]] +name = "dashmap" +version = "6.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c" +dependencies = [ + "cfg-if", + "crossbeam-utils", + "hashbrown 0.14.5", + "lock_api", + "once_cell", + "parking_lot_core", +] + [[package]] name = "data-encoding" version = "2.11.0" @@ -1242,15 +1247,6 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" -[[package]] -name = "foreign-types" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" -dependencies = [ - "foreign-types-shared 0.1.1", -] - [[package]] name = "foreign-types" version = "0.5.0" @@ -1258,7 +1254,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" dependencies = [ "foreign-types-macros", - "foreign-types-shared 0.3.1", + "foreign-types-shared", ] [[package]] @@ -1272,12 +1268,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "foreign-types-shared" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" - [[package]] name = "foreign-types-shared" version = "0.3.1" @@ -1760,29 +1750,13 @@ dependencies = [ "http", "hyper", "hyper-util", - "rustls 0.23.40", + "rustls 0.23.41", "tokio", "tokio-rustls", "tower-service", "webpki-roots 1.0.7", ] -[[package]] -name = "hyper-tls" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" -dependencies = [ - "bytes", - "http-body-util", - "hyper", - "hyper-util", - "native-tls", - "tokio", - "tokio-native-tls", - "tower-service", -] - [[package]] name = "hyper-util" version = "0.1.20" @@ -2419,23 +2393,6 @@ dependencies = [ "version_check", ] -[[package]] -name = "native-tls" -version = "0.2.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" -dependencies = [ - "libc", - "log", - "openssl", - "openssl-probe", - "openssl-sys", - "schannel", - "security-framework", - "security-framework-sys", - "tempfile", -] - [[package]] name = "new_debug_unreachable" version = "1.0.6" @@ -2624,20 +2581,6 @@ version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" -[[package]] -name = "openssl" -version = "0.10.80" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" -dependencies = [ - "bitflags 2.13.0", - "cfg-if", - "foreign-types 0.3.2", - "libc", - "openssl-macros", - "openssl-sys", -] - [[package]] name = "openssl-macros" version = "0.1.1" @@ -2649,24 +2592,6 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "openssl-probe" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" - -[[package]] -name = "openssl-sys" -version = "0.9.117" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695" -dependencies = [ - "cc", - "libc", - "pkg-config", - "vcpkg", -] - [[package]] name = "outref" version = "0.5.2" @@ -2954,7 +2879,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.23.40", + "rustls 0.23.41", "socket2", "thiserror 2.0.18", "tokio", @@ -2974,7 +2899,7 @@ dependencies = [ "rand 0.9.4", "ring", "rustc-hash", - "rustls 0.23.40", + "rustls 0.23.41", "rustls-pki-types", "slab", "thiserror 2.0.18", @@ -3147,23 +3072,20 @@ dependencies = [ "http-body-util", "hyper", "hyper-rustls", - "hyper-tls", "hyper-util", "js-sys", "log", "mime_guess", - "native-tls", "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.40", + "rustls 0.23.41", "rustls-pki-types", "serde", "serde_json", "serde_urlencoded", "sync_wrapper", "tokio", - "tokio-native-tls", "tokio-rustls", "tokio-util", "tower", @@ -3256,9 +3178,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.40" +version = "0.23.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f" dependencies = [ "once_cell", "ring", @@ -3329,15 +3251,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "schannel" -version = "0.1.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" -dependencies = [ - "windows-sys 0.61.2", -] - [[package]] name = "scopeguard" version = "1.2.0" @@ -3354,29 +3267,6 @@ dependencies = [ "untrusted", ] -[[package]] -name = "security-framework" -version = "3.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" -dependencies = [ - "bitflags 2.13.0", - "core-foundation", - "core-foundation-sys", - "libc", - "security-framework-sys", -] - -[[package]] -name = "security-framework-sys" -version = "2.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" -dependencies = [ - "core-foundation-sys", - "libc", -] - [[package]] name = "selectors" version = "0.26.0" @@ -4210,23 +4100,13 @@ dependencies = [ "syn 2.0.117", ] -[[package]] -name = "tokio-native-tls" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" -dependencies = [ - "native-tls", - "tokio", -] - [[package]] name = "tokio-rustls" version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ - "rustls 0.23.40", + "rustls 0.23.41", "tokio", ] diff --git a/Cargo.toml b/Cargo.toml index 206a5c1..739a20b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,11 +21,11 @@ path = "src/bin/cli.rs" [dependencies] tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "signal", "time", "fs", "net", "sync", "process"] } axum = { version = "0.7", features = ["macros", "multipart"] } -tower-http = { version = "0.5", features = ["cors", "fs", "trace", "set-header"] } +tower-http = { version = "0.5", features = ["cors", "fs", "trace", "set-header", "catch-panic"] } sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "sqlite", "chrono", "json"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" -reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "multipart", "cookies", "rustls-tls", "native-tls"] } +reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "multipart", "cookies", "rustls-tls"] } dotenvy = "0.15" quick-xml = { version = "0.31", features = ["serialize"] } anyhow = "1.0" @@ -62,6 +62,7 @@ glob = "0.3" walkdir = "2" lru = "0.12" git2 = { version = "0.18", default-features = false, features = ["vendored-libgit2"] } +dashmap = "6" [features] default = [] diff --git a/dashboard/index.html b/dashboard/index.html index 9f5c916..c1ae761 100644 --- a/dashboard/index.html +++ b/dashboard/index.html @@ -2,6 +2,7 @@ + diff --git a/dashboard/src/App.tsx b/dashboard/src/App.tsx index 7da86aa..0df1ae0 100644 --- a/dashboard/src/App.tsx +++ b/dashboard/src/App.tsx @@ -61,7 +61,13 @@ export default function App() { // 2. 局部状态定义 (多组件共用或全局网络进度状态) const [activeTab, setActiveTab] = useState< - 'search' | 'library' | 'reader' | 'citation' | 'sync' | 'observation' | 'agent' + | 'search' + | 'library' + | 'reader' + | 'citation' + | 'sync' + | 'observation' + | 'agent' >(() => { const saved = localStorage.getItem('astro_active_tab'); const validTabs = [ diff --git a/dashboard/src/components/agent/SpecialToolRenderers.tsx b/dashboard/src/components/agent/SpecialToolRenderers.tsx index 832ee31..0584d4f 100644 --- a/dashboard/src/components/agent/SpecialToolRenderers.tsx +++ b/dashboard/src/components/agent/SpecialToolRenderers.tsx @@ -642,7 +642,9 @@ export function VizierResultCard({ metadata }: VizierResultCardProps) { if (val === null || val === undefined) return '—'; if (typeof val === 'number') { // 数字保留合理精度 - return Number.isInteger(val) ? String(val) : val.toFixed(6).replace(/\.?0+$/, ''); + return Number.isInteger(val) + ? String(val) + : val.toFixed(6).replace(/\.?0+$/, ''); } if (typeof val === 'boolean') return val ? 'true' : 'false'; return String(val); @@ -703,11 +705,17 @@ export function VizierResultCard({ metadata }: VizierResultCardProps) { {f.name} {f.unit && ( - [{f.unit}] + + [{f.unit}] + )} ))} @@ -744,7 +752,6 @@ export function VizierResultCard({ metadata }: VizierResultCardProps) { ); } - // ========================================== // 6. 统一观测数据下载结果卡片 (find_observation) // 跨 (LAMOST/Gaia/SDSS/DESI) × (Spectrum/LightCurve/Photometry/Image) diff --git a/dashboard/src/components/agent/ToolCallCard.tsx b/dashboard/src/components/agent/ToolCallCard.tsx index 4d50802..14393e1 100644 --- a/dashboard/src/components/agent/ToolCallCard.tsx +++ b/dashboard/src/components/agent/ToolCallCard.tsx @@ -108,7 +108,12 @@ export function ToolCallCard({ metadata={ metadata as { table_name?: string; - fields: { name: string; description?: string; unit?: string; datatype?: string }[]; + fields: { + name: string; + description?: string; + unit?: string; + datatype?: string; + }[]; rows: unknown[][]; row_count: number; truncated: boolean; diff --git a/dashboard/src/components/observation/ObservationResultCard.tsx b/dashboard/src/components/observation/ObservationResultCard.tsx index 7ddc072..9827fa2 100644 --- a/dashboard/src/components/observation/ObservationResultCard.tsx +++ b/dashboard/src/components/observation/ObservationResultCard.tsx @@ -7,12 +7,7 @@ // 2. （可选）SpecialToolRenderers 的 FindObservationCard 可改为薄封装调用本组件 // // 视觉与 SpecialToolRenderers::FindObservationCard 保持一致，统一数据源主题色。 -import { - Activity, - CheckCircle2, - AlertTriangle, - Download, -} from 'lucide-react'; +import { Activity, CheckCircle2, AlertTriangle, Download } from 'lucide-react'; import { SOURCE_THEME, PRODUCT_LABEL, @@ -119,8 +114,7 @@ export function ObservationResultCard({ result }: ObservationResultCardProps) { {a.band} )} - {a.file_format.toUpperCase()} ·{' '} - {formatFileSize(a.size_bytes)} + {a.file_format.toUpperCase()} · {formatFileSize(a.size_bytes)} { - if (isAuthenticated === true) fetchCapabilities(); + if (isAuthenticated !== true) return; + // eslint-disable-next-line react-hooks/set-state-in-effect -- async fetch, setState in microtask + fetchCapabilities(); }, [isAuthenticated, fetchCapabilities]); // ════════════════════════════════════════════════════ @@ -88,6 +89,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) { const valid = currentSpec.releases; const hardMax = currentSpec.hard_max_radius_deg; const suggested = currentSpec.suggested_max_radius_deg; + // eslint-disable-next-line react-hooks/set-state-in-effect -- derived form sync on spec change setSearchForm((f) => { let next = f; // release 回落（仅在该源区分版本时） @@ -103,6 +105,27 @@ export function useObservation({ isAuthenticated }: UseObservationProps) { }); }, [currentSpec]); + // 当 release 切换后：若该 release 有子版本列表，则校验当前 version 是否在列表里， + // 不在则清空（清空后端用默认子版本）。无子版本概念的源（release_versions 为空）也清空。 + useEffect(() => { + if (!currentSpec) return; + const versionsMap = currentSpec.release_versions ?? {}; + const versions = searchForm.release + ? versionsMap[searchForm.release] + : undefined; + // eslint-disable-next-line react-hooks/set-state-in-effect -- derived version reset on release change + setSearchForm((f) => { + if (!versions || versions.length === 0) { + // 该 release 无子版本概念 + return f.version ? { ...f, version: '' } : f; + } + if (!f.version || !versions.includes(f.version)) { + return { ...f, version: '' }; // 清空 = 用默认（最新公开）子版本 + } + return f; + }); + }, [currentSpec, searchForm.release]); + const [searchResults, setSearchResults] = useState([]); const [searching, setSearching] = useState(false); const [searchError, setSearchError] = useState(null); @@ -131,6 +154,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) { dec, radius: Math.max(0.0001, parseFloat(searchForm.radius) || 0.1), release: searchForm.release || undefined, + version: searchForm.version || undefined, }, }); setSearchResults(res.data ?? []); @@ -169,13 +193,6 @@ export function useObservation({ isAuthenticated }: UseObservationProps) { useState(null); const [downloadError, setDownloadError] = useState(null); - /** 下载选中的候选源（标识符模式） */ - const downloadSelected = useCallback(async () => { - const ids = Array.from(selectedSourceIds); - if (ids.length === 0) return; - await downloadByIds(ids); - }, [selectedSourceIds]); // eslint-disable-line react-hooks/exhaustive-deps - /** 按标识符列表下载 */ const downloadByIds = useCallback( async (sourceIds: string[]) => { @@ -190,6 +207,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) { product: searchForm.product, subtype: searchForm.subtype || undefined, release: searchForm.release || undefined, + version: searchForm.version || undefined, force: false, mode: 'identifiers', source_ids: sourceIds, @@ -205,6 +223,13 @@ export function useObservation({ isAuthenticated }: UseObservationProps) { [searchForm] ); + /** 下载选中的候选源（标识符模式） */ + const downloadSelected = useCallback(async () => { + const ids = Array.from(selectedSourceIds); + if (ids.length === 0) return; + await downloadByIds(ids); + }, [selectedSourceIds, downloadByIds]); + /** 按坐标下载（cone 检索 + 全部下载） */ const downloadByCoordinates = useCallback( async (strategy: 'nearest' | 'all' = 'nearest') => { @@ -224,6 +249,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) { product: searchForm.product, subtype: searchForm.subtype || undefined, release: searchForm.release || undefined, + version: searchForm.version || undefined, force: false, mode: 'coordinates', ra, @@ -265,25 +291,23 @@ export function useObservation({ isAuthenticated }: UseObservationProps) { setLibraryLoading(true); setLibraryError(null); try { - const res = await axios.get<{ items: ObservationRecord[]; total: number }>( - '/api/observation/list', - { - params: { - source: libSource !== 'all' ? libSource : undefined, - product: libProduct !== 'all' ? libProduct : undefined, - q: libSearch.trim() || undefined, - sort: libSort, - limit: libPageSize, - offset: (libPage - 1) * libPageSize, - }, - } - ); + const res = await axios.get<{ + items: ObservationRecord[]; + total: number; + }>('/api/observation/list', { + params: { + source: libSource !== 'all' ? libSource : undefined, + product: libProduct !== 'all' ? libProduct : undefined, + q: libSearch.trim() || undefined, + sort: libSort, + limit: libPageSize, + offset: (libPage - 1) * libPageSize, + }, + }); setLibraryItems(res.data.items ?? []); setLibraryTotal(res.data.total ?? 0); } catch (e: unknown) { - setLibraryError( - extractErrorMessage(e, '加载缓存库失败，请检查后端连接') - ); + setLibraryError(extractErrorMessage(e, '加载缓存库失败，请检查后端连接')); setLibraryItems([]); } finally { setLibraryLoading(false); @@ -300,6 +324,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) { // 任意筛选/分页变更 → 重新请求后端 useEffect(() => { + // eslint-disable-next-line react-hooks/set-state-in-effect -- async fetch, setState in microtask fetchLibrary(); }, [fetchLibrary]); diff --git a/dashboard/src/pages/ObservationPanel.tsx b/dashboard/src/pages/ObservationPanel.tsx index ae1448a..8caf9cb 100644 --- a/dashboard/src/pages/ObservationPanel.tsx +++ b/dashboard/src/pages/ObservationPanel.tsx @@ -25,6 +25,7 @@ import { Database, Send, RotateCw, + Lock, } from 'lucide-react'; import { CustomSelect } from '../components/CustomSelect'; import { ObservationResultCard } from '../components/observation/ObservationResultCard'; @@ -56,11 +57,13 @@ interface ObservationPanelProps { selectNone: () => void; // 下载 downloading: boolean; - downloadResult: import('../components/observation/constants').ObservationBatchResult | null; + downloadResult: + import('../components/observation/constants').ObservationBatchResult | null; downloadError: string | null; setDownloadResult: React.Dispatch< React.SetStateAction< - import('../components/observation/constants').ObservationBatchResult | null + | import('../components/observation/constants').ObservationBatchResult + | null > >; downloadSelected: () => Promise; @@ -195,8 +198,7 @@ function SearchView({ const subtypeOptions = useMemo(() => { const cap = capabilities.find( - (c) => - c.source === searchForm.source && c.product === searchForm.product + (c) => c.source === searchForm.source && c.product === searchForm.product ); return (cap?.subtypes ?? []).map((s) => ({ value: s, label: s })); }, [capabilities, searchForm.source, searchForm.product]); @@ -208,22 +210,42 @@ function SearchView({ // 当前源+产品支持的版本列表（供下拉） const releaseOptions = useMemo(() => { + const authRequired = new Set(currentSpec?.releases_requiring_auth ?? []); return (currentSpec?.releases ?? []).map((r) => ({ value: r, - label: r.toUpperCase(), + label: authRequired.has(r) + ? `${r.toUpperCase()} · 需登录` + : r.toUpperCase(), })); }, [currentSpec]); - // LAMOST MRS 联动约束：MRS（中分辨率）从 DR7 起才支持，DR5/DR6 无 MRS 数据 - const lamostMrsBlocked = useMemo(() => { - if (searchForm.source !== 'lamost' || searchForm.subtype !== 'mrs') - return null; - const rel = searchForm.release; - if (rel === 'dr5' || rel === 'dr6') { - return 'LAMOST 中分辨率（MRS）从 DR7 起才支持，当前版本无 MRS 数据'; + // 当前选中的 release 是否需要登录（Internal DR） + const currentReleaseNeedsAuth = useMemo(() => { + if (!searchForm.release || !currentSpec) return false; + return (currentSpec.releases_requiring_auth ?? []).includes( + searchForm.release + ); + }, [searchForm.release, currentSpec]); + + // 交叉约束：当前 release 是否支持当前 subtype（从 capabilities 派生，不硬编码） + const subtypeBlocked = useMemo(() => { + if (!currentSpec || !searchForm.release || !searchForm.subtype) return null; + const map = currentSpec.subtypes_per_release; + if (!map || Object.keys(map).length === 0) return null; // 无交叉约束 + const valid = map[searchForm.release]; + if (valid && !valid.includes(searchForm.subtype)) { + return `'${searchForm.release.toUpperCase()}' 不支持子类型 '${searchForm.subtype}'`; } return null; - }, [searchForm.source, searchForm.subtype, searchForm.release]); + }, [currentSpec, searchForm.release, searchForm.subtype]); + + // 当前 release 对应的子版本列表（仅 LAMOST 有内容） + const versionOptions = useMemo(() => { + if (!currentSpec || !searchForm.release) return []; + const versionsMap = currentSpec.release_versions ?? {}; + const versions = versionsMap[searchForm.release] ?? []; + return versions.map((v) => ({ value: v, label: v.toUpperCase() })); + }, [currentSpec, searchForm.release]); const handleSubmitIds = () => { const ids = idsText @@ -275,7 +297,7 @@ function SearchView({ .map((c) => c.product); const safeProduct = newProducts.includes(f.product) ? f.product - : newProducts[0] ?? f.product; + : (newProducts[0] ?? f.product); return { ...f, source: v, product: safeProduct, subtype: '' }; }); }} @@ -331,27 +353,67 @@ function SearchView({ } className="w-full" options={[ - { value: '__none__', label: `默认${currentSpec?.default_release ? `（${currentSpec.default_release.toUpperCase()}）` : ''}` }, + { + value: '__none__', + label: `默认${currentSpec?.default_release ? `（${currentSpec.default_release.toUpperCase()}）` : ''}`, + }, ...releaseOptions, ]} /> )} + {/* 子版本（仅 LAMOST 有内容；其他源该 Field 不渲染） */} + {versionOptions.length > 0 && ( + + + setSearchForm((f) => ({ + ...f, + version: v === '__none__' ? '' : v, + })) + } + className="w-full" + options={[ + { value: '__none__', label: '默认（最新公开）' }, + ...versionOptions, + ]} + /> + + )} - {/* LAMOST MRS 版本约束提示 */} - {lamostMrsBlocked && ( + {/* Internal DR 需登录提示 */} + {currentReleaseNeedsAuth && ( +
+ + + {searchForm.release.toUpperCase()} 当前为 Internal 数据发布，需经 + oauth.china-vo.org 登录认证后才能访问，公开检索暂不可用 + +
+ )} + + {/* 交叉约束提示：当前 release 不支持该 subtype */} + {subtypeBlocked && (
- {lamostMrsBlocked} - + {subtypeBlocked} + {currentSpec?.subtypes && currentSpec.subtypes.length > 0 && ( + + )}
)} @@ -410,14 +472,15 @@ function SearchView({ 当前半径 {searchForm.radius}° 超出建议值（≤ - {currentSpec.suggested_max_radius_deg}°），大范围查询可能因主表过大被服务端超时拒绝 + {currentSpec.suggested_max_radius_deg} + °），大范围查询可能因主表过大被服务端超时拒绝 )}