feat: LAMOST DR12-14 与子版本体系接入、观测层安全加固与并发异步化

- LAMOST 新增 DR12/13/14 及子版本(v0/v1.0/v1.1/v2.0)维度,Internal 发布标记需登录认证并前端灰显,release×subtype 交叉约束下沉至 capabilities 统一声明
- ObservationFetcher trait 扩展版本/认证/交叉约束能力声明,version 参数贯穿 client→service→API→Agent tool→前端全链路
- 安全:observation cache SQL 全参数绑定 + LIKE 转义、cone_cache_hash 加长度前缀防碰撞、DESI survey/program 白名单防穿越
- 异步化:persist/cached_files_total_size/maybe_persist_tool_result迁移到 tokio::fs;cancelled_runs 与 session_permission_checkers改用 DashMap;auth 读锁优先 + 60s 节流
- Gaia 去 native-tls 改禁用连接池规避 UnexpectedEof,reqwest 移除 native-tls feature
- 重构:Source/ProductType from_str 集中解析、download 模块拆分为 try_download_pdf/html、AgentRuntime::init 抽取共享逻辑
- 部署:新增 deploy.sh 一键打包推送脚本、catch-panic 启用
This commit is contained in:
Asfmq 2026-07-07 21:16:46 +08:00
parent 2f1fd19d74
commit 2c8d0b8f8b
75 changed files with 2482 additions and 1370 deletions

131
CLAUDE.md
View File

@ -42,6 +42,9 @@ cargo test --lib
# Run a specific test # Run a specific test
cargo test test_name cargo test test_name
# Coverage (requires cargo-llvm-cov)
cargo llvm-cov --fail-under-lines 80
# Frontend (cd dashboard first) # Frontend (cd dashboard first)
npm run dev # HMR dev server on :5173, proxies /api to :8000 npm run dev # HMR dev server on :5173, proxies /api to :8000
npm run build # TypeScript check + Vite production build → dashboard/dist/ npm run build # TypeScript check + Vite production build → dashboard/dist/
@ -62,44 +65,101 @@ src/
├── main.rs # Axum server entry: logging, DB pool, migrations, vec0 table, ├── main.rs # Axum server entry: logging, DB pool, migrations, vec0 table,
│ # client/service construction, route registration, AppState assembly │ # client/service construction, route registration, AppState assembly
├── lib.rs # Config struct + from_env() loading from .env ├── lib.rs # Config struct + from_env() loading from .env
├── api/ # HTTP handlers, AppState, StandardPaper type ├── api/ # HTTP handlers, AppState, re-exports via handlers namespace
│ ├── mod.rs # AppState (shared state), StandardPaper, handlers re-exports │ ├── mod.rs # AppState (shared state), handler re-exports, permission/event types
│ ├── agent.rs # SSE chat_agent endpoint, session CRUD, metrics, audit log │ ├── agent.rs # SSE chat_agent endpoint, session CRUD, metrics, audit log
│ ├── papers.rs # Search, download, parse, translate, embed, citations, library, export │ ├── papers.rs # Search, download, parse, translate, embed, citations, library, export
│ ├── catalog.rs # VizieR TAP catalog queries + cone search endpoints
│ ├── observation.rs # Observation data search/download (LAMOST/Gaia/SDSS/DESI)
│ ├── notes.rs # Highlight/note CRUD │ ├── notes.rs # Highlight/note CRUD
│ ├── sync.rs # Meta-sync and asset-batch endpoints │ ├── sync.rs # Meta-sync and asset-batch endpoints
│ ├── targets.rs # Target query/associate/extract, RAG chat, figure chat │ ├── targets.rs # Target query/associate/extract, RAG chat, figure chat
│ └── helpers.rs # Shared DB helpers, format conversion, path validation │ ├── auth.rs # Login/logout/auth-check (token-based, session memory)
│ ├── permissions.rs # Agent tool permission rule management
│ ├── search.rs # Search history endpoint
│ ├── error.rs # Unified API error type (status code + user-safe message)
│ └── helpers.rs # Shared DB helpers, format conversion, path validation (private)
├── agent/ # ReAct-based research agent (LLM-driven tool-use loop) ├── agent/ # ReAct-based research agent (LLM-driven tool-use loop)
│ ├── tools/ # AgentTool trait, ToolRegistry, 25+ tool implementations per domain file │ ├── tools/ # AgentTool trait, ToolRegistry, tool implementations per domain
│ ├── runtime/ # ReAct loop core: session, context, streaming, executor, token_budget, │ │ ├── mod.rs # AgentTool trait, ToolContext (session_id, sse_tx, silent, etc.)
│ │ # permission, permission_profile, checkpoint, circuit_breaker, error_recovery, │ │ ├── filesystem/ # read_file, grep_files, write, edit
│ │ # hardline, partitioner, file_cache, system_prompt, finalize, denial_tracker │ │ ├── astro/ # search_papers, download_paper, parse_paper, rag_search,
│ │ │ # query_target, save_note, analyze_image, research/*
│ │ ├── ask_user.rs # Permission-gated interactive user question
│ │ ├── background.rs# bg_task_run, bg_task_check (async slow tasks)
│ │ ├── compress.rs # Context compression trigger
│ │ ├── memory.rs # save_memory, load_memory
│ │ ├── persist.rs # Data persistence (used by memory/compaction)
│ │ ├── skill.rs # load_skill
│ │ ├── subagent.rs # Context-isolated sub-agent spawner
│ │ ├── team.rs # spawn_teammate, send_teammate_message, etc.
│ │ └── todo.rs # todo_write (task tracking)
│ ├── runtime/ # ReAct loop core
│ │ ├── mod.rs # AgentRuntime: session create/resume → context → ReAct → finalize
│ │ ├── executor/ # Step executor + helpers (tool dispatch, parallel tool calls)
│ │ ├── streaming.rs # SSE AgentStreamEvent types
│ │ ├── streaming_executor.rs # SSE-emitting executor wrapper
│ │ ├── session.rs # Session CRUD, DB persist, agent_config
│ │ ├── token_budget.rs # Soft/hard token limits, tiktoken counting
│ │ ├── permission.rs # Deny > Allow > Ask rule chain, pattern matching
│ │ ├── permission_profile.rs # Named profiles (research, readonly)
│ │ ├── checkpoint.rs # Rewind/retry state snapshots
│ │ ├── circuit_breaker.rs# CompactionCircuitBreaker (cooldown rate limiting)
│ │ ├── error_recovery.rs # Error classification + retry strategies
│ │ ├── hardline.rs # Terminal conditions (too many denials, loops, etc.)
│ │ ├── partitioner.rs # Message window partitioning for compaction
│ │ ├── file_cache.rs # ReadFileState — file content dedup cache
│ │ ├── system_prompt.rs # System prompt assembly (mode, skills, memory, tools)
│ │ ├── finalize.rs # Post-loop cleanup: memory extraction, audit log
│ │ └── denial_tracker.rs # Consecutive denial detection
│ ├── compact/ # Context compression (micro/auto/manual layers + collapse) │ ├── compact/ # Context compression (micro/auto/manual layers + collapse)
│ ├── memory/ # Persistent memory manager: extraction, dedup, decay, age, guardrails │ ├── memory/ # Persistent memory manager: extraction, dedup, decay, age, guardrails
│ ├── hooks/ # Lifecycle hooks: registry, dispatch, builtins, matcher, traits (PreToolUse/PostToolUse/Stop/etc.) │ ├── hooks/ # Lifecycle hooks: registry, dispatch, builtins, matcher, traits
│ ├── modes/ # Agent session modes: default, deep-research, literature-reader (identity/tools/config presets) │ ├── modes/ # Agent session modes (see Agent Session Modes section below)
│ ├── skills.rs # SkillRegistry: hot-loads SKILL.md files from skills/ directory │ ├── skills.rs # SkillRegistry: hot-loads SKILL.md files from skills/ directory
│ ├── subagent.rs # Context-isolated sub-agent runner (subagent tool) │ ├── subagent.rs # Context-isolated sub-agent runner
│ ├── team/ # Multi-agent team: file-based inbox, lead/teammate coordination │ ├── team/ # Multi-agent team: file-based inbox, lead/teammate coordination
│ ├── background.rs# BgNotificationQueue for async slow-task (download/parse) notifications │ ├── background.rs# BgNotificationQueue for async slow-task notifications
│ ├── task_board.rs# Persistent task board (agent_tasks table) │ ├── task_board.rs# Persistent task board (agent_tasks table)
│ ├── trajectory.rs# Session trajectory recording for audit/debug │ ├── trajectory.rs# Session trajectory recording for audit/debug
│ └── terminal.rs # Escape sequence filter for ANSI-heavy tool outputs │ └── terminal.rs # Escape sequence filter for ANSI-heavy tool outputs
├── clients/ # External API wrappers ├── clients/ # External API wrappers (pure communication, no business logic)
│ ├── llm.rs # LlmClient (OpenAI-compatible chat + streaming), EmbeddingClient │ ├── llm/ # LlmClient (OpenAI-compatible chat + streaming), EmbeddingClient
│ ├── ads.rs # NASA ADS API │ ├── ads.rs # NASA ADS API
│ ├── arxiv.rs # arXiv Atom XML API │ ├── arxiv.rs # arXiv Atom XML API
│ ├── cds/ # CDS services (sesame.rs — Sesame name resolver; vizier.rs — TAP queries)
│ ├── gaia/ # Gaia archive (TAP + DataLink)
│ ├── lamost/ # LAMOST archive (ConeSearch + FITS.gz download)
│ ├── sdss/ # SDSS archive (Data Lab TAP + SAS bulk download)
│ ├── desi/ # DESI archive (Data Lab TAP + HEALPix coadd SAS)
│ ├── vo/ # Virtual Observatory generic client
│ └── qiniu.rs # Qiniu cloud storage │ └── qiniu.rs # Qiniu cloud storage
├── services/ # Business logic ├── services/ # Business logic
│ ├── search.rs # Unified cross-source search (ADS + arXiv dedup) │ ├── search.rs # Unified cross-source search (ADS + arXiv dedup)
│ ├── download.rs # PDF/HTML download with anti-bot measures and fallback chain │ ├── download.rs # PDF/HTML download with anti-bot measures and fallback chain
│ ├── parser/ # HTML/PDF → Markdown parsers (A&A, IOP, ar5iv, generic, PDF via MinerU) │ ├── parser/ # HTML/PDF → Markdown parsers (A&A, IOP, ar5iv, generic, PDF via MinerU)
│ ├── paper/ # Paper lifecycle: model, db (CRUD), ingest (ADS/arXiv→DB), reader (markdown)
│ ├── translation.rs # LLM bilingual translation with Trie-based astronomy glossary │ ├── translation.rs # LLM bilingual translation with Trie-based astronomy glossary
│ ├── rag.rs # Embedding ingest + vector similarity retrieval + LLM answer generation │ ├── rag.rs # Embedding ingest + vector similarity retrieval + LLM answer generation
│ ├── target.rs # Celestial target extraction (IAU name regex) + CDS Sesame lookup │ ├── cds/ # CDS business layer: target.rs (Sesame name resolver), vizier.rs (TAP + cache)
│ ├── chunker.rs # Markdown text chunking for embedding │ ├── observation/ # Cross-source observation data (spectra, light curves, photometry, images)
│ │ ├── mod.rs # 双轴正交架构: Source × ProductType → ObservationFetcher → Registry
│ │ ├── types.rs # Source/ProductType/ProductSpec/Artifact/ObservationProduct/Candidate
│ │ ├── fetcher.rs # ObservationFetcher trait + cone search cache template methods
│ │ ├── registry.rs # ObservationRegistry (fetcher registration, capability discovery)
│ │ ├── dispatch.rs # search_observation + download_observation (unified entry points)
│ │ ├── cache.rs # observation_cache table read/write + file staging
│ │ ├── lamost.rs # LamostSpectrumFetcher
│ │ ├── gaia.rs # GaiaSpectrumFetcher + GaiaLightCurveFetcher
│ │ ├── sdss.rs # SdssSpectrumFetcher + SdssApogeeFetcher
│ │ └── desi.rs # DesiSpectrumFetcher
│ ├── batch/ # Meta-sync (ADS bulk harvest) and asset-batch processing engines │ ├── batch/ # Meta-sync (ADS bulk harvest) and asset-batch processing engines
│ ├── citation.rs # Citation network graph data (references + citations)
│ ├── session.rs # Agent session listing, metrics, audit log (for API consumption)
│ ├── vision.rs # Image analysis via dedicated vision model
│ ├── note.rs # Highlight/note storage and retrieval
│ ├── pipeline.rs # Synchronous paper processing pipeline (download→parse→translate→embed)
│ ├── chunker.rs # Markdown text chunking for embedding
│ ├── query_parser.rs # Advanced search query syntax parser │ ├── query_parser.rs # Advanced search query syntax parser
│ └── logging.rs # Pretty console + rolling file logger │ └── logging.rs # Pretty console + rolling file logger
└── bin/ └── bin/
@ -113,11 +173,18 @@ src/
All handlers access state via `Arc<AppState>`. Key fields: All handlers access state via `Arc<AppState>`. Key fields:
- `db: SqlitePool` — SQLite connection pool (5 max connections, foreign keys enforced) - `db: SqlitePool` — SQLite connection pool (5 max connections, foreign keys enforced)
- `llm: LlmClient` / `embedding: EmbeddingClient` — OpenAI-compatible LLM clients - `llm: LlmClient` / `medium_llm` / `fast_llm` — OpenAI-compatible LLM clients (multi-tier)
- `vision_llm: Option<LlmClient>` — dedicated vision model client
- `embedding: EmbeddingClient` — embedding model client
- `ads: AdsClient` / `arxiv: ArxivClient` — academic search clients - `ads: AdsClient` / `arxiv: ArxivClient` — academic search clients
- `vizier: VizierClient` — CDS VizieR TAP catalog query client
- `lamost` / `gaia` / `sdss` / `desi` — observation data source clients
- `observation_registry: Arc<ObservationRegistry>` — cross-source observation fetcher registry
- `skill_registry: Arc<RwLock<SkillRegistry>>` — hot-reloaded agent skills - `skill_registry: Arc<RwLock<SkillRegistry>>` — hot-reloaded agent skills
- `memory_manager: Arc<MemoryManager>` — persistent agent memory (MEMORY.md + decay) - `memory_manager: Arc<MemoryManager>` — persistent agent memory (MEMORY.md + decay)
- `cancelled_runs: Arc<Mutex<HashSet<String>>>` — agent cancellation tokens - `cancelled_runs: Arc<DashMap<String, ()>>` — agent cancellation tokens
- `session_permission_checkers: Arc<DashMap<String, PermissionChecker>>` — per-session permission rules
- `pending_questions` / `pending_permissions` — interactive agent-user handshake channels
- `harvest_status` / `batch_status` — async batch operation status tracking - `harvest_status` / `batch_status` — async batch operation status tracking
### Agent System Design ### Agent System Design
@ -131,7 +198,7 @@ The agent (`src/agent/`) implements a **ReAct** (Thought → Action → Observat
5. **Tool call ID tracking**: LLM may not return tool_call IDs — `LlmClient` generates UUID fallbacks. `ToolCall` and `ToolResult` SSE events carry matching IDs for precise frontend pairing. 5. **Tool call ID tracking**: LLM may not return tool_call IDs — `LlmClient` generates UUID fallbacks. `ToolCall` and `ToolResult` SSE events carry matching IDs for precise frontend pairing.
6. **ToolContext** (`tools/mod.rs`): Injected into every tool execution — holds `app_state`, `session_id`, `sse_tx` (for intermediate events), `enable_thinking`, `read_file_state` (file cache for dedup), `silent` (sub-agents skip permission prompts). 6. **ToolContext** (`tools/mod.rs`): Injected into every tool execution — holds `app_state`, `session_id`, `sse_tx` (for intermediate events), `enable_thinking`, `read_file_state` (file cache for dedup), `silent` (sub-agents skip permission prompts).
7. **Context compression**: Four layers — micro (placeholder replacement), snip (old-message truncation), auto (LLM summarization), aggro_micro (aggressive placeholder). Protected by `CompactionCircuitBreaker`. Transcripts persisted in `agent_messages` table, not filesystem snapshots. 7. **Context compression**: Four layers — micro (placeholder replacement), snip (old-message truncation), auto (LLM summarization), aggro_micro (aggressive placeholder). Protected by `CompactionCircuitBreaker`. Transcripts persisted in `agent_messages` table, not filesystem snapshots.
8. **Skills** (`skills.rs`): Two-layer loading — system-reminder lists names (~20 tokens each), LLM calls `load_skill` to inject full SKILL.md content 8. **Skills** (`skills.rs`): Two-layer loading — system-reminder lists names (~20 tokens each), LLM calls `load_skill` to inject full SKILL.md content. Skills are organized into subdirectories: `methodology/`, `plotting/`, `presentation/`.
9. **Sub-agents** (`subagent.rs`): `subagent` tool spawns a context-isolated sub-agent with its own ReAct loop. Sub-agent messages (system/user/assistant/tool) are persisted to `agent_messages` with `agent_name` identifier. Returns final summary + activity log. SSE progress forwarded to parent via ToolContext. 9. **Sub-agents** (`subagent.rs`): `subagent` tool spawns a context-isolated sub-agent with its own ReAct loop. Sub-agent messages (system/user/assistant/tool) are persisted to `agent_messages` with `agent_name` identifier. Returns final summary + activity log. SSE progress forwarded to parent via ToolContext.
10. **Memory** (`memory/`): File-based persistent memory (MEMORY.md). `MemoryManager` handles extraction from conversation, dedup, recency decay, age-based pruning, and guardrails. Tools: `save_memory`, `load_memory` (auto-injected in system prompt). 10. **Memory** (`memory/`): File-based persistent memory (MEMORY.md). `MemoryManager` handles extraction from conversation, dedup, recency decay, age-based pruning, and guardrails. Tools: `save_memory`, `load_memory` (auto-injected in system prompt).
11. **Teams** (`team/`): File-based inbox directory per session for lead/teammate message passing 11. **Teams** (`team/`): File-based inbox directory per session for lead/teammate message passing
@ -141,17 +208,29 @@ Environment variables for agent tuning: `AGENT_TOKEN_SOFT_LIMIT` (default 80000)
### Database ### Database
SQLite via `sqlx::sqlite`. Migrations in `migrations/` are auto-run on startup (`sqlx::migrate!("./migrations")`). Key tables: `papers`, `citations_references`, `notes`, `agent_sessions`, `agent_messages`, `agent_tasks`, `agent_audit_log`, `paper_chunks_content`. Vector embeddings use `sqlite-vec` (`vec_paper_chunks` virtual table, auto-registered before any DB connection). SQLite via `sqlx::sqlite`. Migrations in `migrations/` are auto-run on startup (`sqlx::migrate!("./migrations")`). Key tables: `papers`, `citations_references`, `notes`, `agent_sessions`, `agent_messages`, `agent_tasks`, `agent_audit_log`, `paper_chunks_content`, `vizier_cache`, `observation_cache`. Vector embeddings use `sqlite-vec` (`vec_paper_chunks` virtual table, auto-registered before any DB connection).
The embedding dimension is controlled by `EMBEDDING_DIM` env var (default 1536). On dimension mismatch, the vec table and chunk content are dropped and recreated. The embedding dimension is controlled by `EMBEDDING_DIM` env var (default 1536). On dimension mismatch, the vec table and chunk content are dropped and recreated.
### Observation Data System (`src/services/observation/`)
A cross-source astronomical observation data subsystem with a **dual-axis orthogonal** architecture: **Source** (LAMOST/Gaia/SDSS/DESI) × **ProductType** (Spectrum/LightCurve/Photometry/Image). Each valid combination implements an `ObservationFetcher`, registered in `ObservationRegistry`.
New fetchers follow OCP (pure addition, no modification to existing code):
1. Create `{source}.rs`, implement `ObservationFetcher` trait
2. Add `register!(YourFetcher)` in `registry.rs::default()`
3. Add new variants to `Source`/`ProductType` enums in `types.rs` if needed
External callers use `dispatch::search_observation()` (metadata only) and `dispatch::download_observation()` (metadata + file download) via `ObservationRequest`. Results are cached in `observation_cache` table with file staging to `library/`.
### Frontend (dashboard/) ### Frontend (dashboard/)
React 19 + TypeScript + Vite + Tailwind CSS 4. Organized by technical layer (Type-Based): React 19 + TypeScript + Vite + Tailwind CSS 4. Organized by technical layer (Type-Based):
- `pages/` — Page-level panel view components (SearchPanel, LibraryPanel, ReaderPanel, CitationPanel, SyncPanel, ResearchAgentPanel, SettingsPanel) - `pages/` — Page-level panel view components (SearchPanel, LibraryPanel, ReaderPanel, CitationPanel, SyncPanel, ResearchAgentPanel, ObservationPanel, SettingsPanel)
- `components/` — Reusable and layout components (sub-folders: agent, reader, sync, layout, dialogs) - `components/` — Reusable and layout components (sub-folders: agent, reader, sync, layout, dialogs)
- `hooks/` — Global and feature-specific custom stateful React Hooks (e.g., useLibrary, useSearch, useNotes) - `hooks/` — Global and feature-specific custom stateful React Hooks (e.g., useLibrary, useSearch, useNotes, useObservation)
- `types/` — Global TypeScript type definitions (types/index.ts) - `types/` — Global TypeScript type definitions (types/index.ts)
- `utils/` — Common utility helper functions - `utils/` — Common utility helper functions
- `assets/` — Static assets and global stylesheet styles - `assets/` — Static assets and global stylesheet styles
@ -160,7 +239,7 @@ Dependencies: `react-markdown` + `rehype-katex` + `remark-math` for Markdown/LaT
### Obscura In-Process Browser ### Obscura In-Process Browser
The `obscura-inprocess` feature compiles `obscura-browser` and `obscura-net` directly into the binary, eliminating the need for external browser binaries. This is used for bypassing Cloudflare/WAF on PDF download. Enabled via `--features obscura-inprocess`. The `obscura-inprocess` feature compiles `obscura-browser` and `obscura-net` from `libs/obscura/` directly into the binary, eliminating the need for external browser binaries. This is used for bypassing Cloudflare/WAF on PDF download. Enabled via `--features obscura-inprocess`.
### Astronomy Glossary (dictionary.txt) ### Astronomy Glossary (dictionary.txt)
@ -235,6 +314,10 @@ Controlled by env vars:
- `EXTRACT_MEMORY_THROTTLE_TURNS` (default `3`) — extract every N turns - `EXTRACT_MEMORY_THROTTLE_TURNS` (default `3`) — extract every N turns
- `EXTRACT_MEMORY_MAX_STEPS` (default `3`) — sub-agent max steps for extraction - `EXTRACT_MEMORY_MAX_STEPS` (default `3`) — sub-agent max steps for extraction
### Deployment (`deploy.sh`)
The `deploy.sh` script builds the project locally (release-min profile, Composer-based) and deploys to a remote server via rsync + SSH. It handles: source compilation, frontend build, remote directory sync, service restart via docker compose, and health check verification.
## Code Conventions ## Code Conventions
- Use `anyhow` for application errors, `thiserror` for library-style typed errors - Use `anyhow` for application errors, `thiserror` for library-style typed errors
@ -243,3 +326,7 @@ Controlled by env vars:
- API handlers take `State(Arc<AppState>)` and return Axum-compatible responses - API handlers take `State(Arc<AppState>)` and return Axum-compatible responses
- Agent tools implement `AgentTool` trait; new tools register in `ToolRegistry::new()` - Agent tools implement `AgentTool` trait; new tools register in `ToolRegistry::new()`
- Front-end build is triggered by `build.rs` (auto npm install + build when `dashboard/src/` changes) - Front-end build is triggered by `build.rs` (auto npm install + build when `dashboard/src/` changes)
- New observation fetchers follow OCP: add module + register — don't modify existing fetchers or dispatch
- External API clients (`src/clients/`) are pure communication; business logic lives in `src/services/`
- Route registration happens in `src/main.rs`; handler re-exports are maintained in `src/api/mod.rs``handlers` namespace
- Agent hooks (`src/agent/hooks/`) implement lifecycles: PreToolUse, PostToolUse, Stop, etc. — register in `builtins.rs`

168
Cargo.lock generated
View File

@ -138,6 +138,7 @@ dependencies = [
"base64 0.22.1", "base64 0.22.1",
"chrono", "chrono",
"clap", "clap",
"dashmap",
"dotenvy", "dotenvy",
"flate2", "flate2",
"futures-util", "futures-util",
@ -481,7 +482,7 @@ checksum = "2c5e60b8c8d282c86360cab651ded04ab0335a7b5390c8d34145cbeab8cacf5f"
dependencies = [ dependencies = [
"bitflags 2.13.0", "bitflags 2.13.0",
"btls-sys", "btls-sys",
"foreign-types 0.5.0", "foreign-types",
"libc", "libc",
"openssl-macros", "openssl-macros",
] ]
@ -761,16 +762,6 @@ dependencies = [
"url", "url",
] ]
[[package]]
name = "core-foundation"
version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]] [[package]]
name = "core-foundation-sys" name = "core-foundation-sys"
version = "0.8.7" version = "0.8.7"
@ -900,6 +891,20 @@ dependencies = [
"cmov", "cmov",
] ]
[[package]]
name = "dashmap"
version = "6.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6361d5c062261c78a176addb82d4c821ae42bed6089de0e12603cd25de2059c"
dependencies = [
"cfg-if",
"crossbeam-utils",
"hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core",
]
[[package]] [[package]]
name = "data-encoding" name = "data-encoding"
version = "2.11.0" version = "2.11.0"
@ -1242,15 +1247,6 @@ version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
[[package]]
name = "foreign-types"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
dependencies = [
"foreign-types-shared 0.1.1",
]
[[package]] [[package]]
name = "foreign-types" name = "foreign-types"
version = "0.5.0" version = "0.5.0"
@ -1258,7 +1254,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965" checksum = "d737d9aa519fb7b749cbc3b962edcf310a8dd1f4b67c91c4f83975dbdd17d965"
dependencies = [ dependencies = [
"foreign-types-macros", "foreign-types-macros",
"foreign-types-shared 0.3.1", "foreign-types-shared",
] ]
[[package]] [[package]]
@ -1272,12 +1268,6 @@ dependencies = [
"syn 2.0.117", "syn 2.0.117",
] ]
[[package]]
name = "foreign-types-shared"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]] [[package]]
name = "foreign-types-shared" name = "foreign-types-shared"
version = "0.3.1" version = "0.3.1"
@ -1760,29 +1750,13 @@ dependencies = [
"http", "http",
"hyper", "hyper",
"hyper-util", "hyper-util",
"rustls 0.23.40", "rustls 0.23.41",
"tokio", "tokio",
"tokio-rustls", "tokio-rustls",
"tower-service", "tower-service",
"webpki-roots 1.0.7", "webpki-roots 1.0.7",
] ]
[[package]]
name = "hyper-tls"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0"
dependencies = [
"bytes",
"http-body-util",
"hyper",
"hyper-util",
"native-tls",
"tokio",
"tokio-native-tls",
"tower-service",
]
[[package]] [[package]]
name = "hyper-util" name = "hyper-util"
version = "0.1.20" version = "0.1.20"
@ -2419,23 +2393,6 @@ dependencies = [
"version_check", "version_check",
] ]
[[package]]
name = "native-tls"
version = "0.2.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2"
dependencies = [
"libc",
"log",
"openssl",
"openssl-probe",
"openssl-sys",
"schannel",
"security-framework",
"security-framework-sys",
"tempfile",
]
[[package]] [[package]]
name = "new_debug_unreachable" name = "new_debug_unreachable"
version = "1.0.6" version = "1.0.6"
@ -2624,20 +2581,6 @@ version = "1.70.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
[[package]]
name = "openssl"
version = "0.10.80"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967"
dependencies = [
"bitflags 2.13.0",
"cfg-if",
"foreign-types 0.3.2",
"libc",
"openssl-macros",
"openssl-sys",
]
[[package]] [[package]]
name = "openssl-macros" name = "openssl-macros"
version = "0.1.1" version = "0.1.1"
@ -2649,24 +2592,6 @@ dependencies = [
"syn 2.0.117", "syn 2.0.117",
] ]
[[package]]
name = "openssl-probe"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe"
[[package]]
name = "openssl-sys"
version = "0.9.117"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b47e7e6bb2c38cd930d25a23b40fa52e068c10e85f3e03a7f5ba5aaca5713695"
dependencies = [
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]] [[package]]
name = "outref" name = "outref"
version = "0.5.2" version = "0.5.2"
@ -2954,7 +2879,7 @@ dependencies = [
"quinn-proto", "quinn-proto",
"quinn-udp", "quinn-udp",
"rustc-hash", "rustc-hash",
"rustls 0.23.40", "rustls 0.23.41",
"socket2", "socket2",
"thiserror 2.0.18", "thiserror 2.0.18",
"tokio", "tokio",
@ -2974,7 +2899,7 @@ dependencies = [
"rand 0.9.4", "rand 0.9.4",
"ring", "ring",
"rustc-hash", "rustc-hash",
"rustls 0.23.40", "rustls 0.23.41",
"rustls-pki-types", "rustls-pki-types",
"slab", "slab",
"thiserror 2.0.18", "thiserror 2.0.18",
@ -3147,23 +3072,20 @@ dependencies = [
"http-body-util", "http-body-util",
"hyper", "hyper",
"hyper-rustls", "hyper-rustls",
"hyper-tls",
"hyper-util", "hyper-util",
"js-sys", "js-sys",
"log", "log",
"mime_guess", "mime_guess",
"native-tls",
"percent-encoding", "percent-encoding",
"pin-project-lite", "pin-project-lite",
"quinn", "quinn",
"rustls 0.23.40", "rustls 0.23.41",
"rustls-pki-types", "rustls-pki-types",
"serde", "serde",
"serde_json", "serde_json",
"serde_urlencoded", "serde_urlencoded",
"sync_wrapper", "sync_wrapper",
"tokio", "tokio",
"tokio-native-tls",
"tokio-rustls", "tokio-rustls",
"tokio-util", "tokio-util",
"tower", "tower",
@ -3256,9 +3178,9 @@ dependencies = [
[[package]] [[package]]
name = "rustls" name = "rustls"
version = "0.23.40" version = "0.23.41"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" checksum = "6b92b125634d9b795e7beca796cc790df15a7fb38323bf3196fda83292d06b1f"
dependencies = [ dependencies = [
"once_cell", "once_cell",
"ring", "ring",
@ -3329,15 +3251,6 @@ dependencies = [
"winapi-util", "winapi-util",
] ]
[[package]]
name = "schannel"
version = "0.1.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
dependencies = [
"windows-sys 0.61.2",
]
[[package]] [[package]]
name = "scopeguard" name = "scopeguard"
version = "1.2.0" version = "1.2.0"
@ -3354,29 +3267,6 @@ dependencies = [
"untrusted", "untrusted",
] ]
[[package]]
name = "security-framework"
version = "3.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d"
dependencies = [
"bitflags 2.13.0",
"core-foundation",
"core-foundation-sys",
"libc",
"security-framework-sys",
]
[[package]]
name = "security-framework-sys"
version = "2.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]] [[package]]
name = "selectors" name = "selectors"
version = "0.26.0" version = "0.26.0"
@ -4210,23 +4100,13 @@ dependencies = [
"syn 2.0.117", "syn 2.0.117",
] ]
[[package]]
name = "tokio-native-tls"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
dependencies = [
"native-tls",
"tokio",
]
[[package]] [[package]]
name = "tokio-rustls" name = "tokio-rustls"
version = "0.26.4" version = "0.26.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
dependencies = [ dependencies = [
"rustls 0.23.40", "rustls 0.23.41",
"tokio", "tokio",
] ]

View File

@ -21,11 +21,11 @@ path = "src/bin/cli.rs"
[dependencies] [dependencies]
tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "signal", "time", "fs", "net", "sync", "process"] } tokio = { version = "1", features = ["rt", "rt-multi-thread", "macros", "signal", "time", "fs", "net", "sync", "process"] }
axum = { version = "0.7", features = ["macros", "multipart"] } axum = { version = "0.7", features = ["macros", "multipart"] }
tower-http = { version = "0.5", features = ["cors", "fs", "trace", "set-header"] } tower-http = { version = "0.5", features = ["cors", "fs", "trace", "set-header", "catch-panic"] }
sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "sqlite", "chrono", "json"] } sqlx = { version = "0.7", features = ["runtime-tokio-rustls", "sqlite", "chrono", "json"] }
serde = { version = "1.0", features = ["derive"] } serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0" serde_json = "1.0"
reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "multipart", "cookies", "rustls-tls", "native-tls"] } reqwest = { version = "0.12", default-features = false, features = ["json", "stream", "multipart", "cookies", "rustls-tls"] }
dotenvy = "0.15" dotenvy = "0.15"
quick-xml = { version = "0.31", features = ["serialize"] } quick-xml = { version = "0.31", features = ["serialize"] }
anyhow = "1.0" anyhow = "1.0"
@ -62,6 +62,7 @@ glob = "0.3"
walkdir = "2" walkdir = "2"
lru = "0.12" lru = "0.12"
git2 = { version = "0.18", default-features = false, features = ["vendored-libgit2"] } git2 = { version = "0.18", default-features = false, features = ["vendored-libgit2"] }
dashmap = "6"
[features] [features]
default = [] default = []

View File

@ -2,6 +2,7 @@
<html lang="en"> <html lang="en">
<head> <head>
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<meta name="referrer" content="no-referrer" />
<link rel="icon" type="image/svg+xml" href="/favicon.svg" /> <link rel="icon" type="image/svg+xml" href="/favicon.svg" />
<link rel="manifest" href="/manifest.json" /> <link rel="manifest" href="/manifest.json" />
<!-- iOS PWA 独立全屏支持 --> <!-- iOS PWA 独立全屏支持 -->

View File

@ -61,7 +61,13 @@ export default function App() {
// 2. 局部状态定义 (多组件共用或全局网络进度状态) // 2. 局部状态定义 (多组件共用或全局网络进度状态)
const [activeTab, setActiveTab] = useState< const [activeTab, setActiveTab] = useState<
'search' | 'library' | 'reader' | 'citation' | 'sync' | 'observation' | 'agent' | 'search'
| 'library'
| 'reader'
| 'citation'
| 'sync'
| 'observation'
| 'agent'
>(() => { >(() => {
const saved = localStorage.getItem('astro_active_tab'); const saved = localStorage.getItem('astro_active_tab');
const validTabs = [ const validTabs = [

View File

@ -642,7 +642,9 @@ export function VizierResultCard({ metadata }: VizierResultCardProps) {
if (val === null || val === undefined) return '—'; if (val === null || val === undefined) return '—';
if (typeof val === 'number') { if (typeof val === 'number') {
// 数字保留合理精度 // 数字保留合理精度
return Number.isInteger(val) ? String(val) : val.toFixed(6).replace(/\.?0+$/, ''); return Number.isInteger(val)
? String(val)
: val.toFixed(6).replace(/\.?0+$/, '');
} }
if (typeof val === 'boolean') return val ? 'true' : 'false'; if (typeof val === 'boolean') return val ? 'true' : 'false';
return String(val); return String(val);
@ -703,11 +705,17 @@ export function VizierResultCard({ metadata }: VizierResultCardProps) {
<th <th
key={i} key={i}
className="px-2 py-1.5 text-left font-bold text-slate-600 border-b border-slate-200 whitespace-nowrap" className="px-2 py-1.5 text-left font-bold text-slate-600 border-b border-slate-200 whitespace-nowrap"
title={f.description || f.unit ? `${f.description || ''} ${f.unit ? `[${f.unit}]` : ''}`.trim() : undefined} title={
f.description || f.unit
? `${f.description || ''} ${f.unit ? `[${f.unit}]` : ''}`.trim()
: undefined
}
> >
{f.name} {f.name}
{f.unit && ( {f.unit && (
<span className="text-slate-400 font-normal ml-1">[{f.unit}]</span> <span className="text-slate-400 font-normal ml-1">
[{f.unit}]
</span>
)} )}
</th> </th>
))} ))}
@ -744,7 +752,6 @@ export function VizierResultCard({ metadata }: VizierResultCardProps) {
); );
} }
// ========================================== // ==========================================
// 6. 统一观测数据下载结果卡片 (find_observation) // 6. 统一观测数据下载结果卡片 (find_observation)
// 跨 (LAMOST/Gaia/SDSS/DESI) × (Spectrum/LightCurve/Photometry/Image) // 跨 (LAMOST/Gaia/SDSS/DESI) × (Spectrum/LightCurve/Photometry/Image)

View File

@ -108,7 +108,12 @@ export function ToolCallCard({
metadata={ metadata={
metadata as { metadata as {
table_name?: string; table_name?: string;
fields: { name: string; description?: string; unit?: string; datatype?: string }[]; fields: {
name: string;
description?: string;
unit?: string;
datatype?: string;
}[];
rows: unknown[][]; rows: unknown[][];
row_count: number; row_count: number;
truncated: boolean; truncated: boolean;

View File

@ -7,12 +7,7 @@
// 2. 可选SpecialToolRenderers 的 FindObservationCard 可改为薄封装调用本组件 // 2. 可选SpecialToolRenderers 的 FindObservationCard 可改为薄封装调用本组件
// //
// 视觉与 SpecialToolRenderers::FindObservationCard 保持一致,统一数据源主题色。 // 视觉与 SpecialToolRenderers::FindObservationCard 保持一致,统一数据源主题色。
import { import { Activity, CheckCircle2, AlertTriangle, Download } from 'lucide-react';
Activity,
CheckCircle2,
AlertTriangle,
Download,
} from 'lucide-react';
import { import {
SOURCE_THEME, SOURCE_THEME,
PRODUCT_LABEL, PRODUCT_LABEL,
@ -119,8 +114,7 @@ export function ObservationResultCard({ result }: ObservationResultCardProps) {
{a.band} {a.band}
</span> </span>
)} )}
{a.file_format.toUpperCase()} ·{' '} {a.file_format.toUpperCase()} · {formatFileSize(a.size_bytes)}
{formatFileSize(a.size_bytes)}
</span> </span>
<a <a
href={a.file_url} href={a.file_url}

View File

@ -12,11 +12,7 @@
import { useState, useEffect, useCallback, useRef, useMemo } from 'react'; import { useState, useEffect, useCallback, useRef, useMemo } from 'react';
import axios from 'axios'; import axios from 'axios';
import { extractErrorMessage } from '../utils/apiError'; import { extractErrorMessage } from '../utils/apiError';
import type { import type { ObservationRecord, Candidate, CapabilitySpec } from '../types';
ObservationRecord,
Candidate,
CapabilitySpec,
} from '../types';
import type { ObservationBatchResult } from '../components/observation/constants'; import type { ObservationBatchResult } from '../components/observation/constants';
interface UseObservationProps { interface UseObservationProps {
@ -32,6 +28,8 @@ export interface SearchForm {
dec: string; dec: string;
radius: string; radius: string;
release: string; release: string;
/// 数据发布的子版本(仅 LAMOST 有意义);空串表示用该 DR 的默认子版本
version: string;
} }
export const EMPTY_SEARCH_FORM: SearchForm = { export const EMPTY_SEARCH_FORM: SearchForm = {
@ -42,6 +40,7 @@ export const EMPTY_SEARCH_FORM: SearchForm = {
dec: '', dec: '',
radius: '0.1', radius: '0.1',
release: '', release: '',
version: '',
}; };
export function useObservation({ isAuthenticated }: UseObservationProps) { export function useObservation({ isAuthenticated }: UseObservationProps) {
@ -65,7 +64,9 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
}, []); }, []);
useEffect(() => { useEffect(() => {
if (isAuthenticated === true) fetchCapabilities(); if (isAuthenticated !== true) return;
// eslint-disable-next-line react-hooks/set-state-in-effect -- async fetch, setState in microtask
fetchCapabilities();
}, [isAuthenticated, fetchCapabilities]); }, [isAuthenticated, fetchCapabilities]);
// ════════════════════════════════════════════════════ // ════════════════════════════════════════════════════
@ -88,6 +89,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
const valid = currentSpec.releases; const valid = currentSpec.releases;
const hardMax = currentSpec.hard_max_radius_deg; const hardMax = currentSpec.hard_max_radius_deg;
const suggested = currentSpec.suggested_max_radius_deg; const suggested = currentSpec.suggested_max_radius_deg;
// eslint-disable-next-line react-hooks/set-state-in-effect -- derived form sync on spec change
setSearchForm((f) => { setSearchForm((f) => {
let next = f; let next = f;
// release 回落(仅在该源区分版本时) // release 回落(仅在该源区分版本时)
@ -103,6 +105,27 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
}); });
}, [currentSpec]); }, [currentSpec]);
// 当 release 切换后:若该 release 有子版本列表,则校验当前 version 是否在列表里,
// 不在则清空清空后端用默认子版本。无子版本概念的源release_versions 为空)也清空。
useEffect(() => {
if (!currentSpec) return;
const versionsMap = currentSpec.release_versions ?? {};
const versions = searchForm.release
? versionsMap[searchForm.release]
: undefined;
// eslint-disable-next-line react-hooks/set-state-in-effect -- derived version reset on release change
setSearchForm((f) => {
if (!versions || versions.length === 0) {
// 该 release 无子版本概念
return f.version ? { ...f, version: '' } : f;
}
if (!f.version || !versions.includes(f.version)) {
return { ...f, version: '' }; // 清空 = 用默认(最新公开)子版本
}
return f;
});
}, [currentSpec, searchForm.release]);
const [searchResults, setSearchResults] = useState<Candidate[]>([]); const [searchResults, setSearchResults] = useState<Candidate[]>([]);
const [searching, setSearching] = useState(false); const [searching, setSearching] = useState(false);
const [searchError, setSearchError] = useState<string | null>(null); const [searchError, setSearchError] = useState<string | null>(null);
@ -131,6 +154,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
dec, dec,
radius: Math.max(0.0001, parseFloat(searchForm.radius) || 0.1), radius: Math.max(0.0001, parseFloat(searchForm.radius) || 0.1),
release: searchForm.release || undefined, release: searchForm.release || undefined,
version: searchForm.version || undefined,
}, },
}); });
setSearchResults(res.data ?? []); setSearchResults(res.data ?? []);
@ -169,13 +193,6 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
useState<ObservationBatchResult | null>(null); useState<ObservationBatchResult | null>(null);
const [downloadError, setDownloadError] = useState<string | null>(null); const [downloadError, setDownloadError] = useState<string | null>(null);
/** 下载选中的候选源(标识符模式) */
const downloadSelected = useCallback(async () => {
const ids = Array.from(selectedSourceIds);
if (ids.length === 0) return;
await downloadByIds(ids);
}, [selectedSourceIds]); // eslint-disable-line react-hooks/exhaustive-deps
/** 按标识符列表下载 */ /** 按标识符列表下载 */
const downloadByIds = useCallback( const downloadByIds = useCallback(
async (sourceIds: string[]) => { async (sourceIds: string[]) => {
@ -190,6 +207,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
product: searchForm.product, product: searchForm.product,
subtype: searchForm.subtype || undefined, subtype: searchForm.subtype || undefined,
release: searchForm.release || undefined, release: searchForm.release || undefined,
version: searchForm.version || undefined,
force: false, force: false,
mode: 'identifiers', mode: 'identifiers',
source_ids: sourceIds, source_ids: sourceIds,
@ -205,6 +223,13 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
[searchForm] [searchForm]
); );
/** 下载选中的候选源(标识符模式) */
const downloadSelected = useCallback(async () => {
const ids = Array.from(selectedSourceIds);
if (ids.length === 0) return;
await downloadByIds(ids);
}, [selectedSourceIds, downloadByIds]);
/** 按坐标下载cone 检索 + 全部下载) */ /** 按坐标下载cone 检索 + 全部下载) */
const downloadByCoordinates = useCallback( const downloadByCoordinates = useCallback(
async (strategy: 'nearest' | 'all' = 'nearest') => { async (strategy: 'nearest' | 'all' = 'nearest') => {
@ -224,6 +249,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
product: searchForm.product, product: searchForm.product,
subtype: searchForm.subtype || undefined, subtype: searchForm.subtype || undefined,
release: searchForm.release || undefined, release: searchForm.release || undefined,
version: searchForm.version || undefined,
force: false, force: false,
mode: 'coordinates', mode: 'coordinates',
ra, ra,
@ -265,9 +291,10 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
setLibraryLoading(true); setLibraryLoading(true);
setLibraryError(null); setLibraryError(null);
try { try {
const res = await axios.get<{ items: ObservationRecord[]; total: number }>( const res = await axios.get<{
'/api/observation/list', items: ObservationRecord[];
{ total: number;
}>('/api/observation/list', {
params: { params: {
source: libSource !== 'all' ? libSource : undefined, source: libSource !== 'all' ? libSource : undefined,
product: libProduct !== 'all' ? libProduct : undefined, product: libProduct !== 'all' ? libProduct : undefined,
@ -276,14 +303,11 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
limit: libPageSize, limit: libPageSize,
offset: (libPage - 1) * libPageSize, offset: (libPage - 1) * libPageSize,
}, },
} });
);
setLibraryItems(res.data.items ?? []); setLibraryItems(res.data.items ?? []);
setLibraryTotal(res.data.total ?? 0); setLibraryTotal(res.data.total ?? 0);
} catch (e: unknown) { } catch (e: unknown) {
setLibraryError( setLibraryError(extractErrorMessage(e, '加载缓存库失败,请检查后端连接'));
extractErrorMessage(e, '加载缓存库失败,请检查后端连接')
);
setLibraryItems([]); setLibraryItems([]);
} finally { } finally {
setLibraryLoading(false); setLibraryLoading(false);
@ -300,6 +324,7 @@ export function useObservation({ isAuthenticated }: UseObservationProps) {
// 任意筛选/分页变更 → 重新请求后端 // 任意筛选/分页变更 → 重新请求后端
useEffect(() => { useEffect(() => {
// eslint-disable-next-line react-hooks/set-state-in-effect -- async fetch, setState in microtask
fetchLibrary(); fetchLibrary();
}, [fetchLibrary]); }, [fetchLibrary]);

View File

@ -25,6 +25,7 @@ import {
Database, Database,
Send, Send,
RotateCw, RotateCw,
Lock,
} from 'lucide-react'; } from 'lucide-react';
import { CustomSelect } from '../components/CustomSelect'; import { CustomSelect } from '../components/CustomSelect';
import { ObservationResultCard } from '../components/observation/ObservationResultCard'; import { ObservationResultCard } from '../components/observation/ObservationResultCard';
@ -56,11 +57,13 @@ interface ObservationPanelProps {
selectNone: () => void; selectNone: () => void;
// 下载 // 下载
downloading: boolean; downloading: boolean;
downloadResult: import('../components/observation/constants').ObservationBatchResult | null; downloadResult:
import('../components/observation/constants').ObservationBatchResult | null;
downloadError: string | null; downloadError: string | null;
setDownloadResult: React.Dispatch< setDownloadResult: React.Dispatch<
React.SetStateAction< React.SetStateAction<
import('../components/observation/constants').ObservationBatchResult | null | import('../components/observation/constants').ObservationBatchResult
| null
> >
>; >;
downloadSelected: () => Promise<void>; downloadSelected: () => Promise<void>;
@ -195,8 +198,7 @@ function SearchView({
const subtypeOptions = useMemo(() => { const subtypeOptions = useMemo(() => {
const cap = capabilities.find( const cap = capabilities.find(
(c) => (c) => c.source === searchForm.source && c.product === searchForm.product
c.source === searchForm.source && c.product === searchForm.product
); );
return (cap?.subtypes ?? []).map((s) => ({ value: s, label: s })); return (cap?.subtypes ?? []).map((s) => ({ value: s, label: s }));
}, [capabilities, searchForm.source, searchForm.product]); }, [capabilities, searchForm.source, searchForm.product]);
@ -208,22 +210,42 @@ function SearchView({
// 当前源+产品支持的版本列表(供下拉) // 当前源+产品支持的版本列表(供下拉)
const releaseOptions = useMemo(() => { const releaseOptions = useMemo(() => {
const authRequired = new Set(currentSpec?.releases_requiring_auth ?? []);
return (currentSpec?.releases ?? []).map((r) => ({ return (currentSpec?.releases ?? []).map((r) => ({
value: r, value: r,
label: r.toUpperCase(), label: authRequired.has(r)
? `${r.toUpperCase()} · 需登录`
: r.toUpperCase(),
})); }));
}, [currentSpec]); }, [currentSpec]);
// LAMOST MRS 联动约束MRS中分辨率从 DR7 起才支持DR5/DR6 无 MRS 数据 // 当前选中的 release 是否需要登录Internal DR
const lamostMrsBlocked = useMemo(() => { const currentReleaseNeedsAuth = useMemo(() => {
if (searchForm.source !== 'lamost' || searchForm.subtype !== 'mrs') if (!searchForm.release || !currentSpec) return false;
return null; return (currentSpec.releases_requiring_auth ?? []).includes(
const rel = searchForm.release; searchForm.release
if (rel === 'dr5' || rel === 'dr6') { );
return 'LAMOST 中分辨率MRS从 DR7 起才支持,当前版本无 MRS 数据'; }, [searchForm.release, currentSpec]);
// 交叉约束:当前 release 是否支持当前 subtype从 capabilities 派生,不硬编码)
const subtypeBlocked = useMemo(() => {
if (!currentSpec || !searchForm.release || !searchForm.subtype) return null;
const map = currentSpec.subtypes_per_release;
if (!map || Object.keys(map).length === 0) return null; // 无交叉约束
const valid = map[searchForm.release];
if (valid && !valid.includes(searchForm.subtype)) {
return `'${searchForm.release.toUpperCase()}' 不支持子类型 '${searchForm.subtype}'`;
} }
return null; return null;
}, [searchForm.source, searchForm.subtype, searchForm.release]); }, [currentSpec, searchForm.release, searchForm.subtype]);
// 当前 release 对应的子版本列表(仅 LAMOST 有内容)
const versionOptions = useMemo(() => {
if (!currentSpec || !searchForm.release) return [];
const versionsMap = currentSpec.release_versions ?? {};
const versions = versionsMap[searchForm.release] ?? [];
return versions.map((v) => ({ value: v, label: v.toUpperCase() }));
}, [currentSpec, searchForm.release]);
const handleSubmitIds = () => { const handleSubmitIds = () => {
const ids = idsText const ids = idsText
@ -275,7 +297,7 @@ function SearchView({
.map((c) => c.product); .map((c) => c.product);
const safeProduct = newProducts.includes(f.product) const safeProduct = newProducts.includes(f.product)
? f.product ? f.product
: newProducts[0] ?? f.product; : (newProducts[0] ?? f.product);
return { ...f, source: v, product: safeProduct, subtype: '' }; return { ...f, source: v, product: safeProduct, subtype: '' };
}); });
}} }}
@ -331,27 +353,67 @@ function SearchView({
} }
className="w-full" className="w-full"
options={[ options={[
{ value: '__none__', label: `默认${currentSpec?.default_release ? `${currentSpec.default_release.toUpperCase()}` : ''}` }, {
value: '__none__',
label: `默认${currentSpec?.default_release ? `${currentSpec.default_release.toUpperCase()}` : ''}`,
},
...releaseOptions, ...releaseOptions,
]} ]}
/> />
)} )}
</Field> </Field>
{/* 子版本(仅 LAMOST 有内容;其他源该 Field 不渲染) */}
{versionOptions.length > 0 && (
<Field label="子版本">
<CustomSelect
value={searchForm.version || '__none__'}
onChange={(v) =>
setSearchForm((f) => ({
...f,
version: v === '__none__' ? '' : v,
}))
}
className="w-full"
options={[
{ value: '__none__', label: '默认(最新公开)' },
...versionOptions,
]}
/>
</Field>
)}
</div> </div>
{/* LAMOST MRS 版本约束提示 */} {/* Internal DR 需登录提示 */}
{lamostMrsBlocked && ( {currentReleaseNeedsAuth && (
<div className="flex items-center gap-2 text-xs text-rose-700 bg-rose-50 border border-rose-200 rounded-md p-2.5">
<Lock className="w-3.5 h-3.5 shrink-0" />
<span>
{searchForm.release.toUpperCase()} Internal
oauth.china-vo.org 访
</span>
</div>
)}
{/* 交叉约束提示:当前 release 不支持该 subtype */}
{subtypeBlocked && (
<div className="flex items-center gap-2 text-xs text-amber-700 bg-amber-50 border border-amber-200 rounded-md p-2.5"> <div className="flex items-center gap-2 text-xs text-amber-700 bg-amber-50 border border-amber-200 rounded-md p-2.5">
<AlertTriangle className="w-3.5 h-3.5 shrink-0" /> <AlertTriangle className="w-3.5 h-3.5 shrink-0" />
<span>{lamostMrsBlocked}</span> <span>{subtypeBlocked}</span>
{currentSpec?.subtypes && currentSpec.subtypes.length > 0 && (
<button <button
onClick={() => onClick={() => {
setSearchForm((f) => ({ ...f, subtype: 'lrs' })) // 切换到该 release 支持的首个 subtype
} const map = currentSpec?.subtypes_per_release;
const valid =
(map && searchForm.release && map[searchForm.release]) ||
currentSpec?.subtypes;
setSearchForm((f) => ({ ...f, subtype: valid?.[0] ?? '' }));
}}
className="ml-auto px-2 py-0.5 rounded bg-amber-100 hover:bg-amber-200 text-amber-800 font-bold text-[10px]" className="ml-auto px-2 py-0.5 rounded bg-amber-100 hover:bg-amber-200 text-amber-800 font-bold text-[10px]"
> >
LRS
</button> </button>
)}
</div> </div>
)} )}
@ -410,14 +472,15 @@ function SearchView({
<AlertTriangle className="w-3.5 h-3.5 shrink-0" /> <AlertTriangle className="w-3.5 h-3.5 shrink-0" />
<span> <span>
{searchForm.radius}° {searchForm.radius}°
{currentSpec.suggested_max_radius_deg}° {currentSpec.suggested_max_radius_deg}
°
</span> </span>
</div> </div>
)} )}
<div className="flex items-center gap-2 pt-1"> <div className="flex items-center gap-2 pt-1">
<button <button
onClick={runSearch} onClick={runSearch}
disabled={searching} disabled={searching || currentReleaseNeedsAuth}
className="btn-console btn-console-primary px-4 py-2 rounded-md text-xs font-bold flex items-center gap-2 disabled:opacity-50" className="btn-console btn-console-primary px-4 py-2 rounded-md text-xs font-bold flex items-center gap-2 disabled:opacity-50"
> >
{searching ? ( {searching ? (
@ -432,7 +495,7 @@ function SearchView({
</span> </span>
<button <button
onClick={() => downloadByCoordinates('nearest')} onClick={() => downloadByCoordinates('nearest')}
disabled={downloading} disabled={downloading || currentReleaseNeedsAuth}
className="ml-auto px-3 py-2 rounded-md text-xs font-bold flex items-center gap-1.5 bg-emerald-50 text-emerald-700 border border-emerald-200 hover:bg-emerald-100 disabled:opacity-50" className="ml-auto px-3 py-2 rounded-md text-xs font-bold flex items-center gap-1.5 bg-emerald-50 text-emerald-700 border border-emerald-200 hover:bg-emerald-100 disabled:opacity-50"
> >
<Download className="w-3.5 h-3.5" /> <Download className="w-3.5 h-3.5" />
@ -786,7 +849,10 @@ function LibraryView({
<> <>
<div className="grid grid-cols-1 md:grid-cols-2 gap-3"> <div className="grid grid-cols-1 md:grid-cols-2 gap-3">
{libraryItems.map((rec) => ( {libraryItems.map((rec) => (
<LibraryCard key={`${rec.source}|${rec.product}|${rec.source_id}`} rec={rec} /> <LibraryCard
key={`${rec.source}|${rec.product}|${rec.source_id}`}
rec={rec}
/>
))} ))}
</div> </div>
@ -863,7 +929,11 @@ function Field({
compact?: boolean; compact?: boolean;
}) { }) {
return ( return (
<div className={compact ? 'w-full sm:w-36 space-y-1.5 font-bold' : 'space-y-1.5'}> <div
className={
compact ? 'w-full sm:w-36 space-y-1.5 font-bold' : 'space-y-1.5'
}
>
<label className="block text-slate-500 font-bold text-xs">{label}</label> <label className="block text-slate-500 font-bold text-xs">{label}</label>
{children} {children}
</div> </div>
@ -894,7 +964,12 @@ function LibraryCard({ rec }: { rec: ObservationRecord }) {
const theme = SOURCE_THEME[rec.source] ?? SOURCE_THEME.lamost; const theme = SOURCE_THEME[rec.source] ?? SOURCE_THEME.lamost;
const productLabel = PRODUCT_LABEL[rec.product] ?? rec.product; const productLabel = PRODUCT_LABEL[rec.product] ?? rec.product;
let artifacts: Array<{ path: string; format: string; size?: number; band?: string }> = []; let artifacts: Array<{
path: string;
format: string;
size?: number;
band?: string;
}> = [];
try { try {
const parsed = JSON.parse(rec.artifacts_json); const parsed = JSON.parse(rec.artifacts_json);
if (Array.isArray(parsed)) artifacts = parsed; if (Array.isArray(parsed)) artifacts = parsed;
@ -910,7 +985,9 @@ function LibraryCard({ rec }: { rec: ObservationRecord }) {
<div className="bg-white border border-slate-200 rounded-lg p-3.5 shadow-xs hover:shadow-md hover:border-slate-300 transition-all"> <div className="bg-white border border-slate-200 rounded-lg p-3.5 shadow-xs hover:shadow-md hover:border-slate-300 transition-all">
<div className="flex items-center justify-between mb-2 pb-2 border-b border-slate-100"> <div className="flex items-center justify-between mb-2 pb-2 border-b border-slate-100">
<div className="flex items-center gap-1.5"> <div className="flex items-center gap-1.5">
<span className={`px-2 py-0.5 rounded text-[10px] font-bold ${theme.badge}`}> <span
className={`px-2 py-0.5 rounded text-[10px] font-bold ${theme.badge}`}
>
{theme.label} {theme.label}
</span> </span>
<span className="px-2 py-0.5 rounded text-[10px] font-bold bg-slate-200 text-slate-700"> <span className="px-2 py-0.5 rounded text-[10px] font-bold bg-slate-200 text-slate-700">
@ -927,7 +1004,10 @@ function LibraryCard({ rec }: { rec: ObservationRecord }) {
<div className="text-[9px] font-bold text-slate-400 tracking-widest uppercase"> <div className="text-[9px] font-bold text-slate-400 tracking-widest uppercase">
Source ID Source ID
</div> </div>
<div className="font-mono text-xs text-slate-800 font-semibold truncate" title={rec.source_id}> <div
className="font-mono text-xs text-slate-800 font-semibold truncate"
title={rec.source_id}
>
{rec.source_id} {rec.source_id}
</div> </div>
</div> </div>

View File

@ -250,6 +250,12 @@ export interface CapabilitySpec {
subtypes: string[]; subtypes: string[];
releases: string[]; releases: string[];
default_release?: string | null; default_release?: string | null;
/// 每个 release 对应的子版本列表(仅 LAMOST 有内容,如 {dr11: ["v0","v1.0","v1.1","v2.0"]}
release_versions?: Record<string, string[]>;
/// 需要登录认证才能访问的 release 列表(如 LAMOST Internal DR12-14空表示全部公开
releases_requiring_auth?: string[];
/// 每个 release 实际支持的 subtype交叉约束如 LAMOST DR5/6 无 MRS空表示全部支持
subtypes_per_release?: Record<string, string[]>;
suggested_max_radius_deg: number; suggested_max_radius_deg: number;
hard_max_radius_deg: number; hard_max_radius_deg: number;
identifier_format?: string | null; identifier_format?: string | null;

View File

@ -16,10 +16,7 @@ interface AxiosLikeError {
/// ///
/// 优先级response.data 为字符串 → 对象里的 error/message/detail/description /// 优先级response.data 为字符串 → 对象里的 error/message/detail/description
/// → 整体 JSON 序列化兜底 → axios message → fallback /// → 整体 JSON 序列化兜底 → axios message → fallback
export function extractErrorMessage( export function extractErrorMessage(e: unknown, fallback: string): string {
e: unknown,
fallback: string
): string {
const axiosError = e as AxiosLikeError; const axiosError = e as AxiosLikeError;
const data = axiosError?.response?.data; const data = axiosError?.response?.data;
if (typeof data === 'string') return data; if (typeof data === 'string') return data;

86
deploy.sh Executable file
View File

@ -0,0 +1,86 @@
#!/bin/bash
# deploy.sh — AstroResearch 一键本地打包并推送服务器脚本 (Compose 部署版)
# 退出脚本如果发生错误
set -e
# =============================================================================
# 配置参数
# =============================================================================
REMOTE_USER="fmq"
REMOTE_IP="100.66.1.2"
REMOTE_PORT="22"
REMOTE_DIR="/vol1/1000/program/astroresearch"
IMAGE_NAME="astroresearch"
TAG="latest"
TAR_NAME="astroresearch.tar.gz"
echo "=== 1. 开始在本地构建 Docker 镜像 (基于 Dockerfile.modeB) ==="
docker build -f Dockerfile.modeB -t ${IMAGE_NAME}:${TAG} .
echo "=== 2. 将镜像导出并压缩为本地临时文件 ${TAR_NAME} ==="
docker save ${IMAGE_NAME}:${TAG} | gzip > ${TAR_NAME}
echo "=== 3. 上传镜像包到服务器目标目录 ==="
scp -P ${REMOTE_PORT} ${TAR_NAME} ${REMOTE_USER}@${REMOTE_IP}:${REMOTE_DIR}/
echo "=== 4. 连接服务器:载入镜像并使用 Docker Compose 启动 ==="
# 使用 'EOF' 锁死环境,防止本地变量混淆远程环境变量
ssh -p ${REMOTE_PORT} ${REMOTE_USER}@${REMOTE_IP} << 'EOF'
set -e
# 远程运行参数
REMOTE_DIR="/vol1/1000/program/astroresearch"
IMAGE_NAME="astroresearch"
TAG="latest"
TAR_NAME="astroresearch.tar.gz"
echo " -> 正在导入 Docker 镜像..."
docker load < ${REMOTE_DIR}/${TAR_NAME}
echo " -> 清理服务器上的临时镜像压缩包..."
rm -f ${REMOTE_DIR}/${TAR_NAME}
echo " -> 在服务器上写入/更新生产环境的 docker-compose.yml..."
cat > ${REMOTE_DIR}/docker-compose.yml << 'INNER_EOF'
version: '3.8'
services:
astroresearch:
image: astroresearch:latest
container_name: astroresearch
restart: always
ports:
- "8001:8000"
env_file:
- .env
environment:
- DATABASE_URL=sqlite:///app/library/astro_research.db
- LIBRARY_DIR=/app/library
- SKILLS_DIR=/app/skills
- LOG_DIR=/app/logs
- PORT=8000
volumes:
- ./library:/app/library
- ./logs:/app/logs
- ./skills:/app/skills:ro
INNER_EOF
# 进入远程部署目录
cd ${REMOTE_DIR}
echo " -> 停止并移除已有的旧服务容器 (docker compose down)..."
docker compose down 2>/dev/null || true
echo " -> 使用 Docker Compose 启动新服务 (docker compose up -d)..."
docker compose up -d
echo " -> 部署成功!正在等待 3 秒检查服务运行状态..."
sleep 3
docker compose ps
EOF
echo "=== 5. 清理本地临时压缩包 ==="
rm -f ${TAR_NAME}
echo "🎉 所有部署操作已圆满完成!服务已在服务器上通过 Docker Compose 后台运行。"

View File

@ -25,7 +25,7 @@ open http://localhost:8000
### 两种镜像模式对比 ### 两种镜像模式对比
| | Mode A (`Dockerfile`) | Mode B (`Dockerfile.modeB`) | | | Mode A (`Dockerfile`) | Mode B (`Dockerfile.modeB`) |
|---|---|---| | --------------------- | ----------------------- | ------------------------------------------------ |
| **镜像大小** | ~23 MB | ~87 MB | | **镜像大小** | ~23 MB | ~87 MB |
| **基础镜像** | `alpine:3.21` | `distroless/cc-debian12:nonroot` | | **基础镜像** | `alpine:3.21` | `distroless/cc-debian12:nonroot` |
| **libc** | musl (静态链接) | glibc (动态链接) | | **libc** | musl (静态链接) | glibc (动态链接) |
@ -44,11 +44,13 @@ open http://localhost:8000
镜像最小23 MBObscura 作为外部二进制通过 bind mount 注入,可独立更新。 镜像最小23 MBObscura 作为外部二进制通过 bind mount 注入,可独立更新。
**构建:** **构建:**
```bash ```bash
docker build -t astroresearch:latest . docker build -t astroresearch:latest .
``` ```
**运行:** **运行:**
```bash ```bash
docker run -d --name astro -p 8000:8000 --env-file .env \ docker run -d --name astro -p 8000:8000 --env-file .env \
-v ./library:/app/library \ -v ./library:/app/library \
@ -61,6 +63,7 @@ docker run -d --name astro -p 8000:8000 --env-file .env \
> `bin/` 目录需包含编译好的 Obscura 二进制文件(`obscura` 和 `obscura-worker`)。 > `bin/` 目录需包含编译好的 Obscura 二进制文件(`obscura` 和 `obscura-worker`)。
**docker-compose.yml已内置** **docker-compose.yml已内置**
```yaml ```yaml
services: services:
astroresearch: astroresearch:
@ -90,17 +93,20 @@ services:
Obscura (V8 + BoringSSL) 编译在二进制内,单容器零外部二进制依赖。适合对外交付的一键部署包。 Obscura (V8 + BoringSSL) 编译在二进制内,单容器零外部二进制依赖。适合对外交付的一键部署包。
**前置条件:** **前置条件:**
```bash ```bash
mkdir -p libs mkdir -p libs
git clone https://github.com/h4ckf0r0day/obscura libs/obscura git clone https://github.com/h4ckf0r0day/obscura libs/obscura
``` ```
**构建:** **构建:**
```bash ```bash
docker build -f Dockerfile.modeB -t astroresearch-all:latest . docker build -f Dockerfile.modeB -t astroresearch-all:latest .
``` ```
**运行:** **运行:**
```bash ```bash
docker run -d --name astro -p 8000:8000 --env-file .env \ docker run -d --name astro -p 8000:8000 --env-file .env \
-v ./library:/app/library \ -v ./library:/app/library \
@ -124,7 +130,7 @@ docker build --build-arg USE_MIRRORS=0 -t astroresearch:latest .
**加速源:** **加速源:**
| 工具 | 镜像 | | 工具 | 镜像 |
|------|------| | ---------- | -------------------------- |
| npm | `registry.npmmirror.com` | | npm | `registry.npmmirror.com` |
| Alpine apk | `mirrors.aliyun.com` | | Alpine apk | `mirrors.aliyun.com` |
| Debian apt | `mirrors.ustc.edu.cn` | | Debian apt | `mirrors.ustc.edu.cn` |
@ -134,7 +140,7 @@ docker build --build-arg USE_MIRRORS=0 -t astroresearch:latest .
### 持久化数据卷 ### 持久化数据卷
| 容器路径 | 说明 | 推荐权限 | | 容器路径 | 说明 | 推荐权限 |
|----------|------|---------| | ---------------- | ----------------------------------------- | -------------- |
| `/app/library` | SQLite 数据库 + PDF/HTML 全文 | 读写 | | `/app/library` | SQLite 数据库 + PDF/HTML 全文 | 读写 |
| `/app/logs` | 应用日志(仅`LOG_OUTPUTS=file` 时写入) | 读写 | | `/app/logs` | 应用日志(仅`LOG_OUTPUTS=file` 时写入) | 读写 |
| `/app/skills` | Agent 技能文件 (SKILL.md) | 只读 (`:ro`) | | `/app/skills` | Agent 技能文件 (SKILL.md) | 只读 (`:ro`) |
@ -157,21 +163,26 @@ docker build --build-arg USE_MIRRORS=0 -t astroresearch:latest .
### 构建步骤 ### 构建步骤
**步骤 1构建前端** **步骤 1构建前端**
```bash ```bash
cd dashboard cd dashboard
npm install npm install
npm run build npm run build
``` ```
产物位于 `dashboard/dist/` 产物位于 `dashboard/dist/`
**步骤 2编译后端** **步骤 2编译后端**
```bash ```bash
cd .. cd ..
cargo build --release cargo build --release
``` ```
产物位于 `target/release/astroresearch` 产物位于 `target/release/astroresearch`
**步骤 3可选健康检查工具** **步骤 3可选健康检查工具**
```bash ```bash
cargo build --release --bin health_check cargo build --release --bin health_check
``` ```
@ -195,11 +206,12 @@ cp .env.example .env # 编辑填入 API Key
Obscura 作为独立二进制运行,与主进程隔离。适合常规生产环境。 Obscura 作为独立二进制运行,与主进程隔离。适合常规生产环境。
| 部署方式 | 步骤 | | 部署方式 | 步骤 |
|----------|------| | ---------------- | ----------------------------------------------------- |
| **Docker** | `docker compose up -d`(自动 bind mount `./bin` | | **Docker** | `docker compose up -d`(自动 bind mount `./bin` |
| **源码** | 下载 obscura 到`bin/`,赋予执行权限后启动 | | **源码** | 下载 obscura 到`bin/`,赋予执行权限后启动 |
二进制安装(源码部署时): 二进制安装(源码部署时):
```bash ```bash
mkdir -p bin/ mkdir -p bin/
# 下载 obscura 和 obscura-worker 到 bin/,赋予执行权限 # 下载 obscura 和 obscura-worker 到 bin/,赋予执行权限
@ -211,7 +223,7 @@ chmod +x bin/obscura bin/obscura-worker
Obscura (V8 + BoringSSL) 编译进二进制,单文件零外部依赖。 Obscura (V8 + BoringSSL) 编译进二进制,单文件零外部依赖。
| 部署方式 | 构建命令 | | 部署方式 | 构建命令 |
|----------|---------| | ---------------- | ------------------------------------------------------------------ |
| **Docker** | `docker build -f Dockerfile.modeB -t astroresearch-all:latest .` | | **Docker** | `docker build -f Dockerfile.modeB -t astroresearch-all:latest .` |
| **源码** | `cargo build --release --features obscura-inprocess` | | **源码** | `cargo build --release --features obscura-inprocess` |
@ -228,7 +240,7 @@ Obscura (V8 + BoringSSL) 编译进二进制,单文件零外部依赖。
**优化指标 (Mode A, release-min)** **优化指标 (Mode A, release-min)**
| 指标 | release | release-min | 降幅 | | 指标 | release | release-min | 降幅 |
|------|---------|-------------|------| | --------------- | ------- | ----------- | ---- |
| 二进制大小 | 17.0 MB | 8.3 MB | 51% | | 二进制大小 | 17.0 MB | 8.3 MB | 51% |
| 启动 RSS | 34.8 MB | 32.9 MB | 5% | | 启动 RSS | 34.8 MB | 32.9 MB | 5% |
| 虚拟内存 (VSZ) | 1.27 GB | 302 MB | 76% | | 虚拟内存 (VSZ) | 1.27 GB | 302 MB | 76% |
@ -243,6 +255,7 @@ docker build -t astroresearch:latest .
``` ```
**限制 Tokio 线程数:** **限制 Tokio 线程数:**
```bash ```bash
PORT=8000 TOKIO_WORKER_THREADS=1 ./astroresearch PORT=8000 TOKIO_WORKER_THREADS=1 ./astroresearch
``` ```
@ -252,7 +265,7 @@ PORT=8000 TOKIO_WORKER_THREADS=1 ./astroresearch
## 5. 环境变量 ## 5. 环境变量
| 变量名 | 必需 | 默认值 | 说明 | | 变量名 | 必需 | 默认值 | 说明 |
| :--- | :--- | :--- | :--- | | :-------------------------- | :--- | :------------------------------------------ | :------------------------------- |
| `DATABASE_URL` | 否 | `sqlite:///app/library/astro_research.db` | SQLite 连接 URL | | `DATABASE_URL` | 否 | `sqlite:///app/library/astro_research.db` | SQLite 连接 URL |
| `ADS_API_KEY` | 是 | - | NASA ADS API Token | | `ADS_API_KEY` | 是 | - | NASA ADS API Token |
| `LLM_API_KEY` | 是 | - | LLM API Key | | `LLM_API_KEY` | 是 | - | LLM API Key |
@ -277,11 +290,13 @@ PORT=8000 TOKIO_WORKER_THREADS=1 ./astroresearch
## 6. 健康检查与维护 ## 6. 健康检查与维护
**Docker 健康检查Mode A** **Docker 健康检查Mode A**
```bash ```bash
docker ps --filter “health=healthy” --filter “name=astroresearch” docker ps --filter “health=healthy” --filter “name=astroresearch”
``` ```
Mode B (distroless) 无内置 healthcheck可在编排层配置 Mode B (distroless) 无内置 healthcheck可在编排层配置
```yaml ```yaml
# docker-compose 或 k8s # docker-compose 或 k8s
healthcheck: healthcheck:
@ -289,6 +304,7 @@ healthcheck:
``` ```
**源码部署健康检查:** **源码部署健康检查:**
```bash ```bash
# 只读扫描 # 只读扫描
./health_check ./health_check

View File

@ -250,7 +250,7 @@ pub fn rough_estimate_tokens(messages: &[ChatMessage]) -> usize {
messages messages
.iter() .iter()
.map(|m| { .map(|m| {
let content_len = m.text().map_or(0, |c| estimate_tokens_from_text(c)); let content_len = m.text().map_or(0, estimate_tokens_from_text);
content_len + 4 // 消息结构 overhead 约 4 token content_len + 4 // 消息结构 overhead 约 4 token
}) })
.sum() .sum()
@ -697,7 +697,6 @@ fn spawn_memory_extraction(
memory_manager: std::sync::Arc<tokio::sync::Mutex<crate::agent::memory::MemoryManager>>, memory_manager: std::sync::Arc<tokio::sync::Mutex<crate::agent::memory::MemoryManager>>,
app_state: std::sync::Arc<crate::api::AppState>, app_state: std::sync::Arc<crate::api::AppState>,
) { ) {
let sid = session_id.to_string(); let sid = session_id.to_string();
let mem_mgr = memory_manager.clone(); let mem_mgr = memory_manager.clone();
let app = app_state.clone(); let app = app_state.clone();

View File

@ -2,7 +2,6 @@
// //
// 内置 Hooks — CancellationHook、MetricsHook、AuditLogHook。 // 内置 Hooks — CancellationHook、MetricsHook、AuditLogHook。
use std::collections::HashSet;
use std::sync::Arc; use std::sync::Arc;
use tokio::sync::Mutex; use tokio::sync::Mutex;
@ -48,11 +47,11 @@ impl ContextDeduplicator {
/// 取消检查 Hook — 在每次工具执行前检查用户是否中止了会话。 /// 取消检查 Hook — 在每次工具执行前检查用户是否中止了会话。
pub struct CancellationHook { pub struct CancellationHook {
cancelled_runs: Arc<Mutex<HashSet<String>>>, cancelled_runs: Arc<dashmap::DashMap<String, ()>>,
} }
impl CancellationHook { impl CancellationHook {
pub fn new(cancelled_runs: Arc<Mutex<HashSet<String>>>) -> Self { pub fn new(cancelled_runs: Arc<dashmap::DashMap<String, ()>>) -> Self {
CancellationHook { cancelled_runs } CancellationHook { cancelled_runs }
} }
} }
@ -68,8 +67,7 @@ impl AgentHook for CancellationHook {
} }
async fn pre_tool_use(&self, ctx: &PreToolUseContext) -> PreToolUseAction { async fn pre_tool_use(&self, ctx: &PreToolUseContext) -> PreToolUseAction {
let runs = self.cancelled_runs.lock().await; if self.cancelled_runs.contains_key(&ctx.session_id) {
if runs.contains(&ctx.session_id) {
warn!( warn!(
"[CancellationHook] 会话 {} 已被用户取消,阻止工具 {} 执行", "[CancellationHook] 会话 {} 已被用户取消,阻止工具 {} 执行",
ctx.session_id, ctx.tool_name ctx.session_id, ctx.tool_name
@ -83,8 +81,7 @@ impl AgentHook for CancellationHook {
async fn on_session_stop(&self, ctx: &SessionStopContext<'_>) { async fn on_session_stop(&self, ctx: &SessionStopContext<'_>) {
// 清理取消状态 // 清理取消状态
let mut runs = self.cancelled_runs.lock().await; self.cancelled_runs.remove(&ctx.session_id);
runs.remove(&ctx.session_id);
} }
} }

View File

@ -201,9 +201,8 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_cancellation_hook_blocks_when_cancelled() { async fn test_cancellation_hook_blocks_when_cancelled() {
let mut cancelled = HashSet::new(); let cancelled_runs = Arc::new(dashmap::DashMap::new());
cancelled.insert("test_session".to_string()); cancelled_runs.insert("test_session".to_string(), ());
let cancelled_runs = Arc::new(tokio::sync::Mutex::new(cancelled));
let hook = CancellationHook::new(cancelled_runs); let hook = CancellationHook::new(cancelled_runs);
let ctx = PreToolUseContext { let ctx = PreToolUseContext {
@ -219,7 +218,7 @@ mod tests {
#[tokio::test] #[tokio::test]
async fn test_cancellation_hook_allows_when_not_cancelled() { async fn test_cancellation_hook_allows_when_not_cancelled() {
let cancelled_runs = Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())); let cancelled_runs = Arc::new(dashmap::DashMap::new());
let hook = CancellationHook::new(cancelled_runs); let hook = CancellationHook::new(cancelled_runs);
let ctx = PreToolUseContext { let ctx = PreToolUseContext {

View File

@ -52,7 +52,7 @@ impl HookRegistry {
/// 使得 AgentRuntime.get_metrics() 能查询到实际运行数据。 /// 使得 AgentRuntime.get_metrics() 能查询到实际运行数据。
pub fn with_builtins( pub fn with_builtins(
db: SqlitePool, db: SqlitePool,
cancelled_runs: Arc<tokio::sync::Mutex<std::collections::HashSet<String>>>, cancelled_runs: Arc<dashmap::DashMap<String, ()>>,
metrics_data: Option<Arc<tokio::sync::Mutex<MetricsData>>>, metrics_data: Option<Arc<tokio::sync::Mutex<MetricsData>>>,
) -> Self { ) -> Self {
let mut registry = Self::new(); let mut registry = Self::new();

View File

@ -138,6 +138,7 @@ pub(super) async fn process_single_result(
max_output_chars, max_output_chars,
&tool_results_dir, &tool_results_dir,
) )
.await
}; };
// PostToolUse hook // PostToolUse hook

View File

@ -538,8 +538,7 @@ pub async fn execute_parallel(
let cancel_handle = tokio::spawn(async move { let cancel_handle = tokio::spawn(async move {
loop { loop {
tokio::time::sleep(std::time::Duration::from_millis(250)).await; tokio::time::sleep(std::time::Duration::from_millis(250)).await;
let locked = app_state_ref.cancelled_runs.lock().await; if app_state_ref.cancelled_runs.contains_key(&sid_ref) {
if locked.contains(&sid_ref) {
cancel_flag.store(true, Ordering::SeqCst); cancel_flag.store(true, Ordering::SeqCst);
return; return;
} }

View File

@ -96,10 +96,12 @@ pub struct AgentRuntime {
} }
impl AgentRuntime { impl AgentRuntime {
/// 创建新的运行时实例 /// 共享初始化逻辑:根据 config + mode 构建完整运行时。
pub fn new(app_state: Arc<AppState>) -> Self { fn init(
let mut config = AgentConfig::default(); app_state: Arc<AppState>,
let mode_registry = ModeRegistry::builtins(); mut config: AgentConfig,
mode_registry: ModeRegistry,
) -> Self {
let mode = mode_registry.get(&config.mode).copied().unwrap_or_else(|| { let mode = mode_registry.get(&config.mode).copied().unwrap_or_else(|| {
tracing::warn!("[AgentRuntime] 未知模式 '{}',回退到默认模式", config.mode); tracing::warn!("[AgentRuntime] 未知模式 '{}',回退到默认模式", config.mode);
mode_registry mode_registry
@ -123,14 +125,13 @@ impl AgentRuntime {
tool_registry.add_tool(Box::new(crate::agent::tools::memory::SaveMemoryTool::new( tool_registry.add_tool(Box::new(crate::agent::tools::memory::SaveMemoryTool::new(
app_state.memory_manager.clone(), app_state.memory_manager.clone(),
))); )));
// 替换 DelegateResearchTool 为带有 permission_checker 的版本SSE 通道通过 ToolContext 注入) // 替换 DelegateResearchTool 为带有 permission_checker 的版本
tool_registry.replace_tool(Box::new( tool_registry.replace_tool(Box::new(
crate::agent::tools::subagent::SubAgentTool::new_with_hooks( crate::agent::tools::subagent::SubAgentTool::new_with_hooks(
None, None,
permission_checker.clone(), permission_checker.clone(),
), ),
)); ));
// 会话级权限检查器将在 run_react_loop 中按 session_id 注册
// 视觉模型可用时注册 analyze_image 工具 // 视觉模型可用时注册 analyze_image 工具
if app_state.vision_llm.is_some() { if app_state.vision_llm.is_some() {
@ -140,9 +141,9 @@ impl AgentRuntime {
// ── 应用模式的工具集过滤 ── // ── 应用模式的工具集过滤 ──
apply_mode_tool_filter(&mut tool_registry, mode); apply_mode_tool_filter(&mut tool_registry, mode);
// 初始化 checkpoint 管理器 // 初始化 checkpoint 管理器(存储在 library_dir/.checkpoints 下)
let checkpoint_enabled = true; let checkpoint_enabled = true;
let checkpoint_store = app_state.config.library_dir.join("..").join(".checkpoints"); let checkpoint_store = app_state.config.library_dir.join(".checkpoints");
let checkpoint_manager = Arc::new(checkpoint::CheckpointManager::new( let checkpoint_manager = Arc::new(checkpoint::CheckpointManager::new(
std::fs::canonicalize(&checkpoint_store).unwrap_or(checkpoint_store), std::fs::canonicalize(&checkpoint_store).unwrap_or(checkpoint_store),
checkpoint_enabled, checkpoint_enabled,
@ -168,78 +169,25 @@ impl AgentRuntime {
} }
} }
/// 创建新的运行时实例
pub fn new(app_state: Arc<AppState>) -> Self {
let config = AgentConfig::default();
let mode_registry = ModeRegistry::builtins();
Self::init(app_state, config, mode_registry)
}
/// 创建带自定义配置的运行时实例 /// 创建带自定义配置的运行时实例
pub fn with_config(app_state: Arc<AppState>, mut config: AgentConfig) -> Self { pub fn with_config(app_state: Arc<AppState>, config: AgentConfig) -> Self {
let mode_registry = ModeRegistry::builtins(); let mode_registry = ModeRegistry::builtins();
let mode = mode_registry.get(&config.mode).copied().unwrap_or_else(|| { Self::init(app_state, config, mode_registry)
tracing::warn!("[AgentRuntime] 未知模式 '{}',回退到默认模式", config.mode);
mode_registry
.get(ModeRegistry::default_id())
.copied()
.unwrap_or(&modes::default::DEFAULT_MODE)
});
// 合并模式配置预设
apply_mode_config(&mut config, mode);
let queue = Arc::new(BgNotificationQueue::new());
let metrics_data = Arc::new(tokio::sync::Mutex::new(super::hooks::MetricsData::default()));
let permission_checker = Arc::new(permission::PermissionChecker::from_config(&config));
let denial_tracker = Arc::new(std::sync::Mutex::new(denial_tracker::DenialTracker::new(
config.denial_max_consecutive,
config.denial_max_total,
)));
let skill_registry = app_state.skill_registry.clone();
let mut tool_registry = ToolRegistry::new_with_queue(Some(queue.clone()), skill_registry);
tool_registry.add_tool(Box::new(crate::agent::tools::memory::SaveMemoryTool::new(
app_state.memory_manager.clone(),
)));
tool_registry.replace_tool(Box::new(
crate::agent::tools::subagent::SubAgentTool::new_with_hooks(
None,
permission_checker.clone(),
),
));
// 会话级权限检查器将在 run_react_loop 中按 session_id 注册
// 视觉模型可用时注册 analyze_image 工具
if app_state.vision_llm.is_some() {
tool_registry.add_tool(Box::new(crate::agent::tools::astro::AnalyzeImageTool));
}
// ── 应用模式的工具集过滤 ──
apply_mode_tool_filter(&mut tool_registry, mode);
// 初始化 checkpoint 管理器
let checkpoint_enabled = true;
let checkpoint_store = app_state.config.library_dir.join("..").join(".checkpoints");
let checkpoint_manager = Arc::new(checkpoint::CheckpointManager::new(
std::fs::canonicalize(&checkpoint_store).unwrap_or(checkpoint_store),
checkpoint_enabled,
));
AgentRuntime {
app_state,
config,
tool_registry,
bg_notification_queue: queue,
metrics_data,
compaction_breaker: Arc::new(std::sync::Mutex::new(
circuit_breaker::CompactionCircuitBreaker::new(),
)),
permission_checker,
denial_tracker,
read_file_state: Arc::new(std::sync::Mutex::new(file_cache::FileStateCache::new())),
prompt_cache: std::sync::Mutex::new(SystemPromptCache::new()),
collapse_log: Arc::new(std::sync::Mutex::new(compact::collapse::CollapseLog::new())),
checkpoint_manager,
mode,
mode_registry,
}
} }
/// 返回当前运行指标快照(锁异常时返回默认值并记录警告) /// 返回当前运行指标快照(锁异常时返回默认值并记录警告)
pub fn get_metrics(&self) -> super::hooks::MetricsData { pub fn get_metrics(&self) -> super::hooks::MetricsData {
self.metrics_data.try_lock().map(|m| m.clone()).unwrap_or_else(|_| { self.metrics_data
.try_lock()
.map(|m| m.clone())
.unwrap_or_else(|_| {
warn!("[AgentRuntime] 获取指标锁失败,返回默认值"); warn!("[AgentRuntime] 获取指标锁失败,返回默认值");
super::hooks::MetricsData::default() super::hooks::MetricsData::default()
}) })
@ -456,7 +404,7 @@ impl AgentRuntime {
// Phase 4: 会话收尾(传入实际的终止原因 + trajectory 导出参数) // Phase 4: 会话收尾(传入实际的终止原因 + trajectory 导出参数)
let system_prompt = self.system_prompt(); let system_prompt = self.system_prompt();
let model_name = self.app_state.llm.model(); let model_name = self.app_state.llm.model().await;
finalize::finalize_turn( finalize::finalize_turn(
db, db,
&session_info.session_id, &session_info.session_id,
@ -491,12 +439,10 @@ impl AgentRuntime {
let turn_index = session_info.turn_index; let turn_index = session_info.turn_index;
// 注册当前会话的权限检查器(如不存在则从全局配置初始化) // 注册当前会话的权限检查器(如不存在则从全局配置初始化)
{ self.app_state
let mut checkers = self.app_state.session_permission_checkers.write().await; .session_permission_checkers
checkers
.entry(sid.clone()) .entry(sid.clone())
.or_insert_with(|| (*self.permission_checker).clone()); .or_insert_with(|| (*self.permission_checker).clone());
}
let tool_defs = self.tool_registry.definitions(); let tool_defs = self.tool_registry.definitions();
let mut duplicate_detector = DuplicateDetector::default(); let mut duplicate_detector = DuplicateDetector::default();
@ -522,10 +468,7 @@ impl AgentRuntime {
self.checkpoint_manager.new_turn(); self.checkpoint_manager.new_turn();
// 检查用户取消 // 检查用户取消
let is_cancelled = { let is_cancelled = self.app_state.cancelled_runs.remove(sid).is_some();
let mut cancelled = self.app_state.cancelled_runs.lock().await;
cancelled.remove(sid)
};
if is_cancelled { if is_cancelled {
warn!("[AgentRuntime] 用户手动中止了会话 {} 的智能体执行", sid); warn!("[AgentRuntime] 用户手动中止了会话 {} 的智能体执行", sid);
@ -868,14 +811,11 @@ impl AgentRuntime {
} }
// 并行执行工具带权限检查、checkpoint 和分区器) // 并行执行工具带权限检查、checkpoint 和分区器)
let session_checker_snapshot = { let session_checker_snapshot = self
self.app_state .app_state
.session_permission_checkers .session_permission_checkers
.read()
.await
.get(sid) .get(sid)
.cloned() .map(|r| r.value().clone());
};
let exec_result = executor::execute_parallel( let exec_result = executor::execute_parallel(
&prepared_calls, &prepared_calls,
&self.tool_registry, &self.tool_registry,
@ -947,8 +887,7 @@ impl AgentRuntime {
} }
if exec_result.was_cancelled { if exec_result.was_cancelled {
let mut locked = self.app_state.cancelled_runs.lock().await; self.app_state.cancelled_runs.remove(sid);
locked.remove(sid);
warn!( warn!(
"[AgentRuntime] 工具执行期间被用户手动中止,会话 ID: {}", "[AgentRuntime] 工具执行期间被用户手动中止,会话 ID: {}",
sid sid
@ -1016,8 +955,7 @@ impl AgentRuntime {
match output.status { match output.status {
StreamStatus::Success => return Some(output), StreamStatus::Success => return Some(output),
StreamStatus::Cancelled => { StreamStatus::Cancelled => {
let mut cancelled = self.app_state.cancelled_runs.lock().await; self.app_state.cancelled_runs.remove(session_id);
cancelled.remove(session_id);
warn!( warn!(
"[AgentRuntime] 流式调用期间被用户手动中止,会话 ID: {}", "[AgentRuntime] 流式调用期间被用户手动中止,会话 ID: {}",
session_id session_id
@ -1077,7 +1015,7 @@ impl AgentRuntime {
"[AgentRuntime] 连续 {} 次过载,切换到备用模型: {}", "[AgentRuntime] 连续 {} 次过载,切换到备用模型: {}",
consecutive_overloads, fallback consecutive_overloads, fallback
); );
llm.set_model(fallback.clone()); llm.set_model(fallback.clone()).await;
consecutive_overloads = 0; consecutive_overloads = 0;
} else if !self.app_state.config.llm_fallback_chain.is_empty() { } else if !self.app_state.config.llm_fallback_chain.is_empty() {
let idx = ((consecutive_overloads as usize - 3) let idx = ((consecutive_overloads as usize - 3)
@ -1088,15 +1026,14 @@ impl AgentRuntime {
"[AgentRuntime] 连续 {} 次过载,从链中切换: {}", "[AgentRuntime] 连续 {} 次过载,从链中切换: {}",
consecutive_overloads, alt consecutive_overloads, alt
); );
llm.set_model(alt.clone()); llm.set_model(alt.clone()).await;
consecutive_overloads = 0; consecutive_overloads = 0;
} }
} }
} }
// 检查用户取消 // 检查用户取消
let cancelled = self.app_state.cancelled_runs.lock().await; if self.app_state.cancelled_runs.contains_key(session_id) {
if cancelled.contains(session_id) {
warn!("[AgentRuntime] 退避重试期间被用户取消"); warn!("[AgentRuntime] 退避重试期间被用户取消");
let _ = tx.send(AgentStreamEvent::Error { let _ = tx.send(AgentStreamEvent::Error {
message: "用户已手动中止执行。".to_string(), message: "用户已手动中止执行。".to_string(),
@ -1123,8 +1060,7 @@ impl AgentRuntime {
return Some(retry_output); return Some(retry_output);
} }
StreamStatus::Cancelled => { StreamStatus::Cancelled => {
let mut cancelled = self.app_state.cancelled_runs.lock().await; self.app_state.cancelled_runs.remove(session_id);
cancelled.remove(session_id);
return None; return None;
} }
StreamStatus::Error(_) => { StreamStatus::Error(_) => {
@ -1232,8 +1168,7 @@ impl AgentRuntime {
return Some(retry_output); return Some(retry_output);
} }
StreamStatus::Cancelled => { StreamStatus::Cancelled => {
let mut cancelled = self.app_state.cancelled_runs.lock().await; self.app_state.cancelled_runs.remove(session_id);
cancelled.remove(session_id);
warn!("[AgentRuntime] 恢复期间被用户中止"); warn!("[AgentRuntime] 恢复期间被用户中止");
let _ = tx.send(AgentStreamEvent::Error { let _ = tx.send(AgentStreamEvent::Error {
message: "用户已手动中止执行。".to_string(), message: "用户已手动中止执行。".to_string(),
@ -1400,7 +1335,7 @@ impl AgentRuntime {
}; };
let today = chrono::Utc::now().format("%Y-%m-%d").to_string(); let today = chrono::Utc::now().format("%Y-%m-%d").to_string();
let model_name = self.app_state.llm.model().to_string(); let model_name = self.app_state.llm.model_name_snapshot();
let mut lines = vec![ let mut lines = vec![
"# 环境信息".to_string(), "# 环境信息".to_string(),

View File

@ -83,7 +83,7 @@ pub async fn create_or_resume_session(
) )
.bind(&new_id) .bind(&new_id)
.bind("") .bind("")
.bind(llm.model()) .bind(llm.model().await)
.bind(mode) .bind(mode)
.execute(db) .execute(db)
.await?; .await?;
@ -544,7 +544,9 @@ pub async fn branch_session(db: &SqlitePool, session_id: &str) -> anyhow::Result
let forked_at = last_active_id.unwrap_or(0); let forked_at = last_active_id.unwrap_or(0);
// 4. 创建新会话 // 4. 创建新会话与复制消息(同一个事务内)
let mut tx = db.begin().await?;
let branch_id = uuid::Uuid::new_v4().to_string(); let branch_id = uuid::Uuid::new_v4().to_string();
let branch_title = if title.is_empty() { let branch_title = if title.is_empty() {
format!("分支 (来自 {})", &session_id[..8.min(session_id.len())]) format!("分支 (来自 {})", &session_id[..8.min(session_id.len())])
@ -565,7 +567,7 @@ pub async fn branch_session(db: &SqlitePool, session_id: &str) -> anyhow::Result
.bind(&branch_title) .bind(&branch_title)
.bind(session_id) .bind(session_id)
.bind(serde_json::to_string(&branch_meta).unwrap_or_default()) .bind(serde_json::to_string(&branch_meta).unwrap_or_default())
.execute(db) .execute(&mut *tx)
.await?; .await?;
// 5. 复制所有 active=1 的消息到新会话 // 5. 复制所有 active=1 的消息到新会话
@ -573,7 +575,7 @@ pub async fn branch_session(db: &SqlitePool, session_id: &str) -> anyhow::Result
"SELECT COUNT(*) FROM agent_messages WHERE session_id = ? AND active = 1", "SELECT COUNT(*) FROM agent_messages WHERE session_id = ? AND active = 1",
) )
.bind(session_id) .bind(session_id)
.fetch_one(db) .fetch_one(&mut *tx)
.await?; .await?;
sqlx::query( sqlx::query(
@ -588,9 +590,11 @@ pub async fn branch_session(db: &SqlitePool, session_id: &str) -> anyhow::Result
) )
.bind(&branch_id) .bind(&branch_id)
.bind(session_id) .bind(session_id)
.execute(db) .execute(&mut *tx)
.await?; .await?;
tx.commit().await?;
info!( info!(
"[Session] 分叉完成: parent={}, branch={}, copied={} messages, forked_at={}", "[Session] 分叉完成: parent={}, branch={}, copied={} messages, forked_at={}",
session_id, branch_id, copied, forked_at session_id, branch_id, copied, forked_at

View File

@ -43,7 +43,7 @@ pub async fn process_llm_stream(
tx: &mpsc::UnboundedSender<AgentStreamEvent>, tx: &mpsc::UnboundedSender<AgentStreamEvent>,
step: usize, step: usize,
session_id: &str, session_id: &str,
cancelled_runs: Arc<tokio::sync::Mutex<std::collections::HashSet<String>>>, cancelled_runs: Arc<dashmap::DashMap<String, ()>>,
enable_thinking: bool, enable_thinking: bool,
) -> StreamOutput { ) -> StreamOutput {
// 1. 发起 LLM 流式调用 // 1. 发起 LLM 流式调用
@ -76,8 +76,7 @@ pub async fn process_llm_stream(
let cancel_fut = async { let cancel_fut = async {
loop { loop {
tokio::time::sleep(std::time::Duration::from_millis(250)).await; tokio::time::sleep(std::time::Duration::from_millis(250)).await;
let cancelled = cancelled_runs.lock().await; if cancelled_runs.contains_key(&sid) {
if cancelled.contains(&sid) {
return; return;
} }
} }

View File

@ -590,10 +590,7 @@ mod tests {
ads, ads,
arxiv, arxiv,
vizier, vizier,
lamost: crate::clients::lamost::LamostClient::new( lamost: crate::clients::lamost::LamostClient::new("https://www.lamost.org", 60)
"https://www.lamost.org",
60,
)
.unwrap(), .unwrap(),
gaia: crate::clients::gaia::GaiaClient::new( gaia: crate::clients::gaia::GaiaClient::new(
"https://gea.esac.esa.int/tap-server/tap", "https://gea.esac.esa.int/tap-server/tap",
@ -621,7 +618,7 @@ mod tests {
harvest_status: Arc::new(tokio::sync::Mutex::new(MetaSyncStatus::default())), harvest_status: Arc::new(tokio::sync::Mutex::new(MetaSyncStatus::default())),
batch_status: Arc::new(tokio::sync::Mutex::new(AssetBatchStatus::default())), batch_status: Arc::new(tokio::sync::Mutex::new(AssetBatchStatus::default())),
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)), active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
cancelled_runs: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), cancelled_runs: Arc::new(dashmap::DashMap::new()),
skill_registry: Arc::new(tokio::sync::RwLock::new(SkillRegistry::new(PathBuf::from( skill_registry: Arc::new(tokio::sync::RwLock::new(SkillRegistry::new(PathBuf::from(
"/tmp/sk", "/tmp/sk",
)))), )))),
@ -629,16 +626,16 @@ mod tests {
pending_permissions: Arc::new( pending_permissions: Arc::new(
tokio::sync::Mutex::new(std::collections::HashMap::new()), tokio::sync::Mutex::new(std::collections::HashMap::new()),
), ),
session_permission_checkers: Arc::new(tokio::sync::RwLock::new( session_permission_checkers: Arc::new(dashmap::DashMap::new()),
std::collections::HashMap::new(),
)),
sse_broadcast: None, sse_broadcast: None,
memory_manager: Arc::new(tokio::sync::Mutex::new(MemoryManager::new(PathBuf::from( memory_manager: Arc::new(tokio::sync::Mutex::new(MemoryManager::new(PathBuf::from(
"/tmp/test_mem", "/tmp/test_mem",
)))), )))),
sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())), sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
login_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())), login_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
upload_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())), upload_rate_limiter: Arc::new(
tokio::sync::Mutex::new(std::collections::HashMap::new()),
),
}); });
ToolContext::new(app_state) ToolContext::new(app_state)

View File

@ -26,7 +26,7 @@ impl SkillCreator {
/// ///
/// 返回创建的 skill 名称列表。 /// 返回创建的 skill 名称列表。
/// 如果目标 skill 目录已存在则跳过(不覆盖已有 skill /// 如果目标 skill 目录已存在则跳过(不覆盖已有 skill
pub fn create_from_patterns( pub async fn create_from_patterns(
&self, &self,
patterns: &[pattern_detector::DetectedPattern], patterns: &[pattern_detector::DetectedPattern],
) -> std::io::Result<Vec<String>> { ) -> std::io::Result<Vec<String>> {
@ -46,13 +46,13 @@ impl SkillCreator {
} }
// 创建 skill 目录 // 创建 skill 目录
std::fs::create_dir_all(&skill_dir)?; tokio::fs::create_dir_all(&skill_dir).await?;
// 生成 SKILL.md 内容 // 生成 SKILL.md 内容
let content = self.generate_skill_md(pattern, &skill_name); let content = self.generate_skill_md(pattern, &skill_name);
let md_path = skill_dir.join("SKILL.md"); let md_path = skill_dir.join("SKILL.md");
std::fs::write(&md_path, &content)?; tokio::fs::write(&md_path, &content).await?;
info!( info!(
"[SkillCreator] 已从模式创建 Skill '{}': {}", "[SkillCreator] 已从模式创建 Skill '{}': {}",
@ -217,7 +217,7 @@ impl SelfImprovePipeline {
info!("[SelfImprove] 检测到 {} 个候选模式", patterns.len()); info!("[SelfImprove] 检测到 {} 个候选模式", patterns.len());
// Step 2: 创建 Skills // Step 2: 创建 Skills
let created = self.creator.create_from_patterns(&patterns)?; let created = self.creator.create_from_patterns(&patterns).await?;
// Step 3: Curator 分析 // Step 3: Curator 分析
let curator_report = self.curator.analyze(usage_stats, skill_metas); let curator_report = self.curator.analyze(usage_stats, skill_metas);

View File

@ -340,9 +340,12 @@ impl Curator {
/// 归档指定的 stale skills移动到 archive 目录而非删除)。 /// 归档指定的 stale skills移动到 archive 目录而非删除)。
/// ///
/// 返回成功归档的 skill 名称列表。 /// 返回成功归档的 skill 名称列表。
pub fn archive_stale_skills(&self, skill_names: &[String]) -> std::io::Result<Vec<String>> { pub async fn archive_stale_skills(
&self,
skill_names: &[String],
) -> std::io::Result<Vec<String>> {
// 确保归档目录存在 // 确保归档目录存在
std::fs::create_dir_all(&self.archive_dir)?; tokio::fs::create_dir_all(&self.archive_dir).await?;
let mut archived = Vec::new(); let mut archived = Vec::new();
@ -355,17 +358,16 @@ impl Curator {
let archive_path = self.archive_dir.join(name); let archive_path = self.archive_dir.join(name);
if archive_path.exists() { if archive_path.exists() {
// 避免覆盖已有归档:添加时间戳后缀
let ts = Utc::now().format("%Y%m%d%H%M%S"); let ts = Utc::now().format("%Y%m%d%H%M%S");
let renamed = self.archive_dir.join(format!("{}-{}", name, ts)); let renamed = self.archive_dir.join(format!("{}-{}", name, ts));
info!( info!(
"[Curator] 归档目录已存在,使用新名称: {}", "[Curator] 归档目录已存在,使用新名称: {}",
renamed.display() renamed.display()
); );
std::fs::rename(&skill_dir, &renamed)?; tokio::fs::rename(&skill_dir, &renamed).await?;
archived.push(format!("{}-{}", name, ts)); archived.push(format!("{}-{}", name, ts));
} else { } else {
std::fs::rename(&skill_dir, &archive_path)?; tokio::fs::rename(&skill_dir, &archive_path).await?;
archived.push(name.clone()); archived.push(name.clone());
} }

View File

@ -64,29 +64,39 @@ pub fn inbox_path(team_dir: &Path, agent_name: &str) -> PathBuf {
} }
/// 向收件箱追加一条消息 /// 向收件箱追加一条消息
pub fn append_message(team_dir: &Path, agent_name: &str, msg: &TeamMessage) -> std::io::Result<()> { pub async fn append_message(
team_dir: &Path,
agent_name: &str,
msg: &TeamMessage,
) -> std::io::Result<()> {
let inbox = inbox_path(team_dir, agent_name); let inbox = inbox_path(team_dir, agent_name);
if let Some(parent) = inbox.parent() { if let Some(parent) = inbox.parent() {
std::fs::create_dir_all(parent)?; tokio::fs::create_dir_all(parent).await?;
} }
let line = serde_json::to_string(msg).unwrap_or_default(); let line = serde_json::to_string(msg).unwrap_or_default();
let inbox_clone = inbox.clone();
tokio::task::spawn_blocking(move || {
let mut file = std::fs::OpenOptions::new() let mut file = std::fs::OpenOptions::new()
.create(true) .create(true)
.append(true) .append(true)
.open(&inbox)?; .open(&inbox_clone)?;
file.write_all(line.as_bytes())?; file.write_all(line.as_bytes())?;
file.write_all(b"\n")?; file.write_all(b"\n")?;
Ok(()) Ok(())
})
.await
.unwrap_or_else(|e| Err(std::io::Error::other(e)))
} }
/// 读取并清空收件箱 /// 读取并清空收件箱
pub fn drain_inbox(team_dir: &Path, agent_name: &str) -> Vec<TeamMessage> { pub async fn drain_inbox(team_dir: &Path, agent_name: &str) -> Vec<TeamMessage> {
let inbox = inbox_path(team_dir, agent_name); let inbox = inbox_path(team_dir, agent_name);
if !inbox.exists() { if !inbox.exists() {
return Vec::new(); return Vec::new();
} }
let content = match std::fs::read_to_string(&inbox) { let inbox_clone = inbox.clone();
let content = match tokio::fs::read_to_string(&inbox_clone).await {
Ok(c) => c, Ok(c) => c,
Err(_) => return Vec::new(), Err(_) => return Vec::new(),
}; };
@ -98,7 +108,7 @@ pub fn drain_inbox(team_dir: &Path, agent_name: &str) -> Vec<TeamMessage> {
.collect(); .collect();
// 清空文件 // 清空文件
let _ = std::fs::write(&inbox, ""); let _ = tokio::fs::write(&inbox, "").await;
messages messages
} }

View File

@ -157,9 +157,15 @@ impl TeamManager {
} }
/// 发送消息给指定队友 /// 发送消息给指定队友
pub fn send_message(&self, from: &str, to: &str, content: &str, msg_type: TeamMessageType) { pub async fn send_message(
&self,
from: &str,
to: &str,
content: &str,
msg_type: TeamMessageType,
) {
let msg = TeamMessage::new(from, to, content, msg_type); let msg = TeamMessage::new(from, to, content, msg_type);
if let Err(e) = inbox::append_message(&self.team_dir, to, &msg) { if let Err(e) = inbox::append_message(&self.team_dir, to, &msg).await {
warn!("[TeamManager] 发送消息失败: {}", e); warn!("[TeamManager] 发送消息失败: {}", e);
} else { } else {
info!( info!(
@ -172,7 +178,7 @@ impl TeamManager {
} }
/// 广播消息给所有队友 /// 广播消息给所有队友
pub fn broadcast(&self, from: &str, content: &str) { pub async fn broadcast(&self, from: &str, content: &str) {
info!( info!(
"[TeamManager] 广播消息: {}", "[TeamManager] 广播消息: {}",
content.chars().take(80).collect::<String>() content.chars().take(80).collect::<String>()
@ -180,14 +186,14 @@ impl TeamManager {
let msg = TeamMessage::new(from, "all", content, TeamMessageType::Status); let msg = TeamMessage::new(from, "all", content, TeamMessageType::Status);
for member in &self.config.members { for member in &self.config.members {
if member.name != from { if member.name != from {
let _ = inbox::append_message(&self.team_dir, &member.name, &msg); let _ = inbox::append_message(&self.team_dir, &member.name, &msg).await;
} }
} }
} }
/// 检查并清空指定 agent 的收件箱 /// 检查并清空指定 agent 的收件箱
pub fn check_inbox(&self, agent_name: &str) -> Vec<TeamMessage> { pub async fn check_inbox(&self, agent_name: &str) -> Vec<TeamMessage> {
inbox::drain_inbox(&self.team_dir, agent_name) inbox::drain_inbox(&self.team_dir, agent_name).await
} }
/// 列出所有队友状态 /// 列出所有队友状态

View File

@ -61,14 +61,14 @@ pub async fn run_teammate_loop(
&format!("队友 {} ({}) 已退出。", name, role), &format!("队友 {} ({}) 已退出。", name, role),
TeamMessageType::Status, TeamMessageType::Status,
); );
let _ = inbox::append_message(&team_dir, "lead", &goodbye); let _ = inbox::append_message(&team_dir, "lead", &goodbye).await;
return; return;
} }
// ── IDLE 阶段:检查收件箱 ── // ── IDLE 阶段:检查收件箱 ──
*status.lock().await = MemberStatus::Idle; *status.lock().await = MemberStatus::Idle;
let inbox_msgs = inbox::drain_inbox(&team_dir, &name); let inbox_msgs = inbox::drain_inbox(&team_dir, &name).await;
if inbox_msgs.is_empty() { if inbox_msgs.is_empty() {
// 等待新消息或取消poll 间隔 5 秒,最长 60 秒 // 等待新消息或取消poll 间隔 5 秒,最长 60 秒
@ -84,7 +84,7 @@ pub async fn run_teammate_loop(
} }
// 再次 drain可能因为 pending 标志被唤醒) // 再次 drain可能因为 pending 标志被唤醒)
let inbox_msgs = inbox::drain_inbox(&team_dir, &name); let inbox_msgs = inbox::drain_inbox(&team_dir, &name).await;
if inbox_msgs.is_empty() { if inbox_msgs.is_empty() {
continue; // 超时,没有新消息,继续 idle continue; // 超时,没有新消息,继续 idle
} }
@ -115,7 +115,7 @@ pub async fn run_teammate_loop(
if let Some(ref summary) = result { if let Some(ref summary) = result {
let reply = let reply =
super::inbox::TeamMessage::new(&name, "lead", summary, TeamMessageType::Result); super::inbox::TeamMessage::new(&name, "lead", summary, TeamMessageType::Result);
let _ = inbox::append_message(&team_dir, "lead", &reply); let _ = inbox::append_message(&team_dir, "lead", &reply).await;
info!( info!(
"[Teammate:{}] 任务完成,已发送结果 ({}字符)", "[Teammate:{}] 任务完成,已发送结果 ({}字符)",
name, name,
@ -147,7 +147,7 @@ pub async fn run_teammate_loop(
if let Some(ref summary) = result { if let Some(ref summary) = result {
let reply = let reply =
super::inbox::TeamMessage::new(&name, "lead", summary, TeamMessageType::Result); super::inbox::TeamMessage::new(&name, "lead", summary, TeamMessageType::Result);
let _ = inbox::append_message(&team_dir, "lead", &reply); let _ = inbox::append_message(&team_dir, "lead", &reply).await;
} }
} }
} }

View File

@ -44,7 +44,7 @@ impl AnalyzeImageTool {
json!({ json!({
"image_path": image_path, "image_path": image_path,
"question": question, "question": question,
"model": vision_llm.model(), "model": vision_llm.model().await,
}), }),
), ),
Err(e) => { Err(e) => {
@ -237,7 +237,7 @@ mod tests {
.filter(|e| matches!(e, AgentStreamEvent::TextDelta { .. })) .filter(|e| matches!(e, AgentStreamEvent::TextDelta { .. }))
.count(); .count();
println!("视觉模型: {}", vision_llm.model()); println!("视觉模型: {}", vision_llm.model().await);
println!("SSE 事件总数: {}, TextDelta: {}", events.len(), delta_count); println!("SSE 事件总数: {}, TextDelta: {}", events.len(), delta_count);
println!("分析结果:\n{}", output.content); println!("分析结果:\n{}", output.content);

View File

@ -253,7 +253,14 @@ impl AgentTool for GetCitationNetworkTool {
// 有 query → 引用查找模式:在关联文献中按作者+年份搜索 // 有 query → 引用查找模式:在关联文献中按作者+年份搜索
if let Some(ref q) = query { if let Some(ref q) = query {
match crate::services::citation::search_citations(&state.db, &paper.bibcode, direction, q).await { match crate::services::citation::search_citations(
&state.db,
&paper.bibcode,
direction,
q,
)
.await
{
Ok(results) => { Ok(results) => {
if results.is_empty() { if results.is_empty() {
let dir_label = if direction == "citations" { let dir_label = if direction == "citations" {
@ -315,7 +322,14 @@ impl AgentTool for GetCitationNetworkTool {
} }
} else { } else {
// 无 query → 分页浏览模式 // 无 query → 分页浏览模式
match crate::services::citation::get_citations_paginated(&state.db, &paper.bibcode, direction, sort, offset, limit) match crate::services::citation::get_citations_paginated(
&state.db,
&paper.bibcode,
direction,
sort,
offset,
limit,
)
.await .await
{ {
Ok((rows, total)) => { Ok((rows, total)) => {

View File

@ -11,9 +11,9 @@ pub mod target;
pub mod vizier; pub mod vizier;
pub use library::{GetCitationNetworkTool, SearchLocalLibraryTool}; pub use library::{GetCitationNetworkTool, SearchLocalLibraryTool};
pub use observation::FindObservationTool;
pub use metadata::GetPaperMetadataTool; pub use metadata::GetPaperMetadataTool;
pub use note::SaveNoteTool; pub use note::SaveNoteTool;
pub use observation::FindObservationTool;
pub use paper::{GetPaperContentTool, GetPaperOutlineTool}; pub use paper::{GetPaperContentTool, GetPaperOutlineTool};
pub use rag::RagSearchTool; pub use rag::RagSearchTool;
pub use target::QueryTargetTool; pub use target::QueryTargetTool;

View File

@ -58,6 +58,7 @@ impl AgentTool for SaveNoteTool {
let state = &ctx.app_state; let state = &ctx.app_state;
match crate::services::note::save_note_service(&state.config.library_dir, &title, &content) match crate::services::note::save_note_service(&state.config.library_dir, &title, &content)
.await
{ {
Ok((filename, filepath, size)) => ToolOutput::success( Ok((filename, filepath, size)) => ToolOutput::success(
format!("笔记已保存: {}", filename), format!("笔记已保存: {}", filename),

View File

@ -52,23 +52,26 @@ impl AgentTool for FindObservationTool {
} }
fn parameters(&self) -> serde_json::Value { fn parameters(&self) -> serde_json::Value {
// enum 列表从枚举的 valid_values() 动态生成,新增源/产品时自动同步
let source_enum: Vec<&str> = Source::valid_values().to_vec();
let product_enum: Vec<&str> = ProductType::valid_values().to_vec();
json!({ json!({
"type": "object", "type": "object",
"properties": { "properties": {
"source": { "source": {
"type": "string", "type": "string",
"enum": ["lamost", "gaia", "sdss", "desi"], "enum": source_enum,
"description": "数据源" "description": "数据源"
}, },
"product": { "product": {
"type": "string", "type": "string",
"enum": ["spectrum", "lightcurve", "photometry", "image"], "enum": product_enum,
"description": "产品类型,默认 spectrum", "description": "产品类型,默认 spectrum",
"default": "spectrum" "default": "spectrum"
}, },
"subtype": { "subtype": {
"type": "string", "type": "string",
"description": "产品子类型(可选)。LAMOST spectrum: lrs/mrsGaia spectrum: xp_continuous/xp_sampled/rvsGaia lightcurve: epoch_photometrySDSS spectrum: spec/apstar/aspcapDESI spectrum: coadd" "description": "产品子类型(可选)。各源可选值通过 GET /api/observation/capabilities 查询;LAMOST spectrum: lrs/mrsGaia spectrum: xp_continuous/xp_sampled/rvsGaia lightcurve: epoch_photometrySDSS spectrum: spec/apstar/aspcapDESI spectrum: coadd"
}, },
"ra": { "ra": {
"type": "number", "type": "number",
@ -80,7 +83,7 @@ impl AgentTool for FindObservationTool {
}, },
"radius_deg": { "radius_deg": {
"type": "number", "type": "number",
"description": "检索半径(度),坐标模式用,默认 0.1。各源无硬性 API 上限但建议值LAMOST≤5°Gaia/SDSS/DESI≤1°主表很大超出易超时。硬性上限 30°超出报错。可通过 GET /api/observation/capabilities 查询各源建议值", "description": "检索半径(度),坐标模式用,默认 0.1。建议值与硬上限可通过 GET /api/observation/capabilities 查询LAMOST 建议≤5°其他≤1°硬上限 30°",
"default": 0.1 "default": 0.1
}, },
"strategy": { "strategy": {
@ -96,7 +99,11 @@ impl AgentTool for FindObservationTool {
}, },
"release": { "release": {
"type": "string", "type": "string",
"description": "数据发布版本(可选,留空用各源默认)。可选值与默认值可通过 GET /api/observation/capabilities 查询LAMOST: dr5..dr11默认dr10MRS 需 dr7+Gaia: dr3SDSS spec: dr16/dr17默认dr17SDSS apstar/aspcap: dr17DESI: edr/dr1默认dr1" "description": "数据发布版本(可选,留空用各源默认)。可选值与默认值通过 GET /api/observation/capabilities 查询"
},
"version": {
"type": "string",
"description": "数据发布的子版本(可选,仅 LAMOST 有意义)。留空用该 DR 的默认(最新公开)子版本。可选值通过 GET /api/observation/capabilities 的 release_versions 字段查询"
}, },
"force": { "force": {
"type": "boolean", "type": "boolean",
@ -124,36 +131,41 @@ impl AgentTool for FindObservationTool {
let state = &ctx.app_state; let state = &ctx.app_state;
let source = match args.get("source").and_then(|v| v.as_str()) { let source = match args.get("source").and_then(|v| v.as_str()) {
Some(s) => match s.to_lowercase().as_str() { Some(s) => match Source::from_str(s) {
"lamost" => Source::Lamost, Ok(src) => src,
"gaia" => Source::Gaia, Err(e) => return ToolOutput::error(e),
"sdss" => Source::Sdss,
"desi" => Source::Desi,
other => return ToolOutput::error(format!(
"不支持的 source '{}',可选: {:?}", other, Source::valid_values()
)),
}, },
None => return ToolOutput::error("缺少必需参数 'source'lamost/gaia/sdss/desi"), None => return ToolOutput::error("缺少必需参数 'source'lamost/gaia/sdss/desi"),
}; };
let product_type = match args.get("product").and_then(|v| v.as_str()) { let product_type = match args.get("product").and_then(|v| v.as_str()) {
None | Some("spectrum") => ProductType::Spectrum, None => ProductType::Spectrum,
Some("lightcurve") | Some("light_curve") | Some("lc") => ProductType::LightCurve, Some(s) => match ProductType::from_str(s) {
Some("photometry") => ProductType::Photometry, Ok(p) => p,
Some("image") => ProductType::Image, Err(e) => return ToolOutput::error(e),
Some(other) => return ToolOutput::error(format!( },
"不支持的 product '{}',可选: {:?}", other, ProductType::valid_values()
)),
}; };
let subtype = args.get("subtype") let subtype = args
.get("subtype")
.and_then(|v| v.as_str()) .and_then(|v| v.as_str())
.map(|s| s.to_string()); .map(|s| s.to_string());
let product = ProductSpec { product: product_type, subtype }; let product = ProductSpec {
product: product_type,
subtype,
};
let force = args.get("force").and_then(|v| v.as_bool()).unwrap_or(false); let force = args.get("force").and_then(|v| v.as_bool()).unwrap_or(false);
let release = args.get("release").and_then(|v| v.as_str()).map(|s| s.to_string()); let release = args
.get("release")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
let version = args
.get("version")
.and_then(|v| v.as_str())
.map(|s| s.to_string());
// 标识符模式 vs 坐标模式 // 标识符模式 vs 坐标模式
let request = if let Some(ids) = args.get("source_ids").and_then(|v| v.as_array()) { let request = if let Some(ids) = args.get("source_ids").and_then(|v| v.as_array()) {
let identifiers: Vec<String> = ids.iter() let identifiers: Vec<String> = ids
.iter()
.filter_map(|v| v.as_str().map(|s| s.to_string())) .filter_map(|v| v.as_str().map(|s| s.to_string()))
.collect(); .collect();
if identifiers.is_empty() { if identifiers.is_empty() {
@ -161,23 +173,38 @@ impl AgentTool for FindObservationTool {
} }
info!( info!(
"[FindObservation] by_id source={:?} product={:?} count={}", "[FindObservation] by_id source={:?} product={:?} count={}",
source, product.product, identifiers.len() source,
product.product,
identifiers.len()
); );
ObservationRequest::ByIdentifier { source, product, identifiers, release } ObservationRequest::ByIdentifier {
source,
product,
identifiers,
release,
version,
}
} else { } else {
let ra = match args.get("ra").and_then(|v| v.as_f64()) { let ra = match args.get("ra").and_then(|v| v.as_f64()) {
Some(v) => v, Some(v) => v,
None => return ToolOutput::error( None => {
return ToolOutput::error(
"坐标模式缺少必需参数 'ra'(赤经,度),或改用 source_ids 标识符模式", "坐标模式缺少必需参数 'ra'(赤经,度),或改用 source_ids 标识符模式",
), )
}
}; };
let dec = match args.get("dec").and_then(|v| v.as_f64()) { let dec = match args.get("dec").and_then(|v| v.as_f64()) {
Some(v) => v, Some(v) => v,
None => return ToolOutput::error( None => {
return ToolOutput::error(
"坐标模式缺少必需参数 'dec'(赤纬,度),或改用 source_ids 标识符模式", "坐标模式缺少必需参数 'dec'(赤纬,度),或改用 source_ids 标识符模式",
), )
}
}; };
let radius = args.get("radius_deg").and_then(|v| v.as_f64()).unwrap_or(0.1); let radius = args
.get("radius_deg")
.and_then(|v| v.as_f64())
.unwrap_or(0.1);
let strategy = match args.get("strategy").and_then(|v| v.as_str()) { let strategy = match args.get("strategy").and_then(|v| v.as_str()) {
Some("all") => FindStrategy::All, Some("all") => FindStrategy::All,
_ => FindStrategy::Nearest, _ => FindStrategy::Nearest,
@ -187,7 +214,14 @@ impl AgentTool for FindObservationTool {
source, product.product, ra, dec, radius, strategy source, product.product, ra, dec, radius, strategy
); );
ObservationRequest::ByCoordinates { ObservationRequest::ByCoordinates {
source, product, ra, dec, radius_deg: radius, strategy, release, source,
product,
ra,
dec,
radius_deg: radius,
strategy,
release,
version,
} }
}; };
@ -203,7 +237,11 @@ impl AgentTool for FindObservationTool {
/// 渲染 ObservationBatch 为可读文本(支持多 artifact 展示) /// 渲染 ObservationBatch 为可读文本(支持多 artifact 展示)
fn render_batch(b: &crate::services::observation::ObservationBatch) -> String { fn render_batch(b: &crate::services::observation::ObservationBatch) -> String {
let mut content = format!("{} {} 下载", b.source.display(), b.product.product.display()); let mut content = format!(
"{} {} 下载",
b.source.display(),
b.product.product.display()
);
if let (Some(ra), Some(dec), Some(r)) = (b.ra, b.dec, b.radius_deg) { if let (Some(ra), Some(dec), Some(r)) = (b.ra, b.dec, b.radius_deg) {
content.push_str(&format!("ra={}, dec={}, radius={}°)", ra, dec, r)); content.push_str(&format!("ra={}, dec={}, radius={}°)", ra, dec, r));
} }
@ -219,18 +257,30 @@ fn render_batch(b: &crate::services::observation::ObservationBatch) -> String {
for p in &b.products { for p in &b.products {
// 多 artifact 展示(如 Gaia 光变 G/BP/RP 三波段) // 多 artifact 展示(如 Gaia 光变 G/BP/RP 三波段)
let artifact_summary: Vec<String> = p.artifacts.iter().map(|a| { let artifact_summary: Vec<String> = p
match &a.band { .artifacts
Some(band) => format!("{}波段 {} ({})", .iter()
band, a.file_format.to_uppercase(), format_size(a.size_bytes)), .map(|a| match &a.band {
None => format!("{} ({})", Some(band) => format!(
a.file_format.to_uppercase(), format_size(a.size_bytes)), "{}波段 {} ({})",
} band,
}).collect(); a.file_format.to_uppercase(),
format_size(a.size_bytes)
),
None => format!(
"{} ({})",
a.file_format.to_uppercase(),
format_size(a.size_bytes)
),
})
.collect();
content.push_str(&format!( content.push_str(&format!(
"\n✓ 已下载({}: {} —— {}\n", "\n✓ 已下载({}: {} —— {}\n",
if p.artifacts.iter().all(|a| a.cached) { "缓存" } else { "新下载" } if p.artifacts.iter().all(|a| a.cached) {
.to_string(), "缓存"
} else {
"新下载"
},
p.source_label, p.source_label,
artifact_summary.join(", ") artifact_summary.join(", ")
)); ));
@ -238,7 +288,10 @@ fn render_batch(b: &crate::services::observation::ObservationBatch) -> String {
content.push_str(&format!( content.push_str(&format!(
" {}{}\n", " {}{}\n",
a.file_path, a.file_path,
a.band.as_ref().map(|b| format!(" [{}]", b)).unwrap_or_default() a.band
.as_ref()
.map(|b| format!(" [{}]", b))
.unwrap_or_default()
)); ));
} }
} }

View File

@ -185,10 +185,7 @@ impl AgentTool for CatalogOperationTool {
} }
} }
async fn do_catalog_search( async fn do_catalog_search(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
state: &crate::api::AppState,
args: &serde_json::Value,
) -> ToolOutput {
let keyword = match args.get("keyword").and_then(|v| v.as_str()) { let keyword = match args.get("keyword").and_then(|v| v.as_str()) {
Some(k) => k, Some(k) => k,
None => return ToolOutput::error("search 需要 'keyword' 参数"), None => return ToolOutput::error("search 需要 'keyword' 参数"),
@ -226,10 +223,7 @@ async fn do_catalog_search(
ToolOutput::success(content, json!(items)) ToolOutput::success(content, json!(items))
} }
async fn do_catalog_describe( async fn do_catalog_describe(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
state: &crate::api::AppState,
args: &serde_json::Value,
) -> ToolOutput {
let table = match args.get("table").and_then(|v| v.as_str()) { let table = match args.get("table").and_then(|v| v.as_str()) {
Some(t) => t, Some(t) => t,
None => return ToolOutput::error("describe 需要 'table' 参数"), None => return ToolOutput::error("describe 需要 'table' 参数"),
@ -259,10 +253,7 @@ async fn do_catalog_describe(
ToolOutput::success(content, json!(items)) ToolOutput::success(content, json!(items))
} }
async fn do_catalog_query( async fn do_catalog_query(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
state: &crate::api::AppState,
args: &serde_json::Value,
) -> ToolOutput {
let limit = args let limit = args
.get("limit") .get("limit")
.and_then(|v| v.as_i64()) .and_then(|v| v.as_i64())
@ -275,8 +266,7 @@ async fn do_catalog_query(
limit, limit,
adql.chars().take(150).collect::<String>() adql.chars().take(150).collect::<String>()
); );
crate::services::cds::vizier::query_adql_cached(&state.db, &state.vizier, adql, limit) crate::services::cds::vizier::query_adql_cached(&state.db, &state.vizier, adql, limit).await
.await
} else if let Some(table) = args.get("table").and_then(|v| v.as_str()) { } else if let Some(table) = args.get("table").and_then(|v| v.as_str()) {
let columns: Vec<String> = args let columns: Vec<String> = args
.get("columns") .get("columns")
@ -284,13 +274,7 @@ async fn do_catalog_query(
.map(|c| c.split(',').map(|s| s.trim().to_string()).collect()) .map(|c| c.split(',').map(|s| s.trim().to_string()).collect())
.unwrap_or_default(); .unwrap_or_default();
info!("[CatalogOp:query] table={} limit={}", table, limit); info!("[CatalogOp:query] table={} limit={}", table, limit);
crate::services::cds::vizier::query_table( crate::services::cds::vizier::query_table(&state.db, &state.vizier, table, &columns, limit)
&state.db,
&state.vizier,
table,
&columns,
limit,
)
.await .await
} else { } else {
return ToolOutput::error("query 需要 'adql' 或 'table' 参数之一"); return ToolOutput::error("query 需要 'adql' 或 'table' 参数之一");
@ -305,10 +289,7 @@ async fn do_catalog_query(
} }
} }
async fn do_catalog_cone( async fn do_catalog_cone(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
state: &crate::api::AppState,
args: &serde_json::Value,
) -> ToolOutput {
let coords = match args.get("coords").and_then(|v| v.as_object()) { let coords = match args.get("coords").and_then(|v| v.as_object()) {
Some(c) => c, Some(c) => c,
None => return ToolOutput::error("cone 需要 'coords' 参数(含 ra/dec"), None => return ToolOutput::error("cone 需要 'coords' 参数(含 ra/dec"),
@ -338,7 +319,11 @@ async fn do_catalog_cone(
info!( info!(
"[CatalogOp:cone] ra={} dec={} radius={}° table={} strategy={}", "[CatalogOp:cone] ra={} dec={} radius={}° table={} strategy={}",
ra, dec, radius, table, if nearest { "nearest" } else { "all" } ra,
dec,
radius,
table,
if nearest { "nearest" } else { "all" }
); );
match crate::services::cds::vizier::cone_search( match crate::services::cds::vizier::cone_search(
@ -365,10 +350,7 @@ async fn do_catalog_cone(
} }
} }
async fn do_catalog_export( async fn do_catalog_export(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
state: &crate::api::AppState,
args: &serde_json::Value,
) -> ToolOutput {
let limit = args let limit = args
.get("limit") .get("limit")
.and_then(|v| v.as_i64()) .and_then(|v| v.as_i64())
@ -384,13 +366,16 @@ async fn do_catalog_export(
} }
let catalog = crate::services::cds::vizier::VizierCatalog::new(&state.db, &state.vizier); let catalog = crate::services::cds::vizier::VizierCatalog::new(&state.db, &state.vizier);
let result = match catalog.export_to_file( let result = match catalog
.export_to_file(
state.config.library_dir.to_str().unwrap_or("."), state.config.library_dir.to_str().unwrap_or("."),
adql, adql,
table, table,
columns, columns,
limit, limit,
).await { )
.await
{
Ok(r) => r, Ok(r) => r,
Err(e) => return ToolOutput::error(format!("导出失败: {}", e)), Err(e) => return ToolOutput::error(format!("导出失败: {}", e)),
}; };
@ -410,10 +395,7 @@ async fn do_catalog_export(
) )
} }
async fn do_catalog_lookup( async fn do_catalog_lookup(state: &crate::api::AppState, args: &serde_json::Value) -> ToolOutput {
state: &crate::api::AppState,
args: &serde_json::Value,
) -> ToolOutput {
let bibcode = match args.get("bibcode").and_then(|v| v.as_str()) { let bibcode = match args.get("bibcode").and_then(|v| v.as_str()) {
Some(b) => b, Some(b) => b,
None => return ToolOutput::error("lookup 需要 'bibcode' 参数"), None => return ToolOutput::error("lookup 需要 'bibcode' 参数"),
@ -424,11 +406,8 @@ async fn do_catalog_lookup(
.unwrap_or(10) .unwrap_or(10)
.clamp(1, 30) as usize; .clamp(1, 30) as usize;
let catalog = crate::services::cds::vizier::VizierCatalog::with_ads( let catalog =
&state.db, crate::services::cds::vizier::VizierCatalog::with_ads(&state.db, &state.vizier, &state.ads);
&state.vizier,
&state.ads,
);
let results = match catalog.lookup(bibcode, limit).await { let results = match catalog.lookup(bibcode, limit).await {
Ok(r) => r, Ok(r) => r,
Err(e) => return ToolOutput::error(format!("查找失败: {}", e)), Err(e) => return ToolOutput::error(format!("查找失败: {}", e)),

View File

@ -245,7 +245,12 @@ impl AgentTool for FileEditTool {
); );
} }
let original = match std::fs::read_to_string(&file_path) { let read_path = file_path.clone();
let original =
match tokio::task::spawn_blocking(move || std::fs::read_to_string(&read_path))
.await
.unwrap_or_else(|e| Err(std::io::Error::other(e)))
{
Ok(c) => c, Ok(c) => c,
Err(e) => return ToolOutput::error(format!("读取文件失败: {}", e)), Err(e) => return ToolOutput::error(format!("读取文件失败: {}", e)),
}; };
@ -317,7 +322,12 @@ impl AgentTool for FileEditTool {
); );
} }
match std::fs::write(&file_path, &modified) { let write_path = file_path.clone();
let write_content = modified.clone();
match tokio::task::spawn_blocking(move || std::fs::write(&write_path, &write_content))
.await
.unwrap_or_else(|e| Err(std::io::Error::other(e)))
{
Ok(()) => { Ok(()) => {
let count = if replace_all { match_count } else { 1 }; let count = if replace_all { match_count } else { 1 };
info!("[FileEdit] {} 处替换: {}", count, file_path.display()); info!("[FileEdit] {} 处替换: {}", count, file_path.display());

View File

@ -89,20 +89,27 @@ impl AgentTool for GrepFilesTool {
Err(e) => return ToolOutput::error(format!("正则表达式无效: {}", e)), Err(e) => return ToolOutput::error(format!("正则表达式无效: {}", e)),
}; };
let search_re = re.clone();
let search_inc = include_pattern.map(|s| s.to_string());
let search_p = search_path.clone();
let (results, count) = tokio::task::spawn_blocking(move || {
let mut results = Vec::new(); let mut results = Vec::new();
let mut count = 0; let mut count = 0;
if search_p.is_file() {
if search_path.is_file() { count += Self::search_file(&search_p, &search_re, &mut results);
count += Self::search_file(&search_path, &re, &mut results); } else if search_p.is_dir() {
} else if search_path.is_dir() {
count += Self::search_dir( count += Self::search_dir(
&search_path, &search_p,
&re, &search_re,
include_pattern, search_inc.as_deref(),
max_results, max_results,
&mut results, &mut results,
); );
} }
(results, count)
})
.await
.unwrap_or_else(|_| (Vec::new(), 0));
if results.is_empty() { if results.is_empty() {
ToolOutput::success( ToolOutput::success(

View File

@ -113,18 +113,25 @@ impl AgentTool for ReadFileTool {
} }
// ── 去重检查结束 ── // ── 去重检查结束 ──
// 预检文件大小,避免 OOM // 预检文件大小 + 读取文件内容spawn_blocking 避免阻塞 tokio
const MAX_READ_BYTES: u64 = 10 * 1024 * 1024; // 10MB const MAX_READ_BYTES: u64 = 10 * 1024 * 1024; // 10MB
if let Ok(meta) = std::fs::metadata(&path) { let read_path = path.clone();
if meta.len() > MAX_READ_BYTES { let read_result = tokio::task::spawn_blocking(move || {
return ToolOutput::error(format!( let meta = std::fs::metadata(&read_path).ok();
if let Some(ref m) = meta {
if m.len() > MAX_READ_BYTES {
return Err(format!(
"文件过大 ({} MB),超过 10MB 限制。请使用 read_file 的 offset/limit 参数分段读取。", "文件过大 ({} MB),超过 10MB 限制。请使用 read_file 的 offset/limit 参数分段读取。",
meta.len() / (1024 * 1024) m.len() / (1024 * 1024)
)); ));
} }
} }
std::fs::read_to_string(&read_path).map_err(|e| format!("读取文件失败: {}", e))
})
.await
.unwrap_or_else(|e| Err(format!("任务执行失败: {}", e)));
match std::fs::read_to_string(&path) { match read_result {
Ok(content) => { Ok(content) => {
let total_lines = content.lines().count(); let total_lines = content.lines().count();
let truncated = if max_lines < total_lines { let truncated = if max_lines < total_lines {
@ -141,8 +148,6 @@ impl AgentTool for ReadFileTool {
content.clone() content.clone()
}; };
// read_file 不截断输出——LLM 明确要求读取完整内容,
// 且 skip_persist 确保不会被二次持久化。
let content_len = truncated.len(); let content_len = truncated.len();
info!( info!(
"[ReadFile] 读取 {}: {} 字符 ({} 行)", "[ReadFile] 读取 {}: {} 字符 ({} 行)",
@ -163,14 +168,13 @@ impl AgentTool for ReadFileTool {
}, },
); );
} }
// ── 缓存更新结束 ──
ToolOutput::success_skip_persist( ToolOutput::success_skip_persist(
truncated, truncated,
json!({"file_path": file_path_str, "total_lines": total_lines}), json!({"file_path": file_path_str, "total_lines": total_lines}),
) )
} }
Err(e) => ToolOutput::error(format!("读取文件失败: {}", e)), Err(e) => ToolOutput::error(e),
} }
} }

View File

@ -69,12 +69,21 @@ impl AgentTool for FileWriteTool {
} }
if let Some(parent) = path.parent() { if let Some(parent) = path.parent() {
if let Err(e) = std::fs::create_dir_all(parent) { let parent = parent.to_path_buf();
if let Err(e) = tokio::task::spawn_blocking(move || std::fs::create_dir_all(&parent))
.await
.unwrap_or_else(|e| Err(std::io::Error::other(e)))
{
return ToolOutput::error(format!("创建父目录失败: {}", e)); return ToolOutput::error(format!("创建父目录失败: {}", e));
} }
} }
match std::fs::write(&path, &content) { let write_path = path.clone();
let write_content = content.clone();
match tokio::task::spawn_blocking(move || std::fs::write(&write_path, &write_content))
.await
.unwrap_or_else(|e| Err(std::io::Error::other(e)))
{
Ok(_) => { Ok(_) => {
let line_count = content.lines().count(); let line_count = content.lines().count();
info!("[FileWrite] 写入 {}: {} 行", file_path_str, line_count); info!("[FileWrite] 写入 {}: {} 行", file_path_str, line_count);

View File

@ -17,7 +17,7 @@ use tracing::info;
/// ///
/// 返回 (最终内容, 持久化文件路径)。 /// 返回 (最终内容, 持久化文件路径)。
/// 如果内容未超过限制,直接返回原内容且持久化路径为 None。 /// 如果内容未超过限制,直接返回原内容且持久化路径为 None。
pub fn maybe_persist_tool_result( pub async fn maybe_persist_tool_result(
content: &str, content: &str,
tool_call_id: &str, tool_call_id: &str,
max_chars: usize, max_chars: usize,
@ -28,8 +28,12 @@ pub fn maybe_persist_tool_result(
} }
// 创建目录(幂等) // 创建目录(幂等)
if let Err(_e) = std::fs::create_dir_all(tool_results_dir) { let dir = tool_results_dir.to_path_buf();
// 无法创建目录则直接截断(不持久化) if tokio::task::spawn_blocking(move || std::fs::create_dir_all(&dir))
.await
.unwrap_or(Ok(()))
.is_err()
{
let truncated: String = content.chars().take(max_chars).collect(); let truncated: String = content.chars().take(max_chars).collect();
return ( return (
format!( format!(
@ -60,19 +64,20 @@ pub fn maybe_persist_tool_result(
let file_path = tool_results_dir.join(format!("{}.txt", safe_id)); let file_path = tool_results_dir.join(format!("{}.txt", safe_id));
// 独占创建——只在文件不存在时写入,保证幂等性 // 独占创建——只在文件不存在时写入,保证幂等性
let written = match std::fs::OpenOptions::new() let fp = file_path.clone();
let content_clone = content.to_string();
let written = tokio::task::spawn_blocking(move || {
match std::fs::OpenOptions::new()
.write(true) .write(true)
.create_new(true) .create_new(true)
.open(&file_path) .open(&fp)
{ {
Ok(_) => { Ok(_) => match std::fs::write(&fp, &content_clone) {
// 文件创建成功,写入内容
match std::fs::write(&file_path, content) {
Ok(_) => { Ok(_) => {
info!( info!(
"[Persist] 工具结果已持久化: {} ({} 字符)", "[Persist] 工具结果已持久化: {} ({} 字符)",
file_path.display(), fp.display(),
content.len() content_clone.len()
); );
true true
} }
@ -80,18 +85,19 @@ pub fn maybe_persist_tool_result(
tracing::warn!("[Persist] 写入失败: {}", e); tracing::warn!("[Persist] 写入失败: {}", e);
false false
} }
} },
}
Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => {
// 文件已存在,无需重复写入 info!("[Persist] 工具结果已存在,跳过: {}", fp.display());
info!("[Persist] 工具结果已存在,跳过: {}", file_path.display());
true true
} }
Err(e) => { Err(e) => {
tracing::warn!("[Persist] 创建文件失败: {}", e); tracing::warn!("[Persist] 创建文件失败: {}", e);
false false
} }
}; }
})
.await
.unwrap_or(false);
if !written { if !written {
// 持久化失败,回退到截断 // 持久化失败,回退到截断
@ -148,19 +154,19 @@ mod tests {
(dir, Cleanup(dir_clone)) (dir, Cleanup(dir_clone))
} }
#[test] #[tokio::test]
fn test_small_content_not_persisted() { async fn test_small_content_not_persisted() {
let (dir, _cleanup) = temp_dir(); let (dir, _cleanup) = temp_dir();
let (content, path) = maybe_persist_tool_result("small result", "call_1", 4000, &dir); let (content, path) = maybe_persist_tool_result("small result", "call_1", 4000, &dir).await;
assert_eq!(content, "small result"); assert_eq!(content, "small result");
assert!(path.is_none()); assert!(path.is_none());
} }
#[test] #[tokio::test]
fn test_large_content_persisted() { async fn test_large_content_persisted() {
let (dir, _cleanup) = temp_dir(); let (dir, _cleanup) = temp_dir();
let large = "x".repeat(5000); let large = "x".repeat(5000);
let (content, path) = maybe_persist_tool_result(&large, "call_2", 100, &dir); let (content, path) = maybe_persist_tool_result(&large, "call_2", 100, &dir).await;
assert!(content.contains("<persisted-output>")); assert!(content.contains("<persisted-output>"));
assert!(content.contains("call_2.txt")); assert!(content.contains("call_2.txt"));
assert!(path.is_some()); assert!(path.is_some());
@ -170,14 +176,14 @@ mod tests {
assert_eq!(written, large); assert_eq!(written, large);
} }
#[test] #[tokio::test]
fn test_idempotent_write() { async fn test_idempotent_write() {
let (dir, _cleanup) = temp_dir(); let (dir, _cleanup) = temp_dir();
let large1 = "a".repeat(5000); let large1 = "a".repeat(5000);
let large2 = "b".repeat(5000); let large2 = "b".repeat(5000);
let (_, path1) = maybe_persist_tool_result(&large1, "call_3", 100, &dir); let (_, path1) = maybe_persist_tool_result(&large1, "call_3", 100, &dir).await;
let (_content2, path2) = maybe_persist_tool_result(&large2, "call_3", 100, &dir); let (_content2, path2) = maybe_persist_tool_result(&large2, "call_3", 100, &dir).await;
assert!(path1.is_some()); assert!(path1.is_some());
assert!(path2.is_some()); assert!(path2.is_some());
@ -185,11 +191,11 @@ mod tests {
assert_eq!(written, large1); assert_eq!(written, large1);
} }
#[test] #[tokio::test]
fn test_preview_at_newline_boundary() { async fn test_preview_at_newline_boundary() {
let (dir, _cleanup) = temp_dir(); let (dir, _cleanup) = temp_dir();
let large = format!("Short line\n{}", "x".repeat(5000)); let large = format!("Short line\n{}", "x".repeat(5000));
let (content, _) = maybe_persist_tool_result(&large, "call_4", 100, &dir); let (content, _) = maybe_persist_tool_result(&large, "call_4", 100, &dir).await;
let preview_start = content.find("preview:").unwrap(); let preview_start = content.find("preview:").unwrap();
let preview_section = &content[preview_start..]; let preview_section = &content[preview_start..];
assert!(preview_section.contains("Short line")); assert!(preview_section.contains("Short line"));

View File

@ -151,7 +151,8 @@ impl AgentTool for SendTeammateMessageTool {
let tm_lock = self.team_manager.lock().await; let tm_lock = self.team_manager.lock().await;
match tm_lock.as_ref() { match tm_lock.as_ref() {
Some(tm) => { Some(tm) => {
tm.send_message("lead", &to, &content, TeamMessageType::Task); tm.send_message("lead", &to, &content, TeamMessageType::Task)
.await;
ToolOutput::success( ToolOutput::success(
format!( format!(
"📤 消息已发送给 {}: {}", "📤 消息已发送给 {}: {}",
@ -214,7 +215,7 @@ impl AgentTool for TeamBroadcastTool {
let tm_lock = self.team_manager.lock().await; let tm_lock = self.team_manager.lock().await;
match tm_lock.as_ref() { match tm_lock.as_ref() {
Some(tm) => { Some(tm) => {
tm.broadcast("lead", &content); tm.broadcast("lead", &content).await;
ToolOutput::success("📢 已广播消息给所有队友。", json!({})) ToolOutput::success("📢 已广播消息给所有队友。", json!({}))
} }
None => ToolOutput::error("团队管理器未初始化"), None => ToolOutput::error("团队管理器未初始化"),
@ -278,7 +279,7 @@ impl AgentTool for CheckTeamInboxTool {
let tm_lock = self.team_manager.lock().await; let tm_lock = self.team_manager.lock().await;
match tm_lock.as_ref() { match tm_lock.as_ref() {
Some(tm) => { Some(tm) => {
let msgs = tm.check_inbox(agent); let msgs = tm.check_inbox(agent).await;
if msgs.is_empty() { if msgs.is_empty() {
ToolOutput::success( ToolOutput::success(
format!("📭 {} 的收件箱为空。", agent), format!("📭 {} 的收件箱为空。", agent),

View File

@ -259,10 +259,7 @@ mod tests {
ads, ads,
arxiv, arxiv,
vizier, vizier,
lamost: crate::clients::lamost::LamostClient::new( lamost: crate::clients::lamost::LamostClient::new("https://www.lamost.org", 60)
"https://www.lamost.org",
60,
)
.unwrap(), .unwrap(),
gaia: crate::clients::gaia::GaiaClient::new( gaia: crate::clients::gaia::GaiaClient::new(
"https://gea.esac.esa.int/tap-server/tap", "https://gea.esac.esa.int/tap-server/tap",
@ -290,7 +287,7 @@ mod tests {
harvest_status: Arc::new(tokio::sync::Mutex::new(MetaSyncStatus::default())), harvest_status: Arc::new(tokio::sync::Mutex::new(MetaSyncStatus::default())),
batch_status: Arc::new(tokio::sync::Mutex::new(AssetBatchStatus::default())), batch_status: Arc::new(tokio::sync::Mutex::new(AssetBatchStatus::default())),
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)), active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
cancelled_runs: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), cancelled_runs: Arc::new(dashmap::DashMap::new()),
skill_registry: Arc::new(tokio::sync::RwLock::new( skill_registry: Arc::new(tokio::sync::RwLock::new(
crate::agent::skills::SkillRegistry::new(PathBuf::from("/tmp/test_skills")), crate::agent::skills::SkillRegistry::new(PathBuf::from("/tmp/test_skills")),
)), )),
@ -298,16 +295,16 @@ mod tests {
pending_permissions: Arc::new( pending_permissions: Arc::new(
tokio::sync::Mutex::new(std::collections::HashMap::new()), tokio::sync::Mutex::new(std::collections::HashMap::new()),
), ),
session_permission_checkers: Arc::new(tokio::sync::RwLock::new( session_permission_checkers: Arc::new(dashmap::DashMap::new()),
std::collections::HashMap::new(),
)),
sse_broadcast: None, sse_broadcast: None,
memory_manager: Arc::new(tokio::sync::Mutex::new( memory_manager: Arc::new(tokio::sync::Mutex::new(
crate::agent::memory::MemoryManager::new(PathBuf::from("/tmp/test_memory")), crate::agent::memory::MemoryManager::new(PathBuf::from("/tmp/test_memory")),
)), )),
sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())), sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
login_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())), login_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
upload_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())), upload_rate_limiter: Arc::new(
tokio::sync::Mutex::new(std::collections::HashMap::new()),
),
}); });
ToolContext { ToolContext {

View File

@ -299,8 +299,7 @@ pub async fn stop_agent(
State(state): State<Arc<AppState>>, State(state): State<Arc<AppState>>,
Path(session_id): Path<String>, Path(session_id): Path<String>,
) -> ApiResult<Json<serde_json::Value>> { ) -> ApiResult<Json<serde_json::Value>> {
let mut cancelled = state.cancelled_runs.lock().await; state.cancelled_runs.insert(session_id.clone(), ());
cancelled.insert(session_id.clone());
info!("已接收并记录手动中止请求,会话 ID: {}", session_id); info!("已接收并记录手动中止请求,会话 ID: {}", session_id);
Ok(Json( Ok(Json(
serde_json::json!({ "status": "stopping", "session_id": session_id }), serde_json::json!({ "status": "stopping", "session_id": session_id }),
@ -412,7 +411,12 @@ pub async fn respond_permission(
match perm_id { match perm_id {
Some(id) => { Some(id) => {
let perm = perms.remove(&id).unwrap(); let perm = match perms.remove(&id) {
Some(p) => p,
None => {
return Err(AppError::internal("权限请求在查找后消失"));
}
};
match perm.response_tx.send(req) { match perm.response_tx.send(req) {
Ok(()) => { Ok(()) => {
info!( info!(

View File

@ -124,14 +124,9 @@ pub async fn login(
let mut limiter = state.login_rate_limiter.lock().await; let mut limiter = state.login_rate_limiter.lock().await;
// 批量清理过期条目,防止内存无限增长 // 批量清理过期条目,防止内存无限增长
limiter.retain(|_, (_, first_fail)| first_fail.elapsed().as_secs() < WINDOW_SECS); limiter.retain(|_, (_, first_fail)| first_fail.elapsed().as_secs() < WINDOW_SECS);
// 容量保护:超过上限时强制清理最旧的一半条目 // 容量保护:超过上限时直接清空(比 O(n log n) 排序更高效,且不影响安全性)
if limiter.len() > MAX_RATE_LIMITER_ENTRIES { if limiter.len() > MAX_RATE_LIMITER_ENTRIES {
let mut entries: Vec<_> = limiter.iter().map(|(k, v)| (k.clone(), v.1)).collect(); limiter.clear();
entries.sort_by_key(|(_, t)| *t);
let cutoff = entries.len() / 2;
for (key, _) in &entries[..cutoff] {
limiter.remove(key);
}
} }
if let Some((count, first_fail)) = limiter.get(&client_ip) { if let Some((count, first_fail)) = limiter.get(&client_ip) {
@ -242,16 +237,42 @@ pub async fn auth_middleware(
let token = extract_auth_token_from_headers(req.headers()); let token = extract_auth_token_from_headers(req.headers());
let mut is_valid = false; let mut is_valid = false;
if let Some(tok) = token { if let Some(ref tok) = token {
// 1. 优先校验内存中的动态 Session // 1. 优先校验内存中的动态 Session
// 先用读锁快速检查会话是否存在,并获取其最后活跃时间
let mut last_active_opt = None;
{
let sessions = state.sessions.read().await;
if let Some(last_active) = sessions.get(tok) {
is_valid = true;
last_active_opt = Some(*last_active);
}
}
// 会话存在时,如果最后活跃时间超过 60 秒,则获取写锁更新最后活跃时间 + 过期清理
if is_valid {
let needs_update = match last_active_opt {
Some(last_active) => {
if let Ok(dur) = chrono::Utc::now()
.signed_duration_since(last_active)
.to_std()
{
dur.as_secs() > 60
} else {
true // 时钟回拨等异常情况,强制更新
}
}
None => true,
};
if needs_update {
let mut sessions = state.sessions.write().await; let mut sessions = state.sessions.write().await;
// 顺便执行过期会话自动清理垃圾收集 // 顺便执行过期会话自动清理垃圾收集
prune_expired_sessions(&mut sessions); prune_expired_sessions(&mut sessions);
if let Some(last_active) = sessions.get_mut(tok) {
if let Some(last_active) = sessions.get_mut(&tok) {
// 会话存在且未过期,更新最后活跃时间
*last_active = chrono::Utc::now(); *last_active = chrono::Utc::now();
is_valid = true; }
}
} }
// 2. 校验书签专属永久 API 密钥(严格隔离限制:仅允许用于特定的书签导入和同步路径) // 2. 校验书签专属永久 API 密钥(严格隔离限制:仅允许用于特定的书签导入和同步路径)
@ -259,7 +280,7 @@ pub async fn auth_middleware(
let path = req.uri().path(); let path = req.uri().path();
if path == "/api/upload" || path == "/api/active_bibcode" { if path == "/api/upload" || path == "/api/active_bibcode" {
let expected_key = get_bookmarklet_api_key(&state.config.admin_password); let expected_key = get_bookmarklet_api_key(&state.config.admin_password);
if tok == expected_key { if *tok == expected_key {
is_valid = true; is_valid = true;
} }
} }

View File

@ -85,18 +85,15 @@ pub struct AppState {
pub harvest_status: Arc<tokio::sync::Mutex<crate::services::batch::MetaSyncStatus>>, pub harvest_status: Arc<tokio::sync::Mutex<crate::services::batch::MetaSyncStatus>>,
pub batch_status: Arc<tokio::sync::Mutex<crate::services::batch::AssetBatchStatus>>, pub batch_status: Arc<tokio::sync::Mutex<crate::services::batch::AssetBatchStatus>>,
pub active_bibcode: Arc<tokio::sync::Mutex<Option<String>>>, pub active_bibcode: Arc<tokio::sync::Mutex<Option<String>>>,
pub cancelled_runs: Arc<Mutex<std::collections::HashSet<String>>>, pub cancelled_runs: Arc<dashmap::DashMap<String, ()>>,
pub skill_registry: Arc<RwLock<SkillRegistry>>, pub skill_registry: Arc<RwLock<SkillRegistry>>,
/// ask_user 工具 — 待回答的问题 /// ask_user 工具 — 待回答的问题
pub pending_questions: Arc<Mutex<HashMap<String, PendingQuestion>>>, pub pending_questions: Arc<Mutex<HashMap<String, PendingQuestion>>>,
/// 权限检查 — 待处理的权限确认请求 /// 权限检查 — 待处理的权限确认请求
pub pending_permissions: Arc<Mutex<HashMap<String, PendingPermission>>>, pub pending_permissions: Arc<Mutex<HashMap<String, PendingPermission>>>,
/// 会话级权限检查器注册表(按 session_id 隔离,支持 API 动态添加/移除规则) /// 会话级权限检查器注册表(按 session_id 隔离,支持 API 动态添加/移除规则)
pub session_permission_checkers: Arc< pub session_permission_checkers:
RwLock< Arc<dashmap::DashMap<String, crate::agent::runtime::permission::PermissionChecker>>,
std::collections::HashMap<String, crate::agent::runtime::permission::PermissionChecker>,
>,
>,
/// SSE 广播通道agent 运行时向所有连接的客户端推送事件) /// SSE 广播通道agent 运行时向所有连接的客户端推送事件)
pub sse_broadcast: Option<broadcast::Sender<AppEvent>>, pub sse_broadcast: Option<broadcast::Sender<AppEvent>>,
/// 项目记忆管理器(跨会话持久化) /// 项目记忆管理器(跨会话持久化)

View File

@ -31,47 +31,15 @@ fn default_radius() -> f64 {
0.1 0.1
} }
fn parse_source(s: &str) -> Result<crate::services::observation::Source, AppError> { /// 把 (source, product, subtype) 字符串参数聚合为 ProductSpec。
use crate::services::observation::Source; /// source/product 的解析统一委托给枚举的 from_str() 方法,不再重复 match。
Ok(match s.to_lowercase().as_str() {
"lamost" => Source::Lamost,
"gaia" => Source::Gaia,
"sdss" => Source::Sdss,
"desi" => Source::Desi,
other => {
return Err(AppError::bad_request(format!(
"不支持的 source '{}',可选: {:?}",
other,
Source::valid_values()
)))
}
})
}
fn parse_product(s: &str) -> Result<crate::services::observation::ProductType, AppError> {
use crate::services::observation::ProductType;
Ok(match s.to_lowercase().as_str() {
"spectrum" => ProductType::Spectrum,
"lightcurve" | "light_curve" | "lc" => ProductType::LightCurve,
"photometry" => ProductType::Photometry,
"image" => ProductType::Image,
other => {
return Err(AppError::bad_request(format!(
"不支持的 product '{}',可选: {:?}",
other,
ProductType::valid_values()
)))
}
})
}
/// 把 (source, product, subtype) 字符串参数聚合为 ProductSpec
fn build_product_spec( fn build_product_spec(
product: &str, product: &str,
subtype: &Option<String>, subtype: &Option<String>,
) -> Result<crate::services::observation::ProductSpec, AppError> { ) -> Result<crate::services::observation::ProductSpec, AppError> {
Ok(crate::services::observation::ProductSpec { Ok(crate::services::observation::ProductSpec {
product: parse_product(product)?, product: crate::services::observation::ProductType::from_str(product)
.map_err(AppError::bad_request)?,
subtype: subtype.clone(), subtype: subtype.clone(),
}) })
} }
@ -91,13 +59,16 @@ pub struct ObservationSearchParams {
#[serde(default = "default_radius")] #[serde(default = "default_radius")]
pub radius: f64, pub radius: f64,
pub release: Option<String>, pub release: Option<String>,
/// 数据发布的子版本(仅 LAMOST 有意义,如 "v2.0"None 时用该 DR 的默认版本
pub version: Option<String>,
} }
pub async fn observation_search( pub async fn observation_search(
State(state): State<std::sync::Arc<AppState>>, State(state): State<std::sync::Arc<AppState>>,
Query(params): Query<ObservationSearchParams>, Query(params): Query<ObservationSearchParams>,
) -> ApiResult<Json<Vec<crate::services::observation::Candidate>>> { ) -> ApiResult<Json<Vec<crate::services::observation::Candidate>>> {
let source = parse_source(&params.source)?; let source = crate::services::observation::Source::from_str(&params.source)
.map_err(AppError::bad_request)?;
let product = build_product_spec(&params.product, &params.subtype)?; let product = build_product_spec(&params.product, &params.subtype)?;
let candidates = crate::services::observation::search_observation( let candidates = crate::services::observation::search_observation(
@ -109,6 +80,7 @@ pub async fn observation_search(
params.dec, params.dec,
params.radius, params.radius,
params.release.as_deref(), params.release.as_deref(),
params.version.as_deref(),
) )
.await .await
.map_err(|e| { .map_err(|e| {
@ -140,6 +112,8 @@ pub struct ObservationDownloadRequest {
pub product: String, pub product: String,
pub subtype: Option<String>, pub subtype: Option<String>,
pub release: Option<String>, pub release: Option<String>,
/// 数据发布的子版本(仅 LAMOST 有意义,如 "v2.0"None 时用该 DR 的默认版本
pub version: Option<String>,
#[serde(default)] #[serde(default)]
pub force: bool, pub force: bool,
#[serde(flatten)] #[serde(flatten)]
@ -168,7 +142,8 @@ pub async fn observation_download(
) -> ApiResult<Json<crate::services::observation::ObservationBatch>> { ) -> ApiResult<Json<crate::services::observation::ObservationBatch>> {
use crate::services::observation::{download_observation, ObservationRequest}; use crate::services::observation::{download_observation, ObservationRequest};
let source = parse_source(&req.source)?; let source = crate::services::observation::Source::from_str(&req.source)
.map_err(AppError::bad_request)?;
let product = build_product_spec(&req.product, &req.subtype)?; let product = build_product_spec(&req.product, &req.subtype)?;
let request = match req.mode { let request = match req.mode {
@ -185,6 +160,7 @@ pub async fn observation_download(
radius_deg, radius_deg,
strategy, strategy,
release: req.release.clone(), release: req.release.clone(),
version: req.version.clone(),
}, },
DownloadMode::Identifiers { source_ids } => { DownloadMode::Identifiers { source_ids } => {
let identifiers: Vec<String> = source_ids let identifiers: Vec<String> = source_ids
@ -200,6 +176,7 @@ pub async fn observation_download(
product, product,
identifiers, identifiers,
release: req.release.clone(), release: req.release.clone(),
version: req.version.clone(),
} }
} }
}; };

View File

@ -350,6 +350,11 @@ pub async fn upload_paper_file(
return Err(AppError::bad_request("bibcode 包含非法字符")); return Err(AppError::bad_request("bibcode 包含非法字符"));
} }
// 防御性安全边界:虽然合法 bibcode 允许双点(..),旧 arXiv 允许斜杠(/),但二者不能共存。
if bibcode.contains("..") && (bibcode.contains('/') || bibcode.contains('\\')) {
return Err(AppError::bad_request("bibcode 包含非法的路径遍历字符"));
}
let updated_paper = crate::services::paper::upload_paper_file_service( let updated_paper = crate::services::paper::upload_paper_file_service(
&state.db, &state.db,
&state.config.library_dir, &state.config.library_dir,

View File

@ -130,19 +130,19 @@ pub async fn update_permission_rules(
}, },
}; };
let mut checkers = state.session_permission_checkers.write().await; let mut entry = state
let checker = checkers .session_permission_checkers
.entry(session_id.clone()) .entry(session_id.clone())
.or_insert_with(PermissionChecker::new); .or_insert_with(PermissionChecker::new);
checker.add_rule_dynamic(session_rule); entry.add_rule_dynamic(session_rule);
Json(PermissionActionResponse { Json(PermissionActionResponse {
success: true, success: true,
message: format!("已添加 {}:{} 规则", req.kind, req.rule), message: format!("已添加 {}:{} 规则", req.kind, req.rule),
}) })
} }
"remove" => { "remove" => {
let mut checkers = state.session_permission_checkers.write().await; let removed =
let removed = if let Some(checker) = checkers.get_mut(&session_id) { if let Some(mut checker) = state.session_permission_checkers.get_mut(&session_id) {
checker.remove_rule_dynamic(&req.rule, &req.kind) checker.remove_rule_dynamic(&req.rule, &req.kind)
} else { } else {
0 0
@ -182,19 +182,15 @@ pub async fn update_permission_mode(
session_id, mode_str session_id, mode_str
); );
let mut checkers = state.session_permission_checkers.write().await; let mut checker = state
if let Some(checker) = checkers.get_mut(&session_id) { .session_permission_checkers
.entry(session_id.clone())
.or_insert_with(PermissionChecker::new);
checker.set_mode(mode); checker.set_mode(mode);
Json(PermissionActionResponse { Json(PermissionActionResponse {
success: true, success: true,
message: format!("权限模式已切换为: {}", mode_str), message: format!("权限模式已切换为: {}", mode_str),
}) })
} else {
Json(PermissionActionResponse {
success: false,
message: format!("会话 {} 不存在或无活跃 Agent", session_id),
})
}
} }
/// 列出指定会话的所有权限规则。 /// 列出指定会话的所有权限规则。
@ -202,9 +198,7 @@ pub async fn list_permission_rules(
Path(session_id): Path<String>, Path(session_id): Path<String>,
State(state): State<Arc<AppState>>, State(state): State<Arc<AppState>>,
) -> Json<PermissionRulesResponse> { ) -> Json<PermissionRulesResponse> {
let checkers = state.session_permission_checkers.read().await; if let Some(checker) = state.session_permission_checkers.get(&session_id) {
if let Some(checker) = checkers.get(&session_id) {
let mut deny_rules = Vec::new(); let mut deny_rules = Vec::new();
let mut allow_rules = Vec::new(); let mut allow_rules = Vec::new();
let mut ask_rules = Vec::new(); let mut ask_rules = Vec::new();

View File

@ -259,13 +259,13 @@ async fn main() -> anyhow::Result<()> {
astroresearch::services::batch::AssetBatchStatus::new(), astroresearch::services::batch::AssetBatchStatus::new(),
)), )),
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)), active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
cancelled_runs: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), cancelled_runs: Arc::new(dashmap::DashMap::new()),
skill_registry: Arc::new(tokio::sync::RwLock::new( skill_registry: Arc::new(tokio::sync::RwLock::new(
astroresearch::agent::skills::SkillRegistry::new(config.skills_dir.clone()), astroresearch::agent::skills::SkillRegistry::new(config.skills_dir.clone()),
)), )),
pending_questions: Arc::new(tokio::sync::Mutex::new(HashMap::new())), pending_questions: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
pending_permissions: Arc::new(tokio::sync::Mutex::new(HashMap::new())), pending_permissions: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
session_permission_checkers: Arc::new(tokio::sync::RwLock::new(HashMap::new())), session_permission_checkers: Arc::new(dashmap::DashMap::new()),
sse_broadcast: { sse_broadcast: {
let (tx, _) = tokio::sync::broadcast::channel(256); let (tx, _) = tokio::sync::broadcast::channel(256);
Some(tx) Some(tx)
@ -631,7 +631,8 @@ async fn main() -> anyhow::Result<()> {
&config.library_dir, &config.library_dir,
&title, &title,
&content, &content,
)?; )
.await?;
println!( println!(
"✅ 笔记已成功保存!\n 文件名: {}\n 物理路径: {}\n 总字符数: {}", "✅ 笔记已成功保存!\n 文件名: {}\n 物理路径: {}\n 总字符数: {}",

View File

@ -112,12 +112,11 @@ impl GaiaClient {
Ok(GaiaClient { Ok(GaiaClient {
client: reqwest::Client::builder() client: reqwest::Client::builder()
// Gaia DataLink 端点data-server/data在传输完 ZIP 后直接断 TCP 而不发送 // Gaia DataLink 端点data-server/data在传输完 ZIP 后直接断 TCP 而不发送
// TLS close_notify 警报。rustls 严格遵守 RFC 5246会把这种"无 close_notify 的 // TLS close_notify 警报。rustls 严格遵守 RFC 5246 会判 UnexpectedEof 致命错误。
// 关闭"判为 UnexpectedEof 致命错误哪怕响应体已完整到达。ESA 官方 Python // 多数 UnexpectedEof 只在复用 idle 连接时触发,禁用连接池可避免:
// 客户端 astroquery 走系统 OpenSSLnative-tls后者容忍此行为故不受影响。 // 每次请求新建连接,响应读完即关闭,不会复用"已被对端不洁关闭"的连接。
// 这里对 Gaia 客户端单独启用 native-tls使其与 astroquery 走相同 TLS 栈, .pool_max_idle_per_host(0)
// 从而正确处理 DataLink 的连接关闭。项目其余 client 仍用默认 rustls。 .pool_idle_timeout(Some(Duration::from_secs(0)))
.use_native_tls()
.redirect(crate::utils::ssrf::safe_redirect_policy()) .redirect(crate::utils::ssrf::safe_redirect_policy())
.timeout(Duration::from_secs(timeout_secs)) .timeout(Duration::from_secs(timeout_secs))
.connect_timeout(Duration::from_secs(15)) .connect_timeout(Duration::from_secs(15))

View File

@ -99,13 +99,17 @@ impl LamostClient {
/// GET {base}/{dr}/{ver}/{voservice|medvoservice}/conesearch?RA={ra}&DEC={dec}&SR={radius_deg} /// GET {base}/{dr}/{ver}/{voservice|medvoservice}/conesearch?RA={ra}&DEC={dec}&SR={radius_deg}
/// - LRS低分辨率→ voservice 子路径 /// - LRS低分辨率→ voservice 子路径
/// - MRS中分辨率→ medvoservice 子路径 /// - MRS中分辨率→ medvoservice 子路径
///
/// 服务端返回 VOTable用共享解析器解析后映射为 LamostSpectrumRow。 /// 服务端返回 VOTable用共享解析器解析后映射为 LamostSpectrumRow。
///
/// `version` 为该 DR 的子版本路径段(如 "v2.0"/"v1.0"),由上层 parse_lamost_version 解析后传入。
pub async fn cone_search( pub async fn cone_search(
&self, &self,
ra: f64, ra: f64,
dec: f64, dec: f64,
radius_deg: f64, radius_deg: f64,
release: crate::services::observation::lamost::LamostRelease, release: crate::services::observation::lamost::LamostRelease,
version: &str,
resolution: crate::services::observation::lamost::LamostResolution, resolution: crate::services::observation::lamost::LamostResolution,
) -> anyhow::Result<LamostConeResult> { ) -> anyhow::Result<LamostConeResult> {
use crate::services::observation::lamost::LamostResolution; use crate::services::observation::lamost::LamostResolution;
@ -125,12 +129,13 @@ impl LamostClient {
"{}/{}/{}/{}/conesearch", "{}/{}/{}/{}/conesearch",
self.base_url, self.base_url,
release.path_segment(), release.path_segment(),
release.version_segment(), version,
service_segment, service_segment,
); );
info!( info!(
"[LAMOST] ConeSearch {} {} ra={:.4} dec={:.4} radius={:.4}°", "[LAMOST] ConeSearch {}/{} {} ra={:.4} dec={:.4} radius={:.4}°",
release.path_segment(), release.path_segment(),
version,
resolution.display(), resolution.display(),
ra, ra,
dec, dec,
@ -172,21 +177,25 @@ impl LamostClient {
/// ///
/// GET {base}/{dr}/{ver}/spectrum/fits/{obsid} → application/gzip /// GET {base}/{dr}/{ver}/spectrum/fits/{obsid} → application/gzip
/// 返回的 bytes 可能是 gzip 压缩magic bytes 0x1f 0x8b由业务层解压。 /// 返回的 bytes 可能是 gzip 压缩magic bytes 0x1f 0x8b由业务层解压。
///
/// `version` 为该 DR 的子版本路径段(如 "v2.0"),由上层 parse_lamost_version 解析后传入。
pub async fn download_fits( pub async fn download_fits(
&self, &self,
obsid: i64, obsid: i64,
release: crate::services::observation::lamost::LamostRelease, release: crate::services::observation::lamost::LamostRelease,
version: &str,
) -> anyhow::Result<Vec<u8>> { ) -> anyhow::Result<Vec<u8>> {
let url = format!( let url = format!(
"{}/{}/{}/spectrum/fits/{}", "{}/{}/{}/spectrum/fits/{}",
self.base_url, self.base_url,
release.path_segment(), release.path_segment(),
release.version_segment(), version,
obsid obsid
); );
info!( info!(
"[LAMOST] 下载 FITS {} obsid={}", "[LAMOST] 下载 FITS {}/{} obsid={}",
release.path_segment(), release.path_segment(),
version,
obsid obsid
); );
@ -366,13 +375,14 @@ mod tests {
async fn test_live_cone_search() { async fn test_live_cone_search() {
use crate::services::observation::lamost::{LamostRelease, LamostResolution}; use crate::services::observation::lamost::{LamostRelease, LamostResolution};
let client = LamostClient::new("https://www.lamost.org", 60).unwrap(); let client = LamostClient::new("https://www.lamost.org", 60).unwrap();
// M31 附近DR10 LRS // M31 附近DR10 v2.0 LRS
let result = client let result = client
.cone_search( .cone_search(
10.6847, 10.6847,
41.2687, 41.2687,
0.1, 0.1,
LamostRelease::Dr10, LamostRelease::Dr10,
"v2.0",
LamostResolution::Lrs, LamostResolution::Lrs,
) )
.await .await
@ -403,6 +413,7 @@ mod tests {
41.2687, 41.2687,
0.1, 0.1,
LamostRelease::Dr10, LamostRelease::Dr10,
"v2.0",
LamostResolution::Lrs, LamostResolution::Lrs,
) )
.await .await
@ -410,7 +421,7 @@ mod tests {
let obsid = cone.rows.first().expect("应有结果").obsid; let obsid = cone.rows.first().expect("应有结果").obsid;
let bytes = client let bytes = client
.download_fits(obsid, LamostRelease::Dr10) .download_fits(obsid, LamostRelease::Dr10, "v2.0")
.await .await
.unwrap(); .unwrap();
println!("===== LAMOST FITS Live (obsid={}) =====", obsid); println!("===== LAMOST FITS Live (obsid={}) =====", obsid);

View File

@ -24,7 +24,7 @@ pub trait ChatCompleter: Send + Sync {
pub struct LlmClient { pub struct LlmClient {
api_key: String, api_key: String,
api_base: String, api_base: String,
model: Arc<std::sync::RwLock<String>>, model: Arc<tokio::sync::RwLock<String>>,
client: Client, client: Client,
} }
@ -46,21 +46,25 @@ impl LlmClient {
Ok(LlmClient { Ok(LlmClient {
api_key, api_key,
api_base, api_base,
model: Arc::new(std::sync::RwLock::new(model)), model: Arc::new(tokio::sync::RwLock::new(model)),
client, client,
}) })
} }
pub fn model(&self) -> String { pub async fn model(&self) -> String {
self.model.read().unwrap_or_else(|e| e.into_inner()).clone() self.model.read().await.clone()
}
/// 同步获取模型名称快照(用于非 async 上下文,如环境信息展示)。
/// 如果锁被持有则返回空字符串。
pub fn model_name_snapshot(&self) -> String {
self.model.try_read().map(|m| m.clone()).unwrap_or_default()
} }
/// 运行时切换模型(用于故障转移)。 /// 运行时切换模型(用于故障转移)。
pub fn set_model(&self, model: String) { pub async fn set_model(&self, model: String) {
if let Ok(mut m) = self.model.write() { tracing::info!("[LlmClient] 模型切换为: {}", model);
*m = model; *self.model.write().await = model;
tracing::info!("[LlmClient] 模型切换为: {}", m);
}
} }
pub fn api_base(&self) -> &str { pub fn api_base(&self) -> &str {
@ -79,7 +83,7 @@ impl LlmClient {
let url = format!("{}/chat/completions", self.api_base); let url = format!("{}/chat/completions", self.api_base);
let payload = serde_json::json!({ let payload = serde_json::json!({
"model": self.model(), "model": self.model().await,
"messages": [ "messages": [
{ {
"role": "system", "role": "system",
@ -143,7 +147,7 @@ impl LlmClient {
let url = format!("{}/chat/completions", self.api_base); let url = format!("{}/chat/completions", self.api_base);
let mut payload = serde_json::json!({ let mut payload = serde_json::json!({
"model": self.model(), "model": self.model().await,
"messages": messages, "messages": messages,
"temperature": 0.3 "temperature": 0.3
}); });
@ -151,7 +155,7 @@ impl LlmClient {
// 前端可控的思考模式开关(仅对千问/DashScope 启用 enable_thinking 参数)(仅对千问/DashScope 启用 enable_thinking 参数) // 前端可控的思考模式开关(仅对千问/DashScope 启用 enable_thinking 参数)(仅对千问/DashScope 启用 enable_thinking 参数)
if enable_thinking if enable_thinking
&& (self.api_base.contains("dashscope.aliyuncs.com") && (self.api_base.contains("dashscope.aliyuncs.com")
|| self.model().to_lowercase().contains("qwen")) || self.model().await.to_lowercase().contains("qwen"))
{ {
if let Some(obj) = payload.as_object_mut() { if let Some(obj) = payload.as_object_mut() {
obj.insert("enable_thinking".to_string(), serde_json::json!(true)); obj.insert("enable_thinking".to_string(), serde_json::json!(true));
@ -224,7 +228,7 @@ impl LlmClient {
]); ]);
let payload = serde_json::json!({ let payload = serde_json::json!({
"model": self.model(), "model": self.model().await,
"messages": [ "messages": [
{ "role": "system", "content": system_prompt }, { "role": "system", "content": system_prompt },
{ "role": "user", "content": user_content } { "role": "user", "content": user_content }
@ -297,7 +301,7 @@ impl LlmClient {
let url = format!("{}/chat/completions", self.api_base); let url = format!("{}/chat/completions", self.api_base);
let mut payload = serde_json::json!({ let mut payload = serde_json::json!({
"model": self.model(), "model": self.model().await,
"messages": messages, "messages": messages,
"temperature": 0.3, "temperature": 0.3,
"stream": true, "stream": true,
@ -307,7 +311,7 @@ impl LlmClient {
// 前端可控的思考模式开关(仅对千问/DashScope 启用 enable_thinking 参数) // 前端可控的思考模式开关(仅对千问/DashScope 启用 enable_thinking 参数)
if enable_thinking if enable_thinking
&& (self.api_base.contains("dashscope.aliyuncs.com") && (self.api_base.contains("dashscope.aliyuncs.com")
|| self.model().to_lowercase().contains("qwen")) || self.model().await.to_lowercase().contains("qwen"))
{ {
if let Some(obj) = payload.as_object_mut() { if let Some(obj) = payload.as_object_mut() {
obj.insert("enable_thinking".to_string(), serde_json::json!(true)); obj.insert("enable_thinking".to_string(), serde_json::json!(true));
@ -564,13 +568,13 @@ mod tests {
ChatMessage, FunctionCall, MessageRole, ToolCall, ToolDefinition, ChatMessage, FunctionCall, MessageRole, ToolCall, ToolDefinition,
}; };
#[test] #[tokio::test]
fn test_llm_client_initialization() { async fn test_llm_client_basics() {
let client = let client =
LlmClient::new("key".to_string(), "base".to_string(), "model".to_string()).unwrap(); LlmClient::new("key".to_string(), "base".to_string(), "model".to_string()).unwrap();
assert_eq!(client.api_key(), "key"); assert_eq!(client.api_key(), "key");
assert_eq!(client.api_base(), "base"); assert_eq!(client.api_base(), "base");
assert_eq!(client.model(), "model"); assert_eq!(client.model().await, "model");
} }
#[test] #[test]

View File

@ -2,6 +2,19 @@
use std::env; use std::env;
use std::path::PathBuf; use std::path::PathBuf;
/// 从环境变量解析 u64 值,解析失败时打印警告并返回默认值。
fn env_u64(key: &str, default: u64) -> u64 {
env::var(key)
.ok()
.and_then(|v| {
v.parse::<u64>().ok().or_else(|| {
tracing::warn!("{} '{}' 不是有效数字,使用默认值 {}", key, v, default);
None
})
})
.unwrap_or(default)
}
// 系统配置结构体,加载并管理从环境变量或 .env 文件读取的参数 // 系统配置结构体,加载并管理从环境变量或 .env 文件读取的参数
#[derive(Clone, Debug)] #[derive(Clone, Debug)]
pub struct Config { pub struct Config {
@ -133,65 +146,25 @@ impl Config {
let vizier_tap_url = env::var("VIZIER_TAP_URL") let vizier_tap_url = env::var("VIZIER_TAP_URL")
.unwrap_or_else(|_| "https://tapvizier.cds.unistra.fr/TAPVizieR/tap".to_string()); .unwrap_or_else(|_| "https://tapvizier.cds.unistra.fr/TAPVizieR/tap".to_string());
let vizier_timeout_secs = env::var("VIZIER_TIMEOUT_SECS") let vizier_timeout_secs = env_u64("VIZIER_TIMEOUT_SECS", 60);
.ok()
.and_then(|v| {
v.parse::<u64>().ok().or_else(|| {
tracing::warn!("VIZIER_TIMEOUT_SECS '{}' 不是有效数字,使用默认值 60", v);
None
})
})
.unwrap_or(60);
let lamost_base_url = let lamost_base_url =
env::var("LAMOST_BASE_URL").unwrap_or_else(|_| "https://www.lamost.org".to_string()); env::var("LAMOST_BASE_URL").unwrap_or_else(|_| "https://www.lamost.org".to_string());
let lamost_timeout_secs = env::var("LAMOST_TIMEOUT_SECS") let lamost_timeout_secs = env_u64("LAMOST_TIMEOUT_SECS", 60);
.ok()
.and_then(|v| {
v.parse::<u64>().ok().or_else(|| {
tracing::warn!("LAMOST_TIMEOUT_SECS '{}' 不是有效数字,使用默认值 60", v);
None
})
})
.unwrap_or(60);
let gaia_tap_url = env::var("GAIA_TAP_URL") let gaia_tap_url = env::var("GAIA_TAP_URL")
.unwrap_or_else(|_| "https://gea.esac.esa.int/tap-server/tap".to_string()); .unwrap_or_else(|_| "https://gea.esac.esa.int/tap-server/tap".to_string());
let gaia_datalink_url = env::var("GAIA_DATALINK_URL") let gaia_datalink_url = env::var("GAIA_DATALINK_URL")
.unwrap_or_else(|_| "https://gea.esac.esa.int/data-server".to_string()); .unwrap_or_else(|_| "https://gea.esac.esa.int/data-server".to_string());
let gaia_timeout_secs = env::var("GAIA_TIMEOUT_SECS") let gaia_timeout_secs = env_u64("GAIA_TIMEOUT_SECS", 90);
.ok()
.and_then(|v| {
v.parse::<u64>().ok().or_else(|| {
tracing::warn!("GAIA_TIMEOUT_SECS '{}' 不是有效数字,使用默认值 90", v);
None
})
})
.unwrap_or(90);
let sdss_tap_url = env::var("SDSS_TAP_URL") let sdss_tap_url = env::var("SDSS_TAP_URL")
.unwrap_or_else(|_| "https://datalab.noirlab.edu/tap/sync".to_string()); .unwrap_or_else(|_| "https://datalab.noirlab.edu/tap/sync".to_string());
let sdss_timeout_secs = env::var("SDSS_TIMEOUT_SECS") let sdss_timeout_secs = env_u64("SDSS_TIMEOUT_SECS", 90);
.ok()
.and_then(|v| {
v.parse::<u64>().ok().or_else(|| {
tracing::warn!("SDSS_TIMEOUT_SECS '{}' 不是有效数字,使用默认值 90", v);
None
})
})
.unwrap_or(90);
let desi_tap_url = env::var("DESI_TAP_URL") let desi_tap_url = env::var("DESI_TAP_URL")
.unwrap_or_else(|_| "https://datalab.noirlab.edu/tap/sync".to_string()); .unwrap_or_else(|_| "https://datalab.noirlab.edu/tap/sync".to_string());
let desi_timeout_secs = env::var("DESI_TIMEOUT_SECS") let desi_timeout_secs = env_u64("DESI_TIMEOUT_SECS", 120);
.ok()
.and_then(|v| {
v.parse::<u64>().ok().or_else(|| {
tracing::warn!("DESI_TIMEOUT_SECS '{}' 不是有效数字,使用默认值 120", v);
None
})
})
.unwrap_or(120);
Config { Config {
database_url, database_url,

View File

@ -290,7 +290,9 @@ async fn main() -> anyhow::Result<()> {
gaia, gaia,
sdss, sdss,
desi, desi,
observation_registry: Arc::new(astroresearch::services::observation::ObservationRegistry::default()), observation_registry: Arc::new(
astroresearch::services::observation::ObservationRegistry::default(),
),
llm, llm,
medium_llm, medium_llm,
fast_llm, fast_llm,
@ -309,7 +311,7 @@ async fn main() -> anyhow::Result<()> {
astroresearch::services::batch::AssetBatchStatus::new(), astroresearch::services::batch::AssetBatchStatus::new(),
)), )),
active_bibcode: Arc::new(tokio::sync::Mutex::new(None)), active_bibcode: Arc::new(tokio::sync::Mutex::new(None)),
cancelled_runs: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), cancelled_runs: Arc::new(dashmap::DashMap::new()),
skill_registry, skill_registry,
pending_questions: Arc::new(tokio::sync::Mutex::new(HashMap::new())), pending_questions: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
pending_permissions: Arc::new(tokio::sync::Mutex::new(HashMap::new())), pending_permissions: Arc::new(tokio::sync::Mutex::new(HashMap::new())),
@ -320,9 +322,7 @@ async fn main() -> anyhow::Result<()> {
memory_manager: Arc::new(tokio::sync::Mutex::new( memory_manager: Arc::new(tokio::sync::Mutex::new(
astroresearch::agent::memory::MemoryManager::new(config.library_dir.clone()), astroresearch::agent::memory::MemoryManager::new(config.library_dir.clone()),
)), )),
session_permission_checkers: Arc::new(tokio::sync::RwLock::new( session_permission_checkers: Arc::new(dashmap::DashMap::new()),
std::collections::HashMap::new(),
)),
sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())), sessions: Arc::new(tokio::sync::RwLock::new(std::collections::HashMap::new())),
login_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())), login_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
upload_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())), upload_rate_limiter: Arc::new(tokio::sync::Mutex::new(std::collections::HashMap::new())),
@ -530,6 +530,7 @@ async fn main() -> anyhow::Result<()> {
.nest_service("/api/files", protected_files) .nest_service("/api/files", protected_files)
.fallback_service(serve_dir) .fallback_service(serve_dir)
.layer(tower_http::trace::TraceLayer::new_for_http()) .layer(tower_http::trace::TraceLayer::new_for_http())
.layer(tower_http::catch_panic::CatchPanicLayer::new())
.layer(SetResponseHeaderLayer::overriding( .layer(SetResponseHeaderLayer::overriding(
axum::http::header::X_CONTENT_TYPE_OPTIONS, axum::http::header::X_CONTENT_TYPE_OPTIONS,
HeaderValue::from_static("nosniff"), HeaderValue::from_static("nosniff"),
@ -575,17 +576,18 @@ async fn pdf_inline_middleware(
req: axum::http::Request<axum::body::Body>, req: axum::http::Request<axum::body::Body>,
next: axum::middleware::Next, next: axum::middleware::Next,
) -> Result<axum::response::Response<axum::body::Body>, axum::http::StatusCode> { ) -> Result<axum::response::Response<axum::body::Body>, axum::http::StatusCode> {
let path = req.uri().path().to_lowercase(); let original_path = req.uri().path().to_string();
let is_pdf = path.ends_with(".pdf"); let is_pdf = original_path.to_lowercase().ends_with(".pdf");
let mut response = next.run(req).await; let mut response = next.run(req).await;
if is_pdf && response.status().is_success() { if is_pdf && response.status().is_success() {
// 获取文件名 // 使用原始路径提取文件名(保留大小写)
let filename = std::path::Path::new(&path) let filename = std::path::Path::new(&original_path)
.file_name() .file_name()
.and_then(|n| n.to_str()) .and_then(|n| n.to_str())
.unwrap_or("document.pdf"); .unwrap_or("document.pdf")
.replace('"', "_");
// 强行设置 Content-Type 为 application/pdf // 强行设置 Content-Type 为 application/pdf
response.headers_mut().insert( response.headers_mut().insert(

View File

@ -220,7 +220,7 @@ pub(super) async fn build_markdown_with_frontmatter(
} }
/// 将 Markdown 内容写入 library_dir 并更新数据库 markdown_path。 /// 将 Markdown 内容写入 library_dir 并更新数据库 markdown_path。
pub(super) fn write_markdown_and_update_db( pub(super) async fn write_markdown_and_update_db(
config: &Config, config: &Config,
bibcode: &str, bibcode: &str,
content: &str, content: &str,
@ -228,9 +228,9 @@ pub(super) fn write_markdown_and_update_db(
let md_filename = format!("{}.md", bibcode); let md_filename = format!("{}.md", bibcode);
let md_dest = config.library_dir.join("Markdown").join(&md_filename); let md_dest = config.library_dir.join("Markdown").join(&md_filename);
if let Some(parent) = md_dest.parent() { if let Some(parent) = md_dest.parent() {
let _ = std::fs::create_dir_all(parent); let _ = tokio::fs::create_dir_all(parent).await;
} }
if std::fs::write(&md_dest, content).is_ok() { if tokio::fs::write(&md_dest, content).await.is_ok() {
Some(format!("Markdown/{}", md_filename)) Some(format!("Markdown/{}", md_filename))
} else { } else {
None None

View File

@ -170,7 +170,7 @@ pub(super) async fn process_single_bibcode(
.await .await
{ {
if let Some(rel_path) = if let Some(rel_path) =
write_markdown_and_update_db(config, &bibcode, &parsed_md) write_markdown_and_update_db(config, &bibcode, &parsed_md).await
{ {
relative_md_path = rel_path; relative_md_path = rel_path;
} }
@ -288,7 +288,8 @@ pub(super) async fn process_single_bibcode(
&config_clone, &config_clone,
&bibcode_clone, &bibcode_clone,
&parsed_md, &parsed_md,
); )
.await;
if let Some(rel_path) = rel_md { if let Some(rel_path) = rel_md {
let _ = sqlx::query("UPDATE papers SET markdown_path = ? WHERE bibcode = ?") let _ = sqlx::query("UPDATE papers SET markdown_path = ? WHERE bibcode = ?")

View File

@ -213,20 +213,36 @@ pub async fn extract_and_cache_targets(
let mut results = Vec::new(); let mut results = Vec::new();
let mut new_targets = Vec::new(); let mut new_targets = Vec::new();
for name in names { if names.is_empty() {
// 优先在 paper_targets 表中查找本地缓存 return results;
let row = sqlx::query_as::<_, TargetDbRow>( }
// 1. 批量查询已存在的本地缓存记录,减少 N+1 数据库往返
let mut cached_map = std::collections::HashMap::new();
for chunk in names.chunks(crate::services::paper::SQLITE_PARAM_LIMIT) {
let placeholders = vec!["?"; chunk.len()].join(", ");
let sql = format!(
"SELECT target_name, ra, dec, parallax, parallax_err, spectral_type, v_magnitude, \ "SELECT target_name, ra, dec, parallax, parallax_err, spectral_type, v_magnitude, \
otype, oname, pm_ra, pm_de, radial_velocity, photometry, aliases \ otype, oname, pm_ra, pm_de, radial_velocity, photometry, aliases \
FROM paper_targets WHERE target_name = ? LIMIT 1", FROM paper_targets WHERE target_name IN ({})",
) placeholders
.bind(&name) );
.fetch_optional(pool) let mut query = sqlx::query_as::<_, TargetDbRow>(&sql);
.await; for name in chunk {
query = query.bind(name);
}
if let Ok(rows) = query.fetch_all(pool).await {
for row in rows {
cached_map.insert(row.target_name.clone(), TargetInfo::from(row));
}
}
}
if let Ok(Some(db_row)) = row { // 2. 遍历检查哪些命中缓存,哪些需要调用 Sesame
for name in names {
if let Some(info) = cached_map.remove(&name) {
info!("天体 {} 命中本地缓存", name); info!("天体 {} 命中本地缓存", name);
results.push(TargetInfo::from(db_row)); results.push(info);
} else { } else {
// 本地缓存未命中,释放连接并外部查询 Sesame // 本地缓存未命中,释放连接并外部查询 Sesame
info!("天体 {} 本地缓存未命中,查询 CDS Sesame...", name); info!("天体 {} 本地缓存未命中,查询 CDS Sesame...", name);

View File

@ -323,7 +323,10 @@ impl<'a> VizierCatalog<'a> {
} else { } else {
sanitize_identifier(table)? sanitize_identifier(table)?
}; };
Ok(format!("SELECT TOP {} {} FROM {}", limit, columns, table_ref)) Ok(format!(
"SELECT TOP {} {} FROM {}",
limit, columns, table_ref
))
} }
/// 导出星表数据到 CSV 文件 /// 导出星表数据到 CSV 文件
@ -357,7 +360,13 @@ impl<'a> VizierCatalog<'a> {
let catalog_name = table.unwrap_or("adql_query"); let catalog_name = table.unwrap_or("adql_query");
let safe_name: String = catalog_name let safe_name: String = catalog_name
.chars() .chars()
.map(|c| if c.is_alphanumeric() || c == '-' || c == '_' { c } else { '_' }) .map(|c| {
if c.is_alphanumeric() || c == '-' || c == '_' {
c
} else {
'_'
}
})
.collect(); .collect();
let ts = chrono::Utc::now().format("%Y%m%d_%H%M%S"); let ts = chrono::Utc::now().format("%Y%m%d_%H%M%S");
@ -367,10 +376,12 @@ impl<'a> VizierCatalog<'a> {
.join(format!("{}.csv", ts)); .join(format!("{}.csv", ts));
if let Some(parent) = output_path.parent() { if let Some(parent) = output_path.parent() {
tokio::fs::create_dir_all(parent).await tokio::fs::create_dir_all(parent)
.await
.map_err(|e| anyhow!("创建目录失败: {}", e))?; .map_err(|e| anyhow!("创建目录失败: {}", e))?;
} }
tokio::fs::write(&output_path, &export.csv).await tokio::fs::write(&output_path, &export.csv)
.await
.map_err(|e| anyhow!("写入文件失败: {}", e))?; .map_err(|e| anyhow!("写入文件失败: {}", e))?;
Ok(ExportToFileResult { Ok(ExportToFileResult {

View File

@ -152,83 +152,74 @@ impl Downloader {
doi: Option<&str>, doi: Option<&str>,
library_dir: &Path, library_dir: &Path,
) -> (Result<PathBuf, String>, Result<PathBuf, String>) { ) -> (Result<PathBuf, String>, Result<PathBuf, String>) {
let base = "https://ui.adsabs.harvard.edu/link_gateway";
let pdf_dest = library_dir.join("PDF").join(format!("{}.pdf", bibcode)); let pdf_dest = library_dir.join("PDF").join(format!("{}.pdf", bibcode));
let html_dest = library_dir.join("HTML").join(format!("{}.html", bibcode)); let html_dest = library_dir.join("HTML").join(format!("{}.html", bibcode));
let mut pdf_res = Err("未尝试任何下载通道".to_string()); let pdf_res = self.try_download_pdf(bibcode, doi, &pdf_dest).await;
let mut html_res = Err("未尝试任何下载通道".to_string()); let html_res = self.try_download_html(bibcode, &html_dest).await;
let mut pdf_errors = Vec::new(); (pdf_res, html_res)
let mut html_errors = Vec::new(); }
/// PDF 多级回退下载PUB_PDF → ADS_PDF → EPRINT_PDF → CrossRef → SCAN
async fn try_download_pdf(
&self,
bibcode: &str,
doi: Option<&str>,
dest: &Path,
) -> Result<PathBuf, String> {
let base = "https://ui.adsabs.harvard.edu/link_gateway";
let mut errors = Vec::new();
// ── PDF 下载 ───────────────────────────────────────────
info!("[下载] 开始 PDF 下载: {}", bibcode); info!("[下载] 开始 PDF 下载: {}", bibcode);
'pdf: {
// 1a. ADS PUB_PDF 网关 // 1a. ADS PUB_PDF 网关
let gw = format!("{}/{}/PUB_PDF", base, bibcode); let gw = format!("{}/{}/PUB_PDF", base, bibcode);
match self.resolve_ads_gateway(&gw).await { match self.resolve_ads_gateway(&gw).await {
Ok(resolved) => { Ok(resolved) => {
let result = if resolved.contains("iopscience.iop.org") { let result = if resolved.contains("iopscience.iop.org") {
// 提取 DOI 路径部分,走 IOP 专属策略
let doi = resolved let doi = resolved
.trim_start_matches("https://iopscience.iop.org/article/") .trim_start_matches("https://iopscience.iop.org/article/")
.trim_end_matches("/pdf") .trim_end_matches("/pdf")
.trim_end_matches('/'); .trim_end_matches('/');
self.download_iop_pdf(doi, &pdf_dest).await self.download_iop_pdf(doi, dest).await
} else if resolved.contains("link.springer.com") } else if resolved.contains("link.springer.com") || resolved.contains("nature.com")
|| resolved.contains("nature.com")
{ {
// Springer/NatureHTML 更可靠PDF 用通用策略 self.download_pdf_direct(&resolved, dest, "Springer").await
self.download_pdf_direct(&resolved, &pdf_dest, "Springer")
.await
} else { } else {
self.download_pdf_direct(&resolved, &pdf_dest, "PUB_PDF") self.download_pdf_direct(&resolved, dest, "PUB_PDF").await
.await
}; };
match result { match result {
Ok(_) => { Ok(_) => return Ok(dest.to_path_buf()),
pdf_res = Ok(pdf_dest.clone());
break 'pdf;
}
Err(e) => { Err(e) => {
let msg = format!("PUB_PDF下载失败: {}", e); let msg = format!("PUB_PDF下载失败: {}", e);
warn!("{}", msg); warn!("{}", msg);
pdf_errors.push(msg); errors.push(msg);
} }
} }
} }
Err(e) => { Err(e) => {
let msg = format!("PUB_PDF网关解析失败: {}", e); let msg = format!("PUB_PDF网关解析失败: {}", e);
warn!("{}", msg); warn!("{}", msg);
pdf_errors.push(msg); errors.push(msg);
} }
} }
// 1b. ADS_PDF 网关 (经典 ADS 整合 PDF 直接通道) // 1b. ADS_PDF 网关
let gw = format!("{}/{}/ADS_PDF", base, bibcode); let gw = format!("{}/{}/ADS_PDF", base, bibcode);
match self.resolve_ads_gateway(&gw).await { match self.resolve_ads_gateway(&gw).await {
Ok(resolved) => { Ok(resolved) => match self.download_pdf_direct(&resolved, dest, "ADS_PDF").await {
match self Ok(_) => return Ok(dest.to_path_buf()),
.download_pdf_direct(&resolved, &pdf_dest, "ADS_PDF")
.await
{
Ok(_) => {
pdf_res = Ok(pdf_dest.clone());
break 'pdf;
}
Err(e) => { Err(e) => {
let msg = format!("ADS_PDF下载失败: {}", e); let msg = format!("ADS_PDF下载失败: {}", e);
warn!("{}", msg); warn!("{}", msg);
pdf_errors.push(msg); errors.push(msg);
}
}
} }
},
Err(e) => { Err(e) => {
let msg = format!("ADS_PDF网关解析失败: {}", e); let msg = format!("ADS_PDF网关解析失败: {}", e);
warn!("{}", msg); warn!("{}", msg);
pdf_errors.push(msg); errors.push(msg);
} }
} }
@ -237,142 +228,129 @@ impl Downloader {
match self.resolve_ads_gateway(&gw).await { match self.resolve_ads_gateway(&gw).await {
Ok(resolved) => { Ok(resolved) => {
match self match self
.download_pdf_direct(&resolved, &pdf_dest, "EPRINT_PDF") .download_pdf_direct(&resolved, dest, "EPRINT_PDF")
.await .await
{ {
Ok(_) => { Ok(_) => return Ok(dest.to_path_buf()),
pdf_res = Ok(pdf_dest.clone());
break 'pdf;
}
Err(e) => { Err(e) => {
let msg = format!("EPRINT_PDF下载失败: {}", e); let msg = format!("EPRINT_PDF下载失败: {}", e);
warn!("{}", msg); warn!("{}", msg);
pdf_errors.push(msg); errors.push(msg);
} }
} }
} }
Err(e) => { Err(e) => {
let msg = format!("EPRINT_PDF网关解析失败: {}", e); let msg = format!("EPRINT_PDF网关解析失败: {}", e);
warn!("{}", msg); warn!("{}", msg);
pdf_errors.push(msg); errors.push(msg);
} }
} }
// 1d. CrossRef API 回退(需要 DOI // 1d. CrossRef API 回退
if let Some(doi_str) = doi { if let Some(doi_str) = doi {
match self.download_crossref_pdf(doi_str, &pdf_dest).await { match self.download_crossref_pdf(doi_str, dest).await {
Ok(_) => { Ok(_) => return Ok(dest.to_path_buf()),
pdf_res = Ok(pdf_dest.clone());
break 'pdf;
}
Err(e) => { Err(e) => {
let msg = format!("CrossRef下载失败: {}", e); let msg = format!("CrossRef下载失败: {}", e);
warn!("{}", msg); warn!("{}", msg);
pdf_errors.push(msg); errors.push(msg);
} }
} }
} }
// 1e. ADS SCAN 扫描版文献直接合并下载 PDF主要针对早期/不可下载直接 PDF 的文献) // 1e. ADS SCAN 扫描版
let scan_url = format!("https://articles.adsabs.harvard.edu/cgi-bin/nph-iarticle_query?bibcode={}&db_key=AST&data_type=PDF_HIGH", bibcode); let scan_url = format!(
match self "https://articles.adsabs.harvard.edu/cgi-bin/nph-iarticle_query?bibcode={}&db_key=AST&data_type=PDF_HIGH",
.download_pdf_direct(&scan_url, &pdf_dest, "ADS_SCAN") bibcode
.await );
{ match self.download_pdf_direct(&scan_url, dest, "ADS_SCAN").await {
Ok(_) => { Ok(_) => return Ok(dest.to_path_buf()),
pdf_res = Ok(pdf_dest.clone());
}
Err(e) => { Err(e) => {
let msg = format!("ADS_SCAN下载失败: {}", e); let msg = format!("ADS_SCAN下载失败: {}", e);
warn!("{}", msg); warn!("{}", msg);
pdf_errors.push(msg); errors.push(msg);
}
} }
} }
if pdf_res.is_err() && !pdf_errors.is_empty() { if errors.is_empty() {
pdf_res = Err(pdf_errors.join("; ")); Err("未尝试任何下载通道".to_string())
} else {
Err(errors.join("; "))
}
} }
// ── HTML 下载 ────────────────────────────────────────── /// HTML 多级回退下载PUB_HTML → EPRINT_HTML
async fn try_download_html(&self, bibcode: &str, dest: &Path) -> Result<PathBuf, String> {
let base = "https://ui.adsabs.harvard.edu/link_gateway";
let mut errors = Vec::new();
info!("[下载] 开始 HTML 下载: {}", bibcode); info!("[下载] 开始 HTML 下载: {}", bibcode);
'html: {
// 2a. ADS PUB_HTML 网关 // 2a. ADS PUB_HTML 网关
let gw = format!("{}/{}/PUB_HTML", base, bibcode); let gw = format!("{}/{}/PUB_HTML", base, bibcode);
match self.resolve_ads_gateway(&gw).await { match self.resolve_ads_gateway(&gw).await {
Ok(resolved) => { Ok(resolved) => {
let result = if resolved.contains("link.springer.com") let result =
|| resolved.contains("nature.com") if resolved.contains("link.springer.com") || resolved.contains("nature.com") {
{
// Springer/Nature 专属 HTML 策略
let doi_part = resolved let doi_part = resolved
.trim_start_matches("https://link.springer.com/article/") .trim_start_matches("https://link.springer.com/article/")
.trim_start_matches("https://www.nature.com/articles/") .trim_start_matches("https://www.nature.com/articles/")
.trim_end_matches('/'); .trim_end_matches('/');
self.download_springer_html(doi_part, &html_dest).await self.download_springer_html(doi_part, dest).await
} else if let Some(arxiv_id) = extract_arxiv_id_from_url(&resolved) { } else if let Some(arxiv_id) = extract_arxiv_id_from_url(&resolved) {
// ADS 网关指向 arXiv abs 页面 → 优先官方 HTMLar5iv 兜底 self.download_arxiv_html_with_fallback(&arxiv_id, dest)
self.download_arxiv_html_with_fallback(&arxiv_id, &html_dest)
.await .await
} else { } else {
self.download_html_direct(&resolved, &html_dest, "PUB_HTML") self.download_html_direct(&resolved, dest, "PUB_HTML").await
.await
}; };
match result { match result {
Ok(_) => { Ok(_) => return Ok(dest.to_path_buf()),
html_res = Ok(html_dest.clone());
break 'html;
}
Err(e) => { Err(e) => {
let msg = format!("PUB_HTML下载失败: {}", e); let msg = format!("PUB_HTML下载失败: {}", e);
warn!("{}", msg); warn!("{}", msg);
html_errors.push(msg); errors.push(msg);
} }
} }
} }
Err(e) => { Err(e) => {
let msg = format!("PUB_HTML网关解析失败: {}", e); let msg = format!("PUB_HTML网关解析失败: {}", e);
warn!("{}", msg); warn!("{}", msg);
html_errors.push(msg); errors.push(msg);
} }
} }
// 2b. ADS EPRINT_HTML 网关(大多数天文论文有 arXiv eprint // 2b. ADS EPRINT_HTML 网关
let gw = format!("{}/{}/EPRINT_HTML", base, bibcode); let gw = format!("{}/{}/EPRINT_HTML", base, bibcode);
match self.resolve_ads_gateway(&gw).await { match self.resolve_ads_gateway(&gw).await {
Ok(resolved) => { Ok(resolved) => {
let result = if let Some(arxiv_id) = extract_arxiv_id_from_url(&resolved) { let result = if let Some(arxiv_id) = extract_arxiv_id_from_url(&resolved) {
self.download_arxiv_html_with_fallback(&arxiv_id, &html_dest) self.download_arxiv_html_with_fallback(&arxiv_id, dest)
.await .await
} else { } else {
self.download_html_direct(&resolved, &html_dest, "EPRINT_HTML") self.download_html_direct(&resolved, dest, "EPRINT_HTML")
.await .await
}; };
match result { match result {
Ok(_) => { Ok(_) => return Ok(dest.to_path_buf()),
html_res = Ok(html_dest.clone());
}
Err(e) => { Err(e) => {
let msg = format!("EPRINT_HTML下载失败: {}", e); let msg = format!("EPRINT_HTML下载失败: {}", e);
warn!("{}", msg); warn!("{}", msg);
html_errors.push(msg); errors.push(msg);
} }
} }
} }
Err(e) => { Err(e) => {
let msg = format!("EPRINT_HTML网关解析失败: {}", e); let msg = format!("EPRINT_HTML网关解析失败: {}", e);
warn!("{}", msg); warn!("{}", msg);
html_errors.push(msg); errors.push(msg);
}
} }
} }
if html_res.is_err() && !html_errors.is_empty() { if errors.is_empty() {
html_res = Err(html_errors.join("; ")); Err("未尝试任何下载通道".to_string())
} else {
Err(errors.join("; "))
} }
(pdf_res, html_res)
} }
/// 高层文献下载服务,处理数据库记录、多级回退下载、数据库路径写回 /// 高层文献下载服务,处理数据库记录、多级回退下载、数据库路径写回

View File

@ -27,7 +27,7 @@ impl Downloader {
info!("[Obscura 后备通道] 启动下载: {}", url); info!("[Obscura 后备通道] 启动下载: {}", url);
if let Some(parent) = dest_path.parent() { if let Some(parent) = dest_path.parent() {
std::fs::create_dir_all(parent)?; tokio::fs::create_dir_all(parent).await?;
} }
#[cfg(feature = "obscura-inprocess")] #[cfg(feature = "obscura-inprocess")]
@ -190,7 +190,7 @@ impl Downloader {
use futures_util::StreamExt; use futures_util::StreamExt;
if let Some(parent) = target_path.parent() { if let Some(parent) = target_path.parent() {
std::fs::create_dir_all(parent)?; tokio::fs::create_dir_all(parent).await?;
} }
let mut file = tokio::fs::File::create(target_path) let mut file = tokio::fs::File::create(target_path)

View File

@ -56,7 +56,11 @@ pub fn init_logging() -> anyhow::Result<Vec<WorkerGuard>> {
let log_dir = env::var("LOG_DIR").unwrap_or_else(|_| "logs".to_string()); let log_dir = env::var("LOG_DIR").unwrap_or_else(|_| "logs".to_string());
fs::create_dir_all(&log_dir).unwrap_or(()); fs::create_dir_all(&log_dir).unwrap_or(());
let file_appender = rolling::daily(log_dir, "astro_research.log"); let file_appender = rolling::RollingFileAppender::builder()
.rotation(rolling::Rotation::DAILY)
.filename_prefix("astro_research")
.filename_suffix("log")
.build(log_dir)?;
let (non_blocking, guard) = tracing_appender::non_blocking(file_appender); let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
guards.push(guard); guards.push(guard);

View File

@ -4,12 +4,11 @@
use std::path::{Path, PathBuf}; use std::path::{Path, PathBuf};
/// 保存研究笔记为本地 Markdown 文件 /// 保存研究笔记为本地 Markdown 文件
pub fn save_note_service( pub async fn save_note_service(
library_dir: &Path, library_dir: &Path,
title: &str, title: &str,
content: &str, content: &str,
) -> anyhow::Result<(String, PathBuf, usize)> { ) -> anyhow::Result<(String, PathBuf, usize)> {
// 文件名安全化
let safe_title: String = title let safe_title: String = title
.chars() .chars()
.map(|c| { .map(|c| {
@ -23,7 +22,7 @@ pub fn save_note_service(
.replace(' ', "_"); .replace(' ', "_");
let notes_dir = library_dir.join("notes"); let notes_dir = library_dir.join("notes");
std::fs::create_dir_all(&notes_dir)?; tokio::fs::create_dir_all(&notes_dir).await?;
let filename = format!("{}.md", safe_title); let filename = format!("{}.md", safe_title);
let filepath = notes_dir.join(&filename); let filepath = notes_dir.join(&filename);
@ -35,7 +34,7 @@ pub fn save_note_service(
content content
); );
std::fs::write(&filepath, &full_content)?; tokio::fs::write(&filepath, &full_content).await?;
Ok((filename, filepath, full_content.len())) Ok((filename, filepath, full_content.len()))
} }
@ -103,13 +102,13 @@ pub async fn delete_highlight_note(db: &sqlx::SqlitePool, id: i64) -> Result<boo
mod tests { mod tests {
use super::*; use super::*;
#[test] #[tokio::test]
fn test_save_note_service() { async fn test_save_note_service() {
let tmp_dir = std::env::temp_dir().join("astro_note_test"); let tmp_dir = std::env::temp_dir().join("astro_note_test");
let title = "Test Note Title 123!"; let title = "Test Note Title 123!";
let content = "Hello world from note service test"; let content = "Hello world from note service test";
let res = save_note_service(&tmp_dir, title, content); let res = save_note_service(&tmp_dir, title, content).await;
assert!(res.is_ok()); assert!(res.is_ok());
let (filename, filepath, size) = res.unwrap(); let (filename, filepath, size) = res.unwrap();
assert_eq!(filename, "Test_Note_Title_123_.md"); assert_eq!(filename, "Test_Note_Title_123_.md");

View File

@ -198,26 +198,39 @@ pub struct CachedPage {
} }
/// 列出已缓存产物(服务端分页 + 筛选 + 排序) /// 列出已缓存产物(服务端分页 + 筛选 + 排序)
///
/// 所有筛选值通过 SQL 参数绑定(`?`)传入,杜绝 SQL 注入与 LIKE 通配符(`%`/`_`)误匹配。
/// 排序方向用白名单(只接受 created/source/product 三个字面量)。
pub async fn list_cached_paged(pool: &SqlitePool, params: &ListCachedParams) -> Result<CachedPage> { pub async fn list_cached_paged(pool: &SqlitePool, params: &ListCachedParams) -> Result<CachedPage> {
let limit = params.limit.unwrap_or(50).clamp(1, 200); let limit = params.limit.unwrap_or(50).clamp(1, 200);
let offset = params.offset.unwrap_or(0).max(0); let offset = params.offset.unwrap_or(0).max(0);
// 动态构造 SQL参数绑定防注入列名/排序方向用白名单) // 动态构造 WHERE 子句(用 ? 占位符)+ 同步收集绑定值
let mut where_clauses: Vec<String> = Vec::new(); // bind_values 顺序必须与占位符出现顺序一致source → product → q → limit → offset
let mut where_parts: Vec<&'static str> = Vec::new();
let mut bind_source: Option<String> = None;
let mut bind_product: Option<String> = None;
let mut bind_q: Option<String> = None; // 已包成 '%q%',且 LIKE 通配符已转义
if let Some(s) = &params.source { if let Some(s) = &params.source {
where_clauses.push(format!("source = '{}'", s.replace('\'', "''"))); where_parts.push("source = ?");
bind_source = Some(s.clone());
} }
if let Some(p) = &params.product { if let Some(p) = &params.product {
where_clauses.push(format!("product = '{}'", p.replace('\'', "''"))); where_parts.push("product = ?");
bind_product = Some(p.clone());
} }
if let Some(q) = &params.q { if let Some(q) = &params.q {
let q = q.replace('\'', "''"); // 转义 LIKE 通配符,避免用户输入的 %/_ 被解释为模式
where_clauses.push(format!("source_id LIKE '%{}%'", q)); let escaped = q.replace('%', "\\%").replace('_', "\\_");
where_parts.push("source_id LIKE ? ESCAPE '\\'");
bind_q = Some(format!("%{}%", escaped));
} }
let where_sql = if where_clauses.is_empty() {
let where_sql = if where_parts.is_empty() {
String::new() String::new()
} else { } else {
format!(" WHERE {}", where_clauses.join(" AND ")) format!(" WHERE {}", where_parts.join(" AND "))
}; };
let order_sql = match params.sort.as_deref() { let order_sql = match params.sort.as_deref() {
@ -226,22 +239,41 @@ pub async fn list_cached_paged(pool: &SqlitePool, params: &ListCachedParams) ->
_ => " ORDER BY created_at DESC", _ => " ORDER BY created_at DESC",
}; };
// count // count绑定 source/product/q不绑 limit/offset
let count_sql = format!("SELECT COUNT(*) as cnt FROM observation_cache{}", where_sql); let count_sql = format!("SELECT COUNT(*) FROM observation_cache{}", where_sql);
let total: i64 = sqlx::query_scalar(&count_sql) let mut count_q = sqlx::query_scalar::<_, i64>(&count_sql);
if let Some(v) = &bind_source {
count_q = count_q.bind(v);
}
if let Some(v) = &bind_product {
count_q = count_q.bind(v);
}
if let Some(v) = &bind_q {
count_q = count_q.bind(v);
}
let total = count_q
.fetch_one(pool) .fetch_one(pool)
.await .await
.map_err(|e| anyhow!("统计 observation_cache 总数失败: {}", e))?; .map_err(|e| anyhow!("统计 observation_cache 总数失败: {}", e))?;
// items // items(绑定 source/product/q/limit/offset
let items_sql = format!( let items_sql = format!(
"SELECT source, product, source_id, ra, dec, artifacts_json, meta_json, created_at \ "SELECT source, product, source_id, ra, dec, artifacts_json, meta_json, created_at \
FROM observation_cache{}{} LIMIT ? OFFSET ?", FROM observation_cache{}{} LIMIT ? OFFSET ?",
where_sql, order_sql where_sql, order_sql
); );
let items = sqlx::query_as::<_, ObservationCacheRow>(&items_sql) let mut items_q = sqlx::query_as::<_, ObservationCacheRow>(&items_sql);
.bind(limit) if let Some(v) = &bind_source {
.bind(offset) items_q = items_q.bind(v);
}
if let Some(v) = &bind_product {
items_q = items_q.bind(v);
}
if let Some(v) = &bind_q {
items_q = items_q.bind(v);
}
items_q = items_q.bind(limit).bind(offset);
let items = items_q
.fetch_all(pool) .fetch_all(pool)
.await .await
.map_err(|e| anyhow!("查询 observation_cache 分页列表失败: {}", e))?; .map_err(|e| anyhow!("查询 observation_cache 分页列表失败: {}", e))?;
@ -259,26 +291,27 @@ pub fn file_url_from_path(rel_path: &str) -> String {
} }
/// 落盘字节到 library_dir/<rel_path>,返回字节数 /// 落盘字节到 library_dir/<rel_path>,返回字节数
pub fn persist_bytes(library_dir: &Path, rel_path: &str, bytes: &[u8]) -> Result<usize> { pub async fn persist_bytes(library_dir: &Path, rel_path: &str, bytes: &[u8]) -> Result<usize> {
let abs_path = library_dir.join(rel_path); let abs_path = library_dir.join(rel_path);
if let Some(parent) = abs_path.parent() { if let Some(parent) = abs_path.parent() {
std::fs::create_dir_all(parent) tokio::fs::create_dir_all(parent)
.await
.map_err(|e| anyhow!("创建观测数据目录失败 {:?}: {}", parent, e))?; .map_err(|e| anyhow!("创建观测数据目录失败 {:?}: {}", parent, e))?;
} }
std::fs::write(&abs_path, bytes) tokio::fs::write(&abs_path, bytes)
.await
.map_err(|e| anyhow!("写入观测数据文件失败 {:?}: {}", abs_path, e))?; .map_err(|e| anyhow!("写入观测数据文件失败 {:?}: {}", abs_path, e))?;
Ok(bytes.len()) Ok(bytes.len())
} }
/// 校验缓存产物中至少有一个文件真实存在;存在则返回其总大小 /// 校验缓存产物中至少有一个文件真实存在;存在则返回其总大小
pub fn cached_files_total_size(library_dir: &Path, artifacts: &[Artifact]) -> Option<usize> { pub async fn cached_files_total_size(library_dir: &Path, artifacts: &[Artifact]) -> Option<usize> {
let mut total = 0; let mut total = 0;
for a in artifacts { for a in artifacts {
let abs_path = library_dir.join(&a.file_path); let abs_path = library_dir.join(&a.file_path);
match std::fs::metadata(&abs_path) { match tokio::fs::metadata(&abs_path).await {
Ok(m) => total += m.len() as usize, Ok(m) => total += m.len() as usize,
Err(_) => { Err(_) => {
// 任一文件缺失即视为缓存不完整
return None; return None;
} }
} }
@ -292,7 +325,10 @@ pub fn cached_files_total_size(library_dir: &Path, artifacts: &[Artifact]) -> Op
/// 记录缓存写入失败但不中断流程 /// 记录缓存写入失败但不中断流程
pub fn log_write_failure(source: &str, product: &str, e: anyhow::Error) { pub fn log_write_failure(source: &str, product: &str, e: anyhow::Error) {
error!("[Observation] 写入 observation_cache 失败 ({}/{}): {}", source, product, e); error!(
"[Observation] 写入 observation_cache 失败 ({}/{}): {}",
source, product, e
);
} }
#[cfg(test)] #[cfg(test)]

View File

@ -22,11 +22,11 @@ use tracing::{info, warn};
// ── 版本枚举 ── // ── 版本枚举 ──
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum DesiRelease { Edr, Dr1 } pub enum DesiRelease {
Edr,
impl Default for DesiRelease { #[default]
fn default() -> Self { Self::Dr1 } Dr1,
} }
impl DesiRelease { impl DesiRelease {
@ -34,16 +34,28 @@ impl DesiRelease {
&["edr", "dr1"] &["edr", "dr1"]
} }
pub fn dr_segment(&self) -> &'static str { pub fn dr_segment(&self) -> &'static str {
match self { Self::Edr => "edr", Self::Dr1 => "dr1" } match self {
Self::Edr => "edr",
Self::Dr1 => "dr1",
}
} }
pub fn specredux_ver(&self) -> &'static str { pub fn specredux_ver(&self) -> &'static str {
match self { Self::Edr => "fuji", Self::Dr1 => "iron" } match self {
Self::Edr => "fuji",
Self::Dr1 => "iron",
}
} }
pub fn datalab_table(&self) -> &'static str { pub fn datalab_table(&self) -> &'static str {
match self { Self::Edr => "desi_edr.zpix", Self::Dr1 => "desi_dr1.zpix" } match self {
Self::Edr => "desi_edr.zpix",
Self::Dr1 => "desi_dr1.zpix",
}
} }
pub fn display(&self) -> &'static str { pub fn display(&self) -> &'static str {
match self { Self::Edr => "EDR", Self::Dr1 => "DR1" } match self {
Self::Edr => "EDR",
Self::Dr1 => "DR1",
}
} }
} }
@ -52,7 +64,13 @@ pub fn parse_desi_release(s: Option<&str>) -> Result<DesiRelease> {
None => DesiRelease::Dr1, None => DesiRelease::Dr1,
Some("edr") => DesiRelease::Edr, Some("edr") => DesiRelease::Edr,
Some("dr1") => DesiRelease::Dr1, Some("dr1") => DesiRelease::Dr1,
Some(other) => return Err(anyhow!("不支持的 DESI release '{}',可选: {:?}", other, DesiRelease::valid_values())), Some(other) => {
return Err(anyhow!(
"不支持的 DESI release '{}',可选: {:?}",
other,
DesiRelease::valid_values()
))
}
}) })
} }
@ -114,8 +132,12 @@ impl ObservationFetcher for DesiSpectrumFetcher {
async fn cone_search_raw( async fn cone_search_raw(
&self, &self,
state: &AppState, state: &AppState,
ra: f64, dec: f64, radius_deg: f64, ra: f64,
release: Option<&str>, _subtype: Option<&str>, dec: f64,
radius_deg: f64,
release: Option<&str>,
_subtype: Option<&str>,
_version: Option<&str>,
) -> Result<Vec<Candidate>> { ) -> Result<Vec<Candidate>> {
validate_cone_params(ra, dec, radius_deg)?; validate_cone_params(ra, dec, radius_deg)?;
let rel = parse_desi_release(release)?; let rel = parse_desi_release(release)?;
@ -128,6 +150,7 @@ impl ObservationFetcher for DesiSpectrumFetcher {
identifier: &str, identifier: &str,
_release: Option<&str>, _release: Option<&str>,
_subtype: Option<&str>, _subtype: Option<&str>,
_version: Option<&str>,
) -> Result<Candidate> { ) -> Result<Candidate> {
// 格式 "{survey}-{program}-{healpix}" // 格式 "{survey}-{program}-{healpix}"
let parts: Vec<&str> = identifier.split('-').collect(); let parts: Vec<&str> = identifier.split('-').collect();
@ -137,7 +160,28 @@ impl ObservationFetcher for DesiSpectrumFetcher {
identifier identifier
)); ));
} }
let healpix: i64 = parts[2].parse() let survey = parts[0];
let program = parts[1];
// 白名单校验survey/program 会拼进文件系统路径和下载 URL
// 必须防止路径穿越(如 "../etc")。仅允许 DESI 实际使用的取值。
const VALID_SURVEYS: &[&str] = &["main", "sv1", "sv2", "sv3", "sv", "backup", "cumulative"];
const VALID_PROGRAMS: &[&str] = &["dark", "bright", "backup", "other"];
if !VALID_SURVEYS.contains(&survey) {
return Err(anyhow!(
"DESI survey '{}' 不在允许列表 {:?} 中",
survey,
VALID_SURVEYS
));
}
if !VALID_PROGRAMS.contains(&program) {
return Err(anyhow!(
"DESI program '{}' 不在允许列表 {:?} 中",
program,
VALID_PROGRAMS
));
}
let healpix: i64 = parts[2]
.parse()
.map_err(|_| anyhow!("healpix 非数字: '{}'", parts[2]))?; .map_err(|_| anyhow!("healpix 非数字: '{}'", parts[2]))?;
if healpix < 0 { if healpix < 0 {
return Err(anyhow!("healpix 应为非负整数,当前: {}", healpix)); return Err(anyhow!("healpix 应为非负整数,当前: {}", healpix));
@ -145,10 +189,12 @@ impl ObservationFetcher for DesiSpectrumFetcher {
Ok(Candidate { Ok(Candidate {
source: Source::Desi, source: Source::Desi,
source_id: identifier.to_string(), source_id: identifier.to_string(),
label: format!("{} {} healpix {}", parts[0], parts[1], healpix), label: format!("{} {} healpix {}", survey, program, healpix),
ra: None, dec: None, distance: None, ra: None,
dec: None,
distance: None,
raw: Some(serde_json::json!({ raw: Some(serde_json::json!({
"survey": parts[0], "program": parts[1], "healpix": healpix "survey": survey, "program": program, "healpix": healpix
})), })),
}) })
} }
@ -159,13 +205,26 @@ impl ObservationFetcher for DesiSpectrumFetcher {
candidate: &Candidate, candidate: &Candidate,
release: Option<&str>, release: Option<&str>,
_subtype: Option<&str>, _subtype: Option<&str>,
_version: Option<&str>,
force: bool, force: bool,
) -> Result<ObservationProduct> { ) -> Result<ObservationProduct> {
let survey = candidate.raw.as_ref().and_then(|v| v.get("survey")).and_then(|v| v.as_str()) let survey = candidate
.raw
.as_ref()
.and_then(|v| v.get("survey"))
.and_then(|v| v.as_str())
.ok_or_else(|| anyhow!("DESI Candidate 缺 survey"))?; .ok_or_else(|| anyhow!("DESI Candidate 缺 survey"))?;
let program = candidate.raw.as_ref().and_then(|v| v.get("program")).and_then(|v| v.as_str()) let program = candidate
.raw
.as_ref()
.and_then(|v| v.get("program"))
.and_then(|v| v.as_str())
.ok_or_else(|| anyhow!("DESI Candidate 缺 program"))?; .ok_or_else(|| anyhow!("DESI Candidate 缺 program"))?;
let healpix = candidate.raw.as_ref().and_then(|v| v.get("healpix")).and_then(|v| v.as_i64()) let healpix = candidate
.raw
.as_ref()
.and_then(|v| v.get("healpix"))
.and_then(|v| v.as_i64())
.ok_or_else(|| anyhow!("DESI Candidate 缺 healpix"))?; .ok_or_else(|| anyhow!("DESI Candidate 缺 healpix"))?;
let rel = parse_desi_release(release)?; let rel = parse_desi_release(release)?;
let product = ProductSpec::with_subtype(ProductType::Spectrum, "coadd"); let product = ProductSpec::with_subtype(ProductType::Spectrum, "coadd");
@ -175,12 +234,21 @@ impl ObservationFetcher for DesiSpectrumFetcher {
// 1) 缓存命中 // 1) 缓存命中
if !force { if !force {
if let Some((artifacts, meta)) = fetch_observation_cache(&state.db, Source::Desi, &product, &cache_key).await? { if let Some((artifacts, meta)) =
if cached_files_total_size(&state.config.library_dir, &artifacts).is_some() { fetch_observation_cache(&state.db, Source::Desi, &product, &cache_key).await?
{
if cached_files_total_size(&state.config.library_dir, &artifacts)
.await
.is_some()
{
info!("[DESI] coadd 缓存命中 ({})", cache_key); info!("[DESI] coadd 缓存命中 ({})", cache_key);
return Ok(ObservationProduct { return Ok(ObservationProduct {
source: Source::Desi, product, source: Source::Desi,
source_id: cache_key, source_label, artifacts, source_meta: meta, product,
source_id: cache_key,
source_label,
artifacts,
source_meta: meta,
}); });
} }
warn!("[DESI] 缓存记录存在但文件缺失,重新下载 ({})", cache_key); warn!("[DESI] 缓存记录存在但文件缺失,重新下载 ({})", cache_key);
@ -189,34 +257,57 @@ impl ObservationFetcher for DesiSpectrumFetcher {
// 2) 下载 coadd FITS无压缩 // 2) 下载 coadd FITS无压缩
tokio::time::sleep(Duration::from_millis(50)).await; tokio::time::sleep(Duration::from_millis(50)).await;
let fits_bytes = state.desi.download_coadd(survey, program, healpix, rel).await?; let fits_bytes = state
.desi
.download_coadd(survey, program, healpix, rel)
.await?;
let rel_path = format!( let rel_path = format!(
"Telescope/desi/{dr}/{survey}-{program}/{healpix}.fits", "Telescope/desi/{dr}/{survey}-{program}/{healpix}.fits",
dr = dr, survey = survey, program = program, healpix = healpix dr = dr,
survey = survey,
program = program,
healpix = healpix
);
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes).await?;
info!(
"[DESI] coadd 已保存 {} → {} ({}B)",
cache_key, rel_path, size
); );
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes)?;
info!("[DESI] coadd 已保存 {} → {} ({}B)", cache_key, rel_path, size);
let meta = serde_json::json!({"survey": survey, "program": program, "healpix": healpix, "dr": dr}); let meta =
serde_json::json!({"survey": survey, "program": program, "healpix": healpix, "dr": dr});
let artifact = Artifact { let artifact = Artifact {
band: None, original_name: None, band: None,
original_name: None,
file_path: rel_path.clone(), file_path: rel_path.clone(),
file_url: file_url_from_path(&rel_path), file_url: file_url_from_path(&rel_path),
file_format: "fits".to_string(), file_format: "fits".to_string(),
size_bytes: size, cached: false, size_bytes: size,
cached: false,
}; };
let meta_str = meta.to_string(); let meta_str = meta.to_string();
if let Err(e) = write_observation_cache( if let Err(e) = write_observation_cache(
&state.db, Source::Desi, &product, &cache_key, None, None, &state.db,
std::slice::from_ref(&artifact), Some(&meta_str), Source::Desi,
).await { &product,
&cache_key,
None,
None,
std::slice::from_ref(&artifact),
Some(&meta_str),
)
.await
{
log_write_failure(Source::Desi.as_str(), "spectrum", e); log_write_failure(Source::Desi.as_str(), "spectrum", e);
} }
Ok(ObservationProduct { Ok(ObservationProduct {
source: Source::Desi, product, source: Source::Desi,
source_id: cache_key, source_label, product,
artifacts: vec![artifact], source_meta: Some(meta), source_id: cache_key,
source_label,
artifacts: vec![artifact],
source_meta: Some(meta),
}) })
} }
} }
@ -226,7 +317,9 @@ fn row_to_candidate(row: &DesiSpectrumRow) -> Candidate {
source: Source::Desi, source: Source::Desi,
source_id: format!("{}-{}-{}", row.survey, row.program, row.healpix), source_id: format!("{}-{}-{}", row.survey, row.program, row.healpix),
label: format!("{} {} healpix {}", row.survey, row.program, row.healpix), label: format!("{} {} healpix {}", row.survey, row.program, row.healpix),
ra: row.ra, dec: row.dec, distance: None, ra: row.ra,
dec: row.dec,
distance: None,
raw: Some(serde_json::json!({ raw: Some(serde_json::json!({
"survey": row.survey, "program": row.program, "healpix": row.healpix "survey": row.survey, "program": row.program, "healpix": row.healpix
})), })),

View File

@ -30,6 +30,7 @@ pub async fn search_observation(
dec: f64, dec: f64,
radius_deg: f64, radius_deg: f64,
release: Option<&str>, release: Option<&str>,
version: Option<&str>,
) -> Result<Vec<Candidate>> { ) -> Result<Vec<Candidate>> {
let subtype = product.subtype.as_deref(); let subtype = product.subtype.as_deref();
let fetcher = registry.get(source, product.product, subtype)?; let fetcher = registry.get(source, product.product, subtype)?;
@ -38,7 +39,7 @@ pub async fn search_observation(
source, product.product, ra, dec, radius_deg source, product.product, ra, dec, radius_deg
); );
let matched = fetcher let matched = fetcher
.cone_search_cached(state, ra, dec, radius_deg, release, subtype) .cone_search_cached(state, ra, dec, radius_deg, release, subtype, version)
.await?; .await?;
Ok(matched) Ok(matched)
} }
@ -54,19 +55,48 @@ pub async fn download_observation(
) -> Result<ObservationBatch> { ) -> Result<ObservationBatch> {
match request { match request {
ObservationRequest::ByCoordinates { ObservationRequest::ByCoordinates {
source, product, ra, dec, radius_deg, strategy, release, source,
product,
ra,
dec,
radius_deg,
strategy,
release,
version,
} => { } => {
download_by_coordinates( download_by_coordinates(
state, registry, *source, product, *ra, *dec, *radius_deg, state,
*strategy, release.as_deref(), force, registry,
).await *source,
product,
*ra,
*dec,
*radius_deg,
*strategy,
release.as_deref(),
version.as_deref(),
force,
)
.await
} }
ObservationRequest::ByIdentifier { ObservationRequest::ByIdentifier {
source, product, identifiers, release, source,
product,
identifiers,
release,
version,
} => { } => {
download_by_identifiers( download_by_identifiers(
state, registry, *source, product, identifiers, release.as_deref(), force, state,
).await registry,
*source,
product,
identifiers,
release.as_deref(),
version.as_deref(),
force,
)
.await
} }
} }
} }
@ -77,9 +107,12 @@ async fn download_by_coordinates(
registry: &ObservationRegistry, registry: &ObservationRegistry,
source: crate::services::observation::types::Source, source: crate::services::observation::types::Source,
product: &ProductSpec, product: &ProductSpec,
ra: f64, dec: f64, radius_deg: f64, ra: f64,
dec: f64,
radius_deg: f64,
strategy: FindStrategy, strategy: FindStrategy,
release: Option<&str>, release: Option<&str>,
version: Option<&str>,
force: bool, force: bool,
) -> Result<ObservationBatch> { ) -> Result<ObservationBatch> {
let subtype = product.subtype.as_deref(); let subtype = product.subtype.as_deref();
@ -89,14 +122,21 @@ async fn download_by_coordinates(
source, product.product, ra, dec, radius_deg, strategy source, product.product, ra, dec, radius_deg, strategy
); );
let matched = fetcher.cone_search_cached(state, ra, dec, radius_deg, release, subtype).await?; let matched = fetcher
.cone_search_cached(state, ra, dec, radius_deg, release, subtype, version)
.await?;
let matched_count = matched.len(); let matched_count = matched.len();
if matched_count == 0 { if matched_count == 0 {
return Ok(ObservationBatch { return Ok(ObservationBatch {
source, product: product.clone(), source,
ra: Some(ra), dec: Some(dec), radius_deg: Some(radius_deg), product: product.clone(),
matched_count: 0, products: Vec::new(), failures: Vec::new(), ra: Some(ra),
dec: Some(dec),
radius_deg: Some(radius_deg),
matched_count: 0,
products: Vec::new(),
failures: Vec::new(),
}); });
} }
@ -109,7 +149,10 @@ async fn download_by_coordinates(
let mut failures = Vec::new(); let mut failures = Vec::new();
for idx in targets { for idx in targets {
let candidate = &matched[idx]; let candidate = &matched[idx];
match fetcher.fetch(state, candidate, release, subtype, force).await { match fetcher
.fetch(state, candidate, release, subtype, version, force)
.await
{
Ok(p) => products.push(p), Ok(p) => products.push(p),
Err(e) => { Err(e) => {
let label = candidate.label.clone(); let label = candidate.label.clone();
@ -123,9 +166,14 @@ async fn download_by_coordinates(
} }
Ok(ObservationBatch { Ok(ObservationBatch {
source, product: product.clone(), source,
ra: Some(ra), dec: Some(dec), radius_deg: Some(radius_deg), product: product.clone(),
matched_count, products, failures, ra: Some(ra),
dec: Some(dec),
radius_deg: Some(radius_deg),
matched_count,
products,
failures,
}) })
} }
@ -137,20 +185,34 @@ async fn download_by_identifiers(
product: &ProductSpec, product: &ProductSpec,
identifiers: &[String], identifiers: &[String],
release: Option<&str>, release: Option<&str>,
version: Option<&str>,
force: bool, force: bool,
) -> Result<ObservationBatch> { ) -> Result<ObservationBatch> {
let subtype = product.subtype.as_deref(); let subtype = product.subtype.as_deref();
let fetcher = registry.get(source, product.product, subtype)?; let fetcher = registry.get(source, product.product, subtype)?;
if identifiers.is_empty() {
return Err(anyhow::anyhow!(
"标识符列表不能为空source={:?} product={:?}",
source,
product.product
));
}
info!( info!(
"[Observation] by_id source={:?} product={:?} count={} release={:?}", "[Observation] by_id source={:?} product={:?} count={} release={:?}",
source, product.product, identifiers.len(), release source,
product.product,
identifiers.len(),
release
); );
let mut products = Vec::with_capacity(identifiers.len()); let mut products = Vec::with_capacity(identifiers.len());
let mut failures = Vec::new(); let mut failures = Vec::new();
for id in identifiers { for id in identifiers {
// 先把标识符解析为 Candidate验证格式 + 提取字段) // 先把标识符解析为 Candidate验证格式 + 提取字段)
let candidate = match fetcher.resolve_identifier(id, release, subtype).await { let candidate = match fetcher
.resolve_identifier(id, release, subtype, version)
.await
{
Ok(c) => c, Ok(c) => c,
Err(e) => { Err(e) => {
warn!("[Observation] 标识符解析失败 {}: {}", id, e); warn!("[Observation] 标识符解析失败 {}: {}", id, e);
@ -161,7 +223,10 @@ async fn download_by_identifiers(
continue; continue;
} }
}; };
match fetcher.fetch(state, &candidate, release, subtype, force).await { match fetcher
.fetch(state, &candidate, release, subtype, version, force)
.await
{
Ok(p) => products.push(p), Ok(p) => products.push(p),
Err(e) => { Err(e) => {
warn!("[Observation] 下载失败 {}: {}", id, e); warn!("[Observation] 下载失败 {}: {}", id, e);
@ -174,10 +239,14 @@ async fn download_by_identifiers(
} }
Ok(ObservationBatch { Ok(ObservationBatch {
source, product: product.clone(), source,
ra: None, dec: None, radius_deg: None, product: product.clone(),
ra: None,
dec: None,
radius_deg: None,
matched_count: products.len() + failures.len(), matched_count: products.len() + failures.len(),
products, failures, products,
failures,
}) })
} }
@ -189,10 +258,15 @@ pub async fn download_one(
product: ProductSpec, product: ProductSpec,
identifier: &str, identifier: &str,
release: Option<&str>, release: Option<&str>,
version: Option<&str>,
force: bool, force: bool,
) -> Result<ObservationProduct> { ) -> Result<ObservationProduct> {
let subtype = product.subtype.as_deref(); let subtype = product.subtype.as_deref();
let fetcher = registry.get(source, product.product, subtype)?; let fetcher = registry.get(source, product.product, subtype)?;
let candidate = fetcher.resolve_identifier(identifier, release, subtype).await?; let candidate = fetcher
fetcher.fetch(state, &candidate, release, subtype, force).await .resolve_identifier(identifier, release, subtype, version)
.await?;
fetcher
.fetch(state, &candidate, release, subtype, version, force)
.await
} }

View File

@ -76,6 +76,33 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
None None
} }
/// 该 release 支持的子版本列表(仅 LAMOST 有意义),返回空表示不区分子版本。
/// `release` 为 None 时返回默认 release 的子版本列表。
fn versions_for_release(&self, _release: Option<&str>) -> Vec<String> {
Vec::new()
}
/// 该 release 的默认子版本None 表示无子版本概念或用默认 release
fn default_version_for_release(&self, _release: Option<&str>) -> Option<String> {
None
}
/// 需要登录认证才能访问的 release 列表(如 LAMOST Internal DR空表示全部公开。
///
/// 前端据此对这些 release 做禁用/灰显处理。
fn releases_requiring_auth(&self) -> Vec<String> {
Vec::new()
}
/// 每个 release 实际支持的 subtype 列表(用于表达 release×subtype 交叉约束)。
///
/// 默认返回空 HashMap表示所有 release 都支持 `subtypes()` 的全部值。
/// 仅当某些 release 不支持某些 subtype 时才需覆写(如 LAMOST DR5/6 不支持 MRS
/// key=release 字符串value=该 release 支持的 subtype 列表。
fn subtypes_per_release(&self) -> std::collections::HashMap<String, Vec<String>> {
std::collections::HashMap::new()
}
/// 坐标模式检索半径的建议上限(度),用于前端表单提示与服务端警告 /// 坐标模式检索半径的建议上限(度),用于前端表单提示与服务端警告
/// ///
/// 各源主表规模差异极大Gaia/SDSS/DESI 主表上亿行,大范围查询易超时), /// 各源主表规模差异极大Gaia/SDSS/DESI 主表上亿行,大范围查询易超时),
@ -109,6 +136,8 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
// ── 检索(各源实现) ── // ── 检索(各源实现) ──
/// 纯 cone 检索(不查缓存)—— 各源实现自己的端点调用与行映射 /// 纯 cone 检索(不查缓存)—— 各源实现自己的端点调用与行映射
///
/// `version` 为该 release 的子版本(仅 LAMOST 有意义,其他源可忽略)。
async fn cone_search_raw( async fn cone_search_raw(
&self, &self,
state: &AppState, state: &AppState,
@ -117,6 +146,7 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
radius_deg: f64, radius_deg: f64,
release: Option<&str>, release: Option<&str>,
subtype: Option<&str>, subtype: Option<&str>,
version: Option<&str>,
) -> Result<Vec<Candidate>>; ) -> Result<Vec<Candidate>>;
/// 标识符模式:验证并归一化为 Candidate不发起下载 /// 标识符模式:验证并归一化为 Candidate不发起下载
@ -125,6 +155,7 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
identifier: &str, identifier: &str,
release: Option<&str>, release: Option<&str>,
subtype: Option<&str>, subtype: Option<&str>,
version: Option<&str>,
) -> Result<Candidate>; ) -> Result<Candidate>;
// ── 下载(各源实现) ── // ── 下载(各源实现) ──
@ -136,6 +167,7 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
candidate: &Candidate, candidate: &Candidate,
release: Option<&str>, release: Option<&str>,
subtype: Option<&str>, subtype: Option<&str>,
version: Option<&str>,
force: bool, force: bool,
) -> Result<ObservationProduct>; ) -> Result<ObservationProduct>;
@ -159,8 +191,9 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
radius_deg: f64, radius_deg: f64,
release: Option<&str>, release: Option<&str>,
subtype: Option<&str>, subtype: Option<&str>,
version: Option<&str>,
) -> Result<Vec<Candidate>> { ) -> Result<Vec<Candidate>> {
let query_hash = self.cone_cache_hash(ra, dec, radius_deg, release, subtype); let query_hash = self.cone_cache_hash(ra, dec, radius_deg, release, subtype, version);
if let Some(cached) = fetch_cone_cache(&state.db, &query_hash).await? { if let Some(cached) = fetch_cone_cache(&state.db, &query_hash).await? {
tracing::info!( tracing::info!(
@ -174,10 +207,12 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
tokio::time::sleep(std::time::Duration::from_millis(50)).await; tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let result = self let result = self
.cone_search_raw(state, ra, dec, radius_deg, release, subtype) .cone_search_raw(state, ra, dec, radius_deg, release, subtype, version)
.await?; .await?;
if let Err(e) = write_cone_cache(&state.db, &query_hash, &self.cone_cache_prefix(), &result).await { if let Err(e) =
write_cone_cache(&state.db, &query_hash, &self.cone_cache_prefix(), &result).await
{
error!( error!(
"[Observation] 写入 cone 缓存失败 ({:?}/{:?}): {}", "[Observation] 写入 cone 缓存失败 ({:?}/{:?}): {}",
self.key().0, self.key().0,
@ -188,7 +223,7 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
Ok(result) Ok(result)
} }
/// 计算 cone 缓存的 sha1 hashprefix + 坐标 + release + subtype /// 计算 cone 缓存的 sha1 hashprefix + 坐标 + release + subtype + version
fn cone_cache_hash( fn cone_cache_hash(
&self, &self,
ra: f64, ra: f64,
@ -196,18 +231,42 @@ pub trait ObservationFetcher: Send + Sync + std::fmt::Debug {
radius_deg: f64, radius_deg: f64,
release: Option<&str>, release: Option<&str>,
subtype: Option<&str>, subtype: Option<&str>,
version: Option<&str>,
) -> String { ) -> String {
// 各字段加长度前缀 + 字段标记字节,避免不同组合拼接后产生相同字节流。
// 例如release="dr1"+subtype=None 与 release="dr"+subtype=Some("1")
// 不加界定会哈希到相同字节序列,导致缓存错误命中。
let mut h = Sha1::new(); let mut h = Sha1::new();
h.update(self.cone_cache_prefix().as_bytes()); let prefix = self.cone_cache_prefix();
h.update((prefix.len() as u64).to_le_bytes());
h.update(prefix.as_bytes());
h.update(ra.to_le_bytes()); h.update(ra.to_le_bytes());
h.update(dec.to_le_bytes()); h.update(dec.to_le_bytes());
h.update(radius_deg.to_le_bytes()); h.update(radius_deg.to_le_bytes());
if let Some(r) = release { match release {
Some(r) => {
h.update([1u8]); // presence marker
h.update((r.len() as u64).to_le_bytes());
h.update(r.as_bytes()); h.update(r.as_bytes());
} }
if let Some(st) = subtype { None => h.update([0u8]),
}
match subtype {
Some(st) => {
h.update([1u8]); // presence marker
h.update((st.len() as u64).to_le_bytes());
h.update(st.as_bytes()); h.update(st.as_bytes());
} }
None => h.update([0u8]),
}
match version {
Some(v) => {
h.update([1u8]); // presence marker
h.update((v.len() as u64).to_le_bytes());
h.update(v.as_bytes());
}
None => h.update([0u8]),
}
format!("{:x}", h.finalize()) format!("{:x}", h.finalize())
} }
} }
@ -224,10 +283,7 @@ struct ConeCacheRow {
expires_at: Option<chrono::DateTime<chrono::Utc>>, expires_at: Option<chrono::DateTime<chrono::Utc>>,
} }
async fn fetch_cone_cache( async fn fetch_cone_cache(pool: &SqlitePool, query_hash: &str) -> Result<Option<Vec<Candidate>>> {
pool: &SqlitePool,
query_hash: &str,
) -> Result<Option<Vec<Candidate>>> {
let row = sqlx::query_as::<_, ConeCacheRow>( let row = sqlx::query_as::<_, ConeCacheRow>(
"SELECT result_json, expires_at FROM vizier_query_cache WHERE query_hash = ? LIMIT 1", "SELECT result_json, expires_at FROM vizier_query_cache WHERE query_hash = ? LIMIT 1",
) )
@ -335,12 +391,22 @@ mod tests {
fn test_pick_nearest_without_distance_uses_coords() { fn test_pick_nearest_without_distance_uses_coords() {
let rows = vec![ let rows = vec![
Candidate { Candidate {
source: Source::Lamost, source_id: String::new(), label: String::new(), source: Source::Lamost,
ra: Some(10.5), dec: Some(41.0), distance: None, raw: None, source_id: String::new(),
label: String::new(),
ra: Some(10.5),
dec: Some(41.0),
distance: None,
raw: None,
}, },
Candidate { Candidate {
source: Source::Lamost, source_id: String::new(), label: String::new(), source: Source::Lamost,
ra: Some(10.51), dec: Some(41.01), distance: None, raw: None, source_id: String::new(),
label: String::new(),
ra: Some(10.51),
dec: Some(41.01),
distance: None,
raw: None,
}, },
]; ];
assert_eq!(pick_nearest(&rows, 10.5, 41.0), 0); assert_eq!(pick_nearest(&rows, 10.5, 41.0), 0);

View File

@ -28,21 +28,34 @@ use tracing::{info, warn};
// ── 版本与解析辅助 ── // ── 版本与解析辅助 ──
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum GaiaRelease { #[default] Dr3 } pub enum GaiaRelease {
#[default]
Dr3,
}
impl GaiaRelease { impl GaiaRelease {
pub fn valid_values() -> &'static [&'static str] { pub fn valid_values() -> &'static [&'static str] {
&["dr3"] &["dr3"]
} }
pub fn dr_str(&self) -> &'static str { "dr3" } pub fn dr_str(&self) -> &'static str {
pub fn tap_table(&self) -> &'static str { "gaiadr3.gaia_source" } "dr3"
pub fn display(&self) -> &'static str { "DR3" } }
pub fn tap_table(&self) -> &'static str {
"gaiadr3.gaia_source"
}
pub fn display(&self) -> &'static str {
"DR3"
}
} }
pub fn parse_gaia_release(s: Option<&str>) -> Result<GaiaRelease> { pub fn parse_gaia_release(s: Option<&str>) -> Result<GaiaRelease> {
match s { match s {
None | Some("dr3") => Ok(GaiaRelease::Dr3), None | Some("dr3") => Ok(GaiaRelease::Dr3),
Some(other) => Err(anyhow!("不支持的 Gaia release '{}',可选: {:?}", other, GaiaRelease::valid_values())), Some(other) => Err(anyhow!(
"不支持的 Gaia release '{}',可选: {:?}",
other,
GaiaRelease::valid_values()
)),
} }
} }
@ -53,7 +66,11 @@ pub fn parse_gaia_retrieval_type_str(s: Option<&str>) -> Result<GaiaRetrievalTyp
Some("XP_SAMPLED") => Ok(GaiaRetrievalType::XpSampled), Some("XP_SAMPLED") => Ok(GaiaRetrievalType::XpSampled),
Some("EPOCH_PHOTOMETRY") => Ok(GaiaRetrievalType::EpochPhotometry), Some("EPOCH_PHOTOMETRY") => Ok(GaiaRetrievalType::EpochPhotometry),
Some("RVS") => Ok(GaiaRetrievalType::Rvs), Some("RVS") => Ok(GaiaRetrievalType::Rvs),
Some(other) => Err(anyhow!("不支持的 Gaia subtype '{}',可选: {:?}", other, GaiaRetrievalType::valid_values())), Some(other) => Err(anyhow!(
"不支持的 Gaia subtype '{}',可选: {:?}",
other,
GaiaRetrievalType::valid_values()
)),
} }
} }
@ -64,7 +81,13 @@ pub fn parse_gaia_retrieval_type(s: &str) -> Result<GaiaRetrievalType> {
"XP_SAMPLED" => XpSampled, "XP_SAMPLED" => XpSampled,
"EPOCH_PHOTOMETRY" => EpochPhotometry, "EPOCH_PHOTOMETRY" => EpochPhotometry,
"RVS" => Rvs, "RVS" => Rvs,
other => return Err(anyhow!("不支持的 Gaia retrieval_type '{}',可选: {:?}", other, GaiaRetrievalType::valid_values())), other => {
return Err(anyhow!(
"不支持的 Gaia retrieval_type '{}',可选: {:?}",
other,
GaiaRetrievalType::valid_values()
))
}
}) })
} }
@ -120,29 +143,52 @@ async fn download_gaia_product(
// 1) 缓存命中 // 1) 缓存命中
if !force { if !force {
if let Some((artifacts, meta)) = fetch_observation_cache(&state.db, Source::Gaia, &product, &cache_key).await? { if let Some((artifacts, meta)) =
if cached_files_total_size(&state.config.library_dir, &artifacts).is_some() { fetch_observation_cache(&state.db, Source::Gaia, &product, &cache_key).await?
info!("[Gaia] 缓存命中 (source_id={}, type={})", source_id, rt_param); {
let artifacts = artifacts.into_iter().map(|mut a| { if cached_files_total_size(&state.config.library_dir, &artifacts)
.await
.is_some()
{
info!(
"[Gaia] 缓存命中 (source_id={}, type={})",
source_id, rt_param
);
let artifacts = artifacts
.into_iter()
.map(|mut a| {
if a.size_bytes == 0 { if a.size_bytes == 0 {
if let Ok(m) = std::fs::metadata(state.config.library_dir.join(&a.file_path)) { if let Ok(m) =
std::fs::metadata(state.config.library_dir.join(&a.file_path))
{
a.size_bytes = m.len() as usize; a.size_bytes = m.len() as usize;
} }
} }
a a
}).collect(); })
.collect();
return Ok(ObservationProduct { return Ok(ObservationProduct {
source: Source::Gaia, product, source: Source::Gaia,
source_id: cache_key, source_label, artifacts, source_meta: meta, product,
source_id: cache_key,
source_label,
artifacts,
source_meta: meta,
}); });
} }
warn!("[Gaia] 缓存记录存在但文件缺失,重新下载 (source_id={})", source_id); warn!(
"[Gaia] 缓存记录存在但文件缺失,重新下载 (source_id={})",
source_id
);
} }
} }
// 2) DataLink 下载 ZIP + 解包取**全部**文件(修复 EPOCH_PHOTOMETRY 多文件丢失) // 2) DataLink 下载 ZIP + 解包取**全部**文件(修复 EPOCH_PHOTOMETRY 多文件丢失)
tokio::time::sleep(Duration::from_millis(50)).await; tokio::time::sleep(Duration::from_millis(50)).await;
let zip_bytes = state.gaia.download_products(&[source_id.to_string()], retrieval_type, "fits").await?; let zip_bytes = state
.gaia
.download_products(&[source_id.to_string()], retrieval_type, "fits")
.await?;
let extracted = extract_all_zip_entries(&zip_bytes, source_id)?; let extracted = extract_all_zip_entries(&zip_bytes, source_id)?;
// 3) 落盘 // 3) 落盘
@ -152,13 +198,21 @@ async fn download_gaia_product(
for ext_file in extracted { for ext_file in extracted {
let rel_path = if product_type == ProductType::LightCurve { let rel_path = if product_type == ProductType::LightCurve {
let safe_name = ext_file.name.replace('/', "_"); let safe_name = ext_file.name.replace('/', "_");
format!("Telescope/gaia/lightcurve/{rt}/{sid}/{name}", format!(
rt = rt_lower, sid = source_id, name = safe_name) "Telescope/gaia/lightcurve/{rt}/{sid}/{name}",
rt = rt_lower,
sid = source_id,
name = safe_name
)
} else { } else {
format!("Telescope/gaia/spectrum/{rt}/{sid}.{ext}", format!(
rt = rt_lower, sid = source_id, ext = ext_file.ext) "Telescope/gaia/spectrum/{rt}/{sid}.{ext}",
rt = rt_lower,
sid = source_id,
ext = ext_file.ext
)
}; };
let size = persist_bytes(&state.config.library_dir, &rel_path, &ext_file.bytes)?; let size = persist_bytes(&state.config.library_dir, &rel_path, &ext_file.bytes).await?;
info!("[Gaia] 已保存 {} ({}B) → {}", ext_file.name, size, rel_path); info!("[Gaia] 已保存 {} ({}B) → {}", ext_file.name, size, rel_path);
artifacts.push(Artifact { artifacts.push(Artifact {
band: parse_band_from_name(&ext_file.name), band: parse_band_from_name(&ext_file.name),
@ -174,22 +228,37 @@ async fn download_gaia_product(
a.file_url = file_url_from_path(&a.file_path); a.file_url = file_url_from_path(&a.file_path);
} }
if artifacts.is_empty() { if artifacts.is_empty() {
return Err(anyhow!("Gaia DataLink ZIP 内无可下载文件条目 (source_id={}, type={})", source_id, rt_param)); return Err(anyhow!(
"Gaia DataLink ZIP 内无可下载文件条目 (source_id={}, type={})",
source_id,
rt_param
));
} }
let meta = serde_json::json!({"source_id": source_id, "retrieval_type": rt_param, "dr": dr}); let meta = serde_json::json!({"source_id": source_id, "retrieval_type": rt_param, "dr": dr});
let meta_str = meta.to_string(); let meta_str = meta.to_string();
if let Err(e) = write_observation_cache( if let Err(e) = write_observation_cache(
&state.db, Source::Gaia, &product, &cache_key, None, None, &state.db,
&artifacts, Some(&meta_str), Source::Gaia,
).await { &product,
&cache_key,
None,
None,
&artifacts,
Some(&meta_str),
)
.await
{
log_write_failure(Source::Gaia.as_str(), product.product.as_str(), e); log_write_failure(Source::Gaia.as_str(), product.product.as_str(), e);
} }
Ok(ObservationProduct { Ok(ObservationProduct {
source: Source::Gaia, product, source: Source::Gaia,
source_id: cache_key, source_label, product,
artifacts, source_meta: Some(meta), source_id: cache_key,
source_label,
artifacts,
source_meta: Some(meta),
}) })
} }
@ -208,30 +277,48 @@ struct ExtractedFile {
/// 旧实现 `extract_first_zip_entry` 只取首个文件,丢失了 BP/RP 波段——本函数修复之。 /// 旧实现 `extract_first_zip_entry` 只取首个文件,丢失了 BP/RP 波段——本函数修复之。
fn extract_all_zip_entries(zip_bytes: &[u8], source_id: &str) -> Result<Vec<ExtractedFile>> { fn extract_all_zip_entries(zip_bytes: &[u8], source_id: &str) -> Result<Vec<ExtractedFile>> {
let cursor = Cursor::new(zip_bytes); let cursor = Cursor::new(zip_bytes);
let mut archive = zip::ZipArchive::new(cursor).map_err(|e| anyhow!("Gaia ZIP 解析失败: {}", e))?; let mut archive =
zip::ZipArchive::new(cursor).map_err(|e| anyhow!("Gaia ZIP 解析失败: {}", e))?;
let mut out = Vec::new(); let mut out = Vec::new();
for i in 0..archive.len() { for i in 0..archive.len() {
let mut file = archive.by_index(i) let mut file = archive
.by_index(i)
.map_err(|e| anyhow!("读取 Gaia ZIP 条目 {} 失败: {}", i, e))?; .map_err(|e| anyhow!("读取 Gaia ZIP 条目 {} 失败: {}", i, e))?;
if file.is_dir() { continue; } if file.is_dir() {
continue;
}
let name = file.name().to_string(); let name = file.name().to_string();
let mut buf = Vec::new(); let mut buf = Vec::new();
file.read_to_end(&mut buf) file.read_to_end(&mut buf)
.map_err(|e| anyhow!("解压 Gaia ZIP 条目失败: {}", e))?; .map_err(|e| anyhow!("解压 Gaia ZIP 条目失败: {}", e))?;
if buf.is_empty() { continue; } if buf.is_empty() {
continue;
}
let lower = name.to_lowercase(); let lower = name.to_lowercase();
let ext = if lower.ends_with(".fits") { "fits" } let ext = if lower.ends_with(".fits") {
else if lower.ends_with(".vot") || lower.ends_with(".xml") { "vot" } "fits"
else if lower.ends_with(".csv") { "csv" } } else if lower.ends_with(".vot") || lower.ends_with(".xml") {
else { "fits" }; "vot"
} else if lower.ends_with(".csv") {
"csv"
} else {
"fits"
};
out.push(ExtractedFile { name, ext, bytes: buf }); out.push(ExtractedFile {
name,
ext,
bytes: buf,
});
} }
if out.is_empty() { if out.is_empty() {
return Err(anyhow!("Gaia ZIP 内无可下载文件条目 (source_id={})", source_id)); return Err(anyhow!(
"Gaia ZIP 内无可下载文件条目 (source_id={})",
source_id
));
} }
Ok(out) Ok(out)
} }
@ -284,12 +371,19 @@ impl ObservationFetcher for GaiaSpectrumFetcher {
async fn cone_search_raw( async fn cone_search_raw(
&self, &self,
state: &AppState, state: &AppState,
ra: f64, dec: f64, radius_deg: f64, ra: f64,
_release: Option<&str>, _subtype: Option<&str>, dec: f64,
radius_deg: f64,
_release: Option<&str>,
_subtype: Option<&str>,
_version: Option<&str>,
) -> Result<Vec<Candidate>> { ) -> Result<Vec<Candidate>> {
validate_cone_params(ra, dec, radius_deg)?; validate_cone_params(ra, dec, radius_deg)?;
let rel = parse_gaia_release(_release)?; let rel = parse_gaia_release(_release)?;
let result = state.gaia.cone_search(ra, dec, radius_deg, 50, true, rel).await?; let result = state
.gaia
.cone_search(ra, dec, radius_deg, 50, true, rel)
.await?;
Ok(result.rows.iter().map(row_to_candidate).collect()) Ok(result.rows.iter().map(row_to_candidate).collect())
} }
@ -298,6 +392,7 @@ impl ObservationFetcher for GaiaSpectrumFetcher {
identifier: &str, identifier: &str,
_release: Option<&str>, _release: Option<&str>,
subtype: Option<&str>, subtype: Option<&str>,
_version: Option<&str>,
) -> Result<Candidate> { ) -> Result<Candidate> {
let (rt_param, gaia_sid) = match identifier.split_once('|') { let (rt_param, gaia_sid) = match identifier.split_once('|') {
Some((rt, s)) => (rt, s), Some((rt, s)) => (rt, s),
@ -315,7 +410,9 @@ impl ObservationFetcher for GaiaSpectrumFetcher {
source: Source::Gaia, source: Source::Gaia,
source_id: gaia_sid.to_string(), source_id: gaia_sid.to_string(),
label: format!("Gaia {}", gaia_sid), label: format!("Gaia {}", gaia_sid),
ra: None, dec: None, distance: None, ra: None,
dec: None,
distance: None,
raw: Some(serde_json::json!({"source_id": gaia_sid, "retrieval_type": rt.as_param()})), raw: Some(serde_json::json!({"source_id": gaia_sid, "retrieval_type": rt.as_param()})),
}) })
} }
@ -326,13 +423,18 @@ impl ObservationFetcher for GaiaSpectrumFetcher {
candidate: &Candidate, candidate: &Candidate,
_release: Option<&str>, _release: Option<&str>,
subtype: Option<&str>, subtype: Option<&str>,
_version: Option<&str>,
force: bool, force: bool,
) -> Result<ObservationProduct> { ) -> Result<ObservationProduct> {
let source_id = candidate.raw.as_ref() let source_id = candidate
.raw
.as_ref()
.and_then(|v| v.get("source_id")) .and_then(|v| v.get("source_id"))
.and_then(|v| v.as_str()) .and_then(|v| v.as_str())
.unwrap_or(&candidate.source_id); .unwrap_or(&candidate.source_id);
let rt = if let Some(rt_str) = candidate.raw.as_ref() let rt = if let Some(rt_str) = candidate
.raw
.as_ref()
.and_then(|v| v.get("retrieval_type")) .and_then(|v| v.get("retrieval_type"))
.and_then(|v| v.as_str()) .and_then(|v| v.as_str())
{ {
@ -381,13 +483,20 @@ impl ObservationFetcher for GaiaLightCurveFetcher {
async fn cone_search_raw( async fn cone_search_raw(
&self, &self,
state: &AppState, state: &AppState,
ra: f64, dec: f64, radius_deg: f64, ra: f64,
_release: Option<&str>, _subtype: Option<&str>, dec: f64,
radius_deg: f64,
_release: Option<&str>,
_subtype: Option<&str>,
_version: Option<&str>,
) -> Result<Vec<Candidate>> { ) -> Result<Vec<Candidate>> {
// 光变 cone 复用 Gaia 主表检索(不强制 xp_only因为光变对所有源都有 // 光变 cone 复用 Gaia 主表检索(不强制 xp_only因为光变对所有源都有
validate_cone_params(ra, dec, radius_deg)?; validate_cone_params(ra, dec, radius_deg)?;
let rel = parse_gaia_release(_release)?; let rel = parse_gaia_release(_release)?;
let result = state.gaia.cone_search(ra, dec, radius_deg, 50, false, rel).await?; let result = state
.gaia
.cone_search(ra, dec, radius_deg, 50, false, rel)
.await?;
Ok(result.rows.iter().map(row_to_candidate).collect()) Ok(result.rows.iter().map(row_to_candidate).collect())
} }
@ -396,6 +505,7 @@ impl ObservationFetcher for GaiaLightCurveFetcher {
identifier: &str, identifier: &str,
_release: Option<&str>, _release: Option<&str>,
_subtype: Option<&str>, _subtype: Option<&str>,
_version: Option<&str>,
) -> Result<Candidate> { ) -> Result<Candidate> {
let sid = identifier.trim(); let sid = identifier.trim();
if sid.is_empty() { if sid.is_empty() {
@ -405,7 +515,9 @@ impl ObservationFetcher for GaiaLightCurveFetcher {
source: Source::Gaia, source: Source::Gaia,
source_id: sid.to_string(), source_id: sid.to_string(),
label: format!("Gaia {}", sid), label: format!("Gaia {}", sid),
ra: None, dec: None, distance: None, ra: None,
dec: None,
distance: None,
raw: Some(serde_json::json!({"source_id": sid, "retrieval_type": "EPOCH_PHOTOMETRY"})), raw: Some(serde_json::json!({"source_id": sid, "retrieval_type": "EPOCH_PHOTOMETRY"})),
}) })
} }
@ -416,13 +528,23 @@ impl ObservationFetcher for GaiaLightCurveFetcher {
candidate: &Candidate, candidate: &Candidate,
_release: Option<&str>, _release: Option<&str>,
_subtype: Option<&str>, _subtype: Option<&str>,
_version: Option<&str>,
force: bool, force: bool,
) -> Result<ObservationProduct> { ) -> Result<ObservationProduct> {
let source_id = candidate.raw.as_ref() let source_id = candidate
.raw
.as_ref()
.and_then(|v| v.get("source_id")) .and_then(|v| v.get("source_id"))
.and_then(|v| v.as_str()) .and_then(|v| v.as_str())
.unwrap_or(&candidate.source_id); .unwrap_or(&candidate.source_id);
download_gaia_product(state, source_id, GaiaRetrievalType::EpochPhotometry, ProductType::LightCurve, force).await download_gaia_product(
state,
source_id,
GaiaRetrievalType::EpochPhotometry,
ProductType::LightCurve,
force,
)
.await
} }
} }
@ -442,14 +564,26 @@ mod tests {
#[test] #[test]
fn test_parse_band_from_name_epoch() { fn test_parse_band_from_name_epoch() {
assert_eq!(parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_G.fits").as_deref(), Some("G")); assert_eq!(
assert_eq!(parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_BP.fits").as_deref(), Some("BP")); parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_G.fits").as_deref(),
assert_eq!(parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_RP.fits").as_deref(), Some("RP")); Some("G")
);
assert_eq!(
parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_BP.fits").as_deref(),
Some("BP")
);
assert_eq!(
parse_band_from_name("65214031805717376_EPOCH_PHOTOMETRY_RP.fits").as_deref(),
Some("RP")
);
} }
#[test] #[test]
fn test_parse_band_from_name_xp() { fn test_parse_band_from_name_xp() {
assert_eq!(parse_band_from_name("65214031805717376_XP_CONTINUOUS.fits"), None); assert_eq!(
parse_band_from_name("65214031805717376_XP_CONTINUOUS.fits"),
None
);
} }
#[test] #[test]
@ -474,17 +608,26 @@ mod tests {
{ {
let mut writer = zip::ZipWriter::new(std::io::Cursor::new(&mut buf)); let mut writer = zip::ZipWriter::new(std::io::Cursor::new(&mut buf));
let opts = zip::write::SimpleFileOptions::default(); let opts = zip::write::SimpleFileOptions::default();
writer.start_file("123_EPOCH_PHOTOMETRY_G.fits", opts).unwrap(); writer
.start_file("123_EPOCH_PHOTOMETRY_G.fits", opts)
.unwrap();
writer.write_all(b"G_DATA").unwrap(); writer.write_all(b"G_DATA").unwrap();
writer.start_file("123_EPOCH_PHOTOMETRY_BP.fits", opts).unwrap(); writer
.start_file("123_EPOCH_PHOTOMETRY_BP.fits", opts)
.unwrap();
writer.write_all(b"BP_DATA").unwrap(); writer.write_all(b"BP_DATA").unwrap();
writer.start_file("123_EPOCH_PHOTOMETRY_RP.fits", opts).unwrap(); writer
.start_file("123_EPOCH_PHOTOMETRY_RP.fits", opts)
.unwrap();
writer.write_all(b"RP_DATA").unwrap(); writer.write_all(b"RP_DATA").unwrap();
writer.finish().unwrap(); writer.finish().unwrap();
} }
let files = extract_all_zip_entries(&buf, "123").unwrap(); let files = extract_all_zip_entries(&buf, "123").unwrap();
assert_eq!(files.len(), 3, "EPOCH_PHOTOMETRY 应返回全部 3 个波段文件"); assert_eq!(files.len(), 3, "EPOCH_PHOTOMETRY 应返回全部 3 个波段文件");
let bands: Vec<_> = files.iter().map(|f| parse_band_from_name(&f.name)).collect(); let bands: Vec<_> = files
.iter()
.map(|f| parse_band_from_name(&f.name))
.collect();
assert!(bands.contains(&Some("G".into()))); assert!(bands.contains(&Some("G".into())));
assert!(bands.contains(&Some("BP".into()))); assert!(bands.contains(&Some("BP".into())));
assert!(bands.contains(&Some("RP".into()))); assert!(bands.contains(&Some("RP".into())));

View File

@ -23,55 +23,116 @@ use tracing::{info, warn};
// ── 版本与分辨率枚举(领域知识) ── // ── 版本与分辨率枚举(领域知识) ──
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum LamostRelease { pub enum LamostRelease {
Dr5, Dr6, Dr7, Dr8, Dr9, Dr10, Dr11, Dr5,
} Dr6,
Dr7,
impl Default for LamostRelease { Dr8,
fn default() -> Self { Self::Dr10 } Dr9,
#[default]
Dr10,
Dr11,
Dr12,
Dr13,
Dr14,
} }
impl LamostRelease { impl LamostRelease {
pub fn valid_values() -> &'static [&'static str] { pub fn valid_values() -> &'static [&'static str] {
&["dr5", "dr6", "dr7", "dr8", "dr9", "dr10", "dr11"] &[
"dr5", "dr6", "dr7", "dr8", "dr9", "dr10", "dr11", "dr12", "dr13", "dr14",
]
} }
pub fn path_segment(&self) -> &'static str { pub fn path_segment(&self) -> &'static str {
match self { match self {
Self::Dr5 => "dr5", Self::Dr6 => "dr6", Self::Dr7 => "dr7", Self::Dr5 => "dr5",
Self::Dr8 => "dr8", Self::Dr9 => "dr9", Self::Dr10 => "dr10", Self::Dr11 => "dr11", Self::Dr6 => "dr6",
Self::Dr7 => "dr7",
Self::Dr8 => "dr8",
Self::Dr9 => "dr9",
Self::Dr10 => "dr10",
Self::Dr11 => "dr11",
Self::Dr12 => "dr12",
Self::Dr13 => "dr13",
Self::Dr14 => "dr14",
} }
} }
pub fn version_segment(&self) -> &'static str { /// 该 DR 支持的全部子版本URL 路径段),按时间从早到晚排序
///
/// 数据来源LAMOST 官网 lmusers 页面的发布版本表格 + 端点实测。
/// v0 通常是早期/内部预览版数据量少或为空v2.0 为最终公开版。
pub fn versions(&self) -> &'static [&'static str] {
match self {
Self::Dr5 | Self::Dr6 => &["v0", "v2"],
Self::Dr7 => &["v0", "v1.2"],
Self::Dr8 | Self::Dr9 => &["v0", "v1.0", "v2.0"],
Self::Dr10 => &["v0", "v1.0", "v2.0"],
Self::Dr11 | Self::Dr12 | Self::Dr13 | Self::Dr14 => &["v0", "v1.0", "v1.1", "v2.0"],
}
}
/// 该 DR 的默认(最新公开)子版本
pub fn default_version(&self) -> &'static str {
match self { match self {
Self::Dr5 | Self::Dr6 => "v2", Self::Dr5 | Self::Dr6 => "v2",
Self::Dr7 => "v1.2", Self::Dr7 => "v1.2",
Self::Dr8 | Self::Dr9 | Self::Dr10 | Self::Dr11 => "v2.0", // v2.0 为各 DR 的最终/最新公开版
Self::Dr8
| Self::Dr9
| Self::Dr10
| Self::Dr11
| Self::Dr12
| Self::Dr13
| Self::Dr14 => "v2.0",
} }
} }
pub fn display(&self) -> &'static str { pub fn display(&self) -> &'static str {
match self { match self {
Self::Dr5 => "DR5", Self::Dr6 => "DR6", Self::Dr7 => "DR7", Self::Dr5 => "DR5",
Self::Dr8 => "DR8", Self::Dr9 => "DR9", Self::Dr10 => "DR10", Self::Dr11 => "DR11", Self::Dr6 => "DR6",
Self::Dr7 => "DR7",
Self::Dr8 => "DR8",
Self::Dr9 => "DR9",
Self::Dr10 => "DR10",
Self::Dr11 => "DR11",
Self::Dr12 => "DR12",
Self::Dr13 => "DR13",
Self::Dr14 => "DR14",
} }
} }
pub fn supports_mrs(&self) -> bool { pub fn supports_mrs(&self) -> bool {
!matches!(self, Self::Dr5 | Self::Dr6) !matches!(self, Self::Dr5 | Self::Dr6)
} }
/// 是否为公开发布(无需登录即可访问 cone search / FITS 下载)。
///
/// LAMOST 的 Internal 数据发布需经 oauth.china-vo.org 完成统一身份认证后才能访问。
/// DR12-14 当前为 Internal公开 cone search 端点会被 302 重定向到登录页。
pub fn is_public(&self) -> bool {
!matches!(self, Self::Dr12 | Self::Dr13 | Self::Dr14)
}
} }
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum LamostResolution { Lrs, Mrs } pub enum LamostResolution {
Lrs,
Mrs,
}
impl LamostResolution { impl LamostResolution {
pub fn valid_values() -> &'static [&'static str] { pub fn valid_values() -> &'static [&'static str] {
&["lrs", "mrs"] &["lrs", "mrs"]
} }
pub fn as_str(&self) -> &'static str { pub fn as_str(&self) -> &'static str {
match self { Self::Lrs => "lrs", Self::Mrs => "mrs" } match self {
Self::Lrs => "lrs",
Self::Mrs => "mrs",
}
} }
pub fn display(&self) -> &'static str { pub fn display(&self) -> &'static str {
match self { Self::Lrs => "LRS", Self::Mrs => "MRS" } match self {
Self::Lrs => "LRS",
Self::Mrs => "MRS",
}
} }
} }
@ -85,16 +146,56 @@ pub fn parse_lamost_release(s: Option<&str>) -> Result<LamostRelease> {
Some("dr9") => LamostRelease::Dr9, Some("dr9") => LamostRelease::Dr9,
Some("dr10") => LamostRelease::Dr10, Some("dr10") => LamostRelease::Dr10,
Some("dr11") => LamostRelease::Dr11, Some("dr11") => LamostRelease::Dr11,
Some(other) => return Err(anyhow!("不支持的 LAMOST release '{}',可选: {:?}", other, LamostRelease::valid_values())), Some("dr12") => LamostRelease::Dr12,
Some("dr13") => LamostRelease::Dr13,
Some("dr14") => LamostRelease::Dr14,
Some(other) => {
return Err(anyhow!(
"不支持的 LAMOST release '{}',可选: {:?}",
other,
LamostRelease::valid_values()
))
}
}) })
} }
/// 解析 LAMOST 子版本参数。None 时返回该 DR 的默认(最新公开)子版本。
pub fn parse_lamost_version(release: LamostRelease, version: Option<&str>) -> Result<String> {
let valid = release.versions();
match version.map(|x| x.trim().to_lowercase()) {
None => Ok(release.default_version().to_string()),
Some(v) => {
if valid.iter().any(|x| x.eq_ignore_ascii_case(&v)) {
// 规范化为官网写法(保留原大小写约定,如 v1.0/v2.0
Ok(valid
.iter()
.find(|x| x.eq_ignore_ascii_case(&v))
.unwrap()
.to_string())
} else {
Err(anyhow!(
"不支持的 LAMOST 子版本 '{}'{:?} 的可选版本: {:?}",
v,
release,
valid
))
}
}
}
}
pub fn parse_lamost_resolution(subtype: Option<&str>) -> Result<LamostResolution> { pub fn parse_lamost_resolution(subtype: Option<&str>) -> Result<LamostResolution> {
Ok(match subtype.map(|x| x.to_lowercase()).as_deref() { Ok(match subtype.map(|x| x.to_lowercase()).as_deref() {
None => LamostResolution::Lrs, None => LamostResolution::Lrs,
Some("lrs") | Some("low") => LamostResolution::Lrs, Some("lrs") | Some("low") => LamostResolution::Lrs,
Some("mrs") | Some("medium") => LamostResolution::Mrs, Some("mrs") | Some("medium") => LamostResolution::Mrs,
Some(other) => return Err(anyhow!("不支持的 LAMOST subtype '{}',可选: {:?}", other, LamostResolution::valid_values())), Some(other) => {
return Err(anyhow!(
"不支持的 LAMOST subtype '{}',可选: {:?}",
other,
LamostResolution::valid_values()
))
}
}) })
} }
@ -143,6 +244,49 @@ impl ObservationFetcher for LamostSpectrumFetcher {
Some("dr10") Some("dr10")
} }
fn versions_for_release(&self, release: Option<&str>) -> Vec<String> {
match parse_lamost_release(release) {
Ok(rel) => rel.versions().iter().map(|s| s.to_string()).collect(),
Err(_) => Vec::new(),
}
}
fn default_version_for_release(&self, release: Option<&str>) -> Option<String> {
parse_lamost_release(release)
.ok()
.map(|rel| rel.default_version().to_string())
}
/// Internal 数据发布DR12-14需经 oauth.china-vo.org 登录,公开 cone search 不可用
fn releases_requiring_auth(&self) -> Vec<String> {
LamostRelease::valid_values()
.iter()
.filter_map(|s| {
parse_lamost_release(Some(s))
.ok()
.filter(|rel| !rel.is_public())
.map(|_| s.to_string())
})
.collect()
}
/// 表达 release×subtype 交叉约束DR5/DR6 仅有 LRS无 MRSDR7+ 有 LRS+MRS。
/// 前端据此在选 DR5/DR6 时隐藏/禁用 MRS 子类型选项。
fn subtypes_per_release(&self) -> std::collections::HashMap<String, Vec<String>> {
LamostRelease::valid_values()
.iter()
.map(|s| {
let rel = parse_lamost_release(Some(s)).unwrap();
let subs: Vec<String> = if rel.supports_mrs() {
vec!["lrs".to_string(), "mrs".to_string()]
} else {
vec!["lrs".to_string()]
};
(s.to_string(), subs)
})
.collect()
}
fn suggested_max_radius_deg(&self) -> f64 { fn suggested_max_radius_deg(&self) -> f64 {
// 与 validate_cone_params 的建议范围一致≤5° 仅警告不拒绝) // 与 validate_cone_params 的建议范围一致≤5° 仅警告不拒绝)
5.0 5.0
@ -155,31 +299,70 @@ impl ObservationFetcher for LamostSpectrumFetcher {
async fn cone_search_raw( async fn cone_search_raw(
&self, &self,
state: &AppState, state: &AppState,
ra: f64, dec: f64, radius_deg: f64, ra: f64,
release: Option<&str>, subtype: Option<&str>, dec: f64,
radius_deg: f64,
release: Option<&str>,
subtype: Option<&str>,
version: Option<&str>,
) -> Result<Vec<Candidate>> { ) -> Result<Vec<Candidate>> {
validate_cone_params(ra, dec, radius_deg)?; validate_cone_params(ra, dec, radius_deg)?;
let rel = parse_lamost_release(release)?; let rel = parse_lamost_release(release)?;
let res = parse_lamost_resolution(subtype)?; let res = parse_lamost_resolution(subtype)?;
if res == LamostResolution::Mrs && !rel.supports_mrs() { let ver = parse_lamost_version(rel, version)?;
return Err(anyhow!("MRS中分辨率从 DR7 起才支持,{:?} 无 MRS 数据", rel)); if !rel.is_public() {
return Err(anyhow!(
"{:?} 当前为 Internal 数据发布,公开 Cone Search 不可用(需经 oauth.china-vo.org 登录认证)",
rel
));
} }
let result = state.lamost.cone_search(ra, dec, radius_deg, rel, res).await?; if res == LamostResolution::Mrs && !rel.supports_mrs() {
Ok(result.rows.iter().map(|r| row_to_candidate(r, res.as_str())).collect()) return Err(anyhow!(
"MRS中分辨率从 DR7 起才支持,{:?} 无 MRS 数据",
rel
));
}
let result = state
.lamost
.cone_search(ra, dec, radius_deg, rel, &ver, res)
.await?;
Ok(result
.rows
.iter()
.map(|r| row_to_candidate(r, res.as_str()))
.collect())
} }
async fn resolve_identifier( async fn resolve_identifier(
&self, &self,
identifier: &str, identifier: &str,
_release: Option<&str>, release: Option<&str>,
subtype: Option<&str>, subtype: Option<&str>,
version: Option<&str>,
) -> Result<Candidate> { ) -> Result<Candidate> {
let obsid: i64 = identifier.trim().parse() let obsid: i64 = identifier
.trim()
.parse()
.map_err(|_| anyhow!("LAMOST source_id 应为纯数字 obsid得到 '{}'", identifier))?; .map_err(|_| anyhow!("LAMOST source_id 应为纯数字 obsid得到 '{}'", identifier))?;
if obsid <= 0 { if obsid <= 0 {
return Err(anyhow!("obsid 应为正整数,当前: {}", obsid)); return Err(anyhow!("obsid 应为正整数,当前: {}", obsid));
} }
let res = parse_lamost_resolution(subtype)?.as_str().to_string(); let rel = parse_lamost_release(release)?;
let res = parse_lamost_resolution(subtype)?;
let ver = parse_lamost_version(rel, version)?;
if !rel.is_public() {
return Err(anyhow!(
"{:?} 当前为 Internal 数据发布,公开访问不可用(需经 oauth.china-vo.org 登录认证)",
rel
));
}
// 与 cone_search_raw 一致的交叉约束MRS 仅从 DR7 起支持
if res == LamostResolution::Mrs && !rel.supports_mrs() {
return Err(anyhow!(
"MRS中分辨率从 DR7 起才支持,{:?} 无 MRS 数据",
rel
));
}
Ok(Candidate { Ok(Candidate {
source: Source::Lamost, source: Source::Lamost,
source_id: obsid.to_string(), source_id: obsid.to_string(),
@ -187,7 +370,9 @@ impl ObservationFetcher for LamostSpectrumFetcher {
ra: None, ra: None,
dec: None, dec: None,
distance: None, distance: None,
raw: Some(serde_json::json!({"obsid": obsid, "resolution": res})), raw: Some(
serde_json::json!({"obsid": obsid, "resolution": res.as_str(), "version": ver}),
),
}) })
} }
@ -197,68 +382,116 @@ impl ObservationFetcher for LamostSpectrumFetcher {
candidate: &Candidate, candidate: &Candidate,
release: Option<&str>, release: Option<&str>,
subtype: Option<&str>, subtype: Option<&str>,
version: Option<&str>,
force: bool, force: bool,
) -> Result<ObservationProduct> { ) -> Result<ObservationProduct> {
let obsid = candidate.raw.as_ref() let obsid = candidate
.raw
.as_ref()
.and_then(|v| v.get("obsid")) .and_then(|v| v.get("obsid"))
.and_then(|v| v.as_i64()) .and_then(|v| v.as_i64())
.ok_or_else(|| anyhow!("LAMOST Candidate 缺 obsid 字段"))?; .ok_or_else(|| anyhow!("LAMOST Candidate 缺 obsid 字段"))?;
let rel = parse_lamost_release(release)?; let rel = parse_lamost_release(release)?;
let res = parse_lamost_resolution(subtype)?; let res = parse_lamost_resolution(subtype)?;
let ver = parse_lamost_version(rel, version)?;
if !rel.is_public() {
return Err(anyhow!(
"{:?} 当前为 Internal 数据发布,公开访问不可用(需经 oauth.china-vo.org 登录认证)",
rel
));
}
// 防御性交叉约束resolve_identifier 已查过;直接调 fetch 时再兜一次)
if res == LamostResolution::Mrs && !rel.supports_mrs() {
return Err(anyhow!(
"MRS中分辨率从 DR7 起才支持,{:?} 无 MRS 数据",
rel
));
}
let product = ProductSpec::with_subtype(ProductType::Spectrum, res.as_str()); let product = ProductSpec::with_subtype(ProductType::Spectrum, res.as_str());
let dr = rel.path_segment(); let dr = rel.path_segment();
let ver = rel.version_segment();
let res_str = res.as_str(); let res_str = res.as_str();
let cache_key = format!("{}|{}|{}|{}", dr, ver, res_str, obsid); let cache_key = format!("{}|{}|{}|{}", dr, ver, res_str, obsid);
let source_label = format!("obsid {}", obsid); let source_label = format!("obsid {}", obsid);
// 1) 缓存命中 // 1) 缓存命中
if !force { if !force {
if let Some((artifacts, meta)) = fetch_observation_cache(&state.db, Source::Lamost, &product, &cache_key).await? { if let Some((artifacts, meta)) =
if let Some(_) = cached_files_total_size(&state.config.library_dir, &artifacts) { fetch_observation_cache(&state.db, Source::Lamost, &product, &cache_key).await?
{
if cached_files_total_size(&state.config.library_dir, &artifacts)
.await
.is_some()
{
info!("[LAMOST] 光谱缓存命中 (obsid={})", obsid); info!("[LAMOST] 光谱缓存命中 (obsid={})", obsid);
return Ok(ObservationProduct { return Ok(ObservationProduct {
source: Source::Lamost, product, source: Source::Lamost,
source_id: cache_key, source_label, artifacts, source_meta: meta, product,
source_id: cache_key,
source_label,
artifacts,
source_meta: meta,
}); });
} }
warn!("[LAMOST] 缓存记录存在但文件缺失,重新下载 (obsid={})", obsid); warn!(
"[LAMOST] 缓存记录存在但文件缺失,重新下载 (obsid={})",
obsid
);
} }
} }
// 2) 下载 + 解压 // 2) 下载 + 解压
tokio::time::sleep(Duration::from_millis(50)).await; tokio::time::sleep(Duration::from_millis(50)).await;
let gz_bytes = state.lamost.download_fits(obsid, rel).await?; let gz_bytes = state.lamost.download_fits(obsid, rel, &ver).await?;
let fits_bytes = decompress_if_gzip(&gz_bytes)?; let fits_bytes = decompress_if_gzip(&gz_bytes)?;
// 3) 落盘 // 3) 落盘
let rel_path = format!( let rel_path = format!(
"Telescope/lamost/spectrum/{res}/{dr}/{ver}/{obsid}.fits", "Telescope/lamost/spectrum/{res}/{dr}/{ver}/{obsid}.fits",
res = res_str, dr = dr, ver = ver, obsid = obsid res = res_str,
dr = dr,
ver = ver,
obsid = obsid
);
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes).await?;
info!(
"[LAMOST] FITS 已保存 obsid={} → {} ({}B)",
obsid, rel_path, size
); );
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes)?;
info!("[LAMOST] FITS 已保存 obsid={} → {} ({}B)", obsid, rel_path, size);
let meta = serde_json::json!({"obsid": obsid, "dr": dr, "version": ver, "resolution": res_str}); let meta =
serde_json::json!({"obsid": obsid, "dr": dr, "version": ver, "resolution": res_str});
let artifact = Artifact { let artifact = Artifact {
band: None, original_name: None, band: None,
original_name: None,
file_path: rel_path.clone(), file_path: rel_path.clone(),
file_url: file_url_from_path(&rel_path), file_url: file_url_from_path(&rel_path),
file_format: "fits".to_string(), file_format: "fits".to_string(),
size_bytes: size, cached: false, size_bytes: size,
cached: false,
}; };
let meta_str = meta.to_string(); let meta_str = meta.to_string();
if let Err(e) = write_observation_cache( if let Err(e) = write_observation_cache(
&state.db, Source::Lamost, &product, &cache_key, None, None, &state.db,
std::slice::from_ref(&artifact), Some(&meta_str), Source::Lamost,
).await { &product,
&cache_key,
None,
None,
std::slice::from_ref(&artifact),
Some(&meta_str),
)
.await
{
log_write_failure(Source::Lamost.as_str(), "spectrum", e); log_write_failure(Source::Lamost.as_str(), "spectrum", e);
} }
Ok(ObservationProduct { Ok(ObservationProduct {
source: Source::Lamost, product, source: Source::Lamost,
source_id: cache_key, source_label, product,
artifacts: vec![artifact], source_meta: Some(meta), source_id: cache_key,
source_label,
artifacts: vec![artifact],
source_meta: Some(meta),
}) })
} }
} }
@ -280,7 +513,9 @@ fn decompress_if_gzip(bytes: &[u8]) -> Result<Vec<u8>> {
if bytes.len() >= 2 && bytes[0] == 0x1f && bytes[1] == 0x8b { if bytes.len() >= 2 && bytes[0] == 0x1f && bytes[1] == 0x8b {
let mut decoder = flate2::read::GzDecoder::new(bytes); let mut decoder = flate2::read::GzDecoder::new(bytes);
let mut out = Vec::new(); let mut out = Vec::new();
decoder.read_to_end(&mut out).map_err(|e| anyhow!("gzip 解压失败: {}", e))?; decoder
.read_to_end(&mut out)
.map_err(|e| anyhow!("gzip 解压失败: {}", e))?;
Ok(out) Ok(out)
} else { } else {
Ok(bytes.to_vec()) Ok(bytes.to_vec())
@ -323,9 +558,18 @@ mod tests {
#[test] #[test]
fn test_row_to_candidate() { fn test_row_to_candidate() {
let row = LamostSpectrumRow { let row = LamostSpectrumRow {
obsid: 438809089, designation: None, ra_obs: Some(10.5), dec_obs: Some(41.0), obsid: 438809089,
z: None, class: None, subclass: None, designation: None,
sn_u: None, sn_g: None, sn_r: None, sn_i: None, sn_z: None, ra_obs: Some(10.5),
dec_obs: Some(41.0),
z: None,
class: None,
subclass: None,
sn_u: None,
sn_g: None,
sn_r: None,
sn_i: None,
sn_z: None,
}; };
let c = row_to_candidate(&row, "lrs"); let c = row_to_candidate(&row, "lrs");
assert_eq!(c.source_id, "438809089"); assert_eq!(c.source_id, "438809089");

View File

@ -64,32 +64,38 @@ impl ObservationRegistry {
product: ProductType, product: ProductType,
subtype: Option<&str>, subtype: Option<&str>,
) -> Result<Arc<dyn ObservationFetcher>> { ) -> Result<Arc<dyn ObservationFetcher>> {
let candidates = self let candidates = self.fetchers.get(&(source, product)).ok_or_else(|| {
.fetchers anyhow!(
.get(&(source, product))
.ok_or_else(|| anyhow!(
"{:?} 暂不支持 {:?} 产品。支持的组合:{}", "{:?} 暂不支持 {:?} 产品。支持的组合:{}",
source, source,
product, product,
self.list_supported_combinations() self.list_supported_combinations()
))?; )
})?;
// subtype 精确匹配 // subtype 精确匹配
if let Some(st) = subtype { if let Some(st) = subtype {
let st_lower = st.to_lowercase(); let st_lower = st.to_lowercase();
for f in candidates { for f in candidates {
if f.subtypes().iter().any(|s| s.eq_ignore_ascii_case(&st_lower)) { if f.subtypes()
.iter()
.any(|s| s.eq_ignore_ascii_case(&st_lower))
{
return Ok(f.clone()); return Ok(f.clone());
} }
} }
// 未找到匹配的 subtype — 报错 // 未找到匹配的 subtype — 报错
let valid: Vec<&str> = candidates.iter() let valid: Vec<&str> = candidates
.iter()
.flat_map(|f| f.subtypes()) .flat_map(|f| f.subtypes())
.copied() .copied()
.collect(); .collect();
return Err(anyhow!( return Err(anyhow!(
"{:?} 的 {:?} 不支持 subtype '{}',可选: {:?}", "{:?} 的 {:?} 不支持 subtype '{}',可选: {:?}",
source, product, st, valid source,
product,
st,
valid
)); ));
} }
@ -134,6 +140,16 @@ impl ObservationRegistry {
subtypes: f.subtypes().iter().map(|s| s.to_string()).collect(), subtypes: f.subtypes().iter().map(|s| s.to_string()).collect(),
releases: f.releases().iter().map(|s| s.to_string()).collect(), releases: f.releases().iter().map(|s| s.to_string()).collect(),
default_release: f.default_release().map(|s| s.to_string()), default_release: f.default_release().map(|s| s.to_string()),
// 每个 release 对应的子版本列表(仅 LAMOST 有内容)
release_versions: f
.releases()
.iter()
.map(|r| (r.to_string(), f.versions_for_release(Some(r))))
.collect(),
// 需要登录认证的 releaseLAMOST Internal DR12-14
releases_requiring_auth: f.releases_requiring_auth(),
// 每个 release 实际支持的 subtype表达交叉约束如 LAMOST DR5/6 无 MRS
subtypes_per_release: f.subtypes_per_release().into_iter().collect(),
suggested_max_radius_deg: f.suggested_max_radius_deg(), suggested_max_radius_deg: f.suggested_max_radius_deg(),
hard_max_radius_deg: f.hard_max_radius_deg(), hard_max_radius_deg: f.hard_max_radius_deg(),
identifier_format: f.identifier_format().map(|s| s.to_string()), identifier_format: f.identifier_format().map(|s| s.to_string()),
@ -163,6 +179,15 @@ pub struct CapabilitySpec {
pub releases: Vec<String>, pub releases: Vec<String>,
/// 默认版本release 参数为空时使用None 表示无版本概念 /// 默认版本release 参数为空时使用None 表示无版本概念
pub default_release: Option<String>, pub default_release: Option<String>,
/// 每个 release 对应的子版本列表(仅 LAMOST 有内容key=release 如 "dr11"value=["v0","v1.0",...]
#[serde(default)]
pub release_versions: std::collections::BTreeMap<String, Vec<String>>,
/// 需要登录认证才能访问的 release 列表(如 LAMOST Internal DR12-14空表示全部公开
#[serde(default)]
pub releases_requiring_auth: Vec<String>,
/// 每个 release 实际支持的 subtype表达交叉约束如 LAMOST DR5/6 无 MRS空表示所有 release 都支持全部 subtypes
#[serde(default)]
pub subtypes_per_release: std::collections::BTreeMap<String, Vec<String>>,
/// 坐标模式检索半径的建议上限(度),超出仅警告不拒绝 /// 坐标模式检索半径的建议上限(度),超出仅警告不拒绝
pub suggested_max_radius_deg: f64, pub suggested_max_radius_deg: f64,
/// 坐标模式检索半径的硬性上限(度),超出会返回错误 /// 坐标模式检索半径的硬性上限(度),超出会返回错误
@ -186,8 +211,12 @@ mod tests {
assert!(reg.get(Source::Gaia, ProductType::LightCurve, None).is_ok()); assert!(reg.get(Source::Gaia, ProductType::LightCurve, None).is_ok());
// SDSS spectrum (specobj 默认 + apogee/aspcap subtype) // SDSS spectrum (specobj 默认 + apogee/aspcap subtype)
assert!(reg.get(Source::Sdss, ProductType::Spectrum, None).is_ok()); assert!(reg.get(Source::Sdss, ProductType::Spectrum, None).is_ok());
assert!(reg.get(Source::Sdss, ProductType::Spectrum, Some("apstar")).is_ok()); assert!(reg
assert!(reg.get(Source::Sdss, ProductType::Spectrum, Some("aspcap")).is_ok()); .get(Source::Sdss, ProductType::Spectrum, Some("apstar"))
.is_ok());
assert!(reg
.get(Source::Sdss, ProductType::Spectrum, Some("aspcap"))
.is_ok());
// DESI spectrum // DESI spectrum
assert!(reg.get(Source::Desi, ProductType::Spectrum, None).is_ok()); assert!(reg.get(Source::Desi, ProductType::Spectrum, None).is_ok());
} }
@ -195,7 +224,9 @@ mod tests {
#[test] #[test]
fn test_unsupported_combination_error() { fn test_unsupported_combination_error() {
let reg = ObservationRegistry::default(); let reg = ObservationRegistry::default();
let err = reg.get(Source::Lamost, ProductType::LightCurve, None).unwrap_err(); let err = reg
.get(Source::Lamost, ProductType::LightCurve, None)
.unwrap_err();
assert!(format!("{}", err).contains("暂不支持")); assert!(format!("{}", err).contains("暂不支持"));
} }
@ -203,7 +234,9 @@ mod tests {
fn test_sdss_subtype_routing() { fn test_sdss_subtype_routing() {
let reg = ObservationRegistry::default(); let reg = ObservationRegistry::default();
// subtype=apstar 应路由到 SdssApogeeFetcher其 subtypes 含 "apstar" // subtype=apstar 应路由到 SdssApogeeFetcher其 subtypes 含 "apstar"
let apstar = reg.get(Source::Sdss, ProductType::Spectrum, Some("apstar")).unwrap(); let apstar = reg
.get(Source::Sdss, ProductType::Spectrum, Some("apstar"))
.unwrap();
assert!(apstar.subtypes().contains(&"apstar")); assert!(apstar.subtypes().contains(&"apstar"));
// 默认(无 subtype应路由到 SdssSpectrumFetcher // 默认(无 subtype应路由到 SdssSpectrumFetcher
let specobj = reg.get(Source::Sdss, ProductType::Spectrum, None).unwrap(); let specobj = reg.get(Source::Sdss, ProductType::Spectrum, None).unwrap();

View File

@ -23,11 +23,13 @@ use tracing::{info, warn};
// ── 版本枚举 ── // ── 版本枚举 ──
#[derive(Debug, Clone, Copy, PartialEq, Eq)] #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
pub enum SdssRelease { Dr16, Dr17, Dr18, Dr19 } pub enum SdssRelease {
Dr16,
impl Default for SdssRelease { #[default]
fn default() -> Self { Self::Dr17 } Dr17,
Dr18,
Dr19,
} }
impl SdssRelease { impl SdssRelease {
@ -36,8 +38,10 @@ impl SdssRelease {
} }
pub fn sas_dr_segment(&self) -> &'static str { pub fn sas_dr_segment(&self) -> &'static str {
match self { match self {
Self::Dr16 => "dr16", Self::Dr17 => "dr17", Self::Dr16 => "dr16",
Self::Dr18 => "dr18", Self::Dr19 => "dr19", Self::Dr17 => "dr17",
Self::Dr18 => "dr18",
Self::Dr19 => "dr19",
} }
} }
pub fn datalab_table(&self) -> Option<&'static str> { pub fn datalab_table(&self) -> Option<&'static str> {
@ -55,8 +59,10 @@ impl SdssRelease {
} }
pub fn display(&self) -> &'static str { pub fn display(&self) -> &'static str {
match self { match self {
Self::Dr16 => "DR16", Self::Dr17 => "DR17", Self::Dr16 => "DR16",
Self::Dr18 => "DR18", Self::Dr19 => "DR19", Self::Dr17 => "DR17",
Self::Dr18 => "DR18",
Self::Dr19 => "DR19",
} }
} }
} }
@ -68,7 +74,13 @@ pub fn parse_sdss_release(s: Option<&str>) -> Result<SdssRelease> {
Some("dr17") => SdssRelease::Dr17, Some("dr17") => SdssRelease::Dr17,
Some("dr18") => SdssRelease::Dr18, Some("dr18") => SdssRelease::Dr18,
Some("dr19") => SdssRelease::Dr19, Some("dr19") => SdssRelease::Dr19,
Some(other) => return Err(anyhow!("不支持的 SDSS release '{}',可选: {:?}", other, SdssRelease::valid_values())), Some(other) => {
return Err(anyhow!(
"不支持的 SDSS release '{}',可选: {:?}",
other,
SdssRelease::valid_values()
))
}
}) })
} }
@ -131,8 +143,12 @@ impl ObservationFetcher for SdssSpectrumFetcher {
async fn cone_search_raw( async fn cone_search_raw(
&self, &self,
state: &AppState, state: &AppState,
ra: f64, dec: f64, radius_deg: f64, ra: f64,
release: Option<&str>, _subtype: Option<&str>, dec: f64,
radius_deg: f64,
release: Option<&str>,
_subtype: Option<&str>,
_version: Option<&str>,
) -> Result<Vec<Candidate>> { ) -> Result<Vec<Candidate>> {
validate_cone_params(ra, dec, radius_deg)?; validate_cone_params(ra, dec, radius_deg)?;
let rel = parse_sdss_release(release)?; let rel = parse_sdss_release(release)?;
@ -145,6 +161,7 @@ impl ObservationFetcher for SdssSpectrumFetcher {
identifier: &str, identifier: &str,
_release: Option<&str>, _release: Option<&str>,
_subtype: Option<&str>, _subtype: Option<&str>,
_version: Option<&str>,
) -> Result<Candidate> { ) -> Result<Candidate> {
let parts: Vec<&str> = identifier.split('-').collect(); let parts: Vec<&str> = identifier.split('-').collect();
if parts.len() != 4 { if parts.len() != 4 {
@ -153,14 +170,22 @@ impl ObservationFetcher for SdssSpectrumFetcher {
identifier identifier
)); ));
} }
let plate: i64 = parts[1].parse().map_err(|_| anyhow!("plate 非数字: '{}'", parts[1]))?; let plate: i64 = parts[1]
let mjd: i64 = parts[2].parse().map_err(|_| anyhow!("mjd 非数字: '{}'", parts[2]))?; .parse()
let fiberid: i64 = parts[3].parse().map_err(|_| anyhow!("fiberid 非数字: '{}'", parts[3]))?; .map_err(|_| anyhow!("plate 非数字: '{}'", parts[1]))?;
let mjd: i64 = parts[2]
.parse()
.map_err(|_| anyhow!("mjd 非数字: '{}'", parts[2]))?;
let fiberid: i64 = parts[3]
.parse()
.map_err(|_| anyhow!("fiberid 非数字: '{}'", parts[3]))?;
Ok(Candidate { Ok(Candidate {
source: Source::Sdss, source: Source::Sdss,
source_id: identifier.to_string(), source_id: identifier.to_string(),
label: format!("plate {} mjd {} fiber {}", plate, mjd, fiberid), label: format!("plate {} mjd {} fiber {}", plate, mjd, fiberid),
ra: None, dec: None, distance: None, ra: None,
dec: None,
distance: None,
raw: Some(serde_json::json!({ raw: Some(serde_json::json!({
"plate": plate, "mjd": mjd, "fiberid": fiberid, "run2d": parts[0] "plate": plate, "mjd": mjd, "fiberid": fiberid, "run2d": parts[0]
})), })),
@ -173,15 +198,32 @@ impl ObservationFetcher for SdssSpectrumFetcher {
candidate: &Candidate, candidate: &Candidate,
release: Option<&str>, release: Option<&str>,
_subtype: Option<&str>, _subtype: Option<&str>,
_version: Option<&str>,
force: bool, force: bool,
) -> Result<ObservationProduct> { ) -> Result<ObservationProduct> {
let plate = candidate.raw.as_ref().and_then(|v| v.get("plate")).and_then(|v| v.as_i64()) let plate = candidate
.raw
.as_ref()
.and_then(|v| v.get("plate"))
.and_then(|v| v.as_i64())
.ok_or_else(|| anyhow!("SDSS Candidate 缺 plate"))?; .ok_or_else(|| anyhow!("SDSS Candidate 缺 plate"))?;
let mjd = candidate.raw.as_ref().and_then(|v| v.get("mjd")).and_then(|v| v.as_i64()) let mjd = candidate
.raw
.as_ref()
.and_then(|v| v.get("mjd"))
.and_then(|v| v.as_i64())
.ok_or_else(|| anyhow!("SDSS Candidate 缺 mjd"))?; .ok_or_else(|| anyhow!("SDSS Candidate 缺 mjd"))?;
let fiberid = candidate.raw.as_ref().and_then(|v| v.get("fiberid")).and_then(|v| v.as_i64()) let fiberid = candidate
.raw
.as_ref()
.and_then(|v| v.get("fiberid"))
.and_then(|v| v.as_i64())
.ok_or_else(|| anyhow!("SDSS Candidate 缺 fiberid"))?; .ok_or_else(|| anyhow!("SDSS Candidate 缺 fiberid"))?;
let run2d = candidate.raw.as_ref().and_then(|v| v.get("run2d")).and_then(|v| v.as_str()) let run2d = candidate
.raw
.as_ref()
.and_then(|v| v.get("run2d"))
.and_then(|v| v.as_str())
.ok_or_else(|| anyhow!("SDSS Candidate 缺 run2d"))?; .ok_or_else(|| anyhow!("SDSS Candidate 缺 run2d"))?;
let rel = parse_sdss_release(release)?; let rel = parse_sdss_release(release)?;
let product = ProductSpec::with_subtype(ProductType::Spectrum, "spec"); let product = ProductSpec::with_subtype(ProductType::Spectrum, "spec");
@ -191,12 +233,21 @@ impl ObservationFetcher for SdssSpectrumFetcher {
// 1) 缓存命中 // 1) 缓存命中
if !force { if !force {
if let Some((artifacts, meta)) = fetch_observation_cache(&state.db, Source::Sdss, &product, &cache_key).await? { if let Some((artifacts, meta)) =
if cached_files_total_size(&state.config.library_dir, &artifacts).is_some() { fetch_observation_cache(&state.db, Source::Sdss, &product, &cache_key).await?
{
if cached_files_total_size(&state.config.library_dir, &artifacts)
.await
.is_some()
{
info!("[SDSS] 光谱缓存命中 ({})", cache_key); info!("[SDSS] 光谱缓存命中 ({})", cache_key);
return Ok(ObservationProduct { return Ok(ObservationProduct {
source: Source::Sdss, product, source: Source::Sdss,
source_id: cache_key, source_label, artifacts, source_meta: meta, product,
source_id: cache_key,
source_label,
artifacts,
source_meta: meta,
}); });
} }
warn!("[SDSS] 缓存记录存在但文件缺失,重新下载 ({})", cache_key); warn!("[SDSS] 缓存记录存在但文件缺失,重新下载 ({})", cache_key);
@ -205,34 +256,57 @@ impl ObservationFetcher for SdssSpectrumFetcher {
// 2) SAS 下载(未压缩 FITS+ 落盘 // 2) SAS 下载(未压缩 FITS+ 落盘
tokio::time::sleep(Duration::from_millis(50)).await; tokio::time::sleep(Duration::from_millis(50)).await;
let fits_bytes = state.sdss.download_fits(plate, mjd, fiberid, run2d, rel).await?; let fits_bytes = state
.sdss
.download_fits(plate, mjd, fiberid, run2d, rel)
.await?;
let rel_path = format!( let rel_path = format!(
"Telescope/sdss/{dr}/spec/{run2d}-{plate}-{mjd}-{fiber:04}.fits", "Telescope/sdss/{dr}/spec/{run2d}-{plate}-{mjd}-{fiber:04}.fits",
dr = dr, run2d = run2d, plate = plate, mjd = mjd, fiber = fiberid dr = dr,
run2d = run2d,
plate = plate,
mjd = mjd,
fiber = fiberid
);
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes).await?;
info!(
"[SDSS] FITS 已保存 {} → {} ({}B)",
cache_key, rel_path, size
); );
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes)?;
info!("[SDSS] FITS 已保存 {} → {} ({}B)", cache_key, rel_path, size);
let meta = serde_json::json!({"plate": plate, "mjd": mjd, "fiberid": fiberid, "run2d": run2d, "dr": dr}); let meta = serde_json::json!({"plate": plate, "mjd": mjd, "fiberid": fiberid, "run2d": run2d, "dr": dr});
let artifact = Artifact { let artifact = Artifact {
band: None, original_name: None, band: None,
original_name: None,
file_path: rel_path.clone(), file_path: rel_path.clone(),
file_url: file_url_from_path(&rel_path), file_url: file_url_from_path(&rel_path),
file_format: "fits".to_string(), file_format: "fits".to_string(),
size_bytes: size, cached: false, size_bytes: size,
cached: false,
}; };
let meta_str = meta.to_string(); let meta_str = meta.to_string();
if let Err(e) = write_observation_cache( if let Err(e) = write_observation_cache(
&state.db, Source::Sdss, &product, &cache_key, None, None, &state.db,
std::slice::from_ref(&artifact), Some(&meta_str), Source::Sdss,
).await { &product,
&cache_key,
None,
None,
std::slice::from_ref(&artifact),
Some(&meta_str),
)
.await
{
log_write_failure(Source::Sdss.as_str(), "spectrum", e); log_write_failure(Source::Sdss.as_str(), "spectrum", e);
} }
Ok(ObservationProduct { Ok(ObservationProduct {
source: Source::Sdss, product, source: Source::Sdss,
source_id: cache_key, source_label, product,
artifacts: vec![artifact], source_meta: Some(meta), source_id: cache_key,
source_label,
artifacts: vec![artifact],
source_meta: Some(meta),
}) })
} }
} }
@ -242,7 +316,9 @@ fn row_to_candidate(row: &SdssSpectrumRow) -> Candidate {
source: Source::Sdss, source: Source::Sdss,
source_id: format!("{}-{}-{}-{}", row.run2d, row.plate, row.mjd, row.fiberid), source_id: format!("{}-{}-{}-{}", row.run2d, row.plate, row.mjd, row.fiberid),
label: format!("plate {} mjd {} fiber {}", row.plate, row.mjd, row.fiberid), label: format!("plate {} mjd {} fiber {}", row.plate, row.mjd, row.fiberid),
ra: row.ra, dec: row.dec, distance: None, ra: row.ra,
dec: row.dec,
distance: None,
raw: Some(serde_json::json!({ raw: Some(serde_json::json!({
"plate": row.plate, "mjd": row.mjd, "fiberid": row.fiberid, "run2d": row.run2d "plate": row.plate, "mjd": row.mjd, "fiberid": row.fiberid, "run2d": row.run2d
})), })),
@ -287,12 +363,19 @@ impl ObservationFetcher for SdssApogeeFetcher {
async fn cone_search_raw( async fn cone_search_raw(
&self, &self,
state: &AppState, state: &AppState,
ra: f64, dec: f64, radius_deg: f64, ra: f64,
release: Option<&str>, _subtype: Option<&str>, dec: f64,
radius_deg: f64,
release: Option<&str>,
_subtype: Option<&str>,
_version: Option<&str>,
) -> Result<Vec<Candidate>> { ) -> Result<Vec<Candidate>> {
validate_cone_params(ra, dec, radius_deg)?; validate_cone_params(ra, dec, radius_deg)?;
let rel = parse_sdss_release(release)?; let rel = parse_sdss_release(release)?;
let result = state.sdss.apogee_cone_search(ra, dec, radius_deg, 50, rel).await?; let result = state
.sdss
.apogee_cone_search(ra, dec, radius_deg, 50, rel)
.await?;
Ok(result.rows.iter().map(apogee_row_to_candidate).collect()) Ok(result.rows.iter().map(apogee_row_to_candidate).collect())
} }
@ -301,6 +384,7 @@ impl ObservationFetcher for SdssApogeeFetcher {
identifier: &str, identifier: &str,
_release: Option<&str>, _release: Option<&str>,
subtype: Option<&str>, subtype: Option<&str>,
_version: Option<&str>,
) -> Result<Candidate> { ) -> Result<Candidate> {
// 格式 "{telescope}|{field}|{apogee_id}" // 格式 "{telescope}|{field}|{apogee_id}"
let parts: Vec<&str> = identifier.splitn(3, '|').collect(); let parts: Vec<&str> = identifier.splitn(3, '|').collect();
@ -315,7 +399,9 @@ impl ObservationFetcher for SdssApogeeFetcher {
source: Source::Sdss, source: Source::Sdss,
source_id: identifier.to_string(), source_id: identifier.to_string(),
label: format!("{} {} {}", parts[0], dt, parts[2]), label: format!("{} {} {}", parts[0], dt, parts[2]),
ra: None, dec: None, distance: None, ra: None,
dec: None,
distance: None,
raw: Some(serde_json::json!({ raw: Some(serde_json::json!({
"apogee_id": parts[2], "telescope": parts[0], "field": parts[1], "data_type": dt "apogee_id": parts[2], "telescope": parts[0], "field": parts[1], "data_type": dt
})), })),
@ -328,16 +414,34 @@ impl ObservationFetcher for SdssApogeeFetcher {
candidate: &Candidate, candidate: &Candidate,
release: Option<&str>, release: Option<&str>,
subtype: Option<&str>, subtype: Option<&str>,
_version: Option<&str>,
force: bool, force: bool,
) -> Result<ObservationProduct> { ) -> Result<ObservationProduct> {
let apogee_id = candidate.raw.as_ref().and_then(|v| v.get("apogee_id")).and_then(|v| v.as_str()) let apogee_id = candidate
.raw
.as_ref()
.and_then(|v| v.get("apogee_id"))
.and_then(|v| v.as_str())
.ok_or_else(|| anyhow!("APOGEE Candidate 缺 apogee_id"))?; .ok_or_else(|| anyhow!("APOGEE Candidate 缺 apogee_id"))?;
let telescope = candidate.raw.as_ref().and_then(|v| v.get("telescope")).and_then(|v| v.as_str()) let telescope = candidate
.raw
.as_ref()
.and_then(|v| v.get("telescope"))
.and_then(|v| v.as_str())
.ok_or_else(|| anyhow!("APOGEE Candidate 缺 telescope"))?; .ok_or_else(|| anyhow!("APOGEE Candidate 缺 telescope"))?;
let field = candidate.raw.as_ref().and_then(|v| v.get("field")).and_then(|v| v.as_str()) let field = candidate
.raw
.as_ref()
.and_then(|v| v.get("field"))
.and_then(|v| v.as_str())
.ok_or_else(|| anyhow!("APOGEE Candidate 缺 field"))?; .ok_or_else(|| anyhow!("APOGEE Candidate 缺 field"))?;
let dt = candidate.raw.as_ref().and_then(|v| v.get("data_type")).and_then(|v| v.as_str()) let dt = candidate
.or(subtype).unwrap_or("apstar"); .raw
.as_ref()
.and_then(|v| v.get("data_type"))
.and_then(|v| v.as_str())
.or(subtype)
.unwrap_or("apstar");
let rel = parse_sdss_release(release)?; let rel = parse_sdss_release(release)?;
let product = ProductSpec::with_subtype(ProductType::Spectrum, dt); let product = ProductSpec::with_subtype(ProductType::Spectrum, dt);
let dr = rel.sas_dr_segment(); let dr = rel.sas_dr_segment();
@ -346,12 +450,21 @@ impl ObservationFetcher for SdssApogeeFetcher {
// 1) 缓存命中 // 1) 缓存命中
if !force { if !force {
if let Some((artifacts, meta)) = fetch_observation_cache(&state.db, Source::Sdss, &product, &cache_key).await? { if let Some((artifacts, meta)) =
if cached_files_total_size(&state.config.library_dir, &artifacts).is_some() { fetch_observation_cache(&state.db, Source::Sdss, &product, &cache_key).await?
{
if cached_files_total_size(&state.config.library_dir, &artifacts)
.await
.is_some()
{
info!("[APOGEE] 缓存命中 ({})", cache_key); info!("[APOGEE] 缓存命中 ({})", cache_key);
return Ok(ObservationProduct { return Ok(ObservationProduct {
source: Source::Sdss, product, source: Source::Sdss,
source_id: cache_key, source_label, artifacts, source_meta: meta, product,
source_id: cache_key,
source_label,
artifacts,
source_meta: meta,
}); });
} }
warn!("[APOGEE] 缓存记录存在但文件缺失,重新下载 ({})", cache_key); warn!("[APOGEE] 缓存记录存在但文件缺失,重新下载 ({})", cache_key);
@ -360,10 +473,18 @@ impl ObservationFetcher for SdssApogeeFetcher {
// 2) 下载 FITS无压缩 // 2) 下载 FITS无压缩
tokio::time::sleep(Duration::from_millis(50)).await; tokio::time::sleep(Duration::from_millis(50)).await;
let fits_bytes = state.sdss.download_apogee_fits(apogee_id, telescope, field, rel, dt).await?; let fits_bytes = state
let safe_id = apogee_id.replace('+', "_").replace('/', "_"); .sdss
let rel_path = format!("Telescope/sdss/{dr}/{dt}/{safe_id}.fits", dr = dr, dt = dt, safe_id = safe_id); .download_apogee_fits(apogee_id, telescope, field, rel, dt)
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes)?; .await?;
let safe_id = apogee_id.replace(['+', '/'], "_");
let rel_path = format!(
"Telescope/sdss/{dr}/{dt}/{safe_id}.fits",
dr = dr,
dt = dt,
safe_id = safe_id
);
let size = persist_bytes(&state.config.library_dir, &rel_path, &fits_bytes).await?;
info!("[APOGEE] 已保存 {} → {} ({}B)", cache_key, rel_path, size); info!("[APOGEE] 已保存 {} → {} ({}B)", cache_key, rel_path, size);
let meta = serde_json::json!({ let meta = serde_json::json!({
@ -371,24 +492,37 @@ impl ObservationFetcher for SdssApogeeFetcher {
"data_type": dt, "dr": dr "data_type": dt, "dr": dr
}); });
let artifact = Artifact { let artifact = Artifact {
band: None, original_name: None, band: None,
original_name: None,
file_path: rel_path.clone(), file_path: rel_path.clone(),
file_url: file_url_from_path(&rel_path), file_url: file_url_from_path(&rel_path),
file_format: "fits".to_string(), file_format: "fits".to_string(),
size_bytes: size, cached: false, size_bytes: size,
cached: false,
}; };
let meta_str = meta.to_string(); let meta_str = meta.to_string();
if let Err(e) = write_observation_cache( if let Err(e) = write_observation_cache(
&state.db, Source::Sdss, &product, &cache_key, None, None, &state.db,
std::slice::from_ref(&artifact), Some(&meta_str), Source::Sdss,
).await { &product,
&cache_key,
None,
None,
std::slice::from_ref(&artifact),
Some(&meta_str),
)
.await
{
log_write_failure(Source::Sdss.as_str(), "apogee", e); log_write_failure(Source::Sdss.as_str(), "apogee", e);
} }
Ok(ObservationProduct { Ok(ObservationProduct {
source: Source::Sdss, product, source: Source::Sdss,
source_id: cache_key, source_label, product,
artifacts: vec![artifact], source_meta: Some(meta), source_id: cache_key,
source_label,
artifacts: vec![artifact],
source_meta: Some(meta),
}) })
} }
} }
@ -398,7 +532,9 @@ fn apogee_row_to_candidate(row: &ApogeeStarRow) -> Candidate {
source: Source::Sdss, source: Source::Sdss,
source_id: format!("{}|{}|{}", row.telescope, row.field, row.apogee_id), source_id: format!("{}|{}|{}", row.telescope, row.field, row.apogee_id),
label: format!("{} {}", row.telescope, row.apogee_id), label: format!("{} {}", row.telescope, row.apogee_id),
ra: row.ra, dec: row.dec, distance: None, ra: row.ra,
dec: row.dec,
distance: None,
raw: Some(serde_json::json!({ raw: Some(serde_json::json!({
"apogee_id": row.apogee_id, "telescope": row.telescope, "field": row.field "apogee_id": row.apogee_id, "telescope": row.telescope, "field": row.field
})), })),

View File

@ -39,6 +39,24 @@ impl Source {
} }
} }
/// 从字符串解析数据源(大小写不敏感)。新增源只需在此加一行 match arm
/// 所有调用方API handler、Agent tool统一用本方法不再各自手写 match。
pub fn from_str(s: &str) -> Result<Self, String> {
Ok(match s.to_lowercase().as_str() {
"lamost" => Self::Lamost,
"gaia" => Self::Gaia,
"sdss" => Self::Sdss,
"desi" => Self::Desi,
other => {
return Err(format!(
"不支持的 source '{}',可选: {:?}",
other,
Self::valid_values()
))
}
})
}
pub fn display(&self) -> &'static str { pub fn display(&self) -> &'static str {
match self { match self {
Self::Lamost => "LAMOST", Self::Lamost => "LAMOST",
@ -81,6 +99,24 @@ impl ProductType {
} }
} }
/// 从字符串解析产品类型(大小写不敏感,接受常见别名)。
/// 别名集中在此一处管理,所有调用方统一用本方法。
pub fn from_str(s: &str) -> Result<Self, String> {
Ok(match s.to_lowercase().as_str() {
"spectrum" | "spec" => Self::Spectrum,
"lightcurve" | "light_curve" | "lc" => Self::LightCurve,
"photometry" => Self::Photometry,
"image" => Self::Image,
other => {
return Err(format!(
"不支持的 product '{}',可选: {:?}",
other,
Self::valid_values()
))
}
})
}
pub fn display(&self) -> &'static str { pub fn display(&self) -> &'static str {
match self { match self {
Self::Spectrum => "光谱", Self::Spectrum => "光谱",
@ -113,11 +149,17 @@ pub struct ProductSpec {
impl ProductSpec { impl ProductSpec {
pub fn new(product: ProductType) -> Self { pub fn new(product: ProductType) -> Self {
Self { product, subtype: None } Self {
product,
subtype: None,
}
} }
pub fn with_subtype(product: ProductType, subtype: impl Into<String>) -> Self { pub fn with_subtype(product: ProductType, subtype: impl Into<String>) -> Self {
Self { product, subtype: Some(subtype.into()) } Self {
product,
subtype: Some(subtype.into()),
}
} }
/// 落盘子目录:有 subtype 则拼 "product/subtype",否则只用 product /// 落盘子目录:有 subtype 则拼 "product/subtype",否则只用 product
@ -227,6 +269,9 @@ pub enum ObservationRequest {
strategy: FindStrategy, strategy: FindStrategy,
#[serde(default)] #[serde(default)]
release: Option<String>, release: Option<String>,
/// 数据发布的子版本(仅 LAMOST 有意义,如 "v2.0"/"v1.0"None 时用该 DR 的默认版本
#[serde(default)]
version: Option<String>,
}, },
ByIdentifier { ByIdentifier {
source: Source, source: Source,
@ -234,6 +279,8 @@ pub enum ObservationRequest {
identifiers: Vec<String>, identifiers: Vec<String>,
#[serde(default)] #[serde(default)]
release: Option<String>, release: Option<String>,
#[serde(default)]
version: Option<String>,
}, },
} }
@ -253,7 +300,10 @@ mod tests {
#[test] #[test]
fn test_product_spec_path() { fn test_product_spec_path() {
assert_eq!(ProductSpec::new(ProductType::Spectrum).path_segment(), "spectrum"); assert_eq!(
ProductSpec::new(ProductType::Spectrum).path_segment(),
"spectrum"
);
assert_eq!( assert_eq!(
ProductSpec::with_subtype(ProductType::Spectrum, "lrs").path_segment(), ProductSpec::with_subtype(ProductType::Spectrum, "lrs").path_segment(),
"spectrum/lrs" "spectrum/lrs"

View File

@ -88,6 +88,12 @@ pub async fn upload_paper_file_service(
.map_err(|e| anyhow::anyhow!("PDF 文件内容校验失败: {}", e))?; .map_err(|e| anyhow::anyhow!("PDF 文件内容校验失败: {}", e))?;
let pdf_filename = format!("{}.pdf", bibcode); let pdf_filename = format!("{}.pdf", bibcode);
let pdf_dest = library_dir.join("PDF").join(&pdf_filename); let pdf_dest = library_dir.join("PDF").join(&pdf_filename);
if pdf_dest
.components()
.any(|c| matches!(c, std::path::Component::ParentDir))
{
anyhow::bail!("非法的文件写入路径bibcode 包含路径穿越字符");
}
if let Some(parent) = pdf_dest.parent() { if let Some(parent) = pdf_dest.parent() {
tokio::fs::create_dir_all(parent) tokio::fs::create_dir_all(parent)
.await .await
@ -104,6 +110,12 @@ pub async fn upload_paper_file_service(
.map_err(|e| anyhow::anyhow!("HTML 文件内容校验失败: {}", e))?; .map_err(|e| anyhow::anyhow!("HTML 文件内容校验失败: {}", e))?;
let html_filename = format!("{}.html", bibcode); let html_filename = format!("{}.html", bibcode);
let html_dest = library_dir.join("HTML").join(&html_filename); let html_dest = library_dir.join("HTML").join(&html_filename);
if html_dest
.components()
.any(|c| matches!(c, std::path::Component::ParentDir))
{
anyhow::bail!("非法的文件写入路径bibcode 包含路径穿越字符");
}
if let Some(parent) = html_dest.parent() { if let Some(parent) = html_dest.parent() {
tokio::fs::create_dir_all(parent) tokio::fs::create_dir_all(parent)
.await .await

View File

@ -85,10 +85,20 @@ pub async fn search_papers(
let mut seen_arxiv: HashSet<String> = HashSet::new(); let mut seen_arxiv: HashSet<String> = HashSet::new();
for r in results { for r in results {
let dup = (!r.bibcode.is_empty() && !seen_bibcodes.insert(r.bibcode.clone())) // 先用引用检查是否重复,仅非重复项才 clone 入 HashSet
|| (!r.doi.is_empty() && !seen_dois.insert(r.doi.clone())) let dup = (!r.bibcode.is_empty() && seen_bibcodes.contains(&r.bibcode))
|| (!r.arxiv_id.is_empty() && !seen_arxiv.insert(r.arxiv_id.clone())); || (!r.doi.is_empty() && seen_dois.contains(&r.doi))
|| (!r.arxiv_id.is_empty() && seen_arxiv.contains(&r.arxiv_id));
if !dup { if !dup {
if !r.bibcode.is_empty() {
seen_bibcodes.insert(r.bibcode.clone());
}
if !r.doi.is_empty() {
seen_dois.insert(r.doi.clone());
}
if !r.arxiv_id.is_empty() {
seen_arxiv.insert(r.arxiv_id.clone());
}
unique_results.push(r); unique_results.push(r);
} }
} }
@ -199,13 +209,15 @@ async fn batch_fetch_existing_papers(
/// 会被替换为空格,确保查询作为纯文本搜索而非布尔表达式执行。 /// 会被替换为空格,确保查询作为纯文本搜索而非布尔表达式执行。
pub fn sanitize_fts5_query(query: &str) -> String { pub fn sanitize_fts5_query(query: &str) -> String {
query query
.replace('"', "\"\"") .replace(['"', '\''], " ")
.replace('\'', "''")
.replace(['*', ':'], " ") .replace(['*', ':'], " ")
.replace(" AND ", " ") .replace(" AND ", " ")
.replace(" OR ", " ") .replace(" OR ", " ")
.replace(" NOT ", " ") .replace(" NOT ", " ")
.replace(" NEAR ", " ") .replace(" NEAR ", " ")
.split_whitespace()
.collect::<Vec<_>>()
.join(" ")
} }
/// 本地文献库 FTS5 全文检索。 /// 本地文献库 FTS5 全文检索。

View File

@ -74,23 +74,22 @@ impl Dictionary {
return Vec::new(); return Vec::new();
} }
// 基础分词清理:保留字母数字及连接符,其余视为空格以进行精确段落划分 // 基础分词清理 + 预计算 lowercase减少后续重复分配
let clean_text = text let clean_text: String = text
.chars() .chars()
.map(|c| { .map(|c| {
if c.is_alphanumeric() || c == '-' || c == '\'' || c == ' ' { if c.is_alphanumeric() || c == '-' || c == '\'' || c == ' ' {
c c.to_ascii_lowercase()
} else { } else {
' ' ' '
} }
}) })
.collect::<String>(); .collect();
let words: Vec<&str> = clean_text.split_whitespace().collect(); let words: Vec<&str> = clean_text.split_whitespace().collect();
let mut matched = HashSet::new(); let mut matched = HashSet::new();
let mut results = Vec::new(); let mut results = Vec::new();
// 天文学词条跨度最大限制(一般多词短语不超过 6 个英文单词)
let max_span = 6; let max_span = 6;
let n = words.len(); let n = words.len();
let mut next_valid_index = 0; let mut next_valid_index = 0;
@ -99,22 +98,19 @@ impl Dictionary {
if i < next_valid_index { if i < next_valid_index {
continue; continue;
} }
// 快速过滤:如果当前单词的 lowercase 不在 first_words 集合中,不可能匹配任何词条 // 已 lowercase 的首词直接查 first_words
let first_word_lower = words[i].to_lowercase(); if !self.first_words.contains(words[i]) {
if !self.first_words.contains(&first_word_lower) {
continue; continue;
} }
for len in (1..=max_span).rev() { for len in (1..=max_span).rev() {
if i + len <= n { if i + len <= n {
let phrase_slice = &words[i..i + len]; let phrase = words[i..i + len].join(" ");
let phrase = phrase_slice.join(" ").to_lowercase();
if self.terms.contains_key(&phrase) { if self.terms.contains_key(&phrase) {
// 避免重复匹配更长名词的子词 (如已匹配 'active galactic nucleus' 就不重复提取其中的 'nucleus')
if !matched.contains(&phrase) { if !matched.contains(&phrase) {
let chinese = self.terms.get(&phrase).unwrap().clone(); let chinese = self.terms.get(&phrase).unwrap().clone();
let original_phrase = phrase_slice.join(" "); let original_phrase = words[i..i + len].join(" ");
results.push((original_phrase, chinese)); results.push((original_phrase, chinese));
matched.insert(phrase); matched.insert(phrase);
@ -248,7 +244,7 @@ pub async fn translate_markdown(
info!( info!(
"正在请求大模型开展中英翻译。所选大模型: {}", "正在请求大模型开展中英翻译。所选大模型: {}",
llm_client.model() llm_client.model().await
); );
let start_time = std::time::Instant::now(); let start_time = std::time::Instant::now();
@ -260,7 +256,7 @@ pub async fn translate_markdown(
let duration = start_time.elapsed(); let duration = start_time.elapsed();
info!( info!(
"LLM 翻译成功。所选大模型: {}, 耗时: {:?}, 译文字符数: {}", "LLM 翻译成功。所选大模型: {}, 耗时: {:?}, 译文字符数: {}",
llm_client.model(), llm_client.model().await,
duration, duration,
translated.len() translated.len()
); );

View File

@ -30,7 +30,7 @@ pub async fn analyze_image_stream_service(
// 流式输出:先发送状态头部(作为第一条 text_delta保证持久化到工具结果中 // 流式输出:先发送状态头部(作为第一条 text_delta保证持久化到工具结果中
let header = format!( let header = format!(
"> 正在使用视觉模型({})分析图片: {}\n\n", "> 正在使用视觉模型({})分析图片: {}\n\n",
vision_llm.model(), vision_llm.model().await,
image_path image_path
); );
if let Some(tx) = sse_tx { if let Some(tx) = sse_tx {